I would like to count how many entreprise are in some category but I'm stuck with the asynchrone concept.
Here's what I already have:
Category.getall(function(err, cat){
if(err) return res.negotiate(err);
catIds = []
for( var iCat in cat){
catIds.push(cat[iCat].id)
// and here I would like do something like
Entreprise.count({category_id: cat[iCat].id}, function(err, nbr){
categoriesOUT.push({categorie: cat, entreprise_number: nbr })
// I know that i can not do it but it's just to help to understand the logic I would like to have.
if(cat.length==iCat){
return res.json({categories: categoriesOUT})
}
})
}
})
There are a couple of ways to handle this. One would be to bring in a promise library like Q. Another would be a single database call that can count up enterprise objects grouped by category_id... however, I think that would go beyond Waterline's normal queries, you would have to use .query or .native or something.
The easiest quick fix for you is to just keep a counter of how many results you have handled. You may get tired of this approach after using it a couple of times, but it would look something like this:
Category.getall(function(err, cat){
if(err) { return res.negotiate(err); }
var catIds = [], categoriesOut = [], processedCategories = 0;
for( var iCat in cat){
catIds.push(cat[iCat].id)
Entreprise.count({category_id: cat[iCat].id}, function(err, nbr) {
if (err) {
categoriesOUT.push({categorie: cat, entreprise_number: 0});
} else {
categoriesOUT.push({categorie: cat, entreprise_number: nbr });
}
processedCategories += 1;
if (processedCategories >= cat.length) {
return res.json({categories: categoriesOUT});
}
});
}
});
Here's how I finaly get it only with MySQL request as suggered by #arbuthnott
(The category field is call domaine here)
Domaine.getall(function(err, domaines){
if(err){return res.negotiate(err)}
var domNames = {}, domContain = {}, domOut = [];
Entreprise.query('SELECT domaine_id, COUNT(*) FROM entreprise GROUP BY domaine_id', function(err, entreprises){
if(err){return res.negotiate(err)}
entreprises = JSON.parse(JSON.stringify(entreprises));
for(var ent of entreprises){
domContain[ent['domaine_id']] = ent['COUNT(*)'];
}
for(var iDom in domaines){
var countAdded = false;
for(var dc in domContain){
if(dc==domaines[iDom].id) {
domaines[iDom].entreprises_count = domContain[dc];
countAdded = true;
}
}
if(!countAdded) domaines[iDom].entreprises_count = 0;
}
res.json({domaines:domaines})
})
})
Related
I'm using Node/Puppeteer in the code below, passing in a large list of URL's for traversal and scraping. It has been difficult to do it asynchronously, though I find that I am getting closer and closer to the answer. I am currently stuck on an issue related to the following error.
UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 17): Error: Protocol error (Target.createTarget): Target closed.
This error occurs once upon every iteration of the while loop. Though I'm not sure what I may be doing incorrectly.
Could someone help me do the following:
1) Diagnose the source of the error.
2) Potentially find a more effective way to traverse a large list of URLs asynchronously.
async function subProc(list, batchSize) {
let subList = null;
let i = 0;
while (list.length > 0) {
let browser = await puppeteer.launch();
subList = list.splice(0, batchSize);
console.log("Master List Size :: " + list.length);
console.log("SubList Size :: " + subList.length);
for (let j = 0; j < subList.length; j++) {
promiseArray.push(new Promise((resolve, reject) => {
resolve(pageScrape(subList[j], browser));
}));
}
Promise.all(promiseArray)
.then(response => {
procArray.concat(response);
});
promiseArray = new Array();
try {
await browser.close();
} catch(ex){
console.log(ex);
}
};
}
async function pageScrape(url, browser) {
let page = await browser.newPage();
await page.goto(url, {
timeout: 0
});
await page.waitFor(1000);
return await page.evaluate(() => {
let appTitle = document.querySelector('').innerText;
let companyName = document.querySelector('').innerText;
let dateListed = document.evaluate("", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.innerText;
let category = document.evaluate("']//a//strong", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.innerText;
/* */
return {
appTitle,
companyName,
dateListed,
category
}
}).then(response => {
let urlData = {
id: subList[j],
appName: response.appTitle,
companyName: response.companyName,
dateListed: response.dateListed,
category: response.category
}
return urlData;
});
};
I figured out the solution to the problem I was having.
Every computer is limited in its processing ability, so instead of iterating through 1000 urls simultaneously you have to break it down into smaller pieces.
By using a PromiseAll, and iterating and scraping 10 urls at a time and storing these values in an array, I was able to throttle the processing required to iterate through all 1000 urls.
processBatch(subData, 10, procArray).then((processed)=>{
for(let i = 0; i < procArray.length; i++){
for(let j = 0; j < procArray[i].length; j++){
results.push(procArray[i][j]);
}
}
function processBatch(masterList, batchSize, procArray){
return Promise.all(masterList.splice(0, batchSize).map(async url =>
{
return singleScrape(url)
})).then((results) => {
if (masterList.length < batchSize) {
console.log('done');
procArray.push(results);
return procArray;
} else {
console.log('MasterList Size :: ' + masterList.length);
procArray.push(results);
return processBatch(masterList, batchSize, procArray);
}
})
}
I'm having quite a bit of trouble with the scope of the Meteor.call procedure. It won't set my scope variable to the result.length
'takeaways': function (userId) {
var len = 0;
Meteor.call('userTakeaways', userId, function (error, result) {
if (error) {
console.log('there was an error finding the number of messages that were takeaways')
} else {
len = result.length; // result.length is 2
}
});
console.log(len); // still 0
return len;
}
Please help!
Thank you :)
len is not a reactive variable. So if the len value changes, it won't update the spacebar value.
So here two approach to solving this problem:
1. using reactive var/session.
//Make sure you have install reactive var package
var len = new ReactiveVar(0);
Template['name'].helpers({
'takeaways': function (userId) {
Meteor.call('userTakeaways', userId, function (error, result) {
if (error) {
console.log('there was an error finding the number of messages that were takeaways')
} else {
len.set(result.length); // result.length is 2
}
});
console.log(len.get()); // You will get 2 when response come from you method call.
return len.get();
}
});
2. Using 'simple:reactive-method' package
takeaways : function(userId){
return ReactiveMethod.call('userTakeaways', userId).length;
}
try adding in the else statement
return len = result.length;
as you can see bellow.
'takeaways': function (userId) {
var len = 0;
Meteor.call('userTakeaways', userId, function (error, result) {
if (error) {
console.log('there was an error finding the number of messages that were takeaways')
} else {
return len = result.length; // result.length is 2
}
});
console.log(len); // still 0
return len;
}
I’m having issues getting two dependant types of data from a PouchDB database.
I have a list of cars that I get like so:
localDB.query(function(doc) {
if (doc.type === ‘list’) {
emit(doc);
}
}, {include_docs : true}).then(function(response) {
console.log(“cars”, response);
// Save Cars List to app
for(var i = 0; i < response.rows.length; i++) {
addToCarsList(response.rows[i].id, response.rows[i].carNumber);
}
console.log(“Cars List: " + carsListToString());
return response;
}).then(function(listRecord) {
listRecord.rows.forEach(function(element, index){
console.log(index + ' -> ', element);
localDB.query(function(doc) {
console.log("filtering with carNb = " + element.carNb);
if (doc.type === 'defect' && doc.listId == getCurrentListId() && doc.carNb == element.carNb ) {
emit(doc);
}
}, {include_docs : false}).then(function(result){
console.log("defects", result);
}).catch(function(err){
console.log("an error has occurred", err);
});
});
}).catch(function(err) {
console.log('error', err);
});
Here's what happens. After getting the list of cars, then for each cars I would like to query the defects and store then in some arrays. Then when all that querying is done, I want to build the UI with the data saved.
But what's happening is that the forEach gets processed quickly and does not wait for the inner async'd localDb.query.
How can I query some documents based on an attribute from a parent query? I looked into promises in the PouchDB doc but I can't understand how to do it.
(please forget about curly quotes and possible lint errors, this code was anonymized by hand and ultra simplified)
The method you are looking for is Promise.all() (execute all promises and return when done).
However, your query is already pretty inefficient. It would be better to create a persistent index, otherwise it has to do a full database scan for every query() (!). You can read up on the PouchDB query guide for details.
I would recommend installing the pouchdb-upsert plugin and then doing:
// helper method
function createDesignDoc(name, mapFunction) {
var ddoc = {
_id: '_design/' + name,
views: {}
};
ddoc.views[name] = { map: mapFunction.toString() };
return ddoc;
}
localDB.putIfNotExists(createDesignDoc('my_index', function (doc) {
emit([doc.type, doc.listId, doc.carNb]);
})).then(function () {
// find all docs with type 'list'
return localDB.query('my_index', {
startkey: ['list'],
endkey: ['list', {}],
include_docs: true
});
}).then(function (response) {
console.log("cars", response);
// Save Cars List to app
for(var i = 0; i < response.rows.length; i++) {
addToCarsList(response.rows[i].id, response.rows[i].carNumber);
}
console.log("Cars List: " + carsListToString());
return response;
}).then(function (listRecord) {
return PouchDB.utils.Promise.all(listRecord.rows.map(function (row) {
// find all docs with the given type, listId, carNb
return localDB.query('my_index', {
key: ['defect', getCurrentListId(), row.doc.carNb],
include_docs: true
});
}));
}).then(function (finalResults) {
console.log(finalResults);
}).catch(function(err){
console.log("an error has occurred", err);
});
I'm using a few tricks here:
emit [doc.type, doc.listId, doc.carNb], which allows us to query by type or by type+listId+carNb.
when querying for just the type, we can do {startkey: ['list'], endkey: ['list', {}]}, which matches just those with the type "list" because {} is the "higher" than strings in CouchDB object collation order.
PouchDB.utils.Promise is a "hidden" API, but it's pretty safe to use if you ask me. It's unlikely we'll change it.
Edit Another option is to use the new pouchdb-find plugin, which offers a simplified query API designed to replace the existing map/reduce query() API.
Another approach would be to pull both the list docs and the defect docs down at the same time then merge them together using a reduce like method that will convert them into an array of objects:
{
_id: 1,
type: 'list',
...
defects: [{
type: 'defect'
listId: 1
...
}]
}
By pulling the list and the defects down in one call you save a several calls to the pouchdb query engine, but you do have to iterate through every result to build your collection of lists objects with and embedded array of defects.
// This is untested code so it may not work, but you should get the idea
var _ = require('underscore');
// order documents results by list then defect
var view = function (doc) {
if (doc.type === 'list') {
emit([doc._id, doc.carNumber, 1);
} else if (doc.type === 'defect') {
emit([doc.listId, doc.carNb, 2])
}
}
localDB.query(view, { include_docs: true })
.then(function(response) {
return _(response.rows)
.reduce(function(m, r) {
if (r.key[2] === 1) {
// initialize
r.doc.defects = [];
m.push(r.doc)
return m;
}
if (r.key[2] === 2) {
var list = _(m).last()
if (list._id === r.key[0] && list.carNumber === r.key[1]) {
list.defects.push(r.doc);
}
return m;
}
}, []);
})
.then(function(lists) {
// bind to UI
});
With couch, we found reducing calls to the couch engine to be more performant, but I don't know if this approach is better for PouchDB, but this should work as a solution, especially if you are wanting to embed several collections into one list document.
Consider this hypothetical example. Html:
{{#each items}}
{{>item}}
{{/each}}
Price = {{itemTotals.totalPrice}}
Qty = {{itemTotals.totalQty}}
Js:
Template.cart.items = function () {
var items = Items.find();
var cartTotals = [];
items.forEach(function(item){
..somecode...
cartTotals.price += item.price;
cartTotals.qty += 1;
});
return items;
}
And of course, we want a function "itemTotals". If we do price = {{items.itemTotals.totalPrice}}, and same for qty, we have to traverse items 3 times. I've tried to store the totals as a reactive variable as follows, but haven't been able to make it work:
Template.cart.rendered = function () {
this.totals = new ReactiveVar([]);
}
Template.cart.items = function () {
var items = Items.find();
var cartTotals = [];
items.forEach(function(item){
...
});
this.totals.set(cartTotals);
return items;
}
Template.cart.totals = function () {
return this.totals.get();
}
I'd like to 1) understand why the reactivity is not happening 2) find out if there is a way to have a template function wait on another - that would be more efficient than loading and reloading the values from a reactive variable
The other option suggested by Walter Zalazar would work, however, using Session forces us to store each value(price, quantity, etc) as a separate variable due to limitations of Session. After some trial and error, I've figured out the following way to store the entire totals array as one reactive variable:
totals = new ReactiveVar([]);
Template.cart.items = function () {
//foreach loop as before, calculate cartTotals, etc
totals.set(cartTotals);
return items;
}
Template.cart.totals = function () {
return totals.get();
}
and in html:
{{totals.totalPrice}}
{{totals.qty}}
you can to try something like this
Your .html
<template name="itemsList">
{{#each items}}
{{>item}}
{{/each}}
Price = {{totalPrice}}
Qty = {{totalQty}}
</template>
Your .js
Template.itemsList.helpers({
totalPrice: function(){
var price= 0;
var items= Items.find({});
items.forEach(function(item){
price+=item.price;
});
return price;
},
totalQty: function(){
return Items.find({}).count();
},
items:function(){
return Items.find({});
}
})
Just works!
Other option...
Tracker.autorun(function () {
Session.set("totalPrice");
Session.set("totalQty");
})
Template.itemsList.helpers({
totalPrice: function(){
return Session.get("totalPrice");
},
totalQty: function(){
return Session.get("totalQty");
},
items:function(){
var price= 0;
var items= Items.find({});
var count= items.count();
items.forEach(function(item){
price+= Number(item.price);
});
Session.set("totalPrice",price);
Session.set("totalQty",count);
return items;
}
});
I'm a bit new to Meteor and something I'm having trouble with is reactive data -- particularly in instances where I need to change the data shown based on a mouse or keyboard event. Doing this kind of stuff the normal js way seems to give me trouble in meteor since everything I change gets re-rendered and reset constantly.
So, I thought I'd see if this would be a case in which I could use Meteor's Deps object, however I can't quite grasp it. Here's the code I'm using:
(function(){
var tenants = [];
var selectedTenant = 0;
var tenantsDep = new Deps.Dependency;
Template.tenantsBlock.tenantsList = function()
{
tenants = [];
var property = $properties.findOne({userId: Meteor.userId(), propertyId: Session.get('property')});
var tenancies = _Utils.resolveTenancies(property, true, null, true);
for(var i = 0; i < tenancies.length; i++)
{
if(tenancies[i].tenancyId == Session.get('tenancy'))
{
tenants = tenants.concat(tenancies[i].otherTenants, tenancies[i].primaryTenant);
}
}
tenants[selectedTenant].selected = 'Selected';
tenantsDep.changed();
return tenants;
};
Template.tenantsBlock.onlyOneTenant = function()
{
tenantsDep.depend();
return tenants.length > 1 ? '' : 'OneChild';
};
Template.tenantsBlock.phoneNumber = function()
{
tenantsDep.depend();
for(var i = 0; i < tenants[selectedTenant].details.length; i++)
if(_Utils.getDynamicContactIconClass(tenants[selectedTenant].details[i].key) == 'Phone')
return tenants[selectedTenant].details[i].value;
return null;
};
Template.tenantsBlock.emailAddress = function()
{
tenantsDep.depend();
for(var i = 0; i < tenants[selectedTenant].details.length; i++)
if(_Utils.getDynamicContactIconClass(tenants[selectedTenant].details[i].key) == 'Email')
return tenants[selectedTenant].details[i].value;
return null;
};
Template.tenantsBlock.addedDate = function()
{
tenantsDep.depend();
return _Utils.timeToDateString(tenants[selectedTenant].created);
};
Template.tenantsBlock.events({
'click .Name': function(e, template)
{
tenantsDep.depend();
var _this = e.currentTarget;
var tenantName = _this.innerHTML;
$(_this).addClass('Selected');
$(_this).siblings().removeClass('Selected');
for(var i = 0; i < tenants.length; i++)
{
if(tenants[i].name == tenantName)
tenants[i].selected = "Selected";
else
tenants[i].selected = '';
}
}
})
})();
^This seemed to be what they were getting at in the meteor documentation (http://docs.meteor.com/#deps_dependency) for dependency.changed() and dependency.depend(), but all this does is give me an infinite loop.
So can I modify the way I declare deps to get this to make data reactive? Is there a better way to do this all together?
UPDATE:
Although I was skeptical to do so, I've been inclined to try to use Session.set/Session.get in a localized way. So, the next time I have to do this, I'll just do
Session.set('tenantsBlock' {tenants: [], selectedTenant: 0});
and then just access this variable from within helpers and event maps related to Template.tenantsBlock. That way they all have real time access to the data and they all get re-run when the data changes. Here's what I converted this script into (sorry these are both so large):
(function()
{
Template.tenantsBlock.created = Template.tenantsBlock.destroyed =function()
{
_Utils.setSession('tenantsBlock', {
tenants: [],
selectedTenant: 0
})
};
Template.tenantsBlock.tenantsList = function()
{
var localContext = Session.get('tenantsBlock');
localContext.tenants = [];
var property = $properties.findOne({userId: Meteor.userId(), propertyId: Session.get('property')});
var tenancies = _Utils.resolveTenancies(property, true, null, true);
for(var i = 0; i < tenancies.length; i++)
{
if(tenancies[i].tenancyId == Session.get('tenancy'))
{
localContext.tenants = localContext.tenants.concat(tenancies[i].otherTenants, tenancies[i].primaryTenant);
break;
}
}
localContext.tenants[localContext.selectedTenant].selected = 'Selected';
Session.set('tenantsBlock', localContext);
return localContext.tenants;
};
Template.tenantsBlock.onlyOneTenant = function()
{
var localContext = Session.get('tenantsBlock');
return localContext.tenants.length > 1 ? '' : 'OneChild';
};
Template.tenantsBlock.phoneNumber = function()
{
var localContext = Session.get('tenantsBlock');
for(var i = 0; i < localContext.tenants[localContext.selectedTenant].details.length; i++)
if(_Utils.getDynamicContactIconClass(localContext.tenants[localContext.selectedTenant].details[i].key) == 'Phone')
return localContext.tenants[localContext.selectedTenant].details[i].value;
return null;
};
Template.tenantsBlock.emailAddress = function()
{
var localContext = Session.get('tenantsBlock');
var selectedTenantDetails = localContext.tenants[localContext.selectedTenant].details;
for(var i = 0; i < selectedTenantDetails.length; i++)
if(_Utils.getDynamicContactIconClass(selectedTenantDetails[i].key) == 'Mail')
return selectedTenantDetails[i].value;
return null;
};
Template.tenantsBlock.addedDate = function()
{
var localContext = Session.get('tenantsBlock');
return _Utils.timeToDateString(localContext.tenants[localContext.selectedTenant].created);
};
Template.tenantsBlock.events({
'click .Name': function(e, template)
{
var localContext = Session.get('tenantsBlock');
var _this = e.currentTarget;
var tenantName = _this.innerHTML;
for(var i = 0; i < localContext.tenants.length; i++)
{
if(localContext.tenants[i].name == tenantName)
{
localContext.tenants[i].selected = 'Selected';
localContext.selectedTenant = i;
}
else
{
localContext.tenants[i].selected = '';
}
}
Session.set('tenantsBlock', localContext);
}
})
})();
You'll have to overcome the old-school way of doing it :) Meteor is a lot simpler than you think. A good rule of thumb is that if you're using jQuery to manipulate any DOM elements, you're probably doing it wrong. Additionally, if you're accessing any data without using the collection API, you'd better have good reason to do so.
In your case, you don't need to code up any manual dependencies at all. Manual dependencies are rarely needed in most Meteor applications.
The first thing you need to do is put all your tenants inside a Meteor.Collection, which will make them easier to work with.
Tenants = new Meteor.Collection("tenants");
Your tenantsBlock template should look something like this (modulo some different html elements):
<template name="tenantsBlock">
<ol>
{{#each tenants}}
<li class="name {{selected}}">
<span>Primary Tenant: {{primaryTenant}}</span>
<span>Other Tenants: {{otherTenants}}</span>
<span>Phone Number: {{phoneNumber}}</span>
<span>Email Address: {{emailAddress}}</span>
<span>Added Date: {{addedDate}}</span>
</li>
{{/each}}
</ol>
</template>
Each document in Tenants should look something like the following:
{
primaryTenant: "Joe Blow",
otherTenants: "Mickey Mouse, Minnie Mouse",
phoneNumber: "555-234-5623",
emailAddress: "joe.blow#foo.com",
addedDate: "2005-10-30T10:45Z"
}
Then, all the code you would need is just for the selection/deselection, and you can delete everything else:
Template.tenantsBlock.tenants = function() {
return Tenants.find();
};
Template.tenantsBlock.selected = function() {
return Session.equals("selectedTenant", this._id);
};
Template.tenantsBlock.events({
'click .name': function(e) {
Session.set("selectedTenant", this._id);
}
});
Once again, I reiterate that you should never be doing DOM manipulations with Javascript when using Meteor. You just update your data and your templates will reactively update if everything is done correctly. Declare how you want your data to look, then change the data and watch the magic.
Meteor has really evolved since I posted this back in 2013. I thought
I should post a modern, superior method.
For a while now you've been able to create a ReactiveVar and now you can append those directly to templates. A ReactiveVar, similar to Session, is a reactive data store. ReactiveVar, however, holds only a single value (of any type).
You can add ReactiveVar to the client side of your project by running this in your terminal from your app's root directory:
$meteor add reactive-var
This javascript shows how you can pass the variable between the template's onCreated, onRendered, onDestroyed, events and helpers.
Template.myTemplate.onCreated = function() {
// Appends a reactive variable to the template instance
this.reactiveData = new ReactiveVar('Default Value');
};
Template.myTemplate.events({
'click .someButton': (e, template) => {
// Changes the value of the reactive variable for only this template instance
template.reactiveData.set('New Value');
},
});
Template.myTemplate.helpers({
theData: () => {
// Automatically updates view when reactive variable changes
return Template.instance().reactiveData.get();
},
});
This is superior for a few reasons:
It scopes the variable only to a single template instance. Particularly useful in cases where you might have a dozen instances of a template on a page, all requiring independent states.
It goes away when the template goes away. Using ReactiveVar or Session variables you will have to clear the variable when the template is destroyed (if it is even destroyed predictably).
It's just cleaner code.
Bonus Points: See ReactiveDict for cases in which you have many instances of a template on a page at once, but need to manage a handful of reactive variables and have those variables persist during the session.