I need to use bulk operations with aggregate in order to delete duplicates with a certain condition in my database. I tried to use rawCollection() but I don't really know how.
Here's the code I need to execute with cron every x hours
function removeDups() {
var count = 0,
collection = Beatmaps.rawCollection(),
bulk = collection.initializeUnorderedBulkOp();
collection.aggregate([
{ '$sort': { 'difficultyrating': -1 }},
{ '$group': { '_id': '$beatmapset_id', 'ids': { '$push': '$_id' }, 'count': { '$sum': 1 }}},
{ '$match': { 'count': { '$gt': 1 }}}
]).forEach(function(doc) {
doc.ids.shift();
bulk.find({'_id': { '$in': doc.ids }}).remove();
count++;
if(count === 100) {
bulk.execute();
bulk = collection.initializeUnorderedBulkOp();
}
});
if(count !== 0) {
bulk.execute();
}
}
but it produces an error: Cannot call method 'forEach' of undefined
So what should I do?
Okay, after a bit of research I found similar question and here's what I did to make this work:
var aggregate = Meteor.wrapAsync(collection.aggregate, collection);
and then
aggregate(parameters).forEach(...);
Related
I would like to count how many entreprise are in some category but I'm stuck with the asynchrone concept.
Here's what I already have:
Category.getall(function(err, cat){
if(err) return res.negotiate(err);
catIds = []
for( var iCat in cat){
catIds.push(cat[iCat].id)
// and here I would like do something like
Entreprise.count({category_id: cat[iCat].id}, function(err, nbr){
categoriesOUT.push({categorie: cat, entreprise_number: nbr })
// I know that i can not do it but it's just to help to understand the logic I would like to have.
if(cat.length==iCat){
return res.json({categories: categoriesOUT})
}
})
}
})
There are a couple of ways to handle this. One would be to bring in a promise library like Q. Another would be a single database call that can count up enterprise objects grouped by category_id... however, I think that would go beyond Waterline's normal queries, you would have to use .query or .native or something.
The easiest quick fix for you is to just keep a counter of how many results you have handled. You may get tired of this approach after using it a couple of times, but it would look something like this:
Category.getall(function(err, cat){
if(err) { return res.negotiate(err); }
var catIds = [], categoriesOut = [], processedCategories = 0;
for( var iCat in cat){
catIds.push(cat[iCat].id)
Entreprise.count({category_id: cat[iCat].id}, function(err, nbr) {
if (err) {
categoriesOUT.push({categorie: cat, entreprise_number: 0});
} else {
categoriesOUT.push({categorie: cat, entreprise_number: nbr });
}
processedCategories += 1;
if (processedCategories >= cat.length) {
return res.json({categories: categoriesOUT});
}
});
}
});
Here's how I finaly get it only with MySQL request as suggered by #arbuthnott
(The category field is call domaine here)
Domaine.getall(function(err, domaines){
if(err){return res.negotiate(err)}
var domNames = {}, domContain = {}, domOut = [];
Entreprise.query('SELECT domaine_id, COUNT(*) FROM entreprise GROUP BY domaine_id', function(err, entreprises){
if(err){return res.negotiate(err)}
entreprises = JSON.parse(JSON.stringify(entreprises));
for(var ent of entreprises){
domContain[ent['domaine_id']] = ent['COUNT(*)'];
}
for(var iDom in domaines){
var countAdded = false;
for(var dc in domContain){
if(dc==domaines[iDom].id) {
domaines[iDom].entreprises_count = domContain[dc];
countAdded = true;
}
}
if(!countAdded) domaines[iDom].entreprises_count = 0;
}
res.json({domaines:domaines})
})
})
I'm having quite a bit of trouble with the scope of the Meteor.call procedure. It won't set my scope variable to the result.length
'takeaways': function (userId) {
var len = 0;
Meteor.call('userTakeaways', userId, function (error, result) {
if (error) {
console.log('there was an error finding the number of messages that were takeaways')
} else {
len = result.length; // result.length is 2
}
});
console.log(len); // still 0
return len;
}
Please help!
Thank you :)
len is not a reactive variable. So if the len value changes, it won't update the spacebar value.
So here two approach to solving this problem:
1. using reactive var/session.
//Make sure you have install reactive var package
var len = new ReactiveVar(0);
Template['name'].helpers({
'takeaways': function (userId) {
Meteor.call('userTakeaways', userId, function (error, result) {
if (error) {
console.log('there was an error finding the number of messages that were takeaways')
} else {
len.set(result.length); // result.length is 2
}
});
console.log(len.get()); // You will get 2 when response come from you method call.
return len.get();
}
});
2. Using 'simple:reactive-method' package
takeaways : function(userId){
return ReactiveMethod.call('userTakeaways', userId).length;
}
try adding in the else statement
return len = result.length;
as you can see bellow.
'takeaways': function (userId) {
var len = 0;
Meteor.call('userTakeaways', userId, function (error, result) {
if (error) {
console.log('there was an error finding the number of messages that were takeaways')
} else {
return len = result.length; // result.length is 2
}
});
console.log(len); // still 0
return len;
}
I'm starting with Meteor, and want to organize my methods...
By example, I have 2 collecations: 'a' and 'b', both have the insert method.. I want to do something like that:
Meteor.methods({
a: {
insert : function(){
console.log("insert in Collection a");
}
},
b: {
insert : function(){
console.log("insert in Collection b");
}
}
});
And then call
Meteor.call('a.insert');
It's possible to do this? Or how can I organize my methods?
I don't wanna make methods like: 'insertA' and 'insertB'
You could use this syntax :
Meteor.methods({
"a.insert": function(){
console.log("insert in Collection a");
}
"b.insert": function(){
console.log("insert in Collection b");
}
});
Which allows you to do Meteor.call("a.insert");.
Building on saimeunt's idea, you could also add a helper function to your code if you are concerned with the elegance of these groups in your code. Then you can use the notation you liked and even nest groups arbitrarily:
var methods = {
a: {
insert : function(){
console.log("insert in Collection a");
}
},
b: {
insert : function(){
console.log("insert in Collection b");
},
b2: {
other2: function() {
console.log("other2");
},
other3: function() {
console.log("other3");
}
}
},
};
function flatten(x, prefix, agg) {
if (typeof(x) == "function") {
agg[prefix] = x;
} else {
// x is a (sub-)group
_.each(x, function(sub, name) {
flatten(sub, prefix + (prefix.length > 0 ? "." : "") + name, agg);
});
}
return agg;
}
Meteor.methods(flatten(methods, "", {}));
You would then call with the dot notation, like:
Meteor.call('b.b2.other2');
I’m having issues getting two dependant types of data from a PouchDB database.
I have a list of cars that I get like so:
localDB.query(function(doc) {
if (doc.type === ‘list’) {
emit(doc);
}
}, {include_docs : true}).then(function(response) {
console.log(“cars”, response);
// Save Cars List to app
for(var i = 0; i < response.rows.length; i++) {
addToCarsList(response.rows[i].id, response.rows[i].carNumber);
}
console.log(“Cars List: " + carsListToString());
return response;
}).then(function(listRecord) {
listRecord.rows.forEach(function(element, index){
console.log(index + ' -> ', element);
localDB.query(function(doc) {
console.log("filtering with carNb = " + element.carNb);
if (doc.type === 'defect' && doc.listId == getCurrentListId() && doc.carNb == element.carNb ) {
emit(doc);
}
}, {include_docs : false}).then(function(result){
console.log("defects", result);
}).catch(function(err){
console.log("an error has occurred", err);
});
});
}).catch(function(err) {
console.log('error', err);
});
Here's what happens. After getting the list of cars, then for each cars I would like to query the defects and store then in some arrays. Then when all that querying is done, I want to build the UI with the data saved.
But what's happening is that the forEach gets processed quickly and does not wait for the inner async'd localDb.query.
How can I query some documents based on an attribute from a parent query? I looked into promises in the PouchDB doc but I can't understand how to do it.
(please forget about curly quotes and possible lint errors, this code was anonymized by hand and ultra simplified)
The method you are looking for is Promise.all() (execute all promises and return when done).
However, your query is already pretty inefficient. It would be better to create a persistent index, otherwise it has to do a full database scan for every query() (!). You can read up on the PouchDB query guide for details.
I would recommend installing the pouchdb-upsert plugin and then doing:
// helper method
function createDesignDoc(name, mapFunction) {
var ddoc = {
_id: '_design/' + name,
views: {}
};
ddoc.views[name] = { map: mapFunction.toString() };
return ddoc;
}
localDB.putIfNotExists(createDesignDoc('my_index', function (doc) {
emit([doc.type, doc.listId, doc.carNb]);
})).then(function () {
// find all docs with type 'list'
return localDB.query('my_index', {
startkey: ['list'],
endkey: ['list', {}],
include_docs: true
});
}).then(function (response) {
console.log("cars", response);
// Save Cars List to app
for(var i = 0; i < response.rows.length; i++) {
addToCarsList(response.rows[i].id, response.rows[i].carNumber);
}
console.log("Cars List: " + carsListToString());
return response;
}).then(function (listRecord) {
return PouchDB.utils.Promise.all(listRecord.rows.map(function (row) {
// find all docs with the given type, listId, carNb
return localDB.query('my_index', {
key: ['defect', getCurrentListId(), row.doc.carNb],
include_docs: true
});
}));
}).then(function (finalResults) {
console.log(finalResults);
}).catch(function(err){
console.log("an error has occurred", err);
});
I'm using a few tricks here:
emit [doc.type, doc.listId, doc.carNb], which allows us to query by type or by type+listId+carNb.
when querying for just the type, we can do {startkey: ['list'], endkey: ['list', {}]}, which matches just those with the type "list" because {} is the "higher" than strings in CouchDB object collation order.
PouchDB.utils.Promise is a "hidden" API, but it's pretty safe to use if you ask me. It's unlikely we'll change it.
Edit Another option is to use the new pouchdb-find plugin, which offers a simplified query API designed to replace the existing map/reduce query() API.
Another approach would be to pull both the list docs and the defect docs down at the same time then merge them together using a reduce like method that will convert them into an array of objects:
{
_id: 1,
type: 'list',
...
defects: [{
type: 'defect'
listId: 1
...
}]
}
By pulling the list and the defects down in one call you save a several calls to the pouchdb query engine, but you do have to iterate through every result to build your collection of lists objects with and embedded array of defects.
// This is untested code so it may not work, but you should get the idea
var _ = require('underscore');
// order documents results by list then defect
var view = function (doc) {
if (doc.type === 'list') {
emit([doc._id, doc.carNumber, 1);
} else if (doc.type === 'defect') {
emit([doc.listId, doc.carNb, 2])
}
}
localDB.query(view, { include_docs: true })
.then(function(response) {
return _(response.rows)
.reduce(function(m, r) {
if (r.key[2] === 1) {
// initialize
r.doc.defects = [];
m.push(r.doc)
return m;
}
if (r.key[2] === 2) {
var list = _(m).last()
if (list._id === r.key[0] && list.carNumber === r.key[1]) {
list.defects.push(r.doc);
}
return m;
}
}, []);
})
.then(function(lists) {
// bind to UI
});
With couch, we found reducing calls to the couch engine to be more performant, but I don't know if this approach is better for PouchDB, but this should work as a solution, especially if you are wanting to embed several collections into one list document.
I just found the Firebase API and really like it. However, I was looking at the Leader board sample and was wondering if add a new column:
userScoreRef.setWithPriority({ name:name, score:newScore, board:myboard }, newScore);
can I use this to separate the return to add it to different boards? Something like:
if (prevScoreName === null) {
if (myBoaard == 'Board1') { $("#leaderboardTable1").append(newScoreRow); }
if (myBoaard == 'Board2') { $("#leaderboardTable2").append(newScoreRow); }
if (myBoaard == 'Board3') { $("#leaderboardTable3").append(newScoreRow); }
if (myBoaard == 'Board4') { $("#leaderboardTable4").append(newScoreRow); }
if (myBoaard == 'Board5') { $("#leaderboardTable5").append(newScoreRow); }
}
else {
var lowerScoreRow = htmlForPath[prevScoreName];
lowerScoreRow.before(newScoreRow);
}
Or is there a better way to do this without rewriting the entire code 5 times?
Thanks
You should get the number of the current board and use that to add the score to different boards, like this:
if (prevScore === null) {
var leaderBoardTable = "leaderboardTable" + myBoard.charAt(myBoard.length-1);
$("#" + leaderBoardTable).append(newScoreRow);
}