Correct way to fan out data in Firebase Queue - firebase

I want to fan out data to 4 different nodes.
Since firebase transaction process can be failed, what is the right approach to ensure that all 4 data will be saved?
Suppose that I have 4 functions for saving data, each will call a transaction.
Edit:
function saveToFirstNode(data, response) {
/*
* #param
* [response] is a callback function
* that indicates whether the save was successful
*/
firebase.database().ref('cars/' + data.carId)
.transaction(function(currentData) {
currentData.distance += data.distance;
return currentData;
}, function(error, commited, snapshot) {
if (error) {
response(false);
} else if (commited) {
response(true)
}
}, false);
}
function saveToSecondNode(data, response) {...}
function saveToThirdNode(data, response) {...}
function saveToFourthNode(data, response) {...}
1. Using the first approach
This approach uses nested callback function and calls progress() for every success in saving.
The downside is, if one of the saving process fails, it will re-save all data from the first to fourth.
var queue = new Queue(ref, options, function(data, progress, resolve, reject) {
saveToFirstNode(data, function(success) {
if (success) {
progress(25);
saveToSecondNode(data, function(success) {
if (success) {
progress(50);
saveToThirdNode(data, function(success) {
if (success) {
progress(75);
saveToFourthNode(data, function(success) {
if(success) {
resolve(data);
} else {
reject();
}
});
} else {
reject();
}
});
} else {
reject();
}
});
} else {
reject();
}
});
});
2. Using the second approach
This approach guarantee all data will be saved. When the saving process fails, it will retry from the failed specs, not from the first to fourth. But do you think it is an overkill just for doing 4 save? Is this the right approach?
Define specs in queue/specs:
{
"save_to_first_node": {
"in_progress_state": "save_to_first_node_in_progress",
"finished_state": "save_to_first_node_finished"
},
"save_to_second_node": {
"start_state": "save_to_first_node_finished",
"in_progress_state": "save_to_second_node_in_progress",
"finished_state": "save_to_second_node_finished"
},
"save_to_third_node": {
"start_state": "save_to_second_node_finished",
"in_progress_state": "save_to_third_node_in_progress",
"finished_state": "save_to_third_node_finished"
},
"save_to_fourth_node": {
"start_state": "save_to_third_node_finished",
"in_progress_state": "save_to_fourth_node_in_progress"
}
}
Queue code:
var saveToFirstNodeOptions = {'specId': 'save_to_first_node'};
var saveToFirstNodeQueue = new Queue(ref, saveToFirstNodeOptions, function(data, progress, resolve, reject) {
saveToFirstNode(data, function(success) {
if (success) resolve(data);
else reject();
});
});
var saveToSecondNodeOptions = {'specId': 'save_to_second_node'};
var saveToSecondNodeQueue = new Queue(ref, saveToSecondNodeOptions, function(data, progress, resolve, reject) {
saveToSecondNode(data, function(success) {
if (success) resolve(data);
else reject();
});
});
var saveToThirdNodeOptions = {'specId': 'save_to_third_node'};
var saveToThirdNodeQueue = new Queue(ref, saveToThirdNodeOptions, function(data, progress, resolve, reject) {
saveToThirdNode(data, function(success) {
if (success) resolve(data);
else reject();
});
});
var saveToFourthNodeOptions = {'specId': 'save_to_fourth_node'};
var saveToFourthNodeQueue = new Queue(ref, saveToFourthNodeOptions, function(data, progress, resolve, reject) {
saveToFourthNode(data, function(success) {
if (success) resolve(data);
else reject();
});
});
Which is the right approach? This question is not preference, as those two are completely difference approach and make huge impact on both performance and effectiveness.

Related

Using Meteor.wrapAsync to wrap a callback inside a method

This Meteor code is giving the error:
Error: Meteor code must always run within a Fiber. Try wrapping callbacks that you pass to non-Meteor libraries with Meteor.bindEnvironment.
I tried Meteor.bindEnvironment for no avail and want to try Meteor.wrapAsync. I could not figure it out from the docs. Could some one please help me with the syntax? thx
Meteor.methods({
'createTransaction':
function (nonceFromTheClient, Payment) {
let user = Meteor.user();
gateway.transaction.sale(
{
arg_object
},
function (err, success) {
if (!err) {
//do stuff here
}
}
);
}
});
Wrap in Meteor.wrapAsync
Meteor.methods({
'createTransaction':
function (nonceFromTheClient, Payment) {
this.unblock();
let user = Meteor.user();
var sale = Meteor.wrapAsync(gateway.transaction.sale);
var res = sale({arg_object});
future.return(res);
return future.wait();
}
});
Or try wrapping it in Fiber
var Fiber = Npm.require('fibers');
Meteor.methods({
'createTransaction': function (nonceFromTheClient, Payment) {
Fiber(function() {
let user = Meteor.user();
gateway.transaction.sale(
{
arg_object
},
function (err, success) {
if (!err) {
//do stuff here
}
}
);
}).run()
}
});
Update: Here's how I handle stripe with Async.runSync and Meteor.bindEnvironment
var stripe = require("stripe")(Meteor.settings.private.StripeKeys.secretKey);
Meteor.methods({
'stripeToken': function() {
this.unblock();
var future = new Future();
var encrypted = CryptoJS.AES.encrypt(Meteor.userId(), userIdEncryptionToken);
future.return(encrypted.toString());
return future.wait();
},
'stripePayment': function(token) {
var userId = Meteor.userId();
var totalPrice = 0;
//calculate total price from collection
totalPrice = Math.ceil(totalPrice * 100) / 100;
userEmail = Meteor.users.findOne({
'_id': userId
}).emails[0].address;
// Create a charge: this will charge the user's card
var now = new Date();
Async.runSync(function(done) {
var charge = stripe.charges.create({
amount: Math.ceil(totalPrice * 100), // Amount in cents // coverting dollars to cents
currency: "usd",
source: token,
receipt_email: userEmail,
description: "Charging"
}, Meteor.bindEnvironment(function(err, charge) {
if (err) {
//handle errors with a switch case for different errors
done();
} else {
//handle res, update order
}
}));
}); // Async.runSync
},
});

Meteor.call and server methods not working properly

I have this code on my meteor app:
// client side
Template.lead.events({
'submit .insertExternalAccountForm': function (event) {
event.preventDefault();
Session.set('mcsStatus', 'Creating external account ...');
var target = {
code: event.target.code.value,
leadId: event.target.leadId.value,
name: event.target.name.value,
username: event.target.username.value,
password: event.target.password.value,
searchSourceId: event.target.searchSourceId.value,
clientId: event.target.clientId.value,
clientUserId: event.target.clientUserId.value
};
var noFormError = true;
if (target.username.length === 0) {
Session.set("username_error", "Field must not be empty");
noFormError = false;
} else {
Session.set("username_error", null);
}
if (target.password.length === 0) {
Session.set("password_error", "password must not be empty");
noFormError = false;
} else {
Session.set("password_error", null);
}
if (!noFormError) {
return noFormError;
}
Meteor.call('createExternalAccount', target, function (err, res) {
if (err) {
console.error(err);
}
console.log('in meteor call');
Router.go('/' + res.domain + '/' + res.externalId);
});
}
});
//server side
var createExternalAccountSync = function (query, external) {
return models.SearchSources.findOne(query).exec()
.then(function (searchsource) {
external.domain = searchsource.source;
var emr = searchsource.source.split('-');
return models.Organization.findOne({externalId: emr[2]}).exec();
}).then(function (org) {
console.log('after org');
external.organizationId = org._id;
return models.AppUser.findOne({clientId: external.clientId, externalId: external.clientUserId }).exec();
}).then(function (user) {
console.log('after app user');
external.userId = user._id;
external.userIds = [user._id];
return new Promise(function (resolve,reject) {
console.log('saveOrUpdate');
models.ExternalAccount.saveOrUpdate(external, function (err, newE) {
if (err) {
console.error(err)
reject(err);
}
resolve(newE)
});
});
})
.catch(function (e) {
console.error(e);
throw new Meteor.Error(e);
});
};
Meteor.methods({'createExternalAccount': function (data) {
var query = {};
var newExternalAccount = new models.ExternalAccount();
newExternalAccount.username = data.username;
newExternalAccount.password = data.password;
newExternalAccount.externalId = data.username;
newExternalAccount.name = data.name;
newExternalAccount.clientId = data.clientId;
newExternalAccount.clientUserId = data.clientUserId;
newExternalAccount._metadata = { leadId: data.leadId };
if (data.code === 'f') {
query.searchSourceId = '5744f0925db77e3e42136924';
} else {
query.searchSourceId = data.searchSourceId;
}
newExternalAccount.searchSourceId = query.searchSourceId;
console.log('creating external account')
createExternalAccountSync(query, newExternalAccount)
.then(function (external) {
console.log('should return to meteor call');
return external;
})
.catch(function (e) {
console.error(e);
throw new Meteor.Error(e);
});
}
});
The problem that I'm having is that the code on the server side, while it's being called properly, is not triggering the client side meteor.call, there's no console.log output or anything. I believe that the Meteor.wrapAsync method is properly used, but still not showing anything on the client side, and not in fact redirecting where I want the user to go after form submission.
UPDATE
The code has being updated to the newest form, but now I'm getting a weird error on the client, and its actually because the meteor.call method on the template returns neither error or result
Exception in delivering result of invoking 'createExternalAccount': http://localhost:3000/app/app.js?hash=c61e16cef6474ef12f0289b3f8662d8a83a184ab:540:40
http://localhost:3000/packages/meteor.js?hash=ae8b8affa9680bf9720bd8f7fa112f13a62f71c3:1105:27
_maybeInvokeCallback#http://localhost:3000/packages/ddp-client.js?hash=27502404fad7fc072e57e8b0b6719f40d92709c7:3557:21
receiveResult#http://localhost:3000/packages/ddp-client.js?hash=27502404fad7fc072e57e8b0b6719f40d92709c7:3577:30
_livedata_result#http://localhost:3000/packages/ddp-client.js?hash=27502404fad7fc072e57e8b0b6719f40d92709c7:4742:22
onMessage#http://localhost:3000/packages/ddp-client.js?hash=27502404fad7fc072e57e8b0b6719f40d92709c7:3385:28
http://localhost:3000/packages/ddp-client.js?hash=27502404fad7fc072e57e8b0b6719f40d92709c7:2736:19
forEach#[native code]
forEach#http://localhost:3000/packages/underscore.js?hash=27b3d669b418de8577518760446467e6ff429b1e:149:18
onmessage#http://localhost:3000/packages/ddp-client.js?hash=27502404fad7fc072e57e8b0b6719f40d92709c7:2735:15
dispatchEvent#http://localhost:3000/packages/ddp-client.js?hash=27502404fad7fc072e57e8b0b6719f40d92709c7:175:27
_dispatchMessage#http://localhost:3000/packages/ddp-client.js?hash=27502404fad7fc072e57e8b0b6719f40d92709c7:1160:23
_didMessage#http://localhost:3000/packages/ddp-client.js?hash=27502404fad7fc072e57e8b0b6719f40d92709c7:1218:34
onmessage#http://localhost:3000/packages/ddp-client.js?hash=27502404fad7fc072e57e8b0b6719f40d92709c7:1365:28
By the code you provided,it could be because you are calling different method.
You defined 'createAccount' but on client side you are calling 'createExternalAccount'

Meteor. Problems with subscribe/publish

i have a problem.
I'm trying to build highcharts graphic.
How it works:
I'm going to my route ('ship.details'), and here i have not problems.
My problem:
subsription to (ships_snapshots_all) not working.
My publish.js:
Meteor.publish("ships_snapshots", function(user, options) {
if(!this.userId) return null;
if(this.userId) {
console.log('subsribed by ' + user);
return ships_snapshots.find({userId: user}, options);
}
});
Meteor.publish("ships_snapshots_all", function() {
return ships_snapshots.find({});
})
My subscribe.js (in lib folder):
Meteor.subscribe('ships_snapshots');
Meteor.subscribe('ships_snapshots_all');
Problem 100% in my subsription, because if i'm installing autopublish all working good. And problem in my router i think.
router.js:
Router.route('/ships/details', {
name: 'ship.details',
loadingTemplate: 'loading',
onBeforeAction: function() {
var shipId = Session.get('currentShipId');
if(!shipId) {
Router.go('user.ships');
} else {
this.next();
}
},
waitOn: function() {
if (Meteor.isClient) {
var getCompare = Meteor.user().profile.wows.compareWith;
console.log(getCompare);
var user2 = Meteor.users.findOne({"profile.wows.nickname": getCompare});
var user2Id = user2._id;
if (getCompare) {
var user2 = Meteor.users.findOne({"profile.wows.nickname": getCompare});
if (user2) {
var user2Id = user2._id;
}
}
if (getCompare) {
var handle = Meteor.subscribe('ships_snapshots', Meteor.user()._id) && Meteor.subscribe('ships_snapshots', user2Id) && Meteor.subscribe('userSearchInfo', getCompare);
Session.set('compareWith', user2);
console.log('user2 _____');
console.log(user2);
return handle
} else {
var handle = Meteor.subscribe('ships_snapshots', Meteor.user()._id) && Meteor.subscribe('ships_snapshots', user2Id);
return handle
}
}, data: function() {
if (handle.ready()) {
var shipname = this.params.shipName;
var obj = {};
var query = ships.findOne();
var shipId = Session.get('currentShipId');
var result;
_.each(Meteor.user().profile.wows.ships, function(row) {
if (row.ship_id === shipId) {
result = row;
}
});
return result;
}
}
});
I think my problem in subscripion for ship_snapshots. Something going wrong here, but i can't to resolve this problem.
What exactly do you mean by "not working"? From your code I would assume that you're always seeing all the ship snapshots.
You shouldn't have the subscribes in /lib if you have them in your router. If you have Meteor.subscribe('ships_snapshots_all'); in /lib then you should always be seeing all the ship snapshots (assuming you're not stopping that subscription anywhere).
Also your subscription to all should be:
Meteor.publish("ships_snapshots", function(user, options) {
if(this.userId) {
console.log('subsribed by ' + user);
return ships_snapshots.find({userId: user}, options);
} else this.ready();
});
You don't want to return null when there is no user, you can just mark the subscription as ready without finding any records. This is not the cause of your problem but just good practice.
Meteor.publish("ships_snapshots", function(user, options) {
if(!this.userId) return null;
if(this.userId) {
console.log('subsribed by ' + user);
return ships_snapshots.find({userId: user._id}, options);
}
});
In your publish script, is user really an id or is it a user object? I changed it to user._id. Please check that.

I want async parallel not to break on error

I'm using caolan/async specifically .parallel() and I'm assembling an object of strings. When there's an error the whole thing exists and doesn't finish, even the processes without errors.
async.parallel({
"color": color,
"brand": brand,
"sku": sku,
}, function(err, result) {
console.log(err);
console.log(result);
});
If brand returns an error, I don't get any information. I'd rather brand:false. How can I achieve this?
async.parallelPlus = function(functions, callback) {
var wrap = function(func) {
return function(callback) {
func(function(err, value) {
if (err) return callback(null, false);
return callback(null, value);
});
}
}
var newFunctions = {};
for (var func in functions) {
newFunctions[func] = wrap(functions[func]);
}
return async.parallel(newFunctions, callback);
}

how to push data back to client in meteor?

I have to make a aggregate query to DB when the user click on a button, however I don't know how to return that result back to the client since I'm doing an asynchronous request, this is part of my code:
//Server side
Meteor.startup(function() {
Meteor.methods({
getAllTotals: function (query){
var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
var error = result = match = pipeline = '';
var group = {
$group: {
_id: null,
wall_clock: {
"$sum": "$wall_clock"
},
mem:{
"$sum": "$mem"
},
cpu:{
"$sum": "$cpu"
},
io:{
"$sum": "$io"
},
vmem:{
"$sum": "$vmem"
},
maxvmem:{
"$sum": "maxvmem"
}
}
};
if(typeof query.submission_time !== "undefined"){
match = {"$match": {submission_time: query.submission_time}};
pipeline = [match, group];
}else{
pipeline = [group];
}
db.collection("GE_qstat_job_monitor").aggregate(
pipeline,
Meteor.bindEnvironment(
function (error, result){
console.log(result); // <<--- this is OK!
},
function(error) {
Meteor._debug( "Error doing aggregation: " + error);
}
)
);
return result; // <<--- this is empty
}
});
}
any suggestion? :-)
Short answer:
Solution you can find here:
How to get an async data in a function with Meteor
Detailed answer
using Meteor._wrapAsync
var aggregateTotal = function(callback){
var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
// ...
db.collection("GE_qstat_job_monitor").aggregate(
pipeline,
function (error, result){
if(error){
callback(error);
}else{
callback(null, result);
}
}
);
}
var aggregateTotalsSync = Meteor._wrapAsync(aggregateTotal);
Meteor.methods({
'getAllTotals': function(){
var result;
try{
result = aggregateTotalsSync();
}catch(e){
console.log("getAllTotals method returned error : " + e);
}finally{
return result;
}
}
});
using Futures (meteorPad example)
//Server side
Meteor.startup(function() {
var Future = Npm.require('fibers/future');
Meteor.methods({
getAllTotals: function (query){
var fut = new Future();
var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
// ...
db.collection("GE_qstat_job_monitor").aggregate(
pipeline,
Meteor.bindEnvironment(
function (error, result){
if(error){
fut.throw(error);
}else{
fut.return(result)
}
},
function (exception){
// caught exception is passed to this callback
fut.throw(exception);
}
)
);
return fut.wait();
}
});
}
Easy but a bit dirty way (but not so much if you think well about your architecture) -> send back the result trough Mongo.
You can even do it without Meteor.methods, with the request creation inserted in the database on the client, an observer on the server that check it and does the async task, and then write back the result in the database.

Resources