I need to measure how long it takes to compute my hash with x number of iterations. Here is code:
function hashPromise(secretPhrase, keylen) {
let hashedKey
var promise = new Promise(function(resolve, reject) {
crypto.pbkdf2(secretPhrase, 'salt', keylen, 256,
'sha256', (err, key) => {
if (err) {
console.log(err)
reject(err);
} else {
hashedKey = key.toString('hex')
let time = new Date().getTime()
console.log('keylen: ' + keylen)
console.log('TIMESTAMP Hash Finished: ' + time)
console.log('hashedKey: ' + hashedKey)
resolve( hashedSSN )
}
})
})
return promise;
}
I am looping through this function with promise.all a couple hundred times, passing in a new secretPhrase each time.
I am using a Firebase Functions http request to trigger this functions. Here are my logs when keylen is 1000:
And here are my logs if keylen is 500000
Would computation time simply be the difference between each 'TIMESTAMP Hash Finished'? Or would it be the time it took the entire function to finish (at top of logs)?
Ie- Computation time using a 500000 keylen is 1526473894031 - 1526473893718 = 313ms ?
Related
I got 2 servers, one for the main app and another one for huge tasks.
User -> Server 1 -> Server 2
Server 1: Main app & Easy tasks
Server 2: Huge Tasks
When I call a server 2's function which takes a long time to answer, server 1 receive undefined when server 2 answer a good result. However, if server2's function takes less than 1 minute to answer, server 1 got the result sent by server 2 and then send it back to the client.
Why it doesn't work only for functions which take more than 1 minute to compute ?
Client :
Meteor.call('reporting.default', params.subReport, params, function(error, result) {
if (result) self.setState({data: result});
else self.setState({data: error.message});
});
Server 1:
Meteor.methods({
'reporting.default'(subReport, params) {
this.unblock();
return Meteor.callWorker('reporting.' + subReport, Meteor.callId(), params).then((result, error) => { if (error) return error; else return result; }).await();
},
});
Meteor.worker = DDP.connect('localhost:' + Meteor.settings.heavyTasksServer.port);
Meteor.callWorker = (method, ...myParameters) => new Promise((resolve, reject) => {
console.log(method + ": REQUEST");
Meteor.worker.call(method, ...myParameters, (err, res) => {
if (err) {
console.log(method + ": ERROR");
reject(err);
}
else {
console.log(method + ": ANSWER");
resolve(res);
}
});
});
Meteor.callId = function () {
const d =new Date();
return d.getUTCFullYear() +""+ (d.getUTCMonth()+1) +""+ d.getUTCDate() +""+ d.getUTCHours() +""+ d.getUTCMinutes() +""+ d.getUTCSeconds() +""+ d.getUTCMilliseconds() + "-" + Meteor.userId();
};
Server 2:
Meteor.methods({
'reporting.clientsAssets'(callId, params) {
this.unblock();
const funcName = "reporting.clientsAssets";
if (canRunQuery(1, callId, arguments, funcName)) {
console.log(funcName + ": START");
const data = reportingClientsAssets(params);
console.log(funcName + ": END");
terminateQuery(callId);
return data;
}
}
});
You could consider an asynchronous model instead of a synchronous one (which is probably timing out).
Let's think about a queuing mechanism... create a collection call jobs (or whatever you prefer), and server 1 creates a record in the job collection with a status of 'ready'.
A timed task (you can use node-cron for this) runs on server 2 say every minute, and looks for jobs with a status of 'ready'. It takes the first one, sets the status to 'running' and then calls the function to do the work.
When that function completes, it sets the status of the task to 'complete'.
You make use of Meteor's reactivity, so that the user can see the status of the job, once it is started, it moves to 'running', and then to 'complete' once it is done. At that point a link may appear so they have access to the data, report or whatever is produced.
No timeout issues with this mechanism, and it's nicely decoupled.
I have a Cloud Function triggered by a pub/sub event. I use sendgrid nodejs api. The main idea is sending my clients a weekly stats email. sendEmail() function run for each client (80 times). But when I check function logs I see that 25-30 of client emails are sent with success but the remaining it gives that error: "socket hang up"
I shortened the whole code to show the main part related sending email. Here is the last part.
// I shortened the whole function as it is a very long function.
// The main and the last part is as below
// I have nearly 80 clients and sendEmail function run for each client.
function calcData(i, data) {
return admin.database().ref('clientUrlClicks/' + data.key)
.orderByChild('date')
.startAt(dateStartEpox)
.endAt(dateEndEpox)
.once('value', urlClickSnap => {
clients[i].clickTotalWeek = urlClickSnap.numChildren();
clients[i].listTotalWeek = 0;
admin.database().ref('clientImpressions/' + data.key)
.orderByKey()
.startAt(dateStart)
.endAt(dateEnd)
.once('value', snap => {
snap.forEach(function(impressionSnap) {
clients[i].listTotalWeek += impressionSnap.val();
})
}).then(resp => {
return sendEmail(i, clients[i]);
}).catch(err => {
console.log(err);
});
}).catch(err => {
clients[i].clickTotalWeek = 0;
console.log(err);
});
}
function sendEmail(i, data) {
var options = {
method: 'POST',
url: 'https://api.sendgrid.com/v3/mail/send',
headers:
{
'content-type': 'application/json',
authorization: 'Bearer ' + sgApiKey
},
body:
{
personalizations:
[{
to: [{ email: data.email, name: data.name }],
dynamic_template_data:
{
dateStart: xxx,
dateEnd: xxx,
}
}],
from: { email: 'info#xxx.com', name: 'xxx' },
reply_to: { email: 'info#xxx.com', name: 'xxx' },
template_id: 'd-f44eeexxxxxxxxxxxxx'
},
json: true
};
request(options, function (error, response, body) {
if (error) {
console.log("err: " + error);
return;
}
return;
});
}
Edit:
In addition to answers below related to "chaining the promises correctly", I also added all emails and personalizations to "personalizations" array as an object on "sendEmail" function. So, instead making a request for each email I make one request. No problem now.
You are not chaining the promises correctly and therefore not returning a final promise at the end of the chaining, which is mandatory for a Cloud Function.
The following set of modifications is a first attempt to solve this problem.
Also, it is not crystal clear how do you call Sendgrid and return the Promise returned by the Sendgrid call. I would suggest that you use the send() method, which returns a Promise, as explained in the doc of the Sendgrid v3 Web API for Node.js, see https://github.com/sendgrid/sendgrid-nodejs/tree/master/packages/mail.
function calcData(i, data) {
//Declare clients aray here
return admin.database().ref('clientUrlClicks/' + data.key)
.orderByChild('date')
.startAt(dateStartEpox)
.endAt(dateEndEpox)
.once('value')
.then(urlClickSnap => {
clients[i].clickTotalWeek = urlClickSnap.numChildren();
clients[i].listTotalWeek = 0;
return admin.database().ref('clientImpressions/' + data.key) //Here you didn't return the promise
.orderByKey()
.startAt(dateStart)
.endAt(dateEnd)
.once('value');
.then(snap => {
snap.forEach(function(impressionSnap) {
clients[i].listTotalWeek += impressionSnap.val();
})
return sendEmail(i, clients[i]);
}).catch(err => {
clients[i].clickTotalWeek = 0;
console.log(err);
return null;
});
}
I see two issues with your code related to promise chaining, which may be causing this problem.
First is that you are using request with callback in your sendEmail function. This will simply not wait for your network call to finish and returns the function. Now this will build up the calls in parallel and before you hit your 80 clients counts the execution of your cloud function will finish. The solution would be to use request-promise-native (https://github.com/request/request-promise-native) library with your request library. So your sendEmail Function will now become
sendEmail (i, data) {
.
.
.
return rpn(options).then((d)=>{return d}).catch((e)=>{return console.log(e)})
}
Other solution is to use sendgrid client for nodejs which will simply return the promise and you don't need to use request. https://github.com/sendgrid/sendgrid-nodejs/tree/master/packages/mail
Second issue is in you call for data read from firebase where you are also using callbacks instead of promises. Correct solution will be:
function calcData(i, data) {
return admin.database().ref('clientUrlClicks/' + data.key)
.orderByChild('date')
.startAt(dateStartEpox)
.endAt(dateEndEpox)
.once('value').then( urlClickSnap => {
clients[i].clickTotalWeek = urlClickSnap.numChildren();
clients[i].listTotalWeek = 0;
return admin.database().ref('clientImpressions/' + data.key)
.orderByKey()
.startAt(dateStart)
.endAt(dateEnd)
.once('value').then( snap => {
snap.forEach(function(impressionSnap) {
clients[i].listTotalWeek += impressionSnap.val();
})
return sendEmail(i, clients[i]);
})
.catch(err => {
console.log(err);
});
}).catch(err => {
clients[i].clickTotalWeek = 0;
console.log(err);
});
}
This will make sure that you function calcData returns after finishing the execution of all the promises chained.
One more thing if you are calling calcData in a loop, then make sure that you store all the promises in an array and after loop call Promise.all(promisesArray), so that you function waits for all the executions to finish.
I want to perform a batch transaction in firestore. I am storing last key in other collection.
i need to get the last key then increase by 1, then create two documents using this key. How can i do this?
let lastDealKeyRef = this.db.collection('counters').doc('dealCounter')
let dealsRef = this.db.collection('deals').doc(id)
let lastDealKey = batch.get(lastDealKeyRef) // here is the problem..
batch.set(dealsRef, dealData)
let contentRef = this.db.collection('contents').doc('deal' + id)
batch.set(contentRef, {'html': '<p>Hello World</p>' + lastDealKey })
batch.commit().then(function () {
console.log('done') })
If you want to read/write data in a single operation you should be using a transaction.
// Set up all references
let lastDealKeyRef = this.db.collection('counters').doc('dealCounter');
let dealsRef = this.db.collection('deals').doc(id);
let contentRef = this.db.collection('contents').doc('deal' + id);
// Begin a transaction
db.runTransaction(function(transaction) {
// Get the data you want to read
return transaction.get(lastDealKeyRef).then(function(lastDealDoc) {
let lastDealData = lastDealDoc.data();
// Set all data
let setDeals = transaction.set(dealsRef, dealData);
let setContent = transaction.set(contentRef, {'html': '<p>Hello World</p>' + lastDealKey });
// Return a promise
return Promise.all([setDeals, setContent]);
});
}).then(function() {
console.log("Transaction success.");
}).catch(function(err) {
console.error("Transaction failure: " + err);
});
You can read more about transactions and batches here:
https://firebase.google.com/docs/firestore/manage-data/transactions
The following code executed when something is written at a certain location in the database.
What I expect from this code : If the number of coins are => 500 to subtract only once 500 coins from the current coins value and to add one ticket to the existing ticket value.
What I am getting in reality: The code recursively subtracts 500 coins until the coin value is lower than 500; It adds more tickets then it should.
Please , can somebody modify my code to work as expected ?
I do not know what I am doing wrong
exports.TransformCoinsIntoTickets1 = functions.database.ref('/AddTickets/{userid}').onWrite(event => {
var user = event.params.userid;
/// first get coins to see if we can lower coin value
var usercoinRef1 = admin.database().ref('coins').child(user);
usercoinRef1.on("value", function(snapshot) {
var numberofcoins = snapshot.val();
console.log("We are in snapshot of coins -> He has coins = " + numberofcoins);
if (numberofcoins >= 500 )
return usercoinRef1.set(numberofcoins-500).then(() => {
var ticketRef1 = admin.database().ref('tickets').child(user);
ticketRef1.on("value", function(snap123) {
var numberoftickets = snap123.val();
return ticketRef1.set(numberoftickets+1).then(() => {
console.log('Tickets Write succeeded!');
});
}, function (error) {
console.log("Error Tickets: " + error.code);
});
console.log('Coins Write succeeded!');
});
}, function (error) {
console.log("Error Coins: " + error.code);
});
//then we write the new coin value if we need to
});
I just realized that instead of
on
i should use
once
So, replacing
.on("value",
with
.once("value",
resolved the problem.
I have an array of objects taken from mongodb. Every element in the array is a post, with author as user_id. Now i wish to find the user info related to the user_id.
Since node uses async methods to find the data from db, the forEach loop finishes before the callbacks finish.
docs.forEach(function(doc, index){
//get the user for this doc
User.find({_id: mongo.BSONPure.ObjectID(doc.user_id)}, {name: 1, username: 1, email: 1}).skip(0).limit(1).toArray(function(err, user){
user = user[0]
if(err) {
throw new Error(err)
} else {
docs[index].user = user
if(doc.for_id) {
User.find({_id: mongo.BSONPure.ObjectID(doc.for_id)}, {name: 1, username: 1, email: 1}).skip(0).limit(1).toArray(function(err, for_user){
for_user = for_user[0]
if(err) {
throw new Error(err)
} else {
docs[index].for_user = for_user
}
})
}
}
})
})
So at the end of this loop, if i send a cb(docs), docs do not have the user and for_user attribute. How do I overcome this?
Use Step for node.js. It will run your functions in serial order
var Step = require('step');
Step( docs.forEach(...), function() { cb(docs); } );
Or if you know the total number of records, you can call the callback when you're done processing the last one. Something like this
var count = docs.count(); // or something
var processed = 0;
docs.forEach(... if (++processed == count) cb(docs); );