Using IBM Watson Text-to-Speech with Firebase Cloud Functions? - firebase

I'm trying to use set up a Firebase Cloud Function to access IBM Watson Text-to-Speech. The problem is writing the returned audiofile to my Firestore database.
This test to return the list of voices worked, logging the response to the Functions log:
exports.test = functions.firestore.document('IBM_Watson_Token/Test_Value').onUpdate((change, context) => {
var textToSpeech = new TextToSpeechV1({
username: 'groucho',
password: 'swordfish'
});
return textToSpeech.listVoices(null, function(error, voices) {
if (error) {
console.log(error);
} else {
console.log(JSON.stringify(voices, null, 2));
}
});
});
Here is the documentation example Node code for returning an audiofile and writing it to the server:
var TextToSpeechV1 = require('watson-developer-cloud/text-to-speech/v1');
var fs = require('fs');
var textToSpeech = new TextToSpeechV1({
username: '{username}',
password: '{password}'
});
var synthesizeParams = {
text: 'Hello world',
accept: 'audio/wav',
voice: 'en-US_AllisonVoice'
};
// Pipe the synthesized text to a file.
textToSpeech.synthesize(synthesizeParams).on('error', function(error) {
console.log(error);
}).pipe(fs.createWriteStream('hello_world.wav'));
Firebase doesn't allow writing files to the server using fs, you have to write to a Firestore database. I changed the last line of the example code to write to Firestore, using a promise:
exports.test = functions.firestore.document('IBM_Watson_Token/Test_Value').onUpdate((change, context) => {
var textToSpeech = new TextToSpeechV1({
username: 'groucho',
password: 'swordfish'
});
var synthesizeParams = {
text: 'Hello world',
accept: 'audio/wav',
voice: 'en-US_AllisonVoice'
};
return textToSpeech.synthesize(synthesizeParams).on('error', function(error) {
console.log(error);
}).then(function (audiofile) {
admin.firestore().collection('IBM_Watson_Token').doc('hello_world').set({
'audiofile': audiofile
})
})
.catch(function (error) {
console.log(error);
});
});
The error message was
TypeError: textToSpeech.synthesize(...).on(...).then is not a function
How do I save the audiofile that comes back from Watson to Firestore?

That would be because the synthesize method is not returning a promise. You will need to use a callback construct that looks like
textToSpeech.synthesize(params, function (err, body, response) {
if (err) {
...
} else {
// body is the audio
...
}
});

Related

FCM very slow and unreliable when sending to a group of recipients through Cloud Function

I have the following Function that:
Listens for document (text message) creation
Grab IDs of members of a group chat
Get the FCM Tokens for each member
With a for-loop, send messages to group members
exports.sendChatMessage = functions.firestore
.document("chats/{mealID}/messages/{messageID}")
.onCreate((snap, context) => {
const data = snap.data();
const mealID = context.params.mealID;
const senderID = data.senderID;
const senderName = data.senderName;
const messageContent = data.content;
var docRef = db.collection("chats").doc(mealID);
docRef
.get()
.then((doc) => {
if (doc.exists) {
const docData = doc.data();
const mealName = docData.name;
const userStatus = docData.userStatus;
var users = docData.to;
var eligibleUsers = users.filter(
(user) => userStatus[user] == "accepted"
);
eligibleUsers.push(docData.from);
// get fcmTokens from eligibleUsers and send the messagme
db.collection("users")
.where("uid", "in", eligibleUsers)
.get()
.then((snapshot) => {
var fcmTokens = [];
var thumbnailPicURL = "";
// get thumbnailpic of the sender and collect fcmTokens
snapshot.forEach((doc) => {
if (doc.data().uid == senderID) {
thumbnailPicURL =
doc.data().thumbnailPicURL == null
? "https://i.imgur.com/8wSudUk.png"
: doc.data().thumbnailPicURL;
} else {
fcmTokens.push(doc.data().fcmToken);
}
});
// send the message fcmTokens
fcmTokens.forEach((token) => {
if (token != "") {
const fcmMessage = {
message: {
token: token,
notification: {
title: mealName,
body: senderName + ": " + messageContent,
image: thumbnailPicURL,
},
apns: {
payload: {
aps: {
category: "MESSAGE_RECEIVED",
},
MEAL_ID: mealID,
},
},
},
};
tokenManger.sendFcmMessage(fcmMessage);
}
});
return true;
});
} else {
// doc.data() will be undefined in this case
console.log("No such document!");
return false;
}
})
.catch((error) => {
console.log("Error getting document:", error);
return false;
});
return true;
});
My send function comes from a helper file that uses the HTTP V1 protocol to build the send-request:
const { google } = require("googleapis");
const https = require("https");
const MESSAGING_SCOPE = "https://www.googleapis.com/auth/firebase.messaging";
const SCOPES = [MESSAGING_SCOPE];
const PROJECT_ID = MY_PROJECT_ID;
const HOST = "fcm.googleapis.com";
const PATH = "/v1/projects/" + PROJECT_ID + "/messages:send";
exports.getAccessToken = () => {
return new Promise(function (resolve, reject) {
const key = require("./service-account.json");
var jwtClient = new google.auth.JWT(
key.client_email,
null,
key.private_key,
SCOPES,
null
);
jwtClient.authorize(function (err, tokens) {
if (err) {
reject(err);
return;
}
resolve(tokens.access_token);
});
});
};
//send message
exports.sendFcmMessage = (fcmMessage) => {
this.getAccessToken().then(function (accessToken) {
var options = {
hostname: HOST,
path: PATH,
method: "POST",
headers: {
Authorization: "Bearer " + accessToken,
},
// … plus the body of your notification or data message
};
var request = https.request(options, function (resp) {
resp.setEncoding("utf8");
resp.on("data", function (data) {
console.log("Message sent to Firebase for delivery, response:");
console.log(data);
});
});
request.on("error", function (err) {
console.log("Unable to send message to Firebase");
console.log(err);
});
request.write(JSON.stringify(fcmMessage));
request.end();
});
};
It worked all fine in the emulator but once deployed, there're significant delays (~3 mins):
I also noticed that the console says the cloud function finishes execution BEFORE sendFcmMessage logs success messages.
I did some research online, it appears that it might have something to do with the usage of Promise but I wasn't sure if that's the sole reason or it has something to do with my for-loop.
The Problem
To summarize the issue, you are creating "floating promises" or starting other asynchronous tasks (like in sendFcmMessage) where you aren't returning a promise because they use callbacks instead.
In a deployed function, as soon as the function returns its result or the Promise chain resolves, all further actions should be treated as if they will never be executed as documented here. An "inactive" function might be terminated at any time, is severely throttled and any network calls you make (like setting data in database or calling out to FCM) may never be executed.
An indicator that you haven't properly chained the promises is when you see the function completion log message ("Function execution took...") before other messages you are logging. When you see this, you need to look at the code you are running and confirm whether you have any "floating promises" or are using callback-based APIs. Once you have changed the callback-based APIs to use promises and then made sure they are all chained together properly, you should see a significant boost in performance.
The fixes
Sending the message data to FCM
In your tokenManger file, getAccessToken() could be reworked slightly and sendFcmMessage should be converted to return a Promise:
exports.getAccessToken = () => {
return new Promise(function (resolve, reject) {
const key = require("./service-account.json");
const jwtClient = new google.auth.JWT(
key.client_email,
null,
key.private_key,
SCOPES,
null
);
jwtClient.authorize(
(err, tokens) => err ? reject(err) : resolve(tokens.access_token)
);
});
};
//send message
exports.sendFcmMessage = (fcmMessage) => {
// CHANGED: return the Promise
return this.getAccessToken().then(function (accessToken) {
const options = {
hostname: HOST,
path: PATH,
method: "POST",
headers: {
Authorization: "Bearer " + accessToken,
},
// … plus the body of your notification or data message
};
// CHANGED: convert to Promise:
return new Promise((resolve, reject) => {
const request = https.request(options, (resp) => {
resp.setEncoding("utf8");
resp.on("data", resolve);
resp.on("error", reject);
});
request.on("error", reject);
request.write(JSON.stringify(fcmMessage));
request.end();
});
});
};
However, the above code was built for googleapis ^52.1.0 and google-auth-library ^6.0.3. The modern versions of these modules are v92.0.0 and v7.11.0 respectively. This means you should really update the code to use these later versions like so:
// Import JWT module directly
const { JWT } = require('google-auth-library');
// FIREBASE_CONFIG is a JSON string available in Cloud Functions
const PROJECT_ID = JSON.parse(process.env.FIREBASE_CONFIG).projectId;
const FCM_ENDPOINT = `https://fcm.googleapis.com/v1/projects/${PROJECT_ID}/messages:send`;
const FCM_SCOPES = ["https://www.googleapis.com/auth/firebase.messaging"];
exports.sendFcmMessage = (fcmMessage) => {
const key = require("./service-account.json"); // consider moving outside of function (so it throws an error during deployment if its missing)
const client = new JWT({
email: key.client_email,
key: key.private_key,
scopes: FCM_SCOPES
});
return client.request({ // <-- this uses `gaxios`, Google's fork of `axios` built for Promise-based APIs
url: FCM_ENDPOINT,
method: "POST",
data: fcmMessage
});
}
Better yet, just use the messaging APIs provided by the Firebase Admin SDKs that handle the details for you. Just feed it the message and tokens as needed.
import { initializeApp } from "firebase-admin/app";
import { getMessaging } from "firebase-admin/messaging";
initializeApp(); // initializes using default credentials provided by Cloud Functions
const fcm = getMessaging();
fcm.send(message) // send to one (uses the given token)
fcm.sendAll(messagesArr) // send to many at once (each message uses the given token)
fcm.sendMulticast(message) // send to many at once (uses a `tokens` array instead of `token`)
The Cloud Function
Updating the main Cloud Function, you'd get:
exports.sendChatMessage = functions.firestore
.document("chats/{mealID}/messages/{messageID}")
.onCreate((snap, context) => {
const mealID = context.params.mealID;
const { senderID, senderName, content: messageContent } = snap.data();
const docRef = db.collection("chats").doc(mealID);
/* --> */ return docRef
.get()
.then((doc) => {
if (!doc.exists) { // CHANGED: Fail fast and avoid else statements
console.log(`Could not find "chat:${mealID}"!`);
return false;
}
const { userStatus, to: users, name: mealName, from: fromUser } = doc.data();
const eligibleUsers = users.filter(
(user) => userStatus[user] == "accepted"
);
eligibleUsers.push(fromUser);
// get fcmTokens from eligibleUsers and send the message
/* --> */ return db.collection("users")
.where("uid", "in", eligibleUsers) // WARNING: This will only work for up to 10 users! You'll need to break it up into chunks of 10 if there are more.
.get()
.then(async (snapshot) => {
const fcmTokens = [];
let thumbnailPicURL = "";
// get thumbnailpic of the sender and collect fcmTokens
snapshot.forEach((doc) => {
if (doc.get("uid") == senderID) {
thumbnailPicURL = doc.get("thumbnailPicURL"); // update with given thumbnail pic
} else {
fcmTokens.push(doc.get("fcmToken"));
}
});
const baseMessage = {
notification: {
title: mealName,
body: senderName + ": " + messageContent,
image: thumbnailPicURL || "https://i.imgur.com/8wSudUk.png", // CHANGED: specified fallback image here
},
apns: {
payload: {
aps: {
category: "MESSAGE_RECEIVED",
},
MEAL_ID: mealID,
},
}
}
// log error if fcmTokens empty?
// ----- OPTION 1 -----
// send the message to each fcmToken
const messagePromises = fcmTokens.map((token) => {
if (!token) // handle "" and undefined
return; // skip
/* --> */ return tokenManger
.sendFcmMessage({
message: { ...baseMessage, token }
})
.catch((err) => { // catch the error here, so as many notifications are sent out as possible
console.error(`Failed to send message to "fcm:${token}"`, err);
})
});
await Promise.all(messagePromises); // wait for all messages to be sent out
// --------------------
// ----- OPTION 2 -----
// send the message to each fcmToken
await getMessaging().sendAll(
fcmTokens.map((token) => ({ ...baseMessage, token }))
);
// --------------------
return true;
})
.catch((error) => {
console.log("Error sending messages:", error);
return false;
});
})
.catch((error) => {
console.log("Error getting document:", error);
return false;
});
});
I found out that the culprit is my queries to db. Like #samthecodingman commented, I was creating floating Promises.
Originally, I have codes like:
db.collection("users")
.where("uid", "in", eligibleUsers)
.get()
.then((snapshot) => {...}
All I needed to do is to return that call:
return db.collection("users")
.where("uid", "in", eligibleUsers)
.get()
.then((snapshot) => {...}
Although it's still not instant delivery, it's much faster now.

Returning a value from an asynchronous callback (Node.JS)

So, I have my route which console.logs 'undefined':
router.get("/validate-pin", async (req, res) => {
// restrict when done
try {
const { userId, pin } = req.query;
const isActivePin = await pinsDB.compareActivePin(userId, pin);
console.log(isActivePin)
return res.status(200).json(isActivePin);
} catch (error) {
console.log(error);
res.status(500).json({ error: "db error: ", error });
}
});
I have my compareActivePin method, which logs out the 'res' parameter, but for some reason doesn't return it:
async function compareActivePin(userId, received_pin) {
const active_pin = await db("account_pins").where({ userId, isActive: true });
const pinIsValidated = bcrypt.compareSync(
received_pin,
active_pin[0].account_pin
);
if (pinIsValidated) {
let skLocation = await db("sks").where({ userId }).select("url");
await readKey(skLocation[0].url, (res) => {
// console.log(res);
return res;
});
} else return false;
}
And I have my readKey method, which actually grabs the data I want my compareActivePin to return. This works like a charm.
const readKey = async (key, callback) => {
const aws = require("aws-sdk");
aws.config.update({
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
region: "us-east-2",
});
const s3 = new aws.S3();
const getParams = { Bucket: process.env.SK_BUCKET, Key: `${key}.txt` };
await s3.getObject(getParams, (err, data) => {
if (err) return err;
return callback(data.Body.toString());
});
};
So, just to recap. When I hit my endpoint, I pass in a userId and pin (strings). This calls the compareActivePin method which validates the pin and then, if the pin is valid, it then calls readKey, which grabs the file from S3 and returns the text within the file.
Like I said, I'm able to log it out to the console from within the readKey callback, but when I try to log it out as the returned value from the route, it comes back undefined.
Hoping someone could point me in the right direction.
Thanks...
I ended up answering my own question. I don't think it's possible to get a return value from the callback, so I ended up paring down the call from the database and sending the response from the readKey function using the router response object, like so:
//CompareActivePin Function
async function compareActivePin(userId, received_pin) {
const active_pin = await db("account_pins").where({ userId, isActive: true });
const pinIsValidated = bcrypt.compareSync(
received_pin,
active_pin[0].account_pin
);
return pinIsValidated;
}
//Router Call
router.get("/validate-pin", async (req, res) => {
// restrict when done
try {
const { userId, pin } = req.query;
const isActivePin = await pinsDB.compareActivePin(userId, pin);
if (isActivePin) {
let skLocation = await skDB.findUrl(userId);
readKeyFunc(skLocation[0].url, (result) => {
return res.status(200).json({ confirmed: isActivePin, key: result });
});
} else return res.status(401).json({ confirmed: isActivePin, key: null });
} catch (error) {
res.status(500).json({ error: "db error: ", error });
}
});
This also goes a long way toward keeping my database methods pure and separating my concerns.
Thanks, StackOverflow!

Firebase Callable function with JWT Authentication and Google Sheets V4 API

I want to implement firebase callable function with JWT Authentication and fetching data from Google Sheet, using Google Sheets V4 API.
For test I tried to use Example Spreadsheet but Sheets API not activated for that Spreadsheet and I cloned it on my own drive and use it for testing.
References:
My code based on solution described in this question How to use Google sheets API while inside a google cloud function and Accessing Google APIs using Service account in Node.JS
Also I have got two important information: "Service Account".json and API Key. I save API Key in api_key.json but didn't find examples how to use it with Google Sheets V4 API:
{
key: "xxxxxx"
}
test() callable function which doesn't require any authentication works fine:
exports.test = functions.https.onCall((data, context) => {
return { text: data.text };
});
Calling test() function somewhere on client (in Browser):
function getTest() {
console.log("clicked getTest()");
var test = firebase.functions().httpsCallable('test');
test({text: '12345'}).then(function(result) {
console.log(result);
}).catch(function(error) {
console.log(error.code);
console.log(error.message);
});
}
Calling getData() somewhere on client (in Browser):
function requestData() {
console.log("clicked requestData()");
//https://firebase.google.com/docs/functions/callable
//getData() function described in functions/index.js
var getData = firebase.functions().httpsCallable('getData');
getData(null).then(function (result) {
// Read result of the Cloud Function.
console.log(result); //<------- Expected rows from Spreadsheet????
}).catch(function(error) {
console.log(error.code);
console.log(error.message);
});
}
**Thank you, F10. I corrected code.
index.js:
'use strict'
const functions = require('firebase-functions');
const { google } = require('googleapis');
var serviceAccount = require("./credentials/owner-service-account-gcloud.json");
function getJwt() {
// Define the required scopes.
var scopes = [
'https://www.googleapis.com/auth/spreadsheets'
];
return new google.auth.JWT(
serviceAccount.client_email,
null,
serviceAccount.private_key,
scopes
);
}
function getSpreadsheetDate(jwt) {
return new Promise((resolve, reject) => {
jwt.authorize((error, access_token) => {
if (error) {
console.log('Error in jwt.authorize: ' + error);
reject(error);
} else {
// access_token ready to use to fetch data and return to client
const sheets = google.sheets({ version: 'v4', access_token });
// set auth as a global default:
google.options({ auth: jwt }); //<----------------------
const request = {
auth: jwt,
spreadsheetId: 'xxxx',
range: 'Class Data!A2:E', //'Class Data!A2:E',
}
sheets.spreadsheets.values.get(request, (err, response) => {
console.log("inside: sheets.spreadsheets.values.get() -------------------------------");
if (err) {
console.log('The Sheets API returned an error: ' + err);
//The API returned an error: Error: API key not valid. Please pass a valid API key.
reject(err);
};
try {
var numRows = response.data.values ? response.data.values.length : 0;
console.log('%d rows retrieved.', numRows);
console.log("response.data:-------------------------------");
console.log(response.data.values);
resolve(response.data.values);
} catch (err) {
console.log("Error processing Sheets API response: " + err);
reject(err);
}
})
}
})
})
}
exports.getData = functions.https.onCall((data, context) => {
console.log("getData()---------------------------");
if (!context.auth) {
throw new functions.https.HttpsError('failed-precondition', 'The function must be called ' + 'while authenticated.');
} else {
console.log("context.auth ------------ OK");
const uid = context.auth.uid;
console.log(uid);
var jwt = getJwt();
console.log("getJwt() --------------- OK");
return getSpreadsheetDate(jwt); //<------------ Requested Spreadsheet's Data
}
})
exports.test = functions.https.onCall((data, context) => {
return { text: data.text };
});
There's a solution that uses googleapis instead of the auth library to do the authentication with JWT. Regarding your token inquiries, you could check the OAuth 2.0 for client-side web applications documentations, which explains the steps to do the authentication.

Uploading a form posted image buffer to Cloud Storage with Firebase Functions

Here's my cloud function. It's supposed to get an http posted image and upload it to storage, returning the url.
exports.uploadImageToEditor = functions.https.onRequest((req, res) => {
const img = JSON.parse(JSON.stringify(req.body));
const bucket = admin.storage().bucket();
return bucket.file('blog/foo.jpg').save(img.data, {
resumable: false,
metadata: {
contentType: 'image/jpeg'
}
})
.then(() => {
return cors(req, res, () => {
res.status(200).send({ "url": bucket.file('foo.jpg').getSignedUrl()});
});
});
});
This is how the image is actually sent in the client:
uploadImage(file, endPoint) {
if (!endPoint) {
throw new Error('Image Endpoint isn`t provided or invalid');
}
const formData = new FormData();
if (file) {
formData.append('file', file);
const req = new HttpRequest('POST', endPoint, formData, {
reportProgress: true
});
return this._http.request(req);
}
else {
throw new Error('Invalid Image');
}
}
I think you're probably looking for the save() method on File in the Admin SDK.
const bucket = admin.storage().bucket()
.file('my-file.jpg').save(blob)
.then(() => { /* ... */ });
You can also get back information about the file this way.
export const uploadImage = async (destination: string, image: Buffer) => {
const file = storage.bucket().file(destination);
await file.save(image, { contentType: yourContentType });
return file.publicUrl();
};
If you get a permission error, go to Firebase storage Rules and add this rule to allow to write in the directory :
service firebase.storage {
match /b/{bucket}/o {
match /blog/{anyPath=**} {
allow read;
allow write;
}
}
}

Meteor login through different server

I want to let my meteor users login through my react-native app.
I have one website built by Meteor/reactjs and the other is an App using android react-native and they both share the same Mongo database.
My meteor website uses bcrypt (meteor accounts-password), and my react-native app where I tried to use SHA256 and bcrypt so when I log in through my react-native app which is connected to the database, the hashed passwords do not match and I get an error.
I'm just getting really confused as to how to hash my react-native password to get it to the match the meteor users' hashed password. Any help would be great, cheers.
This is the front end
sendAjax = () => {
const m = encodeURIComponent(this.state.email);
const p = encodeURIComponent(this.state.password);
const hashDigest = sha256(p);
const requestBody = `email=${m}&password=${hashDigest}&mphone`;
//POST
fetch("http://**************/users/auth", {
method: "POST",
mode: "cors",
headers: {
"Accept": "application/json",
"Content-Type": "application/x-www-form-urlencoded"
},
body: requestBody
}).then(function (res, next) {
console.log("fetch request ", JSON.stringify(res.ok));
if(res.ok){
res.json().then(function (json) {
console.info(json);
console.info(json.verifystate);
if(json.verifystate){
Alert.alert("Login success");
Actions.leaderBoard();
}else{
Alert.alert("Login fail, please try again");
}
});
}else{
Alert.alert('Noted','request failed',[{text: 'confirm', onPress: () => console.log('OK Pressed!')},]);
next();
}
})
.catch(function (e) {
console.log("fetch fail");
Alert.alert('Noted','system error',[{text: 'confirm', onPress: () => console.log('OK Pressed!')},]);
});
}
This is my backend
app.post('/users/auth', function(req, res) {
loginData(db, req.body.email, req.body.password, function(result){
if(result == 1){
console.log("find the result!");
res.send({"verifystate":true});
}else{
console.log('cannot find it');
}
});
});
......
var loginData = function(db, email, myPlaintextPassword, mphone, callback){
var collectionUser = db.collection('users');
bcrypt.genSalt(saltRounds, function(err, salt) {
bcrypt.hash(myPlaintextPassword, salt, function(err, hash) {
var queryStr = {"emails.address": email, "services.password.bcrypt": hash};
collectionUser.count(queryStr, function(err, result) {
if(err)
{
console.log('Error2:'+ err);
}
callback(result);
});
});
});
};

Resources