Factorize code with custom functions in Firebase Cloud Function - firebase

Question

I want to factorize my code in Cloud Functions in order to improve readability and maintenance. The code below works but after waiting for all Promises to complete with Promises.all(), the code timeout.
The things I don't understand is that :
It works great and complete without timeout when toiletJsonObject["fields"]["adresse"] = formatAddress(toiletJsonObject["fields"]["adresse"]) is commented
If it works without the line above, the timeout should be due to the formatAddress() function. However, this function in not an async one and just return a string synchronously. Maybe that's what I misunderstand.
So my questions are :
How to correct my code to avoid timeout?
what's the best way to factorize code with custom functions that are only accessible inside the file and therefore does not need export ?
The entire code :
import * as functions from "firebase-functions";
import * as admin from "firebase-admin";
import fetch from "node-fetch";
admin.initializeApp();
const db = admin.firestore();
export const tempoCF = functions.firestore.document("/tempo/{docId}").onCreate(async () => {
console.log("onCreate")
const settings = { method: "Get" }
const metaUrl = "https://opendata.paris.fr/api/datasets/1.0/sanisettesparis/"
const toiletUpdateDateRef = db.collection('toilets').doc("updateDate")
try {
// Get meta data to check last update date
const metaResponse = await fetch(metaUrl, settings)
const metaJson = await metaResponse.json()
const metaUpdateDate = metaJson["metas"]["modified"]
const lastUpdatedDateDoc = await toiletUpdateDateRef.get()
if (!lastUpdatedDateDoc.exists) {
console.log("No existing date document, create one and add last update date : " + metaUpdateDate)
await fetchDataFromURL()
return toiletUpdateDateRef.set({ "lastUpdateDate": metaUpdateDate })
} else {
const lastUpdateDate = lastUpdatedDateDoc.data()["lastUpdateDate"]
// If date from meta data newer that saved date : get data and update
if (new Date(lastUpdateDate) < new Date(metaUpdateDate)) {
console.log("New data available, update database")
await fetchDataFromURL()
return toiletUpdateDateRef.set({ "lastUpdateDate": metaUpdateDate })
}
else {
console.log("No new data available, do nothing")
return null
}
}
}
catch (error) {
console.log(error);
return null;
}
}
);
async function fetchDataFromURL() {
const dataUrl = "https://opendata.paris.fr/api/records/1.0/search/?dataset=sanisettesparis&q=&rows=-1"
const settings = { method: "Get" }
try {
const response = await fetch(dataUrl, settings)
const json = await response.json()
const promises = []
console.log("fetch data and add toilets to collection")
json["records"].forEach(toiletJsonObject => {
delete toiletJsonObject["fields"]["geo_shape"]
toiletJsonObject["fields"]["adresse"] = formatAddress(toiletJsonObject["fields"]["adresse"])
console.log("after updating adresse field: " + toiletJsonObject["fields"].toString())
const p = db.collection("toilets").doc(toiletJsonObject["recordid"]).set(toiletJsonObject["fields"])
promises.push(p)
})
console.log("finished creating promises. Wait for all to complete")
return Promise.all(promises);
}
catch (error) {
console.log(error);
return null;
}
}
const linkWords = ["de", "des", "du", "le"]
const linkLetters = ["l", "d"]
const firstWordsAddress = ["face", "opposé", "au"]
const alwaysLowerCaseWords = ["ville", "rue"]
function formatAddress(address) {
let processedAddress = ""
if (address != null) {
//if (address.length <= 1) processedAddress = address.toUpperCase();
// Split string into list of words
var wordsList = address.split(' ')
.filter((word) => {
// If there is a word in front of the street number, don't use it
if (firstWordsAddress.includes(word.toLowerCase())) return false
// Else use it
return true
})
var capitalizedList = wordsList.map((word) => {
const lowerCaseWord = word.toLowerCase() //TOSTRING ?
// If current word is a link word, don't capitalize
if (linkWords.includes(lowerCaseWord))
return lowerCaseWord
// If current word is a link letter, add ' char
else if (linkLetters.includes(lowerCaseWord))
return lowerCaseWord + '\''
// If current word should always be in lower case, don't capitalize
else if (alwaysLowerCaseWords.includes(lowerCaseWord))
return word.toLowerCase() //TOSTRING
// Else, capitalize the word
return word[0].toUpperCase() + word.substr(1).toLowerCase()
});
// Always capitalize first word of the address
capitalizedList[0] = capitalizedList[0][0].toUpperCase() + capitalizedList[0].substr(1).toLowerCase()
processedAddress = capitalizedList.join(' ')
processedAddress = processedAddress.replace("\' ", "\'")
processedAddress = processedAddress.trim()
}
return processedAddress
}

Answer 1

Regarding the formatAddress() helper function you defined, there doesn't appear to be an issue with it in it's current form. It can happily run through the entire list of 644 addresses ~210 times per second.
Any timeouts are instead likely to be caused by performing so many database writes in quick succession. When running fetchDataFromURL(), you "spam" the Firestore server with a request for each toilet object you are uploading.
The best-practice approach would be to compile a Batched Write and then commit the result once you've finished processing the data.
As stated in that documentation:
A batched write can contain up to 500 operations. Each operation in the batch counts separately towards your Cloud Firestore usage. Within a write operation, field transforms like serverTimestamp, arrayUnion, and increment each count as an additional operation.
Note: The current list of field transforms includes serverTimestamp, arrayUnion, arrayRemove, and increment. Reference: FieldValue
Creating/deleting/writing a document to Firestore is considered "one operation". Because a field transform requires reading the document, then writing data to that document, it is counted as "two operations".
Because a single batched write is limited to 500 operations, you should split your data up into smaller batched writes so that each batch is less than this 500 operations limit. The easiest way to achieve this would be to use this MultiBatch class (included below) that I've updated from one of my old answers.
If the data you are writing to a Cloud Firestore document is just basic data, use one of multibatch.create(), multibatch.delete(), multibatch.set(), or multibatch.update(). Each time one of these is called, the internal operations counter is increased by 1.
If the data you are writing to Cloud Firestore contains any FieldValue
transforms, use one of multibatch.transformCreate(), multibatch.transformDelete(), multibatch.transformSet(), or multibatch.transformUpdate(). Each time one of these is called, the internal operations counter is increased by 2.
Once the internal counter exceeds 500, it automatically starts a new batched write and adds it to it's internal list.
When you've queued up all your data ready to send off to Firestore, call multibatch.commit().
console.log("Fetching data from third-party server...")
const response = await fetch(dataUrl, settings)
const json = await response.json()
console.log("Data obtained. Parsing as Firestore documents...")
const batch = new MultiBatch(db)
json["records"].forEach(toiletJsonObject => {
delete toiletJsonObject["fields"]["geo_shape"]
toiletJsonObject["fields"]["adresse"] = formatAddress(toiletJsonObject["fields"]["adresse"])
console.log("after updating adresse field: " + toiletJsonObject["fields"].toString())
batch.set(db.collection("toilets").doc(toiletJsonObject["recordid"]), toiletJsonObject["fields"])
})
console.log("Finished parsing. Committing data to Firestore...")
const results = await batch.commit() // see notes about MultiBatch#commit()
console.log("Finished data upload!")
return results;
import { firestore } from "firebase-admin";
/**
* Helper class to compile an expanding `firestore.WriteBatch`.
*
* Using an internal operations counter, this class will automatically start a
* new `firestore.WriteBatch` instance when it detects it has hit the operations
* limit of 500. Once prepared, you can commit the batches together.
*
* Note: `FieldValue` transform operations such as `serverTimestamp`,
* `arrayUnion`, `arrayRemove`, `increment` are counted as two operations. If
* your written data makes use of one of these, you should use the appropriate
* `transformCreate`, `transformSet` or `transformUpdate` method so that the
* internal counter is correctly increased by 2 (the normal versions only
* increase the counter by 1).
*
* If not sure, just use `delete`, `transformCreate`, `transformSet`, or
* `transformUpdate` functions for every operation as this will make sure you
* don't exceed the limit.
*
* #author Samuel Jones [MIT License] (#samthecodingman)
* #see https://stackoverflow.com/a/66692467/3068190
* #see https://firebase.google.com/docs/firestore/manage-data/transactions
* #see https://firebase.google.com/docs/reference/js/firebase.firestore.FieldValue
*/
export class MultiBatch {
constructor(dbRef) {
this.dbRef = dbRef;
this.committed = false;
this.currentBatch = this.dbRef.batch();
this.currentBatchOpCount = 0;
this.batches = [this.currentBatch];
}
_getCurrentBatch(count) {
if (this.committed) throw new Error("MultiBatch already committed.");
if (this.currentBatchOpCount + count > 500) {
// operation limit exceeded, start a new batch
this.currentBatch = this.dbRef.batch();
this.currentBatchOpCount = 0;
this.batches.push(this.currentBatch);
}
this.currentBatchOpCount += count;
return this.currentBatch;
}
/** Creates the document, fails if it exists. */
create(ref, data) {
this._getCurrentBatch(1).create(ref, data);
return this;
}
/**
* Creates the document, fails if it exists.
*
* Used for commands that contain serverTimestamp, arrayUnion, etc
*/
transformCreate(ref, data) {
this._getCurrentBatch(2).create(ref, data);
return this;
}
/** Writes the document, creating/overwriting/etc as applicable. */
set(ref, data, options = undefined) {
this._getCurrentBatch(1).set(ref, data, options);
return this;
}
/**
* Writes the document, creating/overwriting/etc as applicable.
*
* Used for commands that contain serverTimestamp, arrayUnion, etc
*/
transformSet(ref, data, options = undefined) {
this._getCurrentBatch(2).set(ref, data, options);
return this;
}
/** Merges data into the document, failing if the document doesn't exist. */
update(ref, data, ...fieldsOrPrecondition) {
this._getCurrentBatch(1).update(ref, data, ...fieldsOrPrecondition);
return this;
}
/**
* Merges data into the document, failing if the document doesn't exist.
*
* Used for commands that contain serverTimestamp, arrayUnion, etc
*/
transformUpdate(ref, data, ...fieldsOrPrecondition) {
this._getCurrentBatch(2).update(ref, data, ...fieldsOrPrecondition);
return this;
}
/** Used when for basic update operations */
delete(ref) {
this._getCurrentBatch(1).delete(ref);
return this;
}
/**
*
* Commits all of the batches to Firestore.
*
* Note: Unlike normal batch operations, this may cause one or more atomic
* writes. One batch may succeed where others fail. By default, if any batch
* fails, it will fail the whole promise. This can be suppressed by passing in
* a truthy value as the first argument and checking the results returned by
* this method.
*
* #param {boolean} [suppressErrors=false] Whether to suppress errors on a
* per-batch basis.
* #return {firestore.WriteResult[]} array containing an array of
* `WriteResult` objects (and error-batch pairs if `suppressErrors=true`),
* for each batch.
*/
commit(suppressErrors = false) {
this.committed = true;
const mapCallback = suppressErrors
? (batch) => batch.commit().catch((error) => ({ error, batch }))
: (batch) => batch.commit();
return Promise.all(this.batches.map(mapCallback));
}
}