As Firebase’s Firestore only allows 10 items at each query. I need to recursively query users in batches of 10 users.
This is what I've done referencing from Composing Sagas
, but users returns a list of undefined. What have I done wrong? Or is there a better (more efficient) way to query a list of users from Firestore? Note: I have to use Redux-Saga.
export function* watchGetListUsersByUids() {
yield takeEvery(USERS_GET_USERS_BY_UIDS, getListUsersByUids);
}
const getListUsersByUidsAsync = async (list_uid, set) => {
var get_uids = list_uid.slice(set * 10, (set + 1) * 10);
console.log("get_uids", get_uids);
await firestore
.collection("users")
.where("uid", "in", get_uids)
.get()
.then((data) => data.docs.map((doc) => doc.data()))
.catch((error) => error);
};
function* getListUsersByUids({ payload }) {
const { uids } = payload;
var numSet = parseInt(uids.length / 10);
var apiCalls = [];
for (var i = 0; i <= numSet; i++) {
apiCalls.push(call(getListUsersByUidsAsync, uids, i));
}
if (apiCalls.length) {
try {
const users = yield all(apiCalls);
console.log(users); // <-- [undefined, undefined, ...]
} catch (error) {
console.log(error);
}
}
}
Related
I'm trying to make changes to several documents in a cloud function once I receive a callback. My code was working when I only had to update one document, but now I need to update several documents atomically in the same function.
I need to read a certain document and then update other documents based on the information held in an array in the original document. I tried to do this using forEach but I get this error in the console whether I'm using a transaction or a batched write:
Error: Cannot modify a WriteBatch that has been committed.
at WriteBatch.verifyNotCommitted (/workspace/node_modules/#google-cloud/firestore/build/src/write-batch.js:126:19)
at WriteBatch.update (/workspace/node_modules/#google-cloud/firestore/build/src/write-batch.js:315:14)
at loyaltyIds.forEach (/workspace/index.js:323:31)
at process._tickCallback (internal/process/next_tick.js:68:7)
Error: Process exited with code 16
at process.on.code (/layers/google.nodejs.functions-framework/functions-framework/node_modules/#google-cloud/functions-framework/build/src/invoker.js:92:22)
at process.emit (events.js:198:13)
at process.EventEmitter.emit (domain.js:448:20)
at process.exit (internal/process/per_thread.js:168:15)
at sendCrashResponse (/layers/google.nodejs.functions-framework/functions-framework/node_modules/#google-cloud/functions-framework/build/src/logger.js:44:9)
at process.on.err (/layers/google.nodejs.functions-framework/functions-framework/node_modules/#google-cloud/functions-framework/build/src/invoker.js:88:44)
at process.emit (events.js:198:13)
at process.EventEmitter.emit (domain.js:448:20)
at emitPromiseRejectionWarnings (internal/process/promises.js:140:18)
at process._tickCallback (internal/process/next_tick.js:69:34)
And what I end up with is the document outside the for loop is updated but the documents inside the for loop are not - which defeats the purpose of an atomic operation.
It also takes a long time to complete the write operation to Firestore. Where am I going wrong?
Below is what I've tried:
Using batched write:
const txDoc = await txRef.get();
if (txDoc.exists) {
console.log('Transaction Document Found');
const userId = txDoc.data().userId;
const loyaltyIds = txDoc.data().loyaltyIds;
const pointsAwardedMap = txDoc.data().pointsAwarded;
let batch = db.batch();
loyaltyIds.forEach(async lpId => {
// There are 2 elements in the loyaltyIds lis
console.log('Inside for loop');
console.log(lpId);
let cardId = 'u_' + userId + '-l_' + lpId; // 'u_$userId-l_$lpId'
let cardRef = db.collection('users').doc(userId).collection('userLoyaltyCards').doc(cardId);
let lpMap = pointsAwardedMap[lpId];
// Get the user LC doc
let cardDoc = await cardRef.get();
if (cardDoc.exists) {
batch.update(cardRef, {
'pointsBalance': cardDoc.data().pointsBalance + lpMap['points'],
'totalSpend': cardDoc.data().totalSpend + txDoc.data().transactionAmount,
'numberOfPurchases': cardDoc.data().numberOfPurchases + 1,
'pointsEarned': cardDoc.data().pointsEarned + lpMap['points'],
'lastPurchaseDate': admin.database.ServerValue.TIMESTAMP,
});
}
});
// Then we update the tx doc
batch.update(txRef, {
transactionCode: `${receiptNo}`,
transactionType: "purchase",
transactionSuccess: true,
}); // only this gets update
console.log('Firebase Transaction success');
return batch.commit();
} else { return null; }
Using transaction operation:
await db.runTransaction(async t => {
const txDoc = await t.get(txRef);
if (txDoc.exists) {
// userId
// For each lp we update the user loyalty card that goes with it
const userId = txDoc.data().userId;
const loyaltyIds = txDoc.data().loyaltyIds;
const pointsAwardedMap = txDoc.data().pointsAwarded;
// What the pointsAwarded map looks like from the transaction:
// var pointsAwarded = {
// lp1: {
// lpName: 'Jeff',
// lpId: 'lp.lpId',
// points: 'points1',
// cashbackPct: 'lp.cashbackPct',
// vendorId: 'lp.vendorId',
// vendorName: 'lp.vendorName',
// },
// lp2: {
// lpName: 'Susan',
// lpId: 'lp.lpId',
// points: 'points2',
// cashbackPct: 'lp.cashbackPct',
// vendorId: 'lp.vendorId',
// vendorName: 'lp.vendorName',
// },
// };
loyaltyIds.forEach(async (lpId) => {
// We update the user loyalty cards
console.log('Inside for loop');
console.log(lpId);
let cardId = 'u_' + userId + '-l_' + lpId; // 'u_$userId-l_$lpId'
let cardRef = db.collection('users').doc(userId).collection('userLoyaltyCards').doc(cardId);
let lpMap = pointsAwardedMap[lpId];
// Get the user LC doc
let cardDoc = await t.get(cardRef);
// We create the initial loyalty card doc without relying on the cloud function
if (cardDoc.exists) {
// Users LC found, we simply update with this transaction
// `${mpesaReceiptNo}`, this is how to add a var as a field value in firestore
t.update(cardRef, {
'pointsBalance': cardDoc.data().pointsBalance + lpMap['points'],
'totalSpend': cardDoc.data().totalSpend + txDoc.data().transactionAmount,
'numberOfPurchases': cardDoc.data().numberOfPurchases + 1,
'pointsEarned': cardDoc.data().pointsEarned + lpMap['points'],
'lastPurchaseDate': admin.database.ServerValue.TIMESTAMP,
});
}
}); // end of loyalty card update loop
// Then we update the transaction doc
console.log('Transaction Document Found')
t.update(txRef, {
transactionCode: `${mpesaReceiptNo}`,
transactionType: "purchase",
transactionSuccess: true,
});
console.log('Firebase Transaction success');
}
});
UPDATE
I've tried to use a normal for loop but I still get the same errors. I even tried to incorporate the batch.commit statement in the loop so it only executes when the loop completes. Still - same errors.
try {
return txRef.get().then( async txDoc => {
if (txDoc.exists) {
const userId = txDoc.data().userId;
const loyaltyIds = txDoc.data().loyaltyIds;
const pointsAwardedMap = txDoc.data().pointsAwarded;
const batch = db.batch();
// loyaltyIds.forEach(lpId => {
for (let i = 0; i < loyaltyIds.length; i++) {
// We update the user loyalty cards
const lpId = loyaltyIds[i];
console.log('Inside for loop');
console.log(lpId);
const cardId = 'u_' + userId + '-l_' + lpId; // 'u_$userId-l_$lpId'
const cardRef = db.collection('users').doc(userId).collection('userLoyaltyCards').doc(cardId);
const lpMap = pointsAwardedMap[lpId];
// Get the user LC doc
cardRef.get().then(cardDoc => {
// We created the initial loyalty card doc without relying on the cloud function
if (cardDoc.exists) {
console.log('Card found');
// Users LC found, we simply update with this transaction
// `${mpesaReceiptNo}`, this is how to add a var as a field value in firestore
batch.update(cardRef, {
'pointsBalance': cardDoc.data().pointsBalance + lpMap['points'],
'totalSpend': cardDoc.data().totalSpend + txDoc.data().transactionAmount,
'numberOfPurchases': cardDoc.data().numberOfPurchases + 1,
'pointsEarned': cardDoc.data().pointsEarned + lpMap['points'],
'lastPurchaseDate': admin.database.ServerValue.TIMESTAMP,
});
}
});
if (i + 1 == loyaltyIds.length) {
console.log('Loyalty card loop complete, now going to update other things and commit the batch.');
// Update the transaction document
batch.update(txRef, {
transactionCode: `${mpesaReceiptNo}`,
transactionType: "purchase",
transactionSuccess: true,
});
console.log('Committing the batch');
return batch.commit();
}
} // end of for loop
} else {
console.log('Transaction Doc not found, terminating function.');
return null;
}
}).then(function () {
console.log("SUCCESS")
return null;
}
).catch(function (error) {
console.log("UNABLE TO EXECUTE TX BATCH");
console.log(error);
// throw new functions.https.HttpsError('unknown', 'An error occurred when trying to sort the posts.');
return null;
});
I think your problem is related to promises. You must await for the batch.commit(), which was not done in your code. No need to use the await for batch.update(), only for the batch.commit().
Usage of the map with the Promise.all is very important here to ensure you await for all the loop operations to be completed.
I updated your code using awaits, I could not test it since I don't have access to your DB, but I think it should solve your problem with the batch.
try {
const txDoc = await txRef.get();
if (txDoc.exists) {
const userId = txDoc.data().userId;
const loyaltyIds = txDoc.data().loyaltyIds;
const pointsAwardedMap = txDoc.data().pointsAwarded;
const batch = db.batch();
await Promise.all(loyaltyIds.map(async (lpId, i) => {
console.log(lpId);
const cardId = 'u_' + userId + '-l_' + lpId; // 'u_$userId-l_$lpId'
const cardRef = db.collection('users').doc(userId).collection('userLoyaltyCards').doc(cardId);
const lpMap = pointsAwardedMap[lpId];
const cardDoc = await cardRef.get();
if (cardDoc.exists) {
batch.update(cardRef, {
'pointsBalance': cardDoc.data().pointsBalance + lpMap['points'],
'totalSpend': cardDoc.data().totalSpend + txDoc.data().transactionAmount,
'numberOfPurchases': cardDoc.data().numberOfPurchases + 1,
'pointsEarned': cardDoc.data().pointsEarned + lpMap['points'],
'lastPurchaseDate': admin.database.ServerValue.TIMESTAMP,
});
}
if (i + 1 == loyaltyIds.length) {
batch.update(txRef, {
transactionCode: `${mpesaReceiptNo}`,
transactionType: "purchase",
transactionSuccess: true,
});
}
}));
await batch.commit();
return null;
} else {
console.log('Transaction Doc not found, terminating function.');
return null;
}
} catch (error) {
console.log(error);
return null;
}
I want to try some code with firestore emulator before using it in production, I want basically to retrieve a collection documents sort them and set them again in the collection:
I have this error while doing a batch delete :
[Error: [firestore/permission-denied] The caller does not have permission to execute the specified operation.]
the code:
useEffect(() => {
(async () => {
await admin_sortUserRanksDB()
})()
}, [])
const admin_sortUserRanksDB = async () => {
const usersData = await admin_getUserDataDBAndClean()
populateUserCollection(usersData)
}
const admin_getUserDataDBAndClean = async () => {
try {
const querySnapshot = await firestore()
.collection('users')
.orderBy('experience_amount', 'desc')
.get();
let rank = 1;
let newDataUsers = [];
for (const user of querySnapshot.docs) {
const userData = user.data();
userData.rank = rank;
newDataUsers.push(userData)
rank++
}
await deleteUserCollection(querySnapshot)
return newDataUsers;
} catch (error) {
if (!__DEV__) {
crashlytics().log(
`error getUserDataDB()
userActions.js ===>> ${error.message}`
);
}
console.log('error getUserDataDB ', error)
return null
}
}
const deleteUserCollection = async (usersQuerySnapshot) => {
// Create a new batch instance
const batch = firestore().batch();
usersQuerySnapshot.forEach(documentSnapshot => {
batch.delete(documentSnapshot.ref);
});
console.log('==============')
return batch.commit();
}
const populateUserCollection = usersData => {
if (usersData) {
const batch = firestore().batch();
usersData.forEach(doc => {
let docRef = firestore()
.collection('users')
.doc(); //automatically generate unique id
batch.set(docRef, doc);
});
batch
.commit()
.catch(error => {
console.log('error populating users', error)
});
}
}
After posting an issue to react-native-firebase repo i was suggested to modify my rules to be open (only locally) and the batch delete worked.
I used the allow read, write: if true in firestore.rules file
link to issue on GitHub
Say I have this minimal database stored in Cloud Firestore. How could I retrieve the names of subCollection1 and subCollection2?
rootCollection {
aDocument: {
someField: { value: 1 },
anotherField: { value: 2 }
subCollection1: ...,
subCollection2: ...,
}
}
I would expect to be able to just read the ids off of aDocument, but only the fields show up when I get() the document.
rootRef.doc('aDocument').get()
.then(doc =>
// only logs [ "someField", "anotherField" ], no collections
console.log( Object.keys(doc.data()) )
)
It is not currently supported to get a list of (sub)collections from Firestore in the client SDKs (Web, iOS, Android).
In server-side SDKs this functionality does exist. For example, in Node.js you'll be after the ListCollectionIds method:
var firestore = require('firestore.v1beta1');
var client = firestore.v1beta1({
// optional auth parameters.
});
// Iterate over all elements.
var formattedParent = client.anyPathPath("[PROJECT]", "[DATABASE]", "[DOCUMENT]", "[ANY_PATH]");
client.listCollectionIds({parent: formattedParent}).then(function(responses) {
var resources = responses[0];
for (var i = 0; i < resources.length; ++i) {
// doThingsWith(resources[i])
}
})
.catch(function(err) {
console.error(err);
});
It seems like they have added a method called getCollections() to Node.js:
firestore.doc(`/myCollection/myDocument`).getCollections().then(collections => {
for (let collection of collections) {
console.log(`Found collection with id: ${collection.id}`);
}
});
This example prints out all subcollections of the document at /myCollection/myDocument
Isn't this detailed in the documentation?
/**
* Delete a collection, in batches of batchSize. Note that this does
* not recursively delete subcollections of documents in the collection
*/
function deleteCollection(db, collectionRef, batchSize) {
var query = collectionRef.orderBy('__name__').limit(batchSize);
return new Promise(function(resolve, reject) {
deleteQueryBatch(db, query, batchSize, resolve, reject);
});
}
function deleteQueryBatch(db, query, batchSize, resolve, reject) {
query.get()
.then((snapshot) => {
// When there are no documents left, we are done
if (snapshot.size == 0) {
return 0;
}
// Delete documents in a batch
var batch = db.batch();
snapshot.docs.forEach(function(doc) {
batch.delete(doc.ref);
});
return batch.commit().then(function() {
return snapshot.size;
});
}).then(function(numDeleted) {
if (numDeleted <= batchSize) {
resolve();
return;
}
// Recurse on the next process tick, to avoid
// exploding the stack.
process.nextTick(function() {
deleteQueryBatch(db, query, batchSize, resolve, reject);
});
})
.catch(reject);
}
This answer is in the docs
Sadly the docs aren't clear what you import.
Based on the docs, my code ended up looking like this:
import admin, { firestore } from 'firebase-admin'
let collections: string[] = null
const adminRef: firestore.DocumentReference<any> = admin.firestore().doc(path)
const collectionRefs: firestore.CollectionReference[] = await adminRef.listCollections()
collections = collectionRefs.map((collectionRef: firestore.CollectionReference) => collectionRef.id)
This is of course Node.js server side code. As per the docs, this cannot be done on the client.
Say I have this minimal database stored in Cloud Firestore. How could I retrieve the names of subCollection1 and subCollection2?
rootCollection {
aDocument: {
someField: { value: 1 },
anotherField: { value: 2 }
subCollection1: ...,
subCollection2: ...,
}
}
I would expect to be able to just read the ids off of aDocument, but only the fields show up when I get() the document.
rootRef.doc('aDocument').get()
.then(doc =>
// only logs [ "someField", "anotherField" ], no collections
console.log( Object.keys(doc.data()) )
)
It is not currently supported to get a list of (sub)collections from Firestore in the client SDKs (Web, iOS, Android).
In server-side SDKs this functionality does exist. For example, in Node.js you'll be after the ListCollectionIds method:
var firestore = require('firestore.v1beta1');
var client = firestore.v1beta1({
// optional auth parameters.
});
// Iterate over all elements.
var formattedParent = client.anyPathPath("[PROJECT]", "[DATABASE]", "[DOCUMENT]", "[ANY_PATH]");
client.listCollectionIds({parent: formattedParent}).then(function(responses) {
var resources = responses[0];
for (var i = 0; i < resources.length; ++i) {
// doThingsWith(resources[i])
}
})
.catch(function(err) {
console.error(err);
});
It seems like they have added a method called getCollections() to Node.js:
firestore.doc(`/myCollection/myDocument`).getCollections().then(collections => {
for (let collection of collections) {
console.log(`Found collection with id: ${collection.id}`);
}
});
This example prints out all subcollections of the document at /myCollection/myDocument
Isn't this detailed in the documentation?
/**
* Delete a collection, in batches of batchSize. Note that this does
* not recursively delete subcollections of documents in the collection
*/
function deleteCollection(db, collectionRef, batchSize) {
var query = collectionRef.orderBy('__name__').limit(batchSize);
return new Promise(function(resolve, reject) {
deleteQueryBatch(db, query, batchSize, resolve, reject);
});
}
function deleteQueryBatch(db, query, batchSize, resolve, reject) {
query.get()
.then((snapshot) => {
// When there are no documents left, we are done
if (snapshot.size == 0) {
return 0;
}
// Delete documents in a batch
var batch = db.batch();
snapshot.docs.forEach(function(doc) {
batch.delete(doc.ref);
});
return batch.commit().then(function() {
return snapshot.size;
});
}).then(function(numDeleted) {
if (numDeleted <= batchSize) {
resolve();
return;
}
// Recurse on the next process tick, to avoid
// exploding the stack.
process.nextTick(function() {
deleteQueryBatch(db, query, batchSize, resolve, reject);
});
})
.catch(reject);
}
This answer is in the docs
Sadly the docs aren't clear what you import.
Based on the docs, my code ended up looking like this:
import admin, { firestore } from 'firebase-admin'
let collections: string[] = null
const adminRef: firestore.DocumentReference<any> = admin.firestore().doc(path)
const collectionRefs: firestore.CollectionReference[] = await adminRef.listCollections()
collections = collectionRefs.map((collectionRef: firestore.CollectionReference) => collectionRef.id)
This is of course Node.js server side code. As per the docs, this cannot be done on the client.
Say I have this minimal database stored in Cloud Firestore. How could I retrieve the names of subCollection1 and subCollection2?
rootCollection {
aDocument: {
someField: { value: 1 },
anotherField: { value: 2 }
subCollection1: ...,
subCollection2: ...,
}
}
I would expect to be able to just read the ids off of aDocument, but only the fields show up when I get() the document.
rootRef.doc('aDocument').get()
.then(doc =>
// only logs [ "someField", "anotherField" ], no collections
console.log( Object.keys(doc.data()) )
)
It is not currently supported to get a list of (sub)collections from Firestore in the client SDKs (Web, iOS, Android).
In server-side SDKs this functionality does exist. For example, in Node.js you'll be after the ListCollectionIds method:
var firestore = require('firestore.v1beta1');
var client = firestore.v1beta1({
// optional auth parameters.
});
// Iterate over all elements.
var formattedParent = client.anyPathPath("[PROJECT]", "[DATABASE]", "[DOCUMENT]", "[ANY_PATH]");
client.listCollectionIds({parent: formattedParent}).then(function(responses) {
var resources = responses[0];
for (var i = 0; i < resources.length; ++i) {
// doThingsWith(resources[i])
}
})
.catch(function(err) {
console.error(err);
});
It seems like they have added a method called getCollections() to Node.js:
firestore.doc(`/myCollection/myDocument`).getCollections().then(collections => {
for (let collection of collections) {
console.log(`Found collection with id: ${collection.id}`);
}
});
This example prints out all subcollections of the document at /myCollection/myDocument
Isn't this detailed in the documentation?
/**
* Delete a collection, in batches of batchSize. Note that this does
* not recursively delete subcollections of documents in the collection
*/
function deleteCollection(db, collectionRef, batchSize) {
var query = collectionRef.orderBy('__name__').limit(batchSize);
return new Promise(function(resolve, reject) {
deleteQueryBatch(db, query, batchSize, resolve, reject);
});
}
function deleteQueryBatch(db, query, batchSize, resolve, reject) {
query.get()
.then((snapshot) => {
// When there are no documents left, we are done
if (snapshot.size == 0) {
return 0;
}
// Delete documents in a batch
var batch = db.batch();
snapshot.docs.forEach(function(doc) {
batch.delete(doc.ref);
});
return batch.commit().then(function() {
return snapshot.size;
});
}).then(function(numDeleted) {
if (numDeleted <= batchSize) {
resolve();
return;
}
// Recurse on the next process tick, to avoid
// exploding the stack.
process.nextTick(function() {
deleteQueryBatch(db, query, batchSize, resolve, reject);
});
})
.catch(reject);
}
This answer is in the docs
Sadly the docs aren't clear what you import.
Based on the docs, my code ended up looking like this:
import admin, { firestore } from 'firebase-admin'
let collections: string[] = null
const adminRef: firestore.DocumentReference<any> = admin.firestore().doc(path)
const collectionRefs: firestore.CollectionReference[] = await adminRef.listCollections()
collections = collectionRefs.map((collectionRef: firestore.CollectionReference) => collectionRef.id)
This is of course Node.js server side code. As per the docs, this cannot be done on the client.