Lambda nodeJS 4.3 not finishing/executing success callback - firebase

Despite running the log statement immediately above it, my call to callback(null) isn't working. Even tried wrapping it in a try catch block but got nothing.
For reference, here's the full function:
var Firebase = require('firebase');
var request = require('request');
//noinspection AnonymousFunctionJS
/**
*
* #param event - from Lambda
* #param context - from Lambda
* #param callback - from Lambda
*/
exports.handler = function (event, context, callback) {
var AUTOPILOT_API_KEY = getKey(event.stage, 'AUTOPILOT_API_KEY');
var AUTOPILOT_JOURNEY_ID = getKey(event.stage, 'AUTOPILOT_JOURNEY_ID');
activate();
function activate () {
console.log('event:', event);
if (validPayload(event, context)) {
addDefaultPresets(event.uid);
addToAutopilot(event.user, event.uid);
}
}
/**
* checks that the necessary payload has been received
* if YES: returns true and allows process to continue
* if NO: throws context.fail with useful error message(s)
* operating under custom error code naming convention of
* http code + 3 digit ULM error code
* #param event - from Lambda
* #param context - from Lambda
* #returns {boolean} - whether the payload contains the required data
*/
function validPayload (event, context) {
return true; // REDACTED FOR BREVITY
}
/**
* Adds the user to Autopilot as a contact and adds them
* to the journey with trigger id 0001
* #param {Object} user
* #param {string} uid generate by Firebase as unique identifier of registered user
*/
function addToAutopilot (user, uid) {
// REDACTED FOR BREVITY
request({
method: 'POST',
url: 'https://api2.autopilothq.com/v1/trigger/' + AUTOPILOT_JOURNEY_ID + '/contact',
headers: {
'autopilotapikey': AUTOPILOT_API_KEY,
'Content-Type': 'application/json'
},
body: JSON.stringify(payload)
}, function (error, response, body) {
//noinspection MagicNumberJS
if (response.statusCode !== 200) {
errorResponse.status = response.statusCode;
errorResponse.error = {
errorMessage: error,
user: event.user,
response: response,
body: body
};
console.log('should throw ERROR callback');
context.fail(JSON.stringify(errorResponse));
} else {
console.log('should throw SUCCESS callback');
console.log(JSON.stringify({
status: response.statusCode,
message: "User successfully added to Autopilot account & journey"
}));
callback(null);
}
console.log('Finished addToAutopilot()');
});
}
/**
* Adds a collection of presets the the account of the new user
* #param uid {String} - Firebase UID
*/
function addDefaultPresets (uid) {
// REDACTED FOR BREVITY
var presets = ref.child('users').child(uid).child('presets');
console.log('Starting addDefaultPresets()');
activate();
function activate () {
console.info('activating...');
// for each field
fields.forEach(function (field) {
// iterate over each preset
presetData[field].forEach(function (label) {
// and add to firebase db via addDefaultPreset() if unique
presetIsUnique(field, label);
})
});
console.log('Finished addDefaultPresets()');
}
function presetIsUnique (field, label) {
presets.child(field).orderByChild('label')
.equalTo(label)
.once('value', function (snapshot) {
var val = snapshot.val();
if (!val) {
addDefaultPreset(field, label);
} else {
console.error('already exists', field, label);
}
});
}
function addDefaultPreset (field, label) {
//noinspection MagicNumberJS
presets.child(field).push().set({
creator: 'default',
dateAdded: Math.floor(Date.now() / 1000),
label: label
}, setCallback);
}
function setCallback (err) {
if (err) {
console.error(err);
} else {
console.info('added preset');
}
}
}
function getKey (stage, keyId) {
var keys = {
AUTOPILOT_API_KEY: {
staging: 'dev123',
prod: 'prod123'
},
AUTOPILOT_JOURNEY_ID: {
staging: 'XXX',
product: 'XXXX'
},
FIREBASE_URL: {
staging: 'https://staging.firebaseio.com/',
prod: 'https://prod.firebaseio.com/'
}
};
if (stage === 'prod') {
return keys[keyId][stage];
} else {
return keys[keyId]['staging'];
}
}
};

It seems like firebase is keeping something in the event loop. By default, lambda waits until the event loop is empty before terminating. You can set context.callbackWaitsForEmptyEventLoop = false to tell lambda to terminate the function soon after the callback is called, even if there is still items in the event loop.

After hours and hours of debugging here is something that worked for me:
var firebase = require('firebase');
var app = firebase.initializeApp(config);
...
...
...
app.delete().then(function(){
var response = {
statusCode: 200,
body: JSON.stringify({
message: 'Success'
}),
};
callback(null, response);
});
You can read more about delete in the official documentation:
Make the given App unusable and free the resources of all associated
services.
Hope this is helpful!

Related

Firestore: remove document and collections and same time [duplicate]

How can you delete a Document with all it's collections and nested subcollections? (inside the functions environment)
In the RTDB you can ref.child('../someNode).setValue(null) and that completes the desired behavior.
I can think of two ways you could achieve the desired delete behavior, both with tremendously ghastly drawbacks.
Create a 'Super' function that will spider every document and delete them in a batch.
This function would be complicated, brittle to changes, and might take a lengthy execution time.
Add 'onDelete' triggers for each Document type, and make it delete any direct subcollections. You'll call delete on the root document, and the deletion calls will propagate down the 'tree'. This is sluggish, scales atrociously and is costly due to the colossal load of function executions.
Imagine you would have to delete a 'GROUP' and all it's children. It would be deeply chaotic with #1 and pricey with #2 (1 function call per doc)
groups > GROUP > projects > PROJECT > files > FILE > assets > ASSET
> urls > URL
> members > MEMBER
> questions > QUESTION > answers > ANSWER > replies > REPLY
> comments > COMMENT
> resources > RESOURCE > submissions > SUBMISSION
> requests > REQUEST
Is there a superior/favored/cleaner way to delete a document and all it's nested subcollections?
It ought to be possible considering you can do it from the console.
according to firebase documentation:
https://firebase.google.com/docs/firestore/solutions/delete-collections
Deleting collection with nested subcollections might be done easy and neat with node-JS on the server side.
const client = require('firebase-tools');
await client.firestore
.delete(collectionPath, {
project: process.env.GCLOUD_PROJECT,
recursive: true,
yes: true
});
Unfortunately, your analysis is spot on and indeed this use case does require a lot of ceremony. According to official documentation, there is no support for deep deletes in a single shot in firestore neither via client libraries nor rest-api nor the cli tool.
The cli is open sourced and its implementation lives here: https://github.com/firebase/firebase-tools/blob/master/src/firestore/delete.js. They basically implemented option 1. you described in your question, so you can take some inspiration from there.
Both options 1. and 2. are far from ideal situation and to make your solution 100% reliable you will need to keep a persistent queue with deletion tasks, as any error in the long running procedure will leave your system in some ill-defined state.
I would discourage to go with raw option 2. as recursive cloud function calls may very easily went wrong - for example, hitting max. limits.
In case the link changed, below the full source of https://github.com/firebase/firebase-tools/blob/master/src/firestore/delete.js:
"use strict";
var clc = require("cli-color");
var ProgressBar = require("progress");
var api = require("../api");
var firestore = require("../gcp/firestore");
var FirebaseError = require("../error");
var logger = require("../logger");
var utils = require("../utils");
/**
* Construct a new Firestore delete operation.
*
* #constructor
* #param {string} project the Firestore project ID.
* #param {string} path path to a document or collection.
* #param {boolean} options.recursive true if the delete should be recursive.
* #param {boolean} options.shallow true if the delete should be shallow (non-recursive).
* #param {boolean} options.allCollections true if the delete should universally remove all collections and docs.
*/
function FirestoreDelete(project, path, options) {
this.project = project;
this.path = path;
this.recursive = Boolean(options.recursive);
this.shallow = Boolean(options.shallow);
this.allCollections = Boolean(options.allCollections);
// Remove any leading or trailing slashes from the path
if (this.path) {
this.path = this.path.replace(/(^\/+|\/+$)/g, "");
}
this.isDocumentPath = this._isDocumentPath(this.path);
this.isCollectionPath = this._isCollectionPath(this.path);
this.allDescendants = this.recursive;
this.parent = "projects/" + project + "/databases/(default)/documents";
// When --all-collections is passed any other flags or arguments are ignored
if (!options.allCollections) {
this._validateOptions();
}
}
/**
* Validate all options, throwing an exception for any fatal errors.
*/
FirestoreDelete.prototype._validateOptions = function() {
if (this.recursive && this.shallow) {
throw new FirebaseError("Cannot pass recursive and shallow options together.");
}
if (this.isCollectionPath && !this.recursive && !this.shallow) {
throw new FirebaseError("Must pass recursive or shallow option when deleting a collection.");
}
var pieces = this.path.split("/");
if (pieces.length === 0) {
throw new FirebaseError("Path length must be greater than zero.");
}
var hasEmptySegment = pieces.some(function(piece) {
return piece.length === 0;
});
if (hasEmptySegment) {
throw new FirebaseError("Path must not have any empty segments.");
}
};
/**
* Determine if a path points to a document.
*
* #param {string} path a path to a Firestore document or collection.
* #return {boolean} true if the path points to a document, false
* if it points to a collection.
*/
FirestoreDelete.prototype._isDocumentPath = function(path) {
if (!path) {
return false;
}
var pieces = path.split("/");
return pieces.length % 2 === 0;
};
/**
* Determine if a path points to a collection.
*
* #param {string} path a path to a Firestore document or collection.
* #return {boolean} true if the path points to a collection, false
* if it points to a document.
*/
FirestoreDelete.prototype._isCollectionPath = function(path) {
if (!path) {
return false;
}
return !this._isDocumentPath(path);
};
/**
* Construct a StructuredQuery to find descendant documents of a collection.
*
* See:
* https://firebase.google.com/docs/firestore/reference/rest/v1beta1/StructuredQuery
*
* #param {boolean} allDescendants true if subcollections should be included.
* #param {number} batchSize maximum number of documents to target (limit).
* #param {string=} startAfter document name to start after (optional).
* #return {object} a StructuredQuery.
*/
FirestoreDelete.prototype._collectionDescendantsQuery = function(
allDescendants,
batchSize,
startAfter
) {
var nullChar = String.fromCharCode(0);
var startAt = this.parent + "/" + this.path + "/" + nullChar;
var endAt = this.parent + "/" + this.path + nullChar + "/" + nullChar;
var where = {
compositeFilter: {
op: "AND",
filters: [
{
fieldFilter: {
field: {
fieldPath: "__name__",
},
op: "GREATER_THAN_OR_EQUAL",
value: {
referenceValue: startAt,
},
},
},
{
fieldFilter: {
field: {
fieldPath: "__name__",
},
op: "LESS_THAN",
value: {
referenceValue: endAt,
},
},
},
],
},
};
var query = {
structuredQuery: {
where: where,
limit: batchSize,
from: [
{
allDescendants: allDescendants,
},
],
select: {
fields: [{ fieldPath: "__name__" }],
},
orderBy: [{ field: { fieldPath: "__name__" } }],
},
};
if (startAfter) {
query.structuredQuery.startAt = {
values: [{ referenceValue: startAfter }],
before: false,
};
}
return query;
};
/**
* Construct a StructuredQuery to find descendant documents of a document.
* The document itself will not be included
* among the results.
*
* See:
* https://firebase.google.com/docs/firestore/reference/rest/v1beta1/StructuredQuery
*
* #param {boolean} allDescendants true if subcollections should be included.
* #param {number} batchSize maximum number of documents to target (limit).
* #param {string=} startAfter document name to start after (optional).
* #return {object} a StructuredQuery.
*/
FirestoreDelete.prototype._docDescendantsQuery = function(allDescendants, batchSize, startAfter) {
var query = {
structuredQuery: {
limit: batchSize,
from: [
{
allDescendants: allDescendants,
},
],
select: {
fields: [{ fieldPath: "__name__" }],
},
orderBy: [{ field: { fieldPath: "__name__" } }],
},
};
if (startAfter) {
query.structuredQuery.startAt = {
values: [{ referenceValue: startAfter }],
before: false,
};
}
return query;
};
/**
* Query for a batch of 'descendants' of a given path.
*
* For document format see:
* https://firebase.google.com/docs/firestore/reference/rest/v1beta1/Document
*
* #param {boolean} allDescendants true if subcollections should be included,
* #param {number} batchSize the maximum size of the batch.
* #param {string=} startAfter the name of the document to start after (optional).
* #return {Promise<object[]>} a promise for an array of documents.
*/
FirestoreDelete.prototype._getDescendantBatch = function(allDescendants, batchSize, startAfter) {
var url;
var body;
if (this.isDocumentPath) {
url = this.parent + "/" + this.path + ":runQuery";
body = this._docDescendantsQuery(allDescendants, batchSize, startAfter);
} else {
url = this.parent + ":runQuery";
body = this._collectionDescendantsQuery(allDescendants, batchSize, startAfter);
}
return api
.request("POST", "/v1beta1/" + url, {
auth: true,
data: body,
origin: api.firestoreOrigin,
})
.then(function(res) {
// Return the 'document' property for each element in the response,
// where it exists.
return res.body
.filter(function(x) {
return x.document;
})
.map(function(x) {
return x.document;
});
});
};
/**
* Progress bar shared by the class.
*/
FirestoreDelete.progressBar = new ProgressBar("Deleted :current docs (:rate docs/s)", {
total: Number.MAX_SAFE_INTEGER,
});
/**
* Repeatedly query for descendants of a path and delete them in batches
* until no documents remain.
*
* #return {Promise} a promise for the entire operation.
*/
FirestoreDelete.prototype._recursiveBatchDelete = function() {
var self = this;
// Tunable deletion parameters
var readBatchSize = 7500;
var deleteBatchSize = 250;
var maxPendingDeletes = 15;
var maxQueueSize = deleteBatchSize * maxPendingDeletes * 2;
// All temporary variables for the deletion queue.
var queue = [];
var numPendingDeletes = 0;
var pagesRemaining = true;
var pageIncoming = false;
var lastDocName;
var failures = [];
var retried = {};
var queueLoop = function() {
if (queue.length == 0 && numPendingDeletes == 0 && !pagesRemaining) {
return true;
}
if (failures.length > 0) {
logger.debug("Found " + failures.length + " failed deletes, failing.");
return true;
}
if (queue.length <= maxQueueSize && pagesRemaining && !pageIncoming) {
pageIncoming = true;
self
._getDescendantBatch(self.allDescendants, readBatchSize, lastDocName)
.then(function(docs) {
pageIncoming = false;
if (docs.length == 0) {
pagesRemaining = false;
return;
}
queue = queue.concat(docs);
lastDocName = docs[docs.length - 1].name;
})
.catch(function(e) {
logger.debug("Failed to fetch page after " + lastDocName, e);
pageIncoming = false;
});
}
if (numPendingDeletes > maxPendingDeletes) {
return false;
}
if (queue.length == 0) {
return false;
}
var toDelete = [];
var numToDelete = Math.min(deleteBatchSize, queue.length);
for (var i = 0; i < numToDelete; i++) {
toDelete.push(queue.shift());
}
numPendingDeletes++;
firestore
.deleteDocuments(self.project, toDelete)
.then(function(numDeleted) {
FirestoreDelete.progressBar.tick(numDeleted);
numPendingDeletes--;
})
.catch(function(e) {
// For server errors, retry if the document has not yet been retried.
if (e.status >= 500 && e.status < 600) {
logger.debug("Server error deleting doc batch", e);
// Retry each doc up to one time
toDelete.forEach(function(doc) {
if (retried[doc.name]) {
logger.debug("Failed to delete doc " + doc.name + " multiple times.");
failures.push(doc.name);
} else {
retried[doc.name] = true;
queue.push(doc);
}
});
} else {
logger.debug("Fatal error deleting docs ", e);
failures = failures.concat(toDelete);
}
numPendingDeletes--;
});
return false;
};
return new Promise(function(resolve, reject) {
var intervalId = setInterval(function() {
if (queueLoop()) {
clearInterval(intervalId);
if (failures.length == 0) {
resolve();
} else {
reject("Failed to delete documents " + failures);
}
}
}, 0);
});
};
/**
* Delete everything under a given path. If the path represents
* a document the document is deleted and then all descendants
* are deleted.
*
* #return {Promise} a promise for the entire operation.
*/
FirestoreDelete.prototype._deletePath = function() {
var self = this;
var initialDelete;
if (this.isDocumentPath) {
var doc = { name: this.parent + "/" + this.path };
initialDelete = firestore.deleteDocument(doc).catch(function(err) {
logger.debug("deletePath:initialDelete:error", err);
if (self.allDescendants) {
// On a recursive delete, we are insensitive to
// failures of the initial delete
return Promise.resolve();
}
// For a shallow delete, this error is fatal.
return utils.reject("Unable to delete " + clc.cyan(this.path));
});
} else {
initialDelete = Promise.resolve();
}
return initialDelete.then(function() {
return self._recursiveBatchDelete();
});
};
/**
* Delete an entire database by finding and deleting each collection.
*
* #return {Promise} a promise for all of the operations combined.
*/
FirestoreDelete.prototype.deleteDatabase = function() {
var self = this;
return firestore
.listCollectionIds(this.project)
.catch(function(err) {
logger.debug("deleteDatabase:listCollectionIds:error", err);
return utils.reject("Unable to list collection IDs");
})
.then(function(collectionIds) {
var promises = [];
logger.info("Deleting the following collections: " + clc.cyan(collectionIds.join(", ")));
for (var i = 0; i < collectionIds.length; i++) {
var collectionId = collectionIds[i];
var deleteOp = new FirestoreDelete(self.project, collectionId, {
recursive: true,
});
promises.push(deleteOp.execute());
}
return Promise.all(promises);
});
};
/**
* Check if a path has any children. Useful for determining
* if deleting a path will affect more than one document.
*
* #return {Promise<boolean>} a promise that retruns true if the path has
* children and false otherwise.
*/
FirestoreDelete.prototype.checkHasChildren = function() {
return this._getDescendantBatch(true, 1).then(function(docs) {
return docs.length > 0;
});
};
/**
* Run the delete operation.
*/
FirestoreDelete.prototype.execute = function() {
var verifyRecurseSafe;
if (this.isDocumentPath && !this.recursive && !this.shallow) {
verifyRecurseSafe = this.checkHasChildren().then(function(multiple) {
if (multiple) {
return utils.reject("Document has children, must specify -r or --shallow.", { exit: 1 });
}
});
} else {
verifyRecurseSafe = Promise.resolve();
}
var self = this;
return verifyRecurseSafe.then(function() {
return self._deletePath();
});
};
module.exports = FirestoreDelete;
For those who don't want or can't use cloud functions, I found a recursiveDelete function in the admin sdk:
https://googleapis.dev/nodejs/firestore/latest/Firestore.html#recursiveDelete
// Recursively delete a reference and log the references of failures.
const bulkWriter = firestore.bulkWriter();
bulkWriter
.onWriteError((error) => {
if (error.failedAttempts < MAX_RETRY_ATTEMPTS) {
return true;
} else {
console.log('Failed write at document: ', error.documentRef.path);
return false;
}
});
await firestore.recursiveDelete(docRef, bulkWriter);
i don't know how much helpful for you but test it and compare the execution time which i get use it from fire store doc
/** Delete a collection in batches to avoid out-of-memory errors.
* Batch size may be tuned based on document size (atmost 1MB) and application requirements.
*/
void deleteCollection(CollectionReference collection, int batchSize) {
try {
// retrieve a small batch of documents to avoid out-of-memory errors
ApiFuture<QuerySnapshot> future = collection.limit(batchSize).get();
int deleted = 0;
// future.get() blocks on document retrieval
List<QueryDocumentSnapshot> documents = future.get().getDocuments();
for (QueryDocumentSnapshot document : documents) {
document.getReference().delete();
++deleted;
}
if (deleted >= batchSize) {
// retrieve and delete another batch
deleteCollection(collection, batchSize);
}
} catch (Exception e) {
System.err.println("Error deleting collection : " + e.getMessage());
}
}
As mentioned above, you need to write good bit of code for this. For each document that is to be deleted you need to check if it has one or more collections. If it does, then you need to queue those up for deletion too. I wrote the code below to do this. It's not tested to be scalable to large data sets, which is fine for me as I'm using it to clean up after small scale integration tests. If you need something more scalable, feel free to take this as a starting point and play around with batching more.
class FirebaseDeleter {
constructor(database, collections) {
this._database = database;
this._pendingCollections = [];
}
run() {
return new Promise((resolve, reject) => {
this._callback = resolve;
this._database.getCollections().then(collections => {
this._pendingCollections = collections;
this._processNext();
});
});
}
_processNext() {
const collections = this._pendingCollections;
this._pendingCollections = [];
const promises = collections.map(collection => {
return this.deleteCollection(collection, 10000);
});
Promise.all(promises).then(() => {
if (this._pendingCollections.length == 0) {
this._callback();
} else {
process.nextTick(() => {
this._processNext();
});
}
});
}
deleteCollection(collectionRef, batchSize) {
var query = collectionRef;
return new Promise((resolve, reject) => {
this.deleteQueryBatch(query, batchSize, resolve, reject);
});
}
deleteQueryBatch(query, batchSize, resolve, reject) {
query
.get()
.then(snapshot => {
// When there are no documents left, we are done
if (snapshot.size == 0) {
return 0;
}
// Delete documents in a batch
var batch = this._database.batch();
const collectionPromises = [];
snapshot.docs.forEach(doc => {
collectionPromises.push(
doc.ref.getCollections().then(collections => {
collections.forEach(collection => {
this._pendingCollections.push(collection);
});
})
);
batch.delete(doc.ref);
});
// Wait until we know if all the documents have collections before deleting them.
return Promise.all(collectionPromises).then(() => {
return batch.commit().then(() => {
return snapshot.size;
});
});
})
.then(numDeleted => {
if (numDeleted === 0) {
resolve();
return;
}
// Recurse on the next process tick, to avoid
// exploding the stack.
process.nextTick(() => {
this.deleteQueryBatch(query, batchSize, resolve, reject);
});
})
.catch(reject);
}
}
Solution using Node.js Admin SDK
export const deleteDocument = async (doc: FirebaseFirestore.DocumentReference) => {
const collections = await doc.listCollections()
await Promise.all(collections.map(collection => deleteCollection(collection)))
await doc.delete()
}
export const deleteCollection = async (collection: FirebaseFirestore.CollectionReference) => {
const query = collection.limit(100)
while (true) {
const snap = await query.get()
if (snap.empty) {
return
}
await Promise.all(snap.docs.map(doc => deleteDocument(doc.ref)))
}
}
There is now a simple way delete a document and all of its subcollections using NodeJS.
This was made available in nodejs-firestore version v4.11.0.
From the docs:
recursiveDelete()
Recursively deletes all documents and subcollections at and under the specified level.
import * as admin from 'firebase-admin'
const ref = admin.firestore().doc('my_document')
admin.firestore().recursiveDelete(ref)
You can write a handler which will recursive delete all nested descendants when triggers onDelete Firestore event.
Example of handler:
const deleteDocumentWithDescendants = async (documentSnap: FirebaseFirestore.QueryDocumentSnapshot) => {
return documentSnap.ref.listCollections().then((subCollections) => {
subCollections.forEach((subCollection) => {
return subCollection.get().then((snap) => {
snap.forEach((doc) => {
doc.ref.delete();
deleteDocumentWithDescendants(doc);
});
});
});
});
};
// On any document delete
export const onDocumentDelete = async (documentSnap: FirebaseFirestore.QueryDocumentSnapshot) => {
await deleteDocumentWithDescendants(documentSnap);
};
Tie it up with firestore event:
exports.onDeleteDocument = functions.firestore.document('{collectionId}/{docId}')
.onDelete(onDocumentDelete);
// You can add all the collection hierarchy to object
private collectionsHierarchy = {
groups: [
[
'groups',
'projects',
'files',
'assets',
'urls',
'members'
]
]
};
async deleteDocument(rootDocument: string) {
// if (!rootDocument.startsWith(`groups/${this.groupId()}`)) {
// rootDocument = `groups/${this.groupId()}/${rootDocument}`;
// }
const batchSize: number = 100;
let root = await this.db
.doc(rootDocument)
.get()
.toPromise();
if (!root.exists) {
return;
}
const segments = rootDocument.split('/');
const documentCollection = segments[segments.length - 2];
const allHierarchies = this.collectionsHierarchy[documentCollection];
for (let i = 0; i < allHierarchies.length; i = i + 1) {
const hierarchy = allHierarchies[i];
const collectionIndex = hierarchy.indexOf(documentCollection) + 1;
const nextCollections: [] = hierarchy.slice(collectionIndex);
const stack = [`${root.ref.path}/${nextCollections.shift()}`];
while (stack.length) {
const path = stack.pop();
const collectionRef = this.db.firestore.collection(path);
const query = collectionRef.orderBy('__name__').limit(batchSize);
let deletedIems = await this.deleteQueryBatch(query, batchSize);
const nextCollection = nextCollections.shift();
deletedIems = deletedIems.map(di => `${di}/${nextCollection}`);
stack.push(...deletedIems);
}
}
await root.ref.delete();
}
private async deleteQueryBatch(
query: firebase.firestore.Query,
batchSize: number
) {
let deletedItems: string[] = [];
let snapshot = await query.get();
if (snapshot.size === 0) {
return deletedItems;
}
const batch = this.db.firestore.batch();
snapshot.docs.forEach(doc => {
deletedItems.push(doc.ref.path);
batch.delete(doc.ref);
});
await batch.commit();
if (snapshot.size === 0) {
return deletedItems;
}
const result = await this.deleteQueryBatch(query, batchSize);
return [...deletedItems, ...result];
}
Another solution using Node.js Admin SDK with Batch.
const traverseDocumentRecursively = async (
docRef: FirebaseFirestore.DocumentReference<FirebaseFirestore.DocumentData>,
accumulatedRefs: FirebaseFirestore.DocumentReference<FirebaseFirestore.DocumentData>[],
) => {
const collections = await docRef.listCollections();
if (collections.length > 0) {
for (const collection of collections) {
const snapshot = await collection.get();
for (const doc of snapshot.docs) {
accumulatedRefs.push(doc.ref);
await traverseDocumentRecursively(doc.ref, accumulatedRefs);
}
}
}
};
import { chunk } from 'lodash';
const doc = admin.firestore().collection('users').doc('001');
const accumulatedRefs: FirebaseFirestore.DocumentReference<FirebaseFirestore.DocumentData>[] = [];
await traverseDocumentRecursively(doc, accumulatedRefs);
await Promise.all(
// Each transaction or batch of writes can write to a maximum of 500 documents
chunk(accumulatedRefs, 500).map((chunkedRefs) => {
const batch = admin.firestore().batch();
for (const ref of chunkedRefs) {
batch.delete(ref);
}
return batch.commit();
}),
);
Not sure if this is helpful for anyone here, but I am frequently facing the error "Fatal error deleting docs <list of docs>" when using firebase-tools.firestore.delete method (firebase-tools version 9.22.0).
I am currently handling these deletion failures using the returned error message in order to avoid rewriting the code cited at Oleg Bondarenko's answer. It uses admin.firestore to effectively delete the failed docs.
It's a poor solution since it relies on the error message, but at least it doesn't force us to copy the whole FirestoreDelete code to modify just a few lines of it:
firebase_tools.firestore
.delete(path, {
project: JSON.parse(process.env.FIREBASE_CONFIG!).projectId,
recursive: true,
yes: true,
token: getToken(),
})
.catch((err: Error) => {
if (err.name == "FirebaseError") {
// If recursive delete fails to delete some of the documents,
// parse the failures from the error message and delete it manually
const failedDeletingDocs = err.message.match(
/.*Fatal error deleting docs ([^\.]+)/
);
if (failedDeletingDocs) {
const docs = failedDeletingDocs[1].split(", ");
const docRefs = docs.map((doc) =>
firestore.doc(doc.slice(doc.search(/\(default\)\/documents/) + 19))
);
firestore
.runTransaction(async (t) => {
docRefs.forEach((doc) => t.delete(doc));
return docs;
})
.then((docs) =>
console.log(
"Succesfully deleted docs after failing: " + docs.join(", ")
)
)
.catch((err) => console.error(err));
}
}
});
If you are looking to delete user data, a solution to consider in 2022 is the Delete User Data Firebase Extension.
Once this is active, you can simply delete the user from Firebase Auth to trigger the recursive deletion of the user documents:
import admin from "firebase-admin";
admin.auth().deleteUser(userId);
You can call firebase.firestore().doc("whatever").set() and that will delete everything in that document.
The only way .set does not erase everything is if you set the merge flag to true.
See Firestore Documentation on Add Data
var cityRef = db.collection('cities').doc('BJ');
var setWithMerge = cityRef.set({
capital: true
}, { merge: true });

Modifying function onDndUploadDrop in _AlfDndDocumentUploadMixin.js file

I have alfresco share/repo version 5.2.3
I'm trying to modifiy the function found in _AlfDndDocumentUploadMixin.js with the fix that has been posted on https://github.com/Alfresco/Aikau/blob/1.0.101_hotfixes/aikau/src/main/resources/alfresco/documentlibrary/_AlfDndDocumentUploadMixin.js
I tried extending the attachment-doclib.xml and get the jsonmodul, but it gives me a null pointer.
My code is the following
From attachment-doclib.get.js
var documentServices = model.jsonModel;
for (var i=0; i<documentServices.length; i++)
{
if (documentServices[i] === "alfresco/documentlibrary")
{
documentServices[i] = "js/aikau/1.0.101.10/alfresco/documentlibrary/my-documentlibrary/_AlfDndDocumentUploadMixin-extension";
}
else if (documentServices[i].name === "alfresco/documentlibrary")
{
documentServices[i].name = "js/aikau/1.0.101.10/alfresco/documentlibrary/my-documentlibrary/_AlfDndDocumentUploadMixin-extension";
}
}
from doclib-customizations.xml
<config evaluator="string-compare" condition="WebFramework" replace="false">
<web-framework>
<dojo-pages>
<packages>
<package name="documentlibrary" location="js/aikau/1.0.101.10/alfresco/documentlibrary" />
</packages>
</dojo-pages>
</web-framework>
</config>
From _alfDndDocumentUploadMixin-extended.js
onDndUploadDrop: function alfresco_documentlibrary__AlfDndDocumentUploadMixin__onDndUploadDrop(evt) {
try
{
// Only perform a file upload if the user has *actually* dropped some files!
this.alfLog("log", "Upload drop detected", evt);
if (evt.dataTransfer.files !== undefined && evt.dataTransfer.files !== null && evt.dataTransfer.files.length > 0)
{
this.removeDndHighlight();
var destination = this._currentNode ? this._currentNode.nodeRef : null;
var config = this.getUploadConfig();
var defaultConfig = {
destination: destination,
siteId: null,
containerId: null,
uploadDirectory: null,
updateNodeRef: null,
description: "",
overwrite: false,
thumbnails: "doclib",
username: null
};
var updatedConfig = lang.mixin(defaultConfig, config);
var walkFileSystem = lang.hitch(this, function alfresco_documentlibrary__AlfDndDocumentUploadMixin__onDndUploadDrop__walkFileSystem(directory, callback, error) {
callback.limit = this.dndMaxFileLimit;
callback.pending = callback.pending || 0;
callback.files = callback.files || [];
// get a dir reader and cleanup file path
var reader = directory.createReader(),
relativePath = directory.fullPath.replace(/^\//, "");
var repeatReader = function alfresco_documentlibrary__AlfDndDocumentUploadMixin__onDndUploadDrop__walkFileSystem__repeatReader() {
// about to start an async callback function
callback.pending++;
reader.readEntries(function alfresco_documentlibrary__AlfDndDocumentUploadMixin__onDndUploadDrop__walkFileSystem__repeatReader__readEntries(entries) {
// processing an async callback function
callback.pending--;
array.forEach(entries, function(entry) {
if (entry.isFile)
{
// about to start an async callback function
callback.pending++;
entry.file(function(File) {
// add the relativePath property to each file - this can be used to rebuild the contents of
// a nested tree folder structure if an appropriate API is available to do so
File.relativePath = relativePath;
callback.files.push(File);
if (callback.limit && callback.files.length > callback.limit)
{
throw new Error("Maximum dnd file limit reached: " + callback.limit);
}
// processing an async callback function
if (--callback.pending === 0)
{
// fall out here if last item processed is a file entry
callback(callback.files);
}
}, error);
}
else
{
walkFileSystem(entry, callback, error);
}
});
// the reader API is a little esoteric,from the MDN docs:
// "Continue calling readEntries() until an empty array is returned.
// You have to do this because the API might not return all entries in a single call."
if (entries.length !== 0)
{
repeatReader();
}
// fall out here if last item processed is a dir entry e.g. empty dir
if (callback.pending === 0)
{
callback(callback.files);
}
}, error);
};
repeatReader();
});
var addSelectedFiles = lang.hitch(this, function alfresco_documentlibrary__AlfDndDocumentUploadMixin__onDndUploadDrop__addSelectedFiles(files) {
if (this.dndMaxFileLimit && files.length > this.dndMaxFileLimit)
{
throw new Error("Maximum dnd file limit reached: " + this.dndMaxFileLimit);
}
// Check to see whether or not the generated upload configuration indicates
// that an existing node will be created or not. If node is being updated then
// we need to generate an intermediary step to capture version and comments...
if (updatedConfig.overwrite === false)
{
// Set up a response topic for receiving notifications that the upload has completed...
var responseTopic = this.generateUuid();
this._uploadSubHandle = this.alfSubscribe(responseTopic, lang.hitch(this, this.onFileUploadComplete), true);
this.alfPublish(topics.UPLOAD_REQUEST, {
alfResponseTopic: responseTopic,
files: files,
targetData: updatedConfig
}, true);
}
else
{
// TODO: Check that only one file has been dropped and issue error...
this.publishUpdateRequest(updatedConfig, files);
}
});
var items = evt.dataTransfer.items || [], firstEntry;
// webkitGetAsEntry is a marker for determining FileSystem API support.
// SHA-2164 - Firefox claims support, but different impl. rather than code around differences, fallback.
if (items[0] && items[0].webkitGetAsEntry && !has("ff") && (firstEntry = items[0].webkitGetAsEntry()))
{
walkFileSystem(firstEntry.filesystem.root, function(files) {
addSelectedFiles(files);
}, function() {
// fallback to standard way if error happens
addSelectedFiles(evt.dataTransfer.files);
}
);
}
else
{
// fallback to standard way if no support for filesystem API
addSelectedFiles(evt.dataTransfer.files);
}
}
else
{
this.alfLog("error", "A drop event was detected, but no files were present for upload: ", evt.dataTransfer);
}
}
catch(exception)
{
this.alfLog("error", "The following error occurred when files were dropped onto the Document List: ", exception);
}
// Remove the drag highlight...
this.removeDndHighlight();
// Destroy the overlay node (required for views that will re-render all the contents)...
domConstruct.destroy(this.dragAndDropOverlayNode);
this.dragAndDropOverlayNode = null;
evt.stopPropagation();
evt.preventDefault();
},
/**
* This function publishes an update version request. It will request that a new dialog
* be displayed containing the form controls defined in
* [widgetsForUpdate]{#link module:alfresco/documentlibrary/_AlfDndDocumentUploadMixin#widgetsForUpdate}.
*
* #instance
* #param {object} uploadConfig
*
* #fires ALF_CREATE_FORM_DIALOG_REQUEST
*/
publishUpdateRequest: function alfresco_documentlibrary__AlfDndDocumentUploadMixin__publishUpdateRequest(uploadConfig, files) {
// TODO: Work out the next minor and major increment versions...
// TODO: Localization required...
// Set up a response topic for receiving notifications that the upload has completed...
var responseTopic = this.generateUuid();
this._uploadSubHandle = this.alfSubscribe(responseTopic, lang.hitch(this, this.onFileUploadComplete), true);
// To avoid the issue with processing payloads containing files with native
// code in them, it is necessary to temporarily store the files in the data model...
var filesRef = this.generateUuid();
this.alfSetData(filesRef, files);
this.alfPublish("ALF_CREATE_FORM_DIALOG_REQUEST", {
dialogTitle: "Update",
dialogConfirmationButtonTitle: "Continue Update",
dialogCancellationButtonTitle: "Cancel",
formSubmissionTopic: topics.UPLOAD_REQUEST,
formSubmissionPayloadMixin: {
alfResponseTopic: responseTopic,
filesRefs: filesRef,
targetData: uploadConfig
},
fixedWidth: true,
widgets: lang.clone(this.widgetsForUpdate)
}, true);
},
Expect: All files that are dragged and drop to be uploaded.
Actual: Only one file is uploaded

How can i start a scheduled script with a button?

i want to create a button in a form that executes a scheduled script but nothing that i try works
I have tried the method addButton to call a function that create a task for the scheluded script but it only execute on form load
/**
*#NApiVersion 2.x
*#NScriptType UserEventScript
*/
define(["N/task", "N/ui/serverWidget"], function(task, serverWidget) {
function beforeLoad(context) {
context.form.addButton({
id: "custpage_setTask",
label: "Execute scheduled",
functionName: executeScheduled()
});
}
return {
beforeLoad: beforeLoad
};
function executeScheduled() {
var scriptTask = task.create({
taskType: task.TaskType.SCHEDULED_SCRIPT,
scriptId: 'customscript_sdr_ss_product_shortage',
deploymentId: 'customdeployprodshortsearch'
});
var scriptTaskId = scriptTask.submit();
log.debug('scriptTaskId', scriptTaskId);
}
});
that execute the scheduled script but only in the form loading and then the button does not do anything, please help and thanks for reading
I managed to do it at the end. It was creating 3 scripts, a user event that calls a client script, that do a get request to a suitelet with the function that creates the task for the scheduled script
This is the User event:
/**
*#NApiVersion 2.x
*#NScriptType UserEventScript
*/
define(["N/task", "N/ui/serverWidget"], function(task, serverWidget) {
function beforeLoad(context) {
// internal id of the client script on file cabinet
context.form.clientScriptFileId = 2343;
context.form.addButton({
id: "custpage_setTask",
label: "Execute scheduled",
functionName: 'redirect'
});
}
return {
beforeLoad: beforeLoad,
};
});
This is the client script:
/**
*#NApiVersion 2.x
*#NScriptType ClientScript
*/
define(["N/url", "N/https"], function(url, https) {
function pageInit(context) {}
function redirect() {
var output = url.resolveScript({
scriptId: "customscript_sss_sl_startschedulescript",
deploymentId: "customdeploy_sss_sl_startschedulescript"
});
log.debug('url', output);
var response = https.get({
url: output
});
log.debug('response', response);
}
return {
pageInit: pageInit,
redirect: redirect
};
});
and this is the suitelet:
/**
*#NApiVersion 2.x
*#NScriptType Suitelet
*/
define(["N/task"], function(task) {
function onRequest(context) {
executeScheduled();
}
function executeScheduled() {
var scriptTask = task.create({
taskType: task.TaskType.SCHEDULED_SCRIPT,
scriptId: "customscript_sdr_ss_product_shortage",
deploymentId: "customdeployprodshortsearch"
});
var scriptTaskId = scriptTask.submit();
log.debug("scriptTaskId", scriptTaskId);
}
return {
onRequest: onRequest
};
});
I hope this helps another one with the same problem.
I believe you also need to export the button function.
return {
beforeLoad: beforeLoad,
executeScheduled:executeScheduled
};
You also need to change your button options
context.form.addButton({
id: "custpage_setTask",
label: "Execute scheduled",
functionName: executeScheduled
});

Server methods outside of Meteor.methods

On the client I need a helper method that returns true or false depending on whether the user is eligible for a payment request.
However, I can't really use a Meteor.method for this, because they don't return a value on the client.
Instead, I have done this and would like to know if this poses any security holes or if there is a preferable approach
Server:
...
// Constants
//
_.extend(Payments, {
MINIMUM_REQUIRED_FOR_REQUEST: 100
});
// Public
//
Meteor.methods({
});
canRequestPayment = function(userId) {
var user = Meteor.users.findOne(userId, { fields: { earnings: 1 } });
if (_.isUndefined(user)) { throw new Meteor.Error('user-not-found', 'User not found'); }
return hasEnoughCreditForRequest(user) && hasNoPendingPayments(user);
};
// Private
//
var hasNoPendingPayments = function(user) {
return Payments.find({ userId: user._id, state: 'pending' }).count() === 0;
};
var hasEnoughCreditForRequest = function(user) {
var period = user.earnings.period;
return period >= Payments.MINIMUM_REQUIRED_FOR_REQUEST;
};
As can be seen, I have created two helper methods with var, to mimic private behavior, and then I have the canRequestPayment method which is accessable outside of the file, and that I call on the client instead of a Meteor.method
Client:
Template.payments.helpers({
eligibleForPaymentRequest: function() {
return canRequestPayment(Meteor.userId());
},

Mongoose pre.save() async middleware not working on record creation

I am using keystone#0.2.32. I would like to change the post category to a tree structure. The below code is running well except when I create a category, it goes into a deadlock:
var keystone = require('keystone'),
Types = keystone.Field.Types;
/**
* PostCategory Model
* ==================
*/
var PostCategory = new keystone.List('PostCategory', {
autokey: { from: 'name', path: 'key', unique: true }
});
PostCategory.add({
name: { type: String, required: true },
parent: { type: Types.Relationship, ref: 'PostCategory' },
parentTree: { type: Types.Relationship, ref: 'PostCategory', many: true }
});
PostCategory.relationship({ ref: 'Post', path: 'categories' });
PostCategory.scanTree = function(item, obj, done) {
if(item.parent){
PostCategory.model.find().where('_id', item.parent).exec(function(err, cats) {
if(cats.length){
obj.parentTree.push(cats[0]);
PostCategory.scanTree(cats[0], obj, done);
}
});
}else{
done();
}
}
PostCategory.schema.pre('save', true, function (next, done) { //Parallel middleware, waiting done to be call
if (this.isModified('parent')) {
this.parentTree = [];
if(this.parent != null){
this.parentTree.push(this.parent);
PostCategory.scanTree(this, this, done);
}else
process.nextTick(done);
}else
process.nextTick(done); //here is deadlock.
next();
});
PostCategory.defaultColumns = 'name, parentTree';
PostCategory.register();
Thanks so much.
As I explained on the issue you logged on Keystone here: https://github.com/keystonejs/keystone/issues/759
This appears to be a reproducible bug in mongoose that prevents middleware from resolving when:
Parallel middleware runs that executes a query, followed by
Serial middleware runs that executes a query
Changing Keystone's autokey middleware to run in parallel mode may cause bugs in other use cases, so cannot be done. The answer is to implement your parentTree middleware in serial mode instead of parallel mode.
Also, some other things I noticed:
There is a bug in your middleware, where the first parent is added to the array twice.
The scanTree method would be better implemented as a method on the schama
You can use the findById method for a simpler parent query
The schema method looks like this:
PostCategory.schema.methods.addParents = function(target, done) {
if (this.parent) {
PostCategory.model.findById(this.parent, function(err, parent) {
if (parent) {
target.parentTree.push(parent.id);
parent.addParents(target, done);
}
});
} else {
done();
}
}
And the fixed middleware looks like this:
PostCategory.schema.pre('save', function(done) {
if (this.isModified('parent')) {
this.parentTree = [];
if (this.parent != null) {
PostCategory.scanTree(this, this, done);
} else {
process.nextTick(done);
}
} else {
process.nextTick(done);
}
});
I think it's a bug of keystone.js. I have changed schemaPlugins.js 104 line
from
this.schema.pre('save', function(next) {
to
this.schema.pre('save', true, function(next, done) {
and change from line 124 to the following,
// if has a value and is unmodified or fixed, don't update it
if ((!modified || autokey.fixed) && this.get(autokey.path)) {
process.nextTick(done);
return next();
}
var newKey = utils.slug(values.join(' ')) || this.id;
if (autokey.unique) {
r = getUniqueKey(this, newKey, done);
next();
return r;
} else {
this.set(autokey.path, newKey);
process.nextTick(done);
return next();
}
It works.

Resources