For some reason, when the following scheduled function runs, it only writes a rules.json file to the Cloud Storage bucket, containing some of the rules in our database.
export const generateBackupScheduledFunction = functions.pubsub.schedule('every 1 days').onRun((context) => {
return generateBackup();
});
async function generateBackup() {
const client = new firestore.v1.FirestoreAdminClient()
const projectId = "project-id"
const databaseName = `projects/${projectId}/databases/(default)`
return client.exportDocuments({
name: databaseName,
outputUriPrefix: `gs://${projectId}-backups`,
collectionIds: []
});
}
However, when I run this function from the Firebase functions shell, or Cloud Functions section in Google Cloud, it runs just fine, and creates a backup of our database.
Any idea why this might be happening?
Related
I'm trying to setup automatic backup of my Firestore using instructions here: https://firebase.google.com/docs/firestore/solutions/schedule-export
I get error:
firestoreExpert
g2o6pmdwatdp
TypeError: Cannot read properties of undefined (reading 'charCodeAt')
at peg$parsetemplate (/workspace/node_modules/google-gax/build/src/pathTemplateParser.js:304:17)
at Object.peg$parse [as parse] (/workspace/node_modules/google-gax/build/src/pathTemplateParser.js:633:18)
at new PathTemplate (/workspace/node_modules/google-gax/build/src/pathTemplate.js:55:54)
Any suggestions to debug this?
I've tried looking for errors in my permissions. E.g. I don't know how to check if the service has access to the specific bucket, although the GCL ran OK.
I've also tried looking for errors in the script.
index.js
const firestore = require('#google-cloud/firestore');
const client = new firestore.v1.FirestoreAdminClient();
// Replace BUCKET_NAME
const bucket = 'gs://EDITEDHERE.appspot.com'
exports.scheduledFirestoreExport = (event, context) => {
const databaseName = client.databasePath(
process.env.GCLOUD_PROJECT,
'(default)'
);
return client
.exportDocuments({
name: databaseName,
outputUriPrefix: bucket,
// Leave collectionIds empty to export all collections
// or define a list of collection IDs:
// collectionIds: ['users', 'posts']
collectionIds: [],
})
.then(responses => {
const response = responses[0];
console.log(`Operation Name: ${response['name']}`);
return response;
})
.catch(err => {
console.error(err);
});
};
and package.json
{
"dependencies": {
"#google-cloud/firestore": "^1.3.0"
}
}
I found these great video tutorials
How to schedule firestorm backups and
How To Transfer Firestore Data From One Project To Another
I know that we can now export and import data with the Firestore emulator.
How might one access the imported data in a unit testing environment? That is, when using #firebase/testing, we typically create the test app like so:
const db = firebase.initializeTestApp({ projectId: PROJECT_ID, auth}).firestore();
How can one access the data imported thru the emulator and feed it into this db instance?
My use case is for running unit tests on firestore rules against the imported data (for reference, the desired setup is very similar to this other related question, but instead of relying on a manually built mock object, I'd like to use data coming from the emulator export/import feature)
Here is my test file
/**
* unit tests for firestore security and data validation rules
*
* adapted from https://github.com/firebase/quickstart-testing/blob/master/unit-test-security-rules
*/
const firebase = require("#firebase/testing");
const fs = require("fs");
const path = require("path")
/**
* The emulator will accept any project ID for testing.
*/
const PROJECT_ID = "test-dev-project";
/**
* The FIRESTORE_EMULATOR_HOST environment variable is set automatically
* by "firebase emulators:exec"
*/
const COVERAGE_URL = `http://${process.env.FIRESTORE_EMULATOR_HOST}/emulator/v1/projects/${PROJECT_ID}:ruleCoverage.html`;
/**
* Set up mock data
*/
const mockUser = {
uid: 'alice',
email: 'alice#example.com',
company: 'testCompany'
}
/**
* Creates a new client FirebaseApp with authentication and returns the Firestore db instance.
* #param {object} auth the object to use for authentication (typically {uid: some-uid})
* #param {object} data the test data to use for filling the db
* #return {object} the firestore instance db .
*/
getAuthedDb = async (auth, data) => {
// initialize test app
const app = firebase.initializeTestApp({ projectId: PROJECT_ID, auth });
let db = app.firestore();
return db;
};
beforeEach(async () => {
// Clear the database between tests
await firebase.clearFirestoreData({ projectId: PROJECT_ID });
});
before(async () => {
// Load the rules file before the tests begin
const rules = fs.readFileSync("firestore.rules", "utf8");
await firebase.loadFirestoreRules({ projectId: PROJECT_ID, rules });
});
after(async () => {
// Delete all the FirebaseApp instances created during testing
// Note: this does not affect or clear any data
await Promise.all(firebase.apps().map((app) => app.delete()));
console.log(`View rule coverage information at ${COVERAGE_URL}\n`);
});
function printDoc(doc) {
if (doc.exists) {
console.log(`Document ${doc.id} data:`, doc.data());
} else {
// doc.data() will be undefined in this case
console.log(`No such document: ${doc.ref.path}, parent: ${doc.ref.parent.id}`);
}
}
describe("Test Security Rules", () => {
it("fetches data imported from emulator", async () => {
const db = await getAuthedDb({uid: 'abcd'});
const ref = db.doc("testdata/ac7yWTZekvGJMQj6Zq3M");
await firebase.assertSucceeds(ref.get().then(printDoc));
});
});
When I run firebase emulators:exec --only firestore --import=data/ "yarn run test-rules" (test-rules maps to mocha --timeout 10000 test/test.spec.js), the assert succeeds but the doc "testdata/ac7yWTZekvGJMQj6Zq3M" is not found, yet it exists in the data I'd imported from data/
Turns out this is the problem:
beforeEach(async () => {
// Clear the database between tests
await firebase.clearFirestoreData({ projectId: PROJECT_ID });
});
Before each test is executed it’s going to fully clear out the emulators data. Big mistake on my part, the solution was to remove that code altogether, since my test data doesn't need to change between test cases
I have multiple database instances in my firebase app. I am trying to write into three database instances in firebase cloud functions. My understanding by following this document is no need to initialize multiple apps for each database instance. We can initialize one and pass in the database url. As a side note, I have another function with similar kind of functionality where I have trigger event in one database and write data to other database instance and it works fine.
import * as functions from "firebase-functions";
import * as admin from "firebase-admin";
const app = admin.app();
export const onStart =
functions.database.instance('my-db-1')
.ref('path')
.onCreate(async (snapshot, context) => {
return await onCreate('my-db-1',snapshot,context);
});
export const onStartDb01 = functions.database.instance('my-db-2')
.ref('path')
.onCreate(async (snapshot, context) => {
return await onCreate('my-db-2', snapshot, context);
});
async function onCreate(dbInstance: string, snapshot:
functions.database.DataSnapshot, context: functions.EventContext):
Promise<any> {
const defaultDb = app.database(defaultDbUrl);
const actvDb = app.database(actvDbUrl);
await defaultDb.ref('path')
.once("value")
.then(snap => {
const val = snap.val();
---do something and write back---
});
await actvDb.ref('path')
.once("value")
.then(snap => {
const val = snap.val();
---do something and write back---
});
return true;
}
But when a db event is fired, it logs the error as below
Error: FIREBASE FATAL ERROR: Database initialized multiple times. Please make sure the format of the database URL matches with each database() call.
You'll need to initialize a separate app() for each database instance.
Based on Doug's answer here that should be something like this:
const app1 = admin.initializeApp(functions.config().firebase)
const app2 = admin.initializeApp(functions.config().firebase)
And then:
const defaultDb = app1.database(defaultDbUrl);
const actvDb = app2.database(actvDbUrl);
I am developing a firebase cloud function that writes to a firestore database.
During development I want the function to write to a local database. So I've started a firestore emulator. But the data is still written to the actual database.
How can I configure the cloud functions to use the local database?
This is my setup:
import * as functions from 'firebase-functions';
import * as cors from "cors";
import * as admin from "firebase-admin";
const REGION = "europe-west1";
const COLLECTION_CONTACT_FORM = "contact_form";
const serviceAccount = require("../keys/auth-key.json");
admin.initializeApp({
credential: admin.credential.cert(serviceAccount)
});
const corsMiddleware = cors({origin: true});
export const sendContactForm = functions.region(REGION).https.onRequest((request, response) => corsMiddleware(request, response, async () => {
let {text} = request.body;
let result = await admin.firestore().collection(COLLECTION_CONTACT_FORM).add({text});
response.send((result.id));
}));
This is the console output when starting the emulator:
[1] i firestore: Serving WebChannel traffic on at http://localhost:8081
[1] i firestore: Emulator logging to firestore-debug.log
[1] ✔ functions: Emulator started at http://localhost:5000
[1] ✔ firestore: Emulator started at http://localhost:8080
[1] i functions: Watching "path/functions" for Cloud Functions...
[1] ⚠ functions: Your GOOGLE_APPLICATION_CREDENTIALS environment variable points to path/keys/auth-key.json. Non-emulated services will access production using these credentials. Be careful!
[1] ✔ functions[sendContactForm]: http function initialized (http://localhost:5000/project/europe-west1/sendContactForm).
When triggering the local endpoint, the production database is written to.
The firestore admin initializeApp() will correctly handle switching between local emulator and production database depending on where it is running. So if you simply remove the service account credentials it should work properly:
import * as functions from 'firebase-functions';
import * as cors from "cors";
import * as admin from "firebase-admin";
const REGION = "europe-west1";
const COLLECTION_CONTACT_FORM = "contact_form";
admin.initializeApp();
const corsMiddleware = cors({origin: true});
export const sendContactForm = functions.region(REGION).https.onRequest((request, response) => corsMiddleware(request, response, async () => {
let {text} = request.body;
let result = await admin.firestore().collection(COLLECTION_CONTACT_FORM).add({text});
response.send((result.id));
}));
But if for some reason you're trying to write to a firestore database outside of the one that the project is created in, you can use firestore/grpc separately from the firebase classes and then use the environment to either include your service account credentials or location emulator credentials. A local emulator example:
const {Firestore} = require('#google-cloud/firestore');
const {credentials} = require('#grpc/grpc-js');
const db = new Firestore({
projectId: 'my-project-id',
servicePath: 'localhost',
port: 5100,
sslCreds: credentials.createInsecure(),
customHeaders: {
"Authorization": "Bearer owner"
}
});
await db.collection("mycollection").doc("someid").set({ test: "value" });
Same answer, but with the docId set dynamically.
exports.makeUppercase = functions.firestore.document('Messages/{docId}').onCreate((snap, context) => {
const original = snap.data().original;
functions.logger.log('Uppercasing', context.params.docId, original);
const uppercase = original.toUpperCase();
// return snap.ref.set({ uppercase }, { merge: true });
return admin.firestore().collection('AnotherCollection').doc(context.params.docId).set({ uppercase }, { merge: true });
});
This grabs the docId that was set dynamically and uses it to write to a document with the same name but in a different collection.
Also I left in commented code for writing to the same document in the same collection. Beware that using onUpdate or onWrite instead of onCreate makes an infinite loop as each write triggers the function again!
I did Google around and tried some code but didn't work, since every time I deploy cloud functions to firebase, it takes about 30 secs - 1 min, I think it's a complete waste of time if I continued to try code from the internet
So, I need to write a cloud function like this:
const admin = require('firebase-admin');
module.exports = function (request, response) {
const { message } = request.body;
// Now, store `message` into firebase storage
// path is: /messages/new_message, where `new_message`
// is NOT a folder, but the file that contains `message`
}
I do have a solution, but obviously, it's not a wise choice, I mean, I can always install firebase package, then call initializeApp(...), then firebase.storage().ref().... Is there another way to do this? Could you please write a little code to elaborate it?
You'll want to use the #google-cloud/storage module.
// Creates a GCS client,
const storage = new Storage();
module.exports = function (req, res) {
const { message } = req.body;
const bucket = storage .bucket('projectid.appspot.com');
const file = bucket.file('myFolder/myFilename');
// gcloud supports upload(file) not upload(bytes), so we need to stream.
const uploadStream = file.createWriteStream();
.on('error', (err) => {
res.send(err);
}).on('finish', () => {
res.send('ok');
}
uploadStream.write(data);
uploadStream.end();
}
See my parse-server GCS adapter for an example.