I'm trying to use firebase functions to do maintenance of my database and storage. Basically remove some old entries from one ref/bucket to another after they expire.
The database part works great. However, the storage part, not so much. Here's how I initialize everything in my code:
var functions = require('firebase-functions');
var admin = require("firebase-admin");
var serviceAccount = require('./my-app-bla-bla.json');
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
databaseURL: 'https://my-app.firebaseio.com',
storageBucket: 'gs://my-app.appspot.com'
});
Then in the cron job that cleans the database and storage, I have the following (this is only some small relevant part):
const st = admin.storage();
st.bucket("gs://my-app.appspot.com/old-listings/"+listingKey).create(function(error, bucket, apiResponse) {
if (error) {
console.log("Couldn't create an OldListing bucket: " + error.code);
console.log(apiResponse);
} else {
console.log("Created OldListing bucket");
}
});
This last piece of code triggers the error and gives me the following log:
Couldn't create an OldListing bucket: 400
{ error:
{ errors: [ [Object] ],
code: 400,
message: 'Invalid bucket name: \'my-app.appspot.com/old-listings/SomeUniqueID\'' } }
Because I'm running this code for the first time, the folder old-listings does not exist yet. So I though maybe I should create its bucket on its own first. It gives me the same error.
I also tried using the buckets without the gs link, e.g. st.bucket("old-listings/"+listingKey) instead of st.bucket("gs://my-app.appspot.com/old-listings/"+listingKey). Still gives me the same error.
so what exactly is missing here? What am I doing wrong?
Edit 1
I tried adding the following code snippet at the beginning of my cron function. In an effort to better understand what's going on.
admin.storage().bucket("my-app.appspot.com").exists(function(error, exists) {
if (!error) {
if (exists) {
console.log("Top Bucket Exists");
} else {
console.log("Top Bucket Does Not Exist");
}
} else {
console.log("Top Bucket Error " + error.code);
}
});
admin.storage().bucket("my-app.appspot.com/listings").exists(function(error, exists) {
if (!error) {
if (exists) {
console.log("Listings Bucket Exists");
} else {
console.log("Listings Bucket Does Not Exist");
}
} else {
console.log("Listings Bucket Error " + error.code);
}
});
I get the following in my log:
Top Bucket Exists
Listing Bucket Error undefined
Of course I already have a folder called listings in my firebase storage. So why on earth would the second bucket be undefined?
When you build a name to a bucket, it's not supposed to contain file path components in it, It should just be the unique name of the bucket - the container for all your objects. If you want to reference a file in the bucket, use the file() method on the bucket object to get a File object to deal with.
const st = admin.storage();
const bucket = st.bucket('name-of-your-bucket');
const file = bucket.file('name-of-your-file');
Related
I've started working with firebase storage and firebase functions recently. Right now I've been developing file upload from functions to storage .
I've got it working (upload is done and file appears on the storage section), yet, the image, stays like this forever (loading forever on the right side):
I though that it was an error from my code. Yet, if I open Google Cloud Platform - Storage, the image appears and I can open it and preview it.
In firebase storage, if I open the image (select on it and click open), it returns the following url: https://console.firebase.google.com/u/0/undefined
What may I been doing wrong? Here's the code I'm using:
function uploadImage() {
const newImageData = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOEAAADhCAMAAAAJbSJIAAAAgVBMVEX///8AAAAEBASAgIDr6+vw8PBYWFjU1NTGxsbz8/P29vb8/Py1tbVhYWHd3d1ra2vk5OS/v78pKSlTU1NOTk6Tk5OpqanNzc13d3dKSkplZWWbm5s5OTkfHx+NjY2GhoYcHBw9PT0TExOioqJ7e3soKCiurq5CQkI6OjoXFxcwMDAuPQWoAAAIJ0lEQVR4nO2daXuqPBCGVfYtIbKLgorLKf//B75ga2sPAdmS8F5n7m/VXgyPISGZzExWKwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAVy6wwlCZG/YUl+vamYLoy0lAkORWbdZNN/YUUVf8ju6bomx2Kq+mEBL5K0UVD9QNCdM0Vfds9QSQMnYTWaN1sEicMCRJ9+29QSCqp28HiftiqUkoU0TLaMDLsl8PbrtGWpY8zQ7QYCoE/pe3+ZusHogW9Yln2nxnVPfljW8t4l8go2zPQV7PPkCxa3kqOvOldr52NF4nVKNuYobxPsC1OoxVilu33ZINDQf1Rw2cO+mrOWBMhMD5w0ldziLnrQyeO+mpOfOdzZspZX03KcQGC2I+gNDC3ZkQ7IQLX6x0niW7fZd83ief7Pg7tH0JcfeIlQy+kcllCyk7/O/LUWIsiDbkVv9/bcv0Rqr+MVa//BR0Ob/++AstdKLuK8n4ZZCiKK4e7cikSez2iTjxm2DPjPj8e8wf1zWviXPqpPuHyeuqXb2ZK6WxaqJCuZe7m7qTTp1da6ty7prtbMoOOVro64VktonlmyFZUqB0NybQrkna7l1ndZIhc2k0xbESjdTEf63M7yBQ9bjO2Z+enamvCXcTCphG1zZ3YNSJ9OroJmRkM6UMOZmXPoppzWE4WEX1oY7Xmv1Fs7Zn9nl9gWt+/MTImNU2dprze+6GfmmYlRrYoHZ9dF/whbJrdMTLVdMzEPDZRlOZr48DIVMMQ2wnUN5SpIiNLjaF7z74X1uiNwWbDyNI/qHCd8XBGW1nDLj+F64K9RKtomuWocMvq3fvDjbIk5ahwvc0YGXuS0dbcPBWu1zlLx4mbU23yVbi+s3srkjvdJGeF1WqGTTO6rZsH3BVWjyqae1C1EP0BFaVwvQ60OR1EshZ0GROisJqG23Pt1Gp2q49GqML1OsmK6aMOKbK3OzbCFFZsLwEZ75YySHDpE1olUmFl/XrP02jE5aM0v1972phd2ydDQkvOnjRk4aFL3pDAjiUo/LoTHOuKLMtGzet+lPn4pPpG0eMRQTnLUfi8oTreWc0j7UmUq22x0f9PhbMDCkEhKBQPKJxNYTkgEmYCXiMShZvCXEkHTUXGcPZSpbFS5KZQWq3Qu4XORGIbUTa9eCqsFqt6OGemxSvbUH8sqwUrXK0sl/gM9PnE/XKNCFdYYZozB7VvsPkzU1+Cwhrrph7mGHfOB/X226+1FIU1duztr+Nlnq97L7YbV12SworITqXdvmcw5QvlfielNtU5sDCFNYpO0iJTk2MvbcdEzYqUtIdVLVDhAwvVubIk8D3Po421j88DUufHvvEnL1XhF2Yd7+xqTR6f9wq1XbjCGQCFs/EPKsR8sq6Vxi4bvxVwzCOJBTVXLxzX+BxSrgPKm4arFwM3J1lzQk/D5eunOTjsIqN0h57Gyd0TlbBJYtVa9xFF+NpUedZSM5Ypd6UfifEm4tts5QFkdOvO4RTlLz3GoTZDVpAWxu/WJAI9wn/uRaiPj65x9bC496ixIdjnvXek7Db8HWLfMsnpWWFjAV79w9ZxqlVtzwtXq2XH2Q7I71+Aws/72FyvVz/IsrzlgnmWBX71P5vBV16IQhr3uCUYbxBLVjgPoBAUgkLxgEJQCArFAwpBISgUDzeF1+E7vWMor8IUZi5mFUrzwxa7jRRLnntP0YCqUaNwItG7a3LBUqNT1B5K0fuHboj7RSQM5YjDT7edaIV1HlbeGAwmc82/c8XEK1ytTDkcXKeuEzWUlxf1JRdzOJlq7sXv/YGlKKxwi91pQtBXHfZ12hUNn/mCFNZEaYxPY6YC5QnH9JSwhSl8qCRpFu9Pfb3Yh9M+zlLSmvAmTmGbL/uBoUfkFhY48dvDa30/wUV4I5HembMoLs7b6xGKISOE9Ce5p6qql3//XX3XY88RNVICllvbpM7Hk+WhOab8aps0nzIeRZSoZZQYWaIkyLMNNfnEbprtHAAmQCtlyL6OEq3Q34WRLcqPyb4V+RqlGVuHLMP3FEofXDPrhqsVPYIHM6xtQg86SZgZ1Kj21iWjEjVu3jL5Y3eOgEw3WK3l4vnrRVlx65qTXRFaWlGqJ/Fq1qivVUdKHMvyW3JXMJZPBs9W6BhyZ6YYZlrSW++wXHVIiaCpIg1EpO61F9syePK7fMokt8noekMWInb+rnRLzLgqu/ved3jw8tQePrq6dpp771eUDvPDA+TT25uo2Ho4v4V9n1gjvOXY6+U+P3E4OqC7K75w/DglSZymQftNyUGaxkly+ujtcOVSi7K3xF987KRXdh+jrsKn2OZK57Pl1KTkJLCaD7PejqHjcDwYsXXGyBBWs98WrBtvieWN89FkJucTkTDif9ysgTgKnDwZHMfwY5FGwucQJAqdaS2zKhR1AiIoBIWgEBSCQlAICkEhKASFi1DIokobDV/Y2ePjvG7D4eaA+gcVRnyi9zdj6p/Pg8vncOedqCX+amXxOYA85exle8Xg4XDDYrxQX2g9iyBMYM8uLqEXiHVKyZbbkfFt5GwLCZ9ZhbANIJwrfp3GnU/04xsidpM3X9yb8BcoYJQzEwjvg09M++3ZNyNIbP6bMa1YCjVIchK2IvBFT8PaDaig85bDbmHyHljhaZ7+eDyFS9RXo9jx9K3hMrY57tcPh6SX/fiWPO4vKZ8TeKdg6SRTx4g8qhnRl/p4/oWJIiKpQx7YUpVIJGCrfgqG7CIb9zjSaeNgG7kzhaZyx7JM07Ljy4XWnuXlEtv1P7B9Mv8DltyUV+hIpoIAAAAASUVORK5CYII="
var mimeTypes = require('mimetypes');
var image = newImageData,
mimeType = image.match(/data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,.*/)![1],
fileName = 'test.' + mimeTypes.detectExtension(mimeType),
base64EncodedImageString = image.replace(/^data:image\/\w+;base64,/, ''),
imageBuffer = new Buffer(base64EncodedImageString, 'base64');
// Instantiate the GCP Storage instance
const { Storage } = require('#google-cloud/storage');
const googleCloudStorage = new Storage(firebaseSettings);
const bucket = googleCloudStorage.bucket('projectID.appspot.com');
var file = bucket.file(fileName);
return file.save(imageBuffer, {
metadata: { contentType: mimeType, cacheControl: "public, max-age=300" },
public: true,
validation: 'md5'
}, function (error: any) {
if (error) {
throw 'error';
}
return "https://storage.googleapis.com/share-expanses-dcc9f.appspot.com/" + fileName;
});
}
Thanks for the help
Haven't been able to test the solution given by Firebase, but here's the transcript of the response:
The problem that you are facing could be because of two reasons. The
first one is how you are uploading the files, via the Firebase
Console, using any Admin SDK, or via the gsutil command. If using the
Admin SDK option, the problem is a known issue where the required
metadata doesn’t exist, fortunately there is a workaround, you can try
this script to solve this issue.
Now, the second one is related to the network if you are using
comcast, please, try on a different network to see if this issue is
related to that.
When you save an image to firebase, you need to provide an access token in metadata : firebaseStorageDownloadTokens. It has to be an uuid.
More info can be found here : https://www.sentinelstand.com/article/guide-to-firebase-storage-download-urls-tokens
const { v4: uuid } = require("uuid")
function uploadImage() {
const newImageData = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOEAAADhCAMAAAAJbSJIAAAAgVBMVEX///8AAAAEBASAgIDr6+vw8PBYWFjU1NTGxsbz8/P29vb8/Py1tbVhYWHd3d1ra2vk5OS/v78pKSlTU1NOTk6Tk5OpqanNzc13d3dKSkplZWWbm5s5OTkfHx+NjY2GhoYcHBw9PT0TExOioqJ7e3soKCiurq5CQkI6OjoXFxcwMDAuPQWoAAAIJ0lEQVR4nO2daXuqPBCGVfYtIbKLgorLKf//B75ga2sPAdmS8F5n7m/VXgyPISGZzExWKwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAVy6wwlCZG/YUl+vamYLoy0lAkORWbdZNN/YUUVf8ju6bomx2Kq+mEBL5K0UVD9QNCdM0Vfds9QSQMnYTWaN1sEicMCRJ9+29QSCqp28HiftiqUkoU0TLaMDLsl8PbrtGWpY8zQ7QYCoE/pe3+ZusHogW9Yln2nxnVPfljW8t4l8go2zPQV7PPkCxa3kqOvOldr52NF4nVKNuYobxPsC1OoxVilu33ZINDQf1Rw2cO+mrOWBMhMD5w0ldziLnrQyeO+mpOfOdzZspZX03KcQGC2I+gNDC3ZkQ7IQLX6x0niW7fZd83ief7Pg7tH0JcfeIlQy+kcllCyk7/O/LUWIsiDbkVv9/bcv0Rqr+MVa//BR0Ob/++AstdKLuK8n4ZZCiKK4e7cikSez2iTjxm2DPjPj8e8wf1zWviXPqpPuHyeuqXb2ZK6WxaqJCuZe7m7qTTp1da6ty7prtbMoOOVro64VktonlmyFZUqB0NybQrkna7l1ndZIhc2k0xbESjdTEf63M7yBQ9bjO2Z+enamvCXcTCphG1zZ3YNSJ9OroJmRkM6UMOZmXPoppzWE4WEX1oY7Xmv1Fs7Zn9nl9gWt+/MTImNU2dprze+6GfmmYlRrYoHZ9dF/whbJrdMTLVdMzEPDZRlOZr48DIVMMQ2wnUN5SpIiNLjaF7z74X1uiNwWbDyNI/qHCd8XBGW1nDLj+F64K9RKtomuWocMvq3fvDjbIk5ahwvc0YGXuS0dbcPBWu1zlLx4mbU23yVbi+s3srkjvdJGeF1WqGTTO6rZsH3BVWjyqae1C1EP0BFaVwvQ60OR1EshZ0GROisJqG23Pt1Gp2q49GqML1OsmK6aMOKbK3OzbCFFZsLwEZ75YySHDpE1olUmFl/XrP02jE5aM0v1972phd2ydDQkvOnjRk4aFL3pDAjiUo/LoTHOuKLMtGzet+lPn4pPpG0eMRQTnLUfi8oTreWc0j7UmUq22x0f9PhbMDCkEhKBQPKJxNYTkgEmYCXiMShZvCXEkHTUXGcPZSpbFS5KZQWq3Qu4XORGIbUTa9eCqsFqt6OGemxSvbUH8sqwUrXK0sl/gM9PnE/XKNCFdYYZozB7VvsPkzU1+Cwhrrph7mGHfOB/X226+1FIU1duztr+Nlnq97L7YbV12SworITqXdvmcw5QvlfielNtU5sDCFNYpO0iJTk2MvbcdEzYqUtIdVLVDhAwvVubIk8D3Po421j88DUufHvvEnL1XhF2Yd7+xqTR6f9wq1XbjCGQCFs/EPKsR8sq6Vxi4bvxVwzCOJBTVXLxzX+BxSrgPKm4arFwM3J1lzQk/D5eunOTjsIqN0h57Gyd0TlbBJYtVa9xFF+NpUedZSM5Ypd6UfifEm4tts5QFkdOvO4RTlLz3GoTZDVpAWxu/WJAI9wn/uRaiPj65x9bC496ixIdjnvXek7Db8HWLfMsnpWWFjAV79w9ZxqlVtzwtXq2XH2Q7I71+Aws/72FyvVz/IsrzlgnmWBX71P5vBV16IQhr3uCUYbxBLVjgPoBAUgkLxgEJQCArFAwpBISgUDzeF1+E7vWMor8IUZi5mFUrzwxa7jRRLnntP0YCqUaNwItG7a3LBUqNT1B5K0fuHboj7RSQM5YjDT7edaIV1HlbeGAwmc82/c8XEK1ytTDkcXKeuEzWUlxf1JRdzOJlq7sXv/YGlKKxwi91pQtBXHfZ12hUNn/mCFNZEaYxPY6YC5QnH9JSwhSl8qCRpFu9Pfb3Yh9M+zlLSmvAmTmGbL/uBoUfkFhY48dvDa30/wUV4I5HembMoLs7b6xGKISOE9Ce5p6qql3//XX3XY88RNVICllvbpM7Hk+WhOab8aps0nzIeRZSoZZQYWaIkyLMNNfnEbprtHAAmQCtlyL6OEq3Q34WRLcqPyb4V+RqlGVuHLMP3FEofXDPrhqsVPYIHM6xtQg86SZgZ1Kj21iWjEjVu3jL5Y3eOgEw3WK3l4vnrRVlx65qTXRFaWlGqJ/Fq1qivVUdKHMvyW3JXMJZPBs9W6BhyZ6YYZlrSW++wXHVIiaCpIg1EpO61F9syePK7fMokt8noekMWInb+rnRLzLgqu/ved3jw8tQePrq6dpp771eUDvPDA+TT25uo2Ho4v4V9n1gjvOXY6+U+P3E4OqC7K75w/DglSZymQftNyUGaxkly+ujtcOVSi7K3xF987KRXdh+jrsKn2OZK57Pl1KTkJLCaD7PejqHjcDwYsXXGyBBWs98WrBtvieWN89FkJucTkTDif9ysgTgKnDwZHMfwY5FGwucQJAqdaS2zKhR1AiIoBIWgEBSCQlAICkEhKASFi1DIokobDV/Y2ePjvG7D4eaA+gcVRnyi9zdj6p/Pg8vncOedqCX+amXxOYA85exle8Xg4XDDYrxQX2g9iyBMYM8uLqEXiHVKyZbbkfFt5GwLCZ9ZhbANIJwrfp3GnU/04xsidpM3X9yb8BcoYJQzEwjvg09M++3ZNyNIbP6bMa1YCjVIchK2IvBFT8PaDaig85bDbmHyHljhaZ7+eDyFS9RXo9jx9K3hMrY57tcPh6SX/fiWPO4vKZ8TeKdg6SRTx4g8qhnRl/p4/oWJIiKpQx7YUpVIJGCrfgqG7CIb9zjSaeNgG7kzhaZyx7JM07Ljy4XWnuXlEtv1P7B9Mv8DltyUV+hIpoIAAAAASUVORK5CYII="
var mimeTypes = require('mimetypes');
var image = newImageData,
mimeType = image.match(/data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,.*/)![1],
fileName = 'test.' + mimeTypes.detectExtension(mimeType),
base64EncodedImageString = image.replace(/^data:image\/\w+;base64,/, ''),
imageBuffer = new Buffer(base64EncodedImageString, 'base64');
// Instantiate the GCP Storage instance
const { Storage } = require('#google-cloud/storage');
const googleCloudStorage = new Storage(firebaseSettings);
const bucket = googleCloudStorage.bucket('projectID.appspot.com');
var file = bucket.file(fileName);
return file.save(imageBuffer, {
metadata: {
contentType: mimeType,
cacheControl: "public,
max-age=300",
// THIS IS THE LINE YOU NEED TO ADD
firebaseStorageDownloadTokens: uuid(),
},
public: true,
validation: 'md5'
}, function (error: any) {
if (error) {
throw 'error';
}
return "https://storage.googleapis.com/share-expanses-dcc9f.appspot.com/" + fileName;
});
}
After that you'll need to click on "Create access token"
#jean-smaug answer is almost complete. Based on the page he linked (https://www.sentinelstand.com/article/guide-to-firebase-storage-download-urls-tokens), the only missing thing is to wrap the firebaseStorageDownloadTokens property inside a metadata object. I've just tested it and it's working fine 👌 No need to create access token afterwards.
In my case I added metadata while uploading and it loading as it showed in image but when I'm refresh page after 3 min I found that it upload correctly , so as Cafn explain if it not matter of metadata you should wait until it loaded
$uploadedObject=$bucket->upload($imageFile, [
'name' => 'Image_Name',
"metadata" => [ "contentType"=> 'image/png'],
]);
i have the following problem. I designed my litesql database over 'DB Browser for SQLite' and I'm stuck as soon as a query gets executed. The functions I am exporting are getting imported and used in nativescript-vue.
Webpack also copies the database with *.sqlite ending to the device. The android version I use is 9.
The way I initialize my db is;
var Sqlite = require("nativescript-sqlite");
var db;
export function init() {
if (!Sqlite.exists("test.sqlite")) {
Sqlite.copyDatabase("test.sqlite");
}
new Sqlite("test.sqlite", function(err, dbConnection) {
if (err) {
console.log(err);
return;
}
db = dbConnection;
console.log("connection successful")
});
}
After running the function console shows 'connection successful'. The database is placed in the root of the app folder. That way it should pull the database?
Besides I got another question. How could I hide the database in the production?
So the way I execute the query is:
export function xxxx(**parameter**) {
db.execSQL(
"select random_id from random_table where some_id = ?",
**parameter**,
function(err, result) {
console.log("result 1: " + result + " err: " + err);
}
);
}
The output is:
JS: 'result 1: null err: null'
I'm not even sure if it opens the database in the right way?
If you just want to export DB file from Android / iOS, you may use nativescript-share-file plugin and pass the right path.
Android
const filePath = application.android.context
.getDatabasePath("your-db-name.sqlite")
.getAbsolutePath();
new ShareFile().open({
path: filePath,
});
For iOS the path will be different,
iOS
const filePath = knownFolders.documents().getFile("your-db-name.sqlite").path;
new ShareFile().open({
path: filePath,
});
TL;DR;
Does anyone know if it's possible to use console.log in a Firebase/Google Cloud Function to log entries to Stack Driver using the jsonPayload property so my logs are searchable (currently anything I pass to console.log gets stringified into textPayload).
I have a multi-module project with some code running on Firebase Cloud Functions, and some running in other environments like Google Compute Engine. Simplifying things a little, I essentially have a 'core' module, and then I deploy the 'cloud-functions' module to Cloud Functions, 'backend-service' to GCE, which all depend on 'core' etc.
I'm using bunyan for logging throughout my 'core' module, and when deployed to GCE the logger is configured using '#google-cloud/logging-bunyan' so my logs go to Stack Driver.
Aside: Using this configuration in Google Cloud Functions is causing issues with Error: Endpoint read failed which I think is due to functions not going cold and trying to reuse dead connections, but I'm not 100% sure what the real cause is.
So now I'm trying to log using console.log(arg) where arg is an object, not a string. I want this object to appear in Stack Driver under the jsonPayload but it's being stringified and put into the textPayload field.
It took me awhile, but I finally came across this example in firebase functions samples repository. In the end I settled on something a bit like this:
const Logging = require('#google-cloud/logging');
const logging = new Logging();
const log = logging.log('my-func-logger');
const logMetadata = {
resource: {
type: 'cloud_function',
labels: {
function_name: process.env.FUNCTION_NAME ,
project: process.env.GCLOUD_PROJECT,
region: process.env.FUNCTION_REGION
},
},
};
const logData = { id: 1, score: 100 };
const entry = log.entry(logMetaData, logData);
log.write(entry)
You can add a string severity property value to logMetaData (e.g. "INFO" or "ERROR"). Here is the list of possible values.
Update for available node 10 env vars. These seem to do the trick:
labels: {
function_name: process.env.FUNCTION_TARGET,
project: process.env.GCP_PROJECT,
region: JSON.parse(process.env.FIREBASE_CONFIG).locationId
}
UPDATE: Looks like for Node 10 runtimes they want you to set env values explicitly during deploy. I guess there has been a grace period in place because my deployed functions are still working.
I ran into the same problem, and as stated by comments on #wtk's answer, I would like to add replicating all of the default cloud function logging behavior I could find in the snippet below, including execution_id.
At least for using Cloud Functions with the HTTP Trigger option the following produced correct logs for me. I have not tested for Firebase Cloud Functions
// global
const { Logging } = require("#google-cloud/logging");
const logging = new Logging();
const Log = logging.log("cloudfunctions.googleapis.com%2Fcloud-functions");
const LogMetadata = {
severity: "INFO",
type: "cloud_function",
labels: {
function_name: process.env.FUNCTION_NAME,
project: process.env.GCLOUD_PROJECT,
region: process.env.FUNCTION_REGION
}
};
// per request
const data = { foo: "bar" };
const traceId = req.get("x-cloud-trace-context").split("/")[0];
const metadata = {
...LogMetadata,
severity: 'INFO',
trace: `projects/${process.env.GCLOUD_PROJECT}/traces/${traceId}`,
labels: {
execution_id: req.get("function-execution-id")
}
};
Log.write(Log.entry(metadata, data));
The github link in #wtk's answer should be updated to:
https://github.com/firebase/functions-samples/blob/2f678fb933e416fed9be93e290ae79f5ea463a2b/stripe/functions/index.js#L103
As it refers to the repository as of when the question was answered, and has the following function in it:
// To keep on top of errors, we should raise a verbose error report with Stackdriver rather
// than simply relying on console.error. This will calculate users affected + send you email
// alerts, if you've opted into receiving them.
// [START reporterror]
function reportError(err, context = {}) {
// This is the name of the StackDriver log stream that will receive the log
// entry. This name can be any valid log stream name, but must contain "err"
// in order for the error to be picked up by StackDriver Error Reporting.
const logName = 'errors';
const log = logging.log(logName);
// https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/MonitoredResource
const metadata = {
resource: {
type: 'cloud_function',
labels: {function_name: process.env.FUNCTION_NAME},
},
};
// https://cloud.google.com/error-reporting/reference/rest/v1beta1/ErrorEvent
const errorEvent = {
message: err.stack,
serviceContext: {
service: process.env.FUNCTION_NAME,
resourceType: 'cloud_function',
},
context: context,
};
// Write the error log entry
return new Promise((resolve, reject) => {
log.write(log.entry(metadata, errorEvent), (error) => {
if (error) {
return reject(error);
}
resolve();
});
});
}
// [END reporterror]
Within the same firebase project and using a cloud function (written in node.js), I first download an FTP file (using npm ftp module) and then try to upload it into the firebase storage.
Every attempts failed so far and documentation doesn't help...any expert advices/tips would be greatly appreciated?
The following code uses two different approaches : fs.createWriteStream() and bucket.file().createWriteStream(). Both failed but for different reasons (see error messages in the code).
'use strict'
// [START import]
let admin = require('firebase-admin')
let functions = require('firebase-functions')
const gcpStorage = require('#google-cloud/storage')()
admin.initializeApp(functions.config().firebase)
var FtpClient = require('ftp')
var fs = require('fs')
// [END import]
// [START Configs]
// Firebase Storage is configured with the following rules and grants read write access to everyone
/*
service firebase.storage {
match /b/{bucket}/o {
match /{allPaths=**} {
allow read, write;
}
}
}
*/
// Replace this with your project id, will be use by: const bucket = gcpStorage.bucket(firebaseProjectID)
const firebaseProjectID = 'your_project_id'
// Public FTP server, uploaded files are removed after 48 hours ! Upload new ones when needed for testing
const CONFIG = {
test_ftp: {
source_path: '/48_hour',
ftp: {
host: 'ftp.uconn.edu'
}
}
}
const SOURCE_FTP = CONFIG.test_ftp
// [END Configs]
// [START saveFTPFileWithFSCreateWriteStream]
function saveFTPFileWithFSCreateWriteStream(file_name) {
const ftpSource = new FtpClient()
ftpSource.on('ready', function() {
ftpSource.get(SOURCE_FTP.source_path + '/' + file_name, function(err, stream) {
if (err) throw err
stream.once('close', function() { ftpSource.end() })
stream.pipe(fs.createWriteStream(file_name))
console.log('File downloaded: ', file_name)
})
})
ftpSource.connect(SOURCE_FTP.ftp)
}
// This fails with the following error in firebase console:
// Error: EROFS: read-only file system, open '20170601.tar.gz' at Error (native)
// [END saveFTPFileWithFSCreateWriteStream]
// [START saveFTPFileWithBucketUpload]
function saveFTPFileWithBucketUpload(file_name) {
const bucket = gcpStorage.bucket(firebaseProjectID)
const file = bucket.file(file_name)
const ftpSource = new FtpClient()
ftpSource.on('ready', function() {
ftpSource.get(SOURCE_FTP.source_path + '/' + file_name, function(err, stream) {
if (err) throw err
stream.once('close', function() { ftpSource.end() })
stream.pipe(file.createWriteStream())
console.log('File downloaded: ', file_name)
})
})
ftpSource.connect(SOURCE_FTP.ftp)
}
// [END saveFTPFileWithBucketUpload]
// [START database triggers]
// Listens for new triggers added to /ftp_fs_triggers/:pushId and calls the saveFTPFileWithFSCreateWriteStream
// function to save the file in the default project storage bucket
exports.dbTriggersFSCreateWriteStream = functions.database
.ref('/ftp_fs_triggers/{pushId}')
.onWrite(event => {
const trigger = event.data.val()
const fileName = trigger.file_name // i.e. : trigger.file_name = '20170601.tar.gz'
return saveFTPFileWithFSCreateWriteStream(trigger.file_name)
// This fails with the following error in firebase console:
// Error: EROFS: read-only file system, open '20170601.tar.gz' at Error (native)
})
// Listens for new triggers added to /ftp_bucket_triggers/:pushId and calls the saveFTPFileWithBucketUpload
// function to save the file in the default project storage bucket
exports.dbTriggersBucketUpload = functions.database
.ref('/ftp_bucket_triggers/{pushId}')
.onWrite(event => {
const trigger = event.data.val()
const fileName = trigger.file_name // i.e. : trigger.file_name = '20170601.tar.gz'
return saveFTPFileWithBucketUpload(trigger.file_name)
// This fails with the following error in firebase console:
/*
Error: Uncaught, unspecified "error" event. ([object Object])
at Pumpify.emit (events.js:163:17)
at Pumpify.onerror (_stream_readable.js:579:12)
at emitOne (events.js:96:13)
at Pumpify.emit (events.js:188:7)
at Pumpify.Duplexify._destroy (/user_code/node_modules/#google-cloud/storage/node_modules/duplexify/index.js:184:15)
at /user_code/node_modules/#google-cloud/storage/node_modules/duplexify/index.js:175:10
at _combinedTickCallback (internal/process/next_tick.js:67:7)
at process._tickDomainCallback (internal/process/next_tick.js:122:9)
*/
})
// [END database triggers]
I've finally found the correct way to implement this.
1) Make sure the bucket is correctly referenced. Initially I just used
my project_id without the '.appspot.com' at the end'.
const bucket = gsc.bucket('<project_id>.appspot.com')
2) Create a bucket stream first then pipe the stream from the FTP get call to the bucketWriteStream. Note that file_name will be the name of the saved file (this file does not have to exist beforehand).
ftpSource.get(filePath, function(err, stream) {
if (err) throw err
stream.once('close', function() { ftpSource.end() })
// This didn't work !
//stream.pipe(fs.createWriteStream(fileName))
// This works...
let bucketWriteStream = bucket.file(fileName).createWriteStream()
stream.pipe(bucketWriteStream)
})
Et voilà, works like a charm...
I developed a Firebase Cloud function that processes several manipulations on uploaded images.
My code is based on this documentation article and this Cloud Function example. Hence, it is using Google Cloud Storage package.
It is working fine almost all the time, but sometimes I am getting this error when uploading to or deleting from Storage :
Error: read ECONNRESET
at exports._errnoException (util.js:1026:11)
at TLSWrap.onread (net.js:569:26)
I am using the default bucket of my application, referenced by event.data.bucket.
Let me know if you need additional information or code snippets, even if my code is really close to the Function example I linked before.
I found this GitHub issue, but I checked that I am returning a promise everytime. For example, here is the deletion part that triggers the error :
index.js
exports.exampleFunction = functions.storage.object().onChange(event => {
return f_thumbnails.exampleFunction(event);
});
example_function.js
module.exports = exports = function (_admin, _config) {
admin = _admin;
config = _config;
return {
"exampleFunction": function (event) {
return exampleFunction(event);
}
};
};
const exampleFunction = function (event) {
const gcsSourceFilePath = event.data.name;
const gcsSourceFilePathSplit = gcsSourceFilePath.split('/');
const gcsBaseFolder = gcsSourceFilePathSplit.length > 0 ? gcsSourceFilePathSplit[0] : '';
const gcsSourceFileName = gcsSourceFilePathSplit.pop();
const gceSourceFileDir = gcsSourceFilePathSplit.join('/') + (gcsSourceFilePathSplit.length > 0 ? '/' : '');
// Not an image
if (!event.data.contentType.startsWith('image/')) {
console.log('Not an image !');
return;
}
// Thumbnail
if (gcsSourceFileName.startsWith(config.IMAGES_THUMBNAIL_PREFIX)) {
console.log('Thumbnail !');
return;
}
const bucket = gcs.bucket(event.data.bucket);
const gcsThumbnailFilePath = gceSourceFileDir + config.IMAGES_THUMBNAIL_PREFIX + gcsSourceFileName;
// File deletion
if (event.data.resourceState === 'not_exists') {
console.log('Thumbnail deletion : ' + gcsThumbnailFilePath);
return bucket.file(gcsThumbnailFilePath).delete().then(() => {
console.log('Deleted thumbnail ' + gcsThumbnailFilePath);
});
}
...
This seems to be related to the google-cloud-node library's handling of sockets, and the default socket timeout in the Cloud Functions environment.
One solution verified by a user is to modify the way the library invokes requests, to not keep the socket open forever by specifying forever: false, eg.
var request = require('request').defaults({
timeout: 60000,
gzip: true,
forever: false,
pool: {
maxSockets: Infinity
}
});
This is hardcoded in packages/common/src/utils.js, so you'll need to vendor a copy of the modified library into your project rather than include it as an NPM dependency. See the related public issue for more details on the issue and a link to a fork with the patch applied.