Document DB 5 secs stored procedure execution limit - azure-cosmosdb

I have a stored procedure which return 4000 documents, due to 5 sec execution limit my sproc is not retuning any data. I tried to handled collection accepted value but it is not working as expected.
I tried to set response to "return" in case of 5 sec limit hit, sproc is not setting response.
function getRequirementNodes()
{
var context = getContext();
var response = context.getResponse();
var collection = context.getCollection();
var collectionLink = collection.getSelfLink();
var nodesBatch = [];
var continueToken= true;
var query = { query: 'SELECT * from root '};
getNodes(null)
function getNodes(continuation)
{
var requestOptions = {continuation: continuation};
var accepted = collection.queryDocuments(collectionLink, query, requestOptions,
function(err, documentsRead, responseOptions)
{
if (documentsRead.length > 0)
{
nodesBatch = nodesBatch.concat(documentsRead);
}
else if (responseOptions.continuation)
{
continueToken = responseOptions.continuation
nodesBatch = nodesBatch.concat(documentsRead);
getNodes(responseOptions.continuation);
}
else
{
continueToken= false;
response.setBody(nodesBatch);
}
});
if (!accepted)
{
response.setBody("return");
}
}
}

The script is returning a empty response, because the blocks containing response.setBody() are never called.
I'll explain. Let's break this section of queryDocuments callback down:
if (documentsRead.length > 0) {
nodesBatch = nodesBatch.concat(documentsRead);
} else if (responseOptions.continuation) {
continueToken = responseOptions.continuation
nodesBatch = nodesBatch.concat(documentsRead);
getNodes(responseOptions.continuation);
} else {
continueToken = false;
response.setBody(nodesBatch);
}
Note that if the query has results inside the first page (which it most likely will)... The script will append the query results on to nodesBatch:
if (documentsRead.length > 0) {
nodesBatch = nodesBatch.concat(documentsRead);
}
The script will then complete. The response body is unset (empty), and the script does not issue a follow up query if there is a continuation token.
Assuming the collection isn't empty, then this probably the behavior you are experiencing.
Note: If you are querying a large dataset, it's possible to hit the response size limit (1 MB).
I've re-wrote the script to fix the issue above, and have included a snippet to illustrate how to handle the response size limit:
function getRequirementNodes(continuationToken) {
var context = getContext();
var response = context.getResponse();
var collection = context.getCollection();
var collectionLink = collection.getSelfLink();
var nodesBatch = [];
var lastContinuationToken;
var responseSize = 0;
var query = {
query: 'SELECT * FROM root'
};
getNodes(continuationToken);
function getNodes(continuationToken) {
// Tune the pageSize to fit your dataset.
var requestOptions = {
continuation: continuationToken,
pageSize: 1
};
var accepted = collection.queryDocuments(collectionLink, query, requestOptions,
function(err, documentsRead, responseOptions) {
// The size of the current query response page.
var queryPageSize = JSON.stringify(documentsRead).length;
// DocumentDB has a response size limit of 1 MB.
if (responseSize + queryPageSize < 1024 * 1024) {
// Append query results to nodesBatch.
nodesBatch = nodesBatch.concat(documentsRead);
// Keep track of the response size.
responseSize += queryPageSize;
if (responseOptions.continuation) {
// If there is a continuation token... Run the query again to get the next page of results
lastContinuationToken = responseOptions.continuation;
getNodes(responseOptions.continuation);
} else {
// If there is no continutation token, we are done. Return the response.
response.setBody({
"message": "Query completed succesfully.",
"queryResponse": nodesBatch
});
}
} else {
// If the response size limit reached; run the script again with the lastContinuationToken as a script parameter.
response.setBody({
"message": "Response size limit reached.",
"lastContinuationToken": lastContinuationToken,
"queryResponse": nodesBatch
});
}
});
if (!accepted) {
// If the execution limit reached; run the script again with the lastContinuationToken as a script parameter.
response.setBody({
"message": "Execution limit reached.",
"lastContinuationToken": lastContinuationToken,
"queryResponse": nodesBatch
});
}
}
}

Related

DynamoDB table seed works in cli but not AWS-SDK

I have a table that has more than 25 items and wrote a basic script to break them into sub arrays of 25 items each then loops thru that collection of sub arrays to run a batch write item command in the AWS DynamoDB Client. The issue I am getting is a returned validation error. When I run the same seed file via the aws-cli it seeds the table perfectly. This makes me think it has something to do with my script. See anything I am missing? Thanks in advance!
var { DynamoDB } = require('aws-sdk');
var db = new DynamoDB.DocumentClient({
region: 'localhost',
endpoint: 'http://localhost:8000',
});
const allItems = require('./allItems.json');
const tableName = 'some-table-name';
console.log({ tableName, allItems });
var batches = [];
var currentBatch = [];
var count = 0;
for (let i = 0; i < allItems.length; i++) {
//push item to the current batch
count++;
currentBatch.push(allItems[i]);
if (count === 25) {
batches.push(currentBatch);
currentBatch = [];
}
}
//if there are still items left in the curr batch, add to the collection of batches
if (currentBatch.length > 0 && currentBatch.length !== 25) {
batches.push(currentBatch);
}
var completedRequests = 0;
var errors = false;
//request handler for DynamoDB
function requestHandler(err, data) {
console.log('In the request handler...');
return function (err, data) {
completedRequests++;
errors = errors ? true : err;
//log error
if (errors) {
console.error('Request caused a DB error.');
console.error('ERROR: ' + err);
console.error(JSON.stringify(err, null, 2));
} else {
var res = {
statusCode: 200,
headers: {
'Content-Type': 'application/json',
'Access-Control-Allow-Methods': 'GET,POST,OPTIONS',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Credentials': true,
},
body: JSON.stringify(data),
isBase64Encoded: false,
};
console.log(`Success: returned ${data}`);
return res;
}
if (completedRequests == batches.length) {
return errors;
}
};
}
//Make request
var params;
for (let j = 0; j < batches.length; j++) {
//items go in params.RequestedItems.id array
//format for the items is {PutRequest : {Item: ITEM_OBJECT}}
params = '{"RequestItems": {"' + tableName + '": []}}';
params = JSON.parse(params);
params.RequestItems[tableName] = batches[j];
console.log('before db.batchWriteItem: ', params);
try {
//send to db
db.batchWrite(params, requestHandler(params));
} catch{
console.error(err)
}
}
Here is the formatted request object and the error:
before db.batchWriteItem:
{ RequestItems:
{ 'some-table-name': [ [Object], [Object], [Object], [Object] ] }
}
In the request handler...
Request caused a DB error.
ERROR: ValidationException: Invalid attribute value type
{
"message": "Invalid attribute value type",
"code": "ValidationException",
"time": "2020-08-04T10:51:13.751Z",
"requestId": "dd49628c-6ee9-4275-9349-6edca29636fd",
"statusCode": 400,
"retryable": false,
"retryDelay": 47.94198279972915
}
You are using the DocumentClient in the nodejs code. This will automatically convert the data format used by DynamoDB to a more easily consumable format.
e.g.
{
"id": {
"S": "A string value"
}
}
would become
{
"id": "A string value"
}
The CLI does not perform this data conversion.
You can use the regular DynamoDB client to not perform this conversion in Nodejs. e.g. const db = new Dynamodb()

Parameterize Query while using OFFSET clause

I've used following stored procedure and try to parameterize the query by passing the number parameter. However it gives me error while executing the stored procedure. Any insights is really helpful
function uspGetUsersByPage(number) {
//Set Environment
let context = getContext();
let coll = context.getCollection();
let link = coll.getSelfLink();
let response = context.getResponse();
let query = {
query: 'SELECT * FROM a WHERE a.DocumentType = "UserRole" and a.AuditFields.IsLatest = true and a.AuditFields.IsDeleted = false OFFSET #number LIMIT 20'
, parameters: [{ name: '#number', value: number }]
};
//Execute the query against the collection
let runquery = coll.queryDocuments(link, query, {}, callbackfn);
//Call function to throw an error(if any) or display the output
function callbackfn(err, queryoutput) {
if (err) {
throw err;
}
if (!queryoutput || !queryoutput.length) {
response.setBody(null);
}
else {
response.setBody(queryoutput);
}
};
//Display standard output if query doesnt get any results
if (!runquery) { throw Error('Unable to retrieve requested information'); }
};
Please see my simple test following your description.
Data:
Stored procedure:
function sample(prefix) {
var collection = getContext().getCollection();
var query = {query: "SELECT c.id,c.number FROM c offset #num limit 1", parameters:
[{name: "#num", value: prefix}]};
console.log(query);
var isAccepted = collection.queryDocuments(
collection.getSelfLink(),
query,
function (err, feed, options) {
if (err) throw err;
// Check the feed and if empty, set the body to 'no docs found', 
// else take 1st element from feed
if (!feed || !feed.length) {
var response = getContext().getResponse();
response.setBody('no docs found');
}
else {
var response = getContext().getResponse();
response.setBody(feed);
}
});
if (!isAccepted) throw new Error('The query was not accepted by the server.');
}
Input:
Output:

MeteorJS: CALLBACKS

PROBLEM: I want to parse the elements in a page from another website, glue resulting elements in an object and insert it in a Mongo collection. Before insertion i want to check if my Mongo yet has an identical object. If it does it shall exit the running functions, otherwise i want the script to start parsing the next target.
Example:
I have a function that connects to a webpage and returns its body content
It is parsed
When <a></a> elements are met, another callback is called in which all parsed elements are merged in one object and inserted in a collection
My code :
var Cheerio = Meteor.npmRequire('cheerio');
var lastUrl;
var exit = false;
Meteor.methods({
parsing:function(){
this.unblock();
request("https://example.com/", Meteor.bindEnvironment(function(error, response, body) {
if (!error && response.statusCode == 200) {
$ = Cheerio.load(body);
var k = 1;
$("div.content").each(function() {
var name = $...//parsing
var age = $....//parsing
var url = $...//parsing <a></a> elements
var r = request("https://example.com/"+url, Meteor.bindEnvironment(function(error, response, body) {
lastUrl = response.request.uri.href;// get the last routing link
var metadata = {
name: name,
age: age
url: lastUrl
};
var postExist;
postExist = Posts.findOne(metadata); // return undefined if doesnt exist, AND every time postExist = undefined ??
if (!postExist){
Posts.insert(metadata);// if post doesnt exist (every time go here ??)
}
else {
exit = true; // if exist
}
}));
if (exit === true) return false;
});
}
}));
}
});
Problem 1 : The problem is my function works every time, but it doesn't stop even if the object exists in my collection
Problem 2 : postExist is always undefined
EDIT : The execution must stop and wait until the second request's response.
var url = $...//parsing <a></a> elements
//STOP HERE AND WAIT !!
var r = request("https://example.com/"+url, Meteor.bindEnvironment(function(error, response, body) {
Looks like you want the second request to be synchronous and not asynchronous.
To achieve this, use a future
var Cheerio = Meteor.npmRequire('cheerio');
var Future = Meteor.npmRequire('fibers/future');
var lastUrl;
var exit = false;
Meteor.methods({
parsing:function(){
this.unblock();
request("https://example.com/", Meteor.bindEnvironment(function(error, response, body) {
if (!error && response.statusCode == 200) {
$ = Cheerio.load(body);
var k = 1;
$("div.content").each(function() {
var name = $...//parsing
var age = $....//parsing
var url = $...//parsing <a></a> elements
var fut = new Future();
var r = request("https://example.com/"+url, Meteor.bindEnvironment(function(error, response, body) {
lastUrl = response.request.uri.href;// get the last routing link
var metadata = {
name: name,
age: age
url: lastUrl
};
var postExist;
postExist = Posts.findOne(metadata); // return undefined if doesnt exist
if (!postExist) {
Posts.insert(metadata);// if post doesnt exist (every time go here ??)
fut.return(true);
} else {
fut.return(false);
}
}));
var status = fut.wait();
return status;
});
}
}));
}
});
You can use futures whenever you can't utilize callback functions (e.g. you want the user to wait on the result of a callback before presenting info).
Hopefully that helps,
Elliott
This is the opposite :
postExist = Posts.findOne(metadata); // return undefined if doesnt exist > you're right
if (!postExist){ //=if NOT undefined = if it EXISTS !
Posts.insert(metadata);
}else {
exit = true; // if undefined > if it DOES NOT EXIST !
}
You need to inverse the condition or the code inside

How to avoid blockin while uploading file using Meteor method

I've created a Meteor method to upload a file, it's working well but until the file is fully uploaded, I cannot move around, all subscriptions seem to wait that the upload finishes... is there a way to avoid that ?
Here is the code on the server :
Meteor.publish('product-photo', function (productId) {
return Meteor.photos.find({productId: productId}, {limit: 1});
});
Meteor.methods({
/**
* Creates an photo
* #param obj
* #return {*}
*/
createPhoto: function (obj) {
check(obj, Object);
// Filter attributes
obj = filter(obj, [
'name',
'productId',
'size',
'type',
'url'
]);
// Check user
if (!this.userId) {
throw new Meteor.Error('not-connected');
}
// Check file name
if (typeof obj.name !== 'string' || obj.name.length > 255) {
throw new Meteor.Error('invalid-file-name');
}
// Check file type
if (typeof obj.type !== 'string' || [
'image/gif',
'image/jpg',
'image/jpeg',
'image/png'
].indexOf(obj.type) === -1) {
throw new Meteor.Error('invalid-file-type');
}
// Check file url
if (typeof obj.url !== 'string' || obj.url.length < 1) {
throw new Meteor.Error('invalid-file-url');
}
// Check file size
if (typeof obj.size !== 'number' || obj.size <= 0) {
throw new Meteor.Error('invalid-file-size');
}
// Check file max size
if (obj.size > 1024 * 1024) {
throw new Meteor.Error('file-too-large');
}
// Check if product exists
if (!obj.productId || Meteor.products.find({_id: obj.productId}).count() !== 1) {
throw new Meteor.Error('product-not-found');
}
// Limit the number of photos per user
if (Meteor.photos.find({productId: obj.productId}).count() >= 3) {
throw new Meteor.Error('max-photos-reached');
}
// Resize the photo if the data is in base64
if (typeof obj.url === 'string' && obj.url.indexOf('data:') === 0) {
obj.url = resizeImage(obj.url, 400, 400);
obj.size = obj.url.length;
obj.type = 'image/png';
}
// Add info
obj.createdAt = new Date();
obj.userId = this.userId;
return Meteor.photos.insert(obj);
}
});
And the code on the client :
Template.product.events({
'change [name=photo]': function (ev) {
var self = this;
readFilesAsDataURL(ev, function (event, file) {
var photo = {
name: file.name,
productId: self._id,
size: file.size,
type: file.type,
url: event.target.result
};
Session.set('uploadingPhoto', true);
// Save the file
Meteor.call('createPhoto', photo, function (err, photoId) {
Session.set('uploadingPhoto', false);
if (err) {
displayError(err);
} else {
notify(i18n("Transfert terminé pour {{name}}", photo));
}
});
});
}
});
I finally found the solution myself.
Explication : the code I used was blocking the subscriptions because it was using only one method call to transfer all the file from the first byte to the last one, that leads to block the thread (I think, the one reserved to each users on the server) until the transfer is complete.
Solution : I splitted the file into chunks of about 8KB, and send chunk by chunk, this way the thread or whatever was blocking the subscriptions is free after each chunk transfer.
The final working solution is on that post : How to write a file from an ArrayBuffer in JS
Client Code
// data comes from file.readAsArrayBuffer();
var total = data.byteLength;
var offset = 0;
var upload = function() {
var length = 4096; // chunk size
// adjust the last chunk size
if (offset + length > total) {
length = total - offset;
}
// I am using Uint8Array to create the chunk
// because it can be passed to the Meteor.method natively
var chunk = new Uint8Array(data, offset, length);
if (offset < total) {
// Send the chunk to the server and tell it what file to append to
Meteor.call('uploadFileData', fileId, chunk, function (err, length) {
if (!err) {
offset += length;
upload();
}
}
}
};
upload();
Server code
var fs = Npm.require('fs');
var Future = Npm.require('fibers/future');
Meteor.methods({
uploadFileData: function(fileId, chunk) {
var fut = new Future();
var path = '/uploads/' + fileId;
// I tried that with no success
chunk = String.fromCharCode.apply(null, chunk);
// how to write the chunk that is an Uint8Array to the disk ?
fs.appendFile(path, new Buffer(chunk), function (err) {
if (err) {
fut.throw(err);
} else {
fut.return(chunk.length);
}
});
return fut.wait();
}
});
Improving #Karl's code:
Client
This function breaks the file into chunks and sends them to the server one by one.
function uploadFile(file) {
const reader = new FileReader();
let _offset = 0;
let _total = file.size;
return new Promise((resolve, reject) => {
function readChunk() {
var length = 10 * 1024; // chunk size
// adjust the last chunk size
if (_offset + length > _total) {
length = _total - _offset;
}
if (_offset < _total) {
const slice = file.slice(_offset, _offset + length);
reader.readAsArrayBuffer(slice);
} else {
// EOF
setProgress(100);
resolve(true);
}
}
reader.onload = function readerOnload() {
let buffer = new Uint8Array(reader.result) // convert to binary
Meteor.call('fileUpload', file.name, buffer, _offset,
(error, length) => {
if (error) {
console.log('Oops, unable to import!');
return false;
} else {
_offset += length;
readChunk();
}
}
);
};
reader.onloadend = function readerOnloadend() {
setProgress(100 * _offset / _total);
};
readChunk();
});
}
Server
The server then writes to a file when offset is zero, or appends to its end otherwise, returning a promise, as I used an asynchronous function to write/append in order to avoid blocking the client.
if (Meteor.isServer) {
var fs = require('fs');
var Future = require('fibers/future');
}
Meteor.methods({
// Upload file from client to server
fileUpload(
fileName: string,
fileData: Uint8Array,
offset: number) {
check(fileName, String);
check(fileData, Uint8Array);
check(offset, Number);
console.log(`[x] Received file ${fileName} data length: ${fileData.length}`);
if (Meteor.isServer) {
const fut = new Future();
const filePath = '/tmp/' + fileName;
const buffer = new Buffer(fileData);
const jot = offset === 0 ? fs.writeFile : fs.appendFile;
jot(filePath, buffer, 'binary', (err) => {
if (err) {
fut.throw(err);
} else {
fut.return(buffer.length);
}
});
return fut.wait();
}
}
)};
Usage
uploadFile(file)
.then(() => {
/* do your stuff */
});

This is my code when I insert multi record into my Collection first time it work but when i insert second time it say duplicate _id,

This is my code when I insert multi record into my Collection first time it work well but when i insert second time it say duplicate _id, how to fix it.
//Save Journal Detail
$("#item-list tr").each(function (e) {
//debugger;
var yearMonth = $('#journalDate').val();
var date = moment(yearMonth).format("YYMM");
var prefix = curBranch + "-" + date;
var chartAccountId = $(this).find(".chart-account-id option:selected").val();
if (chartAccountId == "" || chartAccountId == null) {
return;
}
var journalDetailId = idGenerator.genWithPrefix(Acc.Collection.JournalDetail, prefix, 8);
var debit = parseFloat($(this).find(".debit").val());
var credit = parseFloat($(this).find(".credit").val());
if (debit > 0) {
Acc.Collection.JournalDetail.insert({
_id: journalDetailId,
journalId: doc._id,
chartAccountId: chartAccountId,
drCr: debit,
split: "Hello"
});
} else {
Acc.Collection.JournalDetail.insert({
_id: journalDetailId,
journalId: doc._id,
chartAccountId: chartAccountId,
drCr: -credit,
split: "Hello"
});
}
});
return doc;
The problem is likely to be that your $.each() iterations do not wait for your insert calls to finish, therefore the last id in the collection stays the same, and therefore idGenerator.genWithPrefix gives you the same id for multiple successive insert calls.
In the Meteor client, "insert never blocks". So you would have to wait on each insert call before getting the next id via idGenerator.genWithPrefix, or else your next generated id will end up being the same as the previous one you started to insert.
One easy solution could be to transfer that logic to the server (where insert calls are blocking) using a method call, by only giving it the list of documents you wish for it to insert.
On the client:
//Save Journal Detail
var listToInsert = [];
var yearMonth = $('#journalDate').val();
var date = moment(yearMonth).format("YYMM");
var prefix = curBranch + "-" + date;
$("#item-list tr").each(function (e) {
//debugger;
var chartAccountId = $(this).find(".chart-account-id option:selected").val();
if (chartAccountId == "" || chartAccountId == null) {
return;
}
var debit = parseFloat($(this).find(".debit").val());
var credit = parseFloat($(this).find(".credit").val());
listToInsert.push({
journalId: doc._id,
chartAccountId: chartAccountId,
drCr: debit > 0 ? debit : -credit, // simplified with a ternary operator
split: "Hello"
});
});
Meteor.call("insertEntries", listToInsert, prefix, function () {
//done!
});
return doc;
On the server:
Meteor.methods({
'insertEntries': function (listToInsert, prefix) {
for (var i = 0; i < listToInsert.length; i++) {
listToInsert[i]._id = idGenerator.genWithPrefix(Acc.Collection.JournalDetail, prefix, 8);
// Since we're on the server, this insert will be blocking
Acc.Collection.JournalDetail.insert(listToInsert[i]);
}
}
});

Resources