I'm working on MongoDB 2.6.9 and NodeJs 0.10.37 and I have a collection vols which means flights.
According to my previous question, I have a large collection vols with documents like these samples :
> db.vols.findOne()
{
"_id" : ObjectId("5717a5d4578f3f2556f300f2"),
"Orig" : "PTH",
"Dest" : "GMP",
"Flight" : 126,
"Stops" : 0,
"Seats" : 169,
"Ops_Week" : 3,
"Eff_Date" : "2016-04-14",
"Mkt_Al" : "XX",
"Dep_Time" : 1110,
"Thru_Point" : "",
"Arr_Time" : 1600,
"Block_Mins" : 230
}
The next document, refers to a flight with stop.
> db.vols.findOne({Stops:1})
{
"_id" : ObjectId("5717a5d4578f3f2556f301c5"),
"Orig" : "CIM",
"Dest" : "LKR",
"Flight" : 7756,
"Stops" : 1,
"Seats" : 70,
"Ops_Week" : 2,
"Eff_Date" : "2016-04-11",
"Mkt_Al" : "YY",
"Dep_Time" : 1655,
"Thru_Point" : "BHU",
"Arr_Time" : 140,
"Block_Mins" : 345
}
And I have to calculate a score for each document with this formula and insert it into the document as a new field :
I Have tried this algorithm, using the 'async' module with series method but I didn't get my desirable result ...
var mongoose = require('mongoose'),
express = require('express'),
async = require('async'),
Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/test');
var volSchema = new Schema({},{ strict: false, collection: 'vols' }),
Vol = mongoose.model("Vol", volSchema);
mongoose.set('debug', false);
mongoose.connection.on("open", function (err) {
if (err) throw err;
var bulkUpdateOps = Vol.collection.initializeUnorderedBulkOp(),
counter = 0;
Vol.find({}).lean().exec(function (err, docs) {
if (err) throw err;
var locals = {};
docs.forEach(function(doc) {
locals.c1 = 0.3728 + (0.00454 * doc.Seats);
locals.c3 = doc.Ops_Week;
if (doc.Stops == 1) {
async.series([
// Load doc with first leg first
function(callback) {
Vol.findOne({
Mkt_Al: doc.Mkt_Al,
Orig: doc.Orig,
Dest: doc.Thru_Point
}).lean().exec(function (err, flight) {
if (err) return callback(err);
locals.first_leg = flight.Block_Mins;
callback();
});
},
// Load second leg doc
// (won't be called before task 1's "task callback"
// has been called)
function(callback) {
Vol.findOne({
Mkt_Al: doc.Mkt_Al,
Orig: doc.Thru_Point,
Dest: doc.Dest
}).lean().exec(function (err, flight) {
if (err) return callback(err);
locals.second_leg = flight.Block_Mins;
callback();
});
}
], function(err) { // This function gets called after the
// two tasks have called their "task callbacks"
if (err) throw err;
// Here locals will be populated with `first_leg`
// and `second_leg`
// Just like in the previous example
var total_flight = locals.second_leg + locals.first_leg;
locals.c2 = 0.03;
locals.c4 = Math.pow((doc.Block_Mins / total_flight), -0.675);
});
} else {
locals.c2 = 1;
locals.c4 = 1;
}
counter++;
console.log(locals);
bulkUpdateOps.find({ "_id" : doc._id }).updateOne({
"$set": {
"Qsi": (locals.c1 * locals.c2 * locals.c3 * locals.c4)
}
});
if (counter % 500 == 0) {
bulkUpdateOps.execute(function(err, result) {
if (err) throw err;
bulkUpdateOps = Vol.collection.initializeUnorderedBulkOp();
});
}
});
if (counter % 500 != 0) {
bulkUpdateOps.execute(function(err, result) {
if (err) throw err;
console.log(result.nModified);
});
}
});
});
At the end I find myself with a result that is the same in both cases, flight with stop or not.
I think that the secend task wich calculate second legs, waits until the calculation of all first_legs, instead of after each first_leg.
Also the insertion of the score must wait until the end of calculating the score for each document, I think bulkinsert should take place after other calculations.
Now I don't know if I should work with async.series or async.foreach or async.parralel or combinate them ...
Please who can help to achieve the above ...
Related
I have a table that has more than 25 items and wrote a basic script to break them into sub arrays of 25 items each then loops thru that collection of sub arrays to run a batch write item command in the AWS DynamoDB Client. The issue I am getting is a returned validation error. When I run the same seed file via the aws-cli it seeds the table perfectly. This makes me think it has something to do with my script. See anything I am missing? Thanks in advance!
var { DynamoDB } = require('aws-sdk');
var db = new DynamoDB.DocumentClient({
region: 'localhost',
endpoint: 'http://localhost:8000',
});
const allItems = require('./allItems.json');
const tableName = 'some-table-name';
console.log({ tableName, allItems });
var batches = [];
var currentBatch = [];
var count = 0;
for (let i = 0; i < allItems.length; i++) {
//push item to the current batch
count++;
currentBatch.push(allItems[i]);
if (count === 25) {
batches.push(currentBatch);
currentBatch = [];
}
}
//if there are still items left in the curr batch, add to the collection of batches
if (currentBatch.length > 0 && currentBatch.length !== 25) {
batches.push(currentBatch);
}
var completedRequests = 0;
var errors = false;
//request handler for DynamoDB
function requestHandler(err, data) {
console.log('In the request handler...');
return function (err, data) {
completedRequests++;
errors = errors ? true : err;
//log error
if (errors) {
console.error('Request caused a DB error.');
console.error('ERROR: ' + err);
console.error(JSON.stringify(err, null, 2));
} else {
var res = {
statusCode: 200,
headers: {
'Content-Type': 'application/json',
'Access-Control-Allow-Methods': 'GET,POST,OPTIONS',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Credentials': true,
},
body: JSON.stringify(data),
isBase64Encoded: false,
};
console.log(`Success: returned ${data}`);
return res;
}
if (completedRequests == batches.length) {
return errors;
}
};
}
//Make request
var params;
for (let j = 0; j < batches.length; j++) {
//items go in params.RequestedItems.id array
//format for the items is {PutRequest : {Item: ITEM_OBJECT}}
params = '{"RequestItems": {"' + tableName + '": []}}';
params = JSON.parse(params);
params.RequestItems[tableName] = batches[j];
console.log('before db.batchWriteItem: ', params);
try {
//send to db
db.batchWrite(params, requestHandler(params));
} catch{
console.error(err)
}
}
Here is the formatted request object and the error:
before db.batchWriteItem:
{ RequestItems:
{ 'some-table-name': [ [Object], [Object], [Object], [Object] ] }
}
In the request handler...
Request caused a DB error.
ERROR: ValidationException: Invalid attribute value type
{
"message": "Invalid attribute value type",
"code": "ValidationException",
"time": "2020-08-04T10:51:13.751Z",
"requestId": "dd49628c-6ee9-4275-9349-6edca29636fd",
"statusCode": 400,
"retryable": false,
"retryDelay": 47.94198279972915
}
You are using the DocumentClient in the nodejs code. This will automatically convert the data format used by DynamoDB to a more easily consumable format.
e.g.
{
"id": {
"S": "A string value"
}
}
would become
{
"id": "A string value"
}
The CLI does not perform this data conversion.
You can use the regular DynamoDB client to not perform this conversion in Nodejs. e.g. const db = new Dynamodb()
this is my colletion:
{
"_id" : "Kan6btPXwNiF84j8e",
"title" : "Chapter Title 1",
"businessId" : "qmWJ3HtZrpka8dpbM",
"createdBy" : "GfdPfoPTiSwLv8TBR",
"sections" : [
{
"id" : "T5KAfTcCb7pCudT3a",
"type" : "TEXT",
"data" : {
"text" : "<h1>2</h1><h1>asd</h1>"
},
"createdAt" : ISODate("2016-12-03T10:35:59.023Z"),
"updatedAt" : ISODate("2016-12-03T10:35:59.023Z")
}
],
"createdAt" : ISODate("2016-12-02T12:15:16.577Z"),
"updatedAt" : ISODate("2016-12-03T12:54:50.591Z")
}
this is the meteor method I am calling from client side
deleteSection: function (section_id, chapter_id) {
chaptersCollection.update(
{$and: [{_id: chapter_id}, {'sections.id': section_id}]},
{$pull: {'sections': {'id': section_id}}},
function (err, numAffected) {
if (err) {
console.log(err);
return err;
}else{
console.log(numAffected);
}
});
return 'Section Successfully Deleted';
}
in callback function of meteor method, it returns 1 as affected rows. But on server document is not updating.
Any suggestion where am I wrong?
Do you really need $and?
deleteSection: function (section_id, chapter_id) {
chaptersCollection.update(
{_id: chapter_id, 'sections.id': section_id},
{$pull: {'sections': {'id': section_id}}},
function (err) {
if (err) {
console.log(err);
return err;
}else{
console.log('success');
return 'success';
}
});
}
I had a similar issue when I tried to use pull in a project. So instead of using $pull, I handled the array outside of the database and then set the array as the one I handled outside. So maybe you can try something like that as an alternative way
deleteSection: function (section_id, chapter_id){
const oldArray = chaptersCollection.findOne(chapter_id).sections;
const newArray = oldArray.filter(function(section) {
return section.id !== section_id
});
chaptersCollection.update({_id: chapter_id},
{$set: {sections: newArray}},
function (err, numAffected) {
if (err) {
console.log(err);
return err;
}else{
console.log(numAffected);
}
});
return 'Section Successfully Deleted';
}
I have an extended question to this question.
What if the player belong to more than one team?
I have this
json
"Players" : {
"-YRHd4IjrjsBXx__B" : {
"name" : "The best forward",
"creationDate" : "2016-02-26 15:50:39",
"teams" : {
"-KAByPeIz4IjrjsBXx__B" : true,
"-KEFPuCXcqOah_GJwsMCu" : true,
"-KEwuQxvGpYTEJ7YQ58-l" : true,
"-KKF8vPtf8J7cfqFh2PLm" : true
},
},
etc...
}
players-service.js
getPlayers: function(teamid) {
var Players = {};
var teamsIndex = ref.child('teams/' + teamid + '/players/');
var playersIndex = ref.child('players/');
teamsIndex.on('child_added', function(snapshot) {
var playerKey = snapshot.key;
playersIndex.child(playerKey).on('value', function(playersnap){
$timeout(function() {
console.log("key", playerKey);
players[playerKey] = playersnap.val();
});
});
});
teamIndex.on('child_removed', function(snapshot) {
$timeout(function(snapshot) {
delete players[snapshot.key()];
});
});
return players;
}
But it returns a list of object. I know that I could probably query/change the data structure to/in firebase and return it as a $firebaseArray which I prefer as I use angularfire.
You usually structure your data depending on how you want to retrieve them.
From my understanding (correct me if I'm wrong) you want to get all the players in a team. For this purpose I would use this structure:
"Players": {
"player1": {...},
"player2": {...},
"player3": {...}
},
"Teams': {
"team1": {...},
"team2": {...}
},
"TeamPlayers" : {
"team1": {
"player1": true,
"player2": true
},
"team2": {
"player1": true,
"player3": true
}
}
Or using an array
"TeamPlayers" : {
"team1": [
0: "player1",
1: "player2"
]
}
I'm currently creating a research engine for my app.
Until now, I used Publish composite + iron router : The user could had filters to search for some specific set of users.
Now, I want him to be able to look for some keywords too. For that I downloaded and tested the great SearchSource package.
The problem is that the SearchSource server side definition only seems to allow to return one cursor.
How could I combine the two logics ? Even if it's tricky, please, share.
Of course I could make an autorunned subscription where I look for every users loaded on the client and then subscribe to the additionnal documents, but it is not really the most performant and beautifull thing to do.
Some data :
Here is my current Publish Composite for filters :
Meteor.publishComposite("findTalkers", function(page, langs){
//console.log("Find Talkers");
//console.log("page : " + page);
//console.log("langs : " + langs);
if (langs.length)
{
return ({
find: function()
{
if (langs && langs.length)
{
var test = {$in: langs};
preSelectedUsers = [],
selector = {
_id: {$ne: this.userId},
"profile.completed": true,
"profile.firstName": {$exists: true},
"profile.languages.native": {$exists: false},
"profile.languages.lang": test
};
Counts.publish(this, "nbUsers", Meteor.users.find(selector, {
fields: {_id: 1}
}), {noReady: false, nonReactive: true});
if (page > 1)
{
preSelectedUsers = Meteor.users.find(selector, {
sort: {'profile.talkname': 1},
limit: 25,
skip: (25 * (page || 1)),
fields: {_id: 1}
}).fetch();
var i = -1;
while (preSelectedUsers[++i])
preSelectedUsers[i] = preSelectedUsers[i]._id;
}
if (page > 1)
selector._id = {$in: preSelectedUsers};
return Meteor.users.find(selector, {
fields: userFields,
sort: {'profile.talkname': 1},
limit: 25
});
}
},
children: [
{
// Finding user's profile picture if it is not url
find: function(user)
{
if (user && user.profile && user.profile.avatar.type != "url")
return Images.find({_id: user.profile.avatar.data}, {sort: {uploadedAt: -1}, limit: 1});
}
}
]
});
}
else
{
return ({
find: function()
{
return Meteor.users.find({_id: "flush"});
}
});
}
});
Here is my research with SearchSource :
Client :
var searchOptions = {
keepHistory: 1000 * 60 * 5,
localSearch: true
},
SearchSources = {
talkersSearch: new SearchSource('users', ['profile.talkname'], searchOptions)
};
Router.map(function(){
this.route('talkers/:page?',
{
template: "talkers",
onBeforeAction: function(pause){
(Meteor.user() && Meteor.user().profile.completed)
? this.next()
: this.render('/profile');
},
waitOn: function(){
var filters = MatesFilter.find().fetch(),
i = -1;
while (filters[++i])
filters[i] = filters[i].value;
if (filters.length)
{
return Meteor.subscribe("findTalkers", (this.params.page || 1), filters, function(){
Session.set('numberuser', Counts.get("nbUsers"));
});
}
return Meteor.subscribe('myself');
}
});
}
Template.talkers.helpers({
getPackages: function() {
return SearchSources.talkersSearch.getData({
transform: function(matchText, regExp) {
return matchText.replace(regExp, "<b>$&</b>")
},
sort: {isoScore: -1}
});
}
}
Template.talkers.events({
"keyup #header-search": _.throttle(function(e) {
Session.set("matesSearch", $(e.target).val().trim());
console.log("Searching for : " + text);
SearchSources.talkersSearch.search(Session.get("matesSearch"), {
page: (this.params.page || 1),
filters: filters
});
}, 200)
}
SERVER :
SearchSource.defineSource('users', function(searchText, options) {
var options = {sort: {"profile.talkname": -1}, limit: 25};
if(searchText)
{
var regExp = buildRegExp(searchText);
selector = { $or: [
{ "profile.talkname": regExp },
{ "profile.bio": regExp }
] };
return Meteor.users.find(selector, options).fetch();
}
return ;
});
All this Gives me two sources from which I can get users. I'd want to get a mean to merge the two ides (a composition of publication INSIDE the search, for example).
Thanks you.
I've created a Meteor method to upload a file, it's working well but until the file is fully uploaded, I cannot move around, all subscriptions seem to wait that the upload finishes... is there a way to avoid that ?
Here is the code on the server :
Meteor.publish('product-photo', function (productId) {
return Meteor.photos.find({productId: productId}, {limit: 1});
});
Meteor.methods({
/**
* Creates an photo
* #param obj
* #return {*}
*/
createPhoto: function (obj) {
check(obj, Object);
// Filter attributes
obj = filter(obj, [
'name',
'productId',
'size',
'type',
'url'
]);
// Check user
if (!this.userId) {
throw new Meteor.Error('not-connected');
}
// Check file name
if (typeof obj.name !== 'string' || obj.name.length > 255) {
throw new Meteor.Error('invalid-file-name');
}
// Check file type
if (typeof obj.type !== 'string' || [
'image/gif',
'image/jpg',
'image/jpeg',
'image/png'
].indexOf(obj.type) === -1) {
throw new Meteor.Error('invalid-file-type');
}
// Check file url
if (typeof obj.url !== 'string' || obj.url.length < 1) {
throw new Meteor.Error('invalid-file-url');
}
// Check file size
if (typeof obj.size !== 'number' || obj.size <= 0) {
throw new Meteor.Error('invalid-file-size');
}
// Check file max size
if (obj.size > 1024 * 1024) {
throw new Meteor.Error('file-too-large');
}
// Check if product exists
if (!obj.productId || Meteor.products.find({_id: obj.productId}).count() !== 1) {
throw new Meteor.Error('product-not-found');
}
// Limit the number of photos per user
if (Meteor.photos.find({productId: obj.productId}).count() >= 3) {
throw new Meteor.Error('max-photos-reached');
}
// Resize the photo if the data is in base64
if (typeof obj.url === 'string' && obj.url.indexOf('data:') === 0) {
obj.url = resizeImage(obj.url, 400, 400);
obj.size = obj.url.length;
obj.type = 'image/png';
}
// Add info
obj.createdAt = new Date();
obj.userId = this.userId;
return Meteor.photos.insert(obj);
}
});
And the code on the client :
Template.product.events({
'change [name=photo]': function (ev) {
var self = this;
readFilesAsDataURL(ev, function (event, file) {
var photo = {
name: file.name,
productId: self._id,
size: file.size,
type: file.type,
url: event.target.result
};
Session.set('uploadingPhoto', true);
// Save the file
Meteor.call('createPhoto', photo, function (err, photoId) {
Session.set('uploadingPhoto', false);
if (err) {
displayError(err);
} else {
notify(i18n("Transfert terminé pour {{name}}", photo));
}
});
});
}
});
I finally found the solution myself.
Explication : the code I used was blocking the subscriptions because it was using only one method call to transfer all the file from the first byte to the last one, that leads to block the thread (I think, the one reserved to each users on the server) until the transfer is complete.
Solution : I splitted the file into chunks of about 8KB, and send chunk by chunk, this way the thread or whatever was blocking the subscriptions is free after each chunk transfer.
The final working solution is on that post : How to write a file from an ArrayBuffer in JS
Client Code
// data comes from file.readAsArrayBuffer();
var total = data.byteLength;
var offset = 0;
var upload = function() {
var length = 4096; // chunk size
// adjust the last chunk size
if (offset + length > total) {
length = total - offset;
}
// I am using Uint8Array to create the chunk
// because it can be passed to the Meteor.method natively
var chunk = new Uint8Array(data, offset, length);
if (offset < total) {
// Send the chunk to the server and tell it what file to append to
Meteor.call('uploadFileData', fileId, chunk, function (err, length) {
if (!err) {
offset += length;
upload();
}
}
}
};
upload();
Server code
var fs = Npm.require('fs');
var Future = Npm.require('fibers/future');
Meteor.methods({
uploadFileData: function(fileId, chunk) {
var fut = new Future();
var path = '/uploads/' + fileId;
// I tried that with no success
chunk = String.fromCharCode.apply(null, chunk);
// how to write the chunk that is an Uint8Array to the disk ?
fs.appendFile(path, new Buffer(chunk), function (err) {
if (err) {
fut.throw(err);
} else {
fut.return(chunk.length);
}
});
return fut.wait();
}
});
Improving #Karl's code:
Client
This function breaks the file into chunks and sends them to the server one by one.
function uploadFile(file) {
const reader = new FileReader();
let _offset = 0;
let _total = file.size;
return new Promise((resolve, reject) => {
function readChunk() {
var length = 10 * 1024; // chunk size
// adjust the last chunk size
if (_offset + length > _total) {
length = _total - _offset;
}
if (_offset < _total) {
const slice = file.slice(_offset, _offset + length);
reader.readAsArrayBuffer(slice);
} else {
// EOF
setProgress(100);
resolve(true);
}
}
reader.onload = function readerOnload() {
let buffer = new Uint8Array(reader.result) // convert to binary
Meteor.call('fileUpload', file.name, buffer, _offset,
(error, length) => {
if (error) {
console.log('Oops, unable to import!');
return false;
} else {
_offset += length;
readChunk();
}
}
);
};
reader.onloadend = function readerOnloadend() {
setProgress(100 * _offset / _total);
};
readChunk();
});
}
Server
The server then writes to a file when offset is zero, or appends to its end otherwise, returning a promise, as I used an asynchronous function to write/append in order to avoid blocking the client.
if (Meteor.isServer) {
var fs = require('fs');
var Future = require('fibers/future');
}
Meteor.methods({
// Upload file from client to server
fileUpload(
fileName: string,
fileData: Uint8Array,
offset: number) {
check(fileName, String);
check(fileData, Uint8Array);
check(offset, Number);
console.log(`[x] Received file ${fileName} data length: ${fileData.length}`);
if (Meteor.isServer) {
const fut = new Future();
const filePath = '/tmp/' + fileName;
const buffer = new Buffer(fileData);
const jot = offset === 0 ? fs.writeFile : fs.appendFile;
jot(filePath, buffer, 'binary', (err) => {
if (err) {
fut.throw(err);
} else {
fut.return(buffer.length);
}
});
return fut.wait();
}
}
)};
Usage
uploadFile(file)
.then(() => {
/* do your stuff */
});