Force synchronous looping of http requests - asynchronous

I've been struggling with this piece of code for a while and I'm reaching out for help. I have a array of dates and I"m trying to make http requests in order of the array and to write the return information, also in order.
Here is my code:
const dateArray = ['november-22-2019', 'november-25-2019', 'november-26-2019', 'november-27-2019', 'november-29-2019'];
async function loppThroughArray() {
for (const date of dateArray) {
const options = {
url: process.env.URL + date
};
await asyncRequest(options, date);
}
}
async function asyncRequest(options, date) {
request(options, function(error, response, html) {
if (error) {
return;
}
if (response.statusCode !== 200) {
return;
}
const $ = cheerio.load(html);
const content = $('.entry-content');
const formattedContent = content.text()
.split('\t').join('')
.split('\n\n\n').join('')
.split('\n\n').join('');
const dataToBeWritten = '### ' + date + '\n' + formattedContent + '\n\n';
fs.appendFileSync(`./WODs/${currentDate}.md`, dataToBeWritten, 'utf-8', {'flags':'a+'});
fs.appendFileSync(`./WODs/${currentDate}.txt`, dataToBeWritten, 'utf-8', {'flags':'a+'});
});
}
loppThroughArray();

I managed to solve this by using a version of 'request' that uses promises instead of callbacks. See code:
async function loppThroughArray() {
for (const date of dateArray) {
const options = {
url: process.env.URL + date
};
await asyncRequest(options, date);
console.log('fetching: ', date);
}
}
async function asyncRequest(options, date) {
await rp(options)
.then(function(html){
const $ = cheerio.load(html);
const content = $('.entry-content');
const formattedContent = content.text()
.split('\t').join('')
.split('\n\n\n').join('')
.split('\n\n').join('');
const dataToBeWritten = '### ' + date + '\n' + formattedContent + '\n\n';
fs.appendFileSync(`./WODs/${currentDate}.md`, dataToBeWritten, 'utf-8', {'flags':'a+'});
fs.appendFileSync(`./WODs/${currentDate}.txt`, dataToBeWritten, 'utf-8', {'flags':'a+'});
});
}
loppThroughArray();

Related

Small issue in chat system

window.onload = function () {
const user_name =
NAME_ARRAY[Math.round(Math.random() * NAME_ARRAY.length)];
document.getElementById("userName_profile").innerHTML = user_name;
document.getElementById("userName").innerHTML = user_name;
const btn = document.querySelector("#sendMessage");
const textfield = document.querySelector("#messageInput");
const log = document.querySelector("#messageArea");
// +++++++++++++++ For chat messaging system ++++++++++++
NAF.connection.subscribeToDataChannel(
"chat",
(senderId, dataType, data, targetId) => {
console.log("msg", data, "from", senderId, targetId);
messageArea.innerHTML += senderId + ": " + data.txt + "<br>";
}
);
btn.addEventListener("click", (evt) => {
NAF.clientId = user_name;
messageArea.innerHTML +=
NAF.clientId + ": " + textfield.value + "<br>";
NAF.connection.broadcastData("chat", { txt: textfield.value });
document.getElementById("messageInput").value = "";
// NAF.easy = user_name;
});
}
When I've made chat system and working fine. The only, issue is that I want a user name rather than a client id. Please, help with this.

wait on async fuction +

I want to get the last deviceId.
Please try the following code on a smartphone.
https://www.ofima.ch/file1.html
Inside the function "getConnectedDevices" the variable deviceId ok.
But outside is returned a promise and not the variable deviceId.
How can I get the variable deviceId ?
Thanks
Miche
Explanation:
You need to wrap your alert in an async function and use await. Also to take the value from the promise you needed .then(). Hope the below helps.
Original code:
async function getConnectedDevices() {
var index;
const devices = await navigator.mediaDevices.enumerateDevices();
for (var i=0; i<devices.length; i++) {
if (devices[i].kind == "videoinput") {
index = i;
}
}
var deviceId = devices[index].deviceId;
alert('deviceId is ok: ' + deviceId);
return (deviceId);
}
const deviceId = getConnectedDevices();
alert('deviceId is not defined, why ?: ' + deviceId);
New code:
async function getConnectedDevices() {
let index;
const devices = await navigator.mediaDevices.enumerateDevices();
for (let i=0; i < devices.length; i++) {
console.debug(devices[i]);
if (devices[i].kind == "videoinput") {
index = i;
}
}
console.log('deviceId is ok: ', devices[index]);
return devices[index].deviceId;
}
(async() => {
const deviceId = await getConnectedDevices().then();
alert(`deviceId: ${deviceId}`);
})();
And a quick hack for storing the deviceId in the window
console.log('globalDeviceId should be undefined', window.globalDeviceObj);
async function getConnectedDevices() {
let index;
const devices = await navigator.mediaDevices.enumerateDevices();
for (let i = 0; i < devices.length; i++) {
console.debug(devices[i]);
if (devices[i].kind == "videoinput") {
index = i;
}
}
console.log('deviceId is ok', devices[index]);
return devices[index];
}
function getDeviceId() {
(async() => {
window.globalDeviceObj = await getConnectedDevices().then();
console.log(`globalDeviceId set: ${JSON.stringify(window.globalDeviceObj)}`);
})();
}
function tick() {
if(typeof window.globalDeviceObj === 'undefined'){
requestAnimationFrame(tick);
}else {
alert(`globalDeviceId get: ${JSON.stringify(window.globalDeviceObj)}, with deviceId: ${(window.globalDeviceObj.deviceId)}`)
}
}
function init() {
tick();
getDeviceId();
}
init();

navigation after AsyncStorage.setItem: _this3.navigateTo is not a function

Currently, I am implementing a chat. After user pressed a chat button, the app will navigate the user to the Chat component. The chat content will simply store in firebase and chatId is needed to identify which chat belongs to the user.
Since I don't know how to pass props during navigation, I decided to save the CurrentChatId in AsyncStorage. After navigated to the Chat component, it will get the CurrentChatId from AsyncStorage so that I can map the chat content with the firebase.
However, I got the error _this3.navigateTo is not a function with code below:
let ref = FirebaseClient.database().ref('/Chat');
ref.orderByChild("chatId").equalTo(chatId).once("value", function(snapshot) {
chatId = taskId + "_" + user1Id + "_" + user2Id;
if (snapshot.val() == null) {
ref.push({
chatId: chatId,
taskId: taskId,
user1Id: user1Id,
user2Id: user2Id,
})
}
try {
AsyncStorage.setItem("CurrentChatId", chatId).then(res => {
this.navigateTo('chat');
});
} catch (error) {
console.log('AsyncStorage error: ' + error.message);
}
}
The function navigateTo is copied from the demo app of NativeBase
import { actions } from 'react-native-navigation-redux-helpers';
import { closeDrawer } from './drawer';
const {
replaceAt,
popRoute,
pushRoute,
} = actions;
export default function navigateTo(route, homeRoute) {
return (dispatch, getState) => {
const navigation = getState().cardNavigation;
const currentRouteKey = navigation.routes[navigation.routes.length - 1].key;
dispatch(closeDrawer());
if (currentRouteKey !== homeRoute && route !== homeRoute) {
dispatch(replaceAt(currentRouteKey, { key: route, index: 1 }, navigation.key));
} else if (currentRouteKey !== homeRoute && route === homeRoute) {
dispatch(popRoute(navigation.key));
} else if (currentRouteKey === homeRoute && route !== homeRoute) {
dispatch(pushRoute({ key: route, index: 1 }, navigation.key));
}
};
}
You should bind this to the function that contains the try & catch. The best practice is to add this bind the constructor of the the component:
constructor(props) {
super(props);
this.myFunctoin = this.myfuction.bind(this);
}
Finally, I solved the problem. It is really because this.navigateTo('chat'); is inside function(snapshot)
ref.orderByChild("chatId").equalTo(chatId).once("value", function(snapshot) {
chatId = taskId + "_" + user1Id + "_" + user2Id;
if (snapshot.val() == null) {
ref.push({
chatId: chatId,
taskId: taskId,
user1Id: user1Id,
user2Id: user2Id,
})
}
}
try {
AsyncStorage.setItem("CurrentChatId", chatId).then(res => {
this.navigateTo('chat');
});
} catch (error) {
console.log('AsyncStorage error: ' + error.message);
}
Take it out from the function will solve the problem.

httpParamSerializerJQLike in angular2?

How to serialize JSON for Ruby API?
Angular 1
$scope.submitForm = function() {
var data = {"contato": $scope.contato, "id":$scope.contato.id, "_method":'PUT'};
$http.post(
'http://myApi/contatos/' + $scope.contato.id,
**$httpParamSerializerJQLike(data)**,
{
headers: {'Content-Type': 'application/x-www-form-urlencoded'},
datatype: "JSONP"
}).then(function successCallback(response) {
modalContato.show();
setTimeout(function (){
modalContato.hide();
$state.go('contato-detalhe', {"id":$scope.contato.id});
}, 1500);
});
};
Angular2:
insertContato(contato: Contato) {
let headers = new Headers({
'Content-Type': 'application/x-www-form-urlencoded'
});
let options = new RequestOptions({ headers: headers });
this._http
.post(this.urlApi + '/contatos', JSON.stringify(contato), options)
.subscribe(data => {
console.log('Funciona: ' + data.text());
}, error => {
console.log('Erro: ' + error.text())
});
}
"JSON.stringify(contato)"
It does not have the same behavior as $httpParamSerializerJQLike(data).
Json's broken in the server...
Started POST "/contatos" for 127.0.0.1 at 2016-04-13 13:25:55 -0300
Processing by ContatosController#create as HTML
Parameters: {"{\"nome\":\"asd\",\"email\":\"asd#asda.com\",\"telefone\":\"123\"}"=>nil}
Completed 400 Bad Request in 4ms (ActiveRecord: 0.0ms)
Correct is:
Started POST "/contatos" for 127.0.0.1 at 2016-04-12 17:00:24 -0300
Processing by ContatosController#create as JSON
Parameters: {"contato"=>{"nome"=>"felipe", "telefone"=>"5555"}}
Completed 200 OK in 278ms (Views: 0.1ms | ActiveRecord: 229.4ms)
I had a similar problem, i can solve this:
import { Headers, Http, Response, URLSearchParams, RequestOptions } from '#angular/http';
let headers = new Headers({
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': '*/*'});
let options = new RequestOptions({ headers: headers });
let body = new URLSearchParams();
body.set("message", JSON.stringify(m_dataRequest));
body.set("webService", "authService");
return this.http
.post(this.Url, body.toString(), options)
.toPromise()
.then(this.extractData)
.catch(this.handleError);
URLSearchParams normalized the params of the form and the pipes dimiss, this work for me.
I hope this solve your problem.
I'm going to preface this by saying that this may not be the best way to handle this, but it's how I took care of the problem for myself (The Angular 2 docs don't seem to mention x-www-form-urlencoded anywhere).
So if your data is set up like
var data = {"contato": $scope.contato, "id":$scope.contato.id, "_method":'PUT'};
You want to basically convert it to a form yourself.
var encodedData = "contato=" + contato + "&id=" + contato.id + "&_method=PUT";
then you can modify your POST request to look like this
this._http
.post(this.urlApi + '/contatos', encodedData, options)
.subscribe(data => {
console.log('Funciona: ' + data.text());
}, error => {
console.log('Erro: ' + error.text())
});
There's no need to JSON.stringify it since you're not passing json, you're passing form data.
I hope this helps.
I wrote a function in my http.ts provider like so ---
private formatData(data){
let returnData = '';
console.log(data);
let count = 0;
for (let i in data){
if(count == 0){
returnData += i+'='+data[i];
}else{
returnData += '&'+i+'='+data[i];
}
count = count + 1;
console.log(returnData);
}
return returnData;
}
Call it like this.
post('localhost/url',data){
data = this.formatData(data);
}
Just copy the relative codes from angularjs http module
import {
isArray,
forEach,
isObject,
isDate,
isFunction,
isUndefined,
isNumber,
} from 'lodash';
function toJsonReplacer(key, value) {
let val = value;
if (
typeof key === 'string' &&
key.charAt(0) === '$' &&
key.charAt(1) === '$'
) {
val = undefined;
}
return val;
}
function toJson(obj, pretty = undefined) {
if (isUndefined(obj)) return undefined;
if (!isNumber(pretty)) {
pretty = pretty ? 2 : null; // tslint:disable-line no-parameter-reassignment
}
return JSON.stringify(obj, toJsonReplacer, pretty);
}
function serializeValue(v) {
if (isObject(v)) {
return isDate(v) ? v.toISOString() : toJson(v);
}
return v;
}
function forEachSorted(obj, iterator, context = null) {
const keys = Object.keys(obj).sort();
for (let i = 0; i < keys.length; i += 1) {
iterator.call(context, obj[keys[i]], keys[i]);
}
return keys;
}
/**
* This method is intended for encoding *key* or *value* parts of query component. We need a custom
* method because encodeURIComponent is too aggressive and encodes stuff that doesn't have to be
* encoded per http://tools.ietf.org/html/rfc3986:
* query = *( pchar / "/" / "?" )
* pchar = unreserved / pct-encoded / sub-delims / ":" / "#"
* unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
* pct-encoded = "%" HEXDIG HEXDIG
* sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
* / "*" / "+" / "," / ";" / "="
*/
function encodeUriQuery(val, pctEncodeSpaces = undefined) {
return encodeURIComponent(val)
.replace(/%40/gi, '#')
.replace(/%3A/gi, ':')
.replace(/%24/g, '$')
.replace(/%2C/gi, ',')
.replace(/%3B/gi, ';')
.replace(/%20/g, pctEncodeSpaces ? '%20' : '+');
}
export function jQueryLikeParamSerializer(params) {
if (!params) return '';
const parts = [];
serialize(params, '', true);
return parts.join('&');
function serialize(toSerialize, prefix, topLevel = undefined) {
if (isArray(toSerialize)) {
forEach(toSerialize, (value, index) => {
serialize(value, prefix + '[' + (isObject(value) ? index : '') + ']');
});
} else if (isObject(toSerialize) && !isDate(toSerialize)) {
forEachSorted(toSerialize, (value, key) => {
serialize(
value,
prefix + (topLevel ? '' : '[') + key + (topLevel ? '' : ']'),
);
});
} else {
if (isFunction(toSerialize)) {
toSerialize = toSerialize(); // tslint:disable-line no-parameter-reassignment
}
parts.push(
encodeUriQuery(prefix) +
'=' +
(toSerialize == null
? ''
: encodeUriQuery(serializeValue(toSerialize))),
);
}
}
}
I've met the similar issue when was upgrading from angular 1.x
Here is my solution which also process nested JSON objects:
function Json2FormEncoded(json_obj) {
let path = arguments[1] || '';
let s = '', p = '';
for (let i in json_obj) {
p = path == '' ? i : path + '[' + i + ']';
s = s ? s + "&" : s;
if (typeof json_obj[i] == 'object') {
s += Json2FormEncoded(json_obj[i], p);
} else {
s += p + '=' + encodeURIComponent(json_obj[i]);
}
}
return s;
}
Hope you'll find it useful!
Also check it here

How to avoid blockin while uploading file using Meteor method

I've created a Meteor method to upload a file, it's working well but until the file is fully uploaded, I cannot move around, all subscriptions seem to wait that the upload finishes... is there a way to avoid that ?
Here is the code on the server :
Meteor.publish('product-photo', function (productId) {
return Meteor.photos.find({productId: productId}, {limit: 1});
});
Meteor.methods({
/**
* Creates an photo
* #param obj
* #return {*}
*/
createPhoto: function (obj) {
check(obj, Object);
// Filter attributes
obj = filter(obj, [
'name',
'productId',
'size',
'type',
'url'
]);
// Check user
if (!this.userId) {
throw new Meteor.Error('not-connected');
}
// Check file name
if (typeof obj.name !== 'string' || obj.name.length > 255) {
throw new Meteor.Error('invalid-file-name');
}
// Check file type
if (typeof obj.type !== 'string' || [
'image/gif',
'image/jpg',
'image/jpeg',
'image/png'
].indexOf(obj.type) === -1) {
throw new Meteor.Error('invalid-file-type');
}
// Check file url
if (typeof obj.url !== 'string' || obj.url.length < 1) {
throw new Meteor.Error('invalid-file-url');
}
// Check file size
if (typeof obj.size !== 'number' || obj.size <= 0) {
throw new Meteor.Error('invalid-file-size');
}
// Check file max size
if (obj.size > 1024 * 1024) {
throw new Meteor.Error('file-too-large');
}
// Check if product exists
if (!obj.productId || Meteor.products.find({_id: obj.productId}).count() !== 1) {
throw new Meteor.Error('product-not-found');
}
// Limit the number of photos per user
if (Meteor.photos.find({productId: obj.productId}).count() >= 3) {
throw new Meteor.Error('max-photos-reached');
}
// Resize the photo if the data is in base64
if (typeof obj.url === 'string' && obj.url.indexOf('data:') === 0) {
obj.url = resizeImage(obj.url, 400, 400);
obj.size = obj.url.length;
obj.type = 'image/png';
}
// Add info
obj.createdAt = new Date();
obj.userId = this.userId;
return Meteor.photos.insert(obj);
}
});
And the code on the client :
Template.product.events({
'change [name=photo]': function (ev) {
var self = this;
readFilesAsDataURL(ev, function (event, file) {
var photo = {
name: file.name,
productId: self._id,
size: file.size,
type: file.type,
url: event.target.result
};
Session.set('uploadingPhoto', true);
// Save the file
Meteor.call('createPhoto', photo, function (err, photoId) {
Session.set('uploadingPhoto', false);
if (err) {
displayError(err);
} else {
notify(i18n("Transfert terminé pour {{name}}", photo));
}
});
});
}
});
I finally found the solution myself.
Explication : the code I used was blocking the subscriptions because it was using only one method call to transfer all the file from the first byte to the last one, that leads to block the thread (I think, the one reserved to each users on the server) until the transfer is complete.
Solution : I splitted the file into chunks of about 8KB, and send chunk by chunk, this way the thread or whatever was blocking the subscriptions is free after each chunk transfer.
The final working solution is on that post : How to write a file from an ArrayBuffer in JS
Client Code
// data comes from file.readAsArrayBuffer();
var total = data.byteLength;
var offset = 0;
var upload = function() {
var length = 4096; // chunk size
// adjust the last chunk size
if (offset + length > total) {
length = total - offset;
}
// I am using Uint8Array to create the chunk
// because it can be passed to the Meteor.method natively
var chunk = new Uint8Array(data, offset, length);
if (offset < total) {
// Send the chunk to the server and tell it what file to append to
Meteor.call('uploadFileData', fileId, chunk, function (err, length) {
if (!err) {
offset += length;
upload();
}
}
}
};
upload();
Server code
var fs = Npm.require('fs');
var Future = Npm.require('fibers/future');
Meteor.methods({
uploadFileData: function(fileId, chunk) {
var fut = new Future();
var path = '/uploads/' + fileId;
// I tried that with no success
chunk = String.fromCharCode.apply(null, chunk);
// how to write the chunk that is an Uint8Array to the disk ?
fs.appendFile(path, new Buffer(chunk), function (err) {
if (err) {
fut.throw(err);
} else {
fut.return(chunk.length);
}
});
return fut.wait();
}
});
Improving #Karl's code:
Client
This function breaks the file into chunks and sends them to the server one by one.
function uploadFile(file) {
const reader = new FileReader();
let _offset = 0;
let _total = file.size;
return new Promise((resolve, reject) => {
function readChunk() {
var length = 10 * 1024; // chunk size
// adjust the last chunk size
if (_offset + length > _total) {
length = _total - _offset;
}
if (_offset < _total) {
const slice = file.slice(_offset, _offset + length);
reader.readAsArrayBuffer(slice);
} else {
// EOF
setProgress(100);
resolve(true);
}
}
reader.onload = function readerOnload() {
let buffer = new Uint8Array(reader.result) // convert to binary
Meteor.call('fileUpload', file.name, buffer, _offset,
(error, length) => {
if (error) {
console.log('Oops, unable to import!');
return false;
} else {
_offset += length;
readChunk();
}
}
);
};
reader.onloadend = function readerOnloadend() {
setProgress(100 * _offset / _total);
};
readChunk();
});
}
Server
The server then writes to a file when offset is zero, or appends to its end otherwise, returning a promise, as I used an asynchronous function to write/append in order to avoid blocking the client.
if (Meteor.isServer) {
var fs = require('fs');
var Future = require('fibers/future');
}
Meteor.methods({
// Upload file from client to server
fileUpload(
fileName: string,
fileData: Uint8Array,
offset: number) {
check(fileName, String);
check(fileData, Uint8Array);
check(offset, Number);
console.log(`[x] Received file ${fileName} data length: ${fileData.length}`);
if (Meteor.isServer) {
const fut = new Future();
const filePath = '/tmp/' + fileName;
const buffer = new Buffer(fileData);
const jot = offset === 0 ? fs.writeFile : fs.appendFile;
jot(filePath, buffer, 'binary', (err) => {
if (err) {
fut.throw(err);
} else {
fut.return(buffer.length);
}
});
return fut.wait();
}
}
)};
Usage
uploadFile(file)
.then(() => {
/* do your stuff */
});

Resources