I am using a CasperJS script as a web service, accessing it from a node server. What I did not manage to do is to make Casper be 'multithread'. If I make two simultaneously requests to Casper from postman the result will be something scrambled between both requests for one response, and the second will be null. I saw that PhantomJS has a page principle, but I did not find anything similar for Casper.
Can i call Casper's web service with multiple requests at the same time and get correct/coherent responses?
Is there some configuration needed for the web server to allow me to do this?
Should the request be done in a 'special manner'? Are there any caveats regarding this that i should be aware of?
If it can only function sequentially, would starting multiple servers on the same machine but different ports solve the issue?
Here is the casper web service i am talking about. When I make a request like
locahost:1338/?query=name.name it will crawl for that query on the
specified url. My problem comes when I make 2 parallel requests with different queries.
//includes web server modules
"use strict";
var port = 1338;
var server = require('webserver').create();
var url = 'url to scrap';
//start web server
var service = server.listen(port, function(request, response) {
var arr1 = [];
var arr2 = [];
var arr3 = [];
var casper = require("casper").create({
verbose: true,
logLevel: 'error',
pageSettings: {
loadImages: false,
loadPlugins: false,
userAgent: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36'
},
clientScripts: ["vendor/jquery-1.12.1.js"]
});
casper.start(url, function() {
}, function(){
console.log(url + " not found");
return;
});
casper.waitForSelector('.cssClass', function() {
}, function(){
console.log("not found");
return;
});
casper.then(function() {
var query = getQuery(request.url);
casper.sendKeys('.cssClass', query);
casper.click('.cssClass');
casper.waitForSelector('.cssClass', function(){
arr1 = this.evaluate(function(){
var nodeList = document.querySelectorAll(".cssClass");
return Array.prototype.map.call(nodeList, function(node){
return node.textContent;
});
});
}, function(){
console.log("not found");
return;
});
casper.then(function(){
if(names.length > 0)
{
casper.waitForSelector('.cssClass', function(){
arr2 = this.evaluate(function(){
var nodeList = document.querySelectorAll(".cssClass");
return Array.prototype.map.call(nodeList, function(node){
return node.textContent;
});
});
console.log("found");
}, function(){
console.log("not found");
return;
});
casper.waitForSelector('.cssClass', function(){
arr3 = this.evaluate(function(){
var nodeList = document.querySelectorAll(".cssClass");
return Array.prototype.map.call(nodeList, function(node){
return node.src;
});
});
console.log("found");
}, function(){
console.log("not found");
return;
});
}
});
});
casper.run(function() {
response.statusCode = 200;
response.write(JSON.stringify({p1: arr1, p2: arr2, p3: arr3}));
response.close();
});
});
console.log('\nServer running at http://localhost:' + port+'/');
Related
I've been trying to get my refresh token to work for a while now, and I hope I'm close. My token refreshes and triggers a subsequent 200 call to whatever call caused the 401, but my the data on my page doesn't refresh.
When an access token expires, the following happens:
After the 401, the GetListofCompanyNames returns 200 with a list of names using the correct updated access token. However, my dropdown does not refresh.
My interceptor:
app.factory('authInterceptorService',['$q', '$location', 'localStorageService', '$injector', function($q, $location, localStorageService, $injector) {
return {
request: function(config) {
config.headers = config.headers || {};
var authData = localStorageService.get('authorizationData');
if (authData) {
config.headers.Authorization = 'Bearer ' + authData.token;
}
return config;
},
responseError: function(rejection) {
//var promise = $q.reject(rejection);
var authService = $injector.get('authService');
if (rejection.status === 401) {
// refresh the token
authService.refreshToken().then(function() {
// retry the request
var $http = $injector.get('$http');
return $http(rejection.config);
});
}
if (rejection.status === 400) {
authService.logOut();
$location.path('/login');
}
return $q.reject(rejection);
}
};
}
]);
My return statement on the 401 rejection looks suspect here, but I'm not sure what to replace it with. Thereby my question is: How can I get my page to refresh it's data when I make the new call?
Update:
This gets me past when the 200 returns and I can get a dropdown to refresh, but I lose any state on the page (ex. selected dropdown) with the below.
authService.refreshToken().then(function() {
var $state = $injector.get('$state');
$state.reload();
});
Back to the drawing board!
Try putting up your retry call in $timeout, it should work.
Here's the updated code:
app.factory('authInterceptorService',['$q', '$location', 'localStorageService', '$injector', function($q, $location, localStorageService, $injector) {
return {
request: function(config) {
config.headers = config.headers || {};
var authData = localStorageService.get('authorizationData');
if (authData) {
config.headers.Authorization = 'Bearer ' + authData.token;
}
return config;
},
responseError: function(rejection) {
//var promise = $q.reject(rejection);
var authService = $injector.get('authService');
if (rejection.status === 401) {
// refresh the token
authService.refreshToken().then(function() {
// retry the request
return $timeout(function() {
var $http = $injector.get('$http');
return $http(rejection.config);
}});
}
if (rejection.status === 400) {
authService.logOut();
$location.path('/login');
}
return $q.reject(rejection);
}
};
}
]);
$timeout returns a promise that is completed with what is returned
from the function parameter, so we can conveniently just return the
$http call wrapped in $timeout.
Thanks.
I think you may want to change up how you go about this. One way to go about this would be to inject the $rootScope into your authInterceptorService and then once you successfully refresh the token, call something like $rootScope.broadcast('tokenRefreshed').
I don't quite know how you have set up the view and controller that handles your dropdown, but I would set up a listener for that 'tokenRefreshed' event. From here, you can do another call to GetListofCompanyNames. If you do it this way you can easily control and ensure that the model gets updated.
My final solution:
app.factory('authInterceptorService', ['$q', '$location', 'localStorageService', '$injector', function($q, $location, localStorageService, $injector) {
var $http;
var retryHttpRequest = function(config, deferred) {
$http = $http || $injector.get('$http');
$http(config).then(function(response) {
deferred.resolve(response);
},
function(response) {
deferred.reject(response);
});
}
return {
request: function(config) {
config.headers = config.headers || {};
var authData = localStorageService.get('authorizationData');
if (authData) {
config.headers.Authorization = 'Bearer ' + authData.token;
}
return config;
},
responseError: function(rejection) {
var deferred = $q.defer();
if (rejection.status === 401) {
var authService = $injector.get('authService');
authService.refreshToken().then(function() {
retryHttpRequest(rejection.config, deferred);
},
function () {
authService.logOut();
$location.path('/login');
deferred.reject(rejection);
});
} else {
deferred.reject(rejection);
}
return deferred.promise;
}
};
}
]);
Copied almost 1 for 1 from https://github.com/tjoudeh/AngularJSAuthentication/blob/master/AngularJSAuthentication.Web/app/services/authInterceptorService.js .
This one transparently handles all requests and refreshes them when necessary. It logs out users when the refresh token is expired and passes errors along to the controllers by properly rejecting them. However, it doesn't seem to work with multiple in flight requests, I'll look into that when I get a use case for it in my system.
I have a test case where my code make request to server and check if the filename with specified date exists. Since GET request is async call, how can I make sure that I have the filename from the server before I check if it is a specified date?
Here's excerpt of my code :
var re = new RInterface();
it('data show exists', function() {
target.each(function(ele){
browser.actions().mouseMove(ele).perform();
re.get(function(result){
expect(result).toEqual(true);
});
});
});
RInterface.js
var Service = function() {
var serv = this;
var uname = atob(settings.username);
var pwd = atob(settings.password);
var url = 'https://' + uname + ':' + pwd + '#' + settings.Url + '/' + settings.format + '/' + settings.period;
var completeURL = url;
var today = DateString();
serv.get = function(callback) {
var dataStrAry = [];
var count = 0;
request(completeURL, function (error, response, body) {
if (!error && response.statusCode == 200) {
var serverData = JSON.parse(body);
var split = serverData[serverData.length-1].Name.split(" ");
var target = split[split.length-1].split(".")[0];
// Check if the file with current date is available
// If it is, then assume data is saved on the server
if(target == today) {
console.log("equal");
callback(true);
}
else {
console.log("not equal");
callback(false);
}
}
else {
console.log("errror call");
callback(false);
return;
}
});
};
So, re.get is where I make GET request to the server and I passed callback function to be called at the end of get request. The problem, I think is protractor complete executing the test before my code gets data from the server. How do I force protractor to wait so that I can check the returned data? My current workaround is put the get request inside beforeEach and seems that protractor forces test to wait for it finish executing.
You need to handle the result of your request with a Promise if you want the control flow to wait for it:
var re = new RInterface();
it('data show exists', function() {
target.each(function(ele){
browser.actions().mouseMove(ele).perform();
expect(re.get()).toEqual(true);
});
});
var Service = function() {
...
this.get = function() {
var defer = protractor.promise.defer();
...
request(completeURL, function (error, response, body) {
if (!error && response.statusCode == 200) {
var result = ...
defer.fulfill(result);
} else {
defer.reject(error);
}
});
return defer.promise;
};
};
You can make it easier by using browser.wait, just wrap the request function (which returns a promise) like
browser.wait(request...).then(result => {
// here continue execution with the result of the request
}
This will make the browser wait for your promise.
I'm attempting to build a simple Alexa skill to return data from an API using the [Node.js ASK] (https://developer.amazon.com/public/community/post/Tx213D2XQIYH864/Announcing-the-Alexa-Skills-Kit-for-Node-js). I have put the http get within a handler, but Alexa completes the handler before the callback asynchronously returns the API data.
I have been searching for answers, and my thoughts are currently:
not use node.js
figure out a way to synchronously get the data
Something simple I am missing
Core of the code:
exports.handler = function(event, context, callback) {
var alexa = Alexa.handler(event, context);
alexa.registerHandlers(handler);
alexa.execute();
};
var handler = Alexa.CreateStateHandler(states.x, {
'intent': function() {
var options = {
host: baseURL,
path: pathURL
};
callback = function(response) {
var str = "";
response.on('data', function(piece) {
str += piece;
});
response.on('end', function() {
//does not get executed
this.emit(':tell', str, "test");
});
}
http.request(options, callback).end();
//this does get executed if I leave this here
this.emit(':tell'...);
};
I think you are having a scope issue.
try ...
response.on('end',() => {
this.emit(':tell', str, "test");
});
I attempting to use ASP and vb.net to run a node.js file that returns a port number from Bungie's API. The issue I am having is that I can use this application locally, but not if I were to publish and deploy the code. The way I do this in Visual Studio is open the Package Manager Console, type in "node proxy.js," and copy the returned value (proxy.js is the node file I'm executing). Then I'd take that port number and add it into my code.
Obviously, this isn't the ideal way to execute this file. So, my question is: is there a way I can execute this file from the code behind in VB.NET? Or, is there a way that I can take my node file and execute it in VB? Here is the proxy.js file I'm working with:
var https = require('https');
var http = require('http');
var BUNGIE = {
host: 'www.bungie.net',
port: 443
};
function copyHeaderFrom(source) {
return function (target, k) {
if (typeof target[k.toLowerCase()] === 'undefined') {
target[k] = source[k];
}
return target;
};
}
http.createServer(function (req, res) {
var outboundData = {
method: req.method,
host: BUNGIE.host,
port: BUNGIE.port,
path: req.url,
headers: req.headers
};
console.log('req.headers.cookie:', req.headers.cookie);
outboundData.headers.host = BUNGIE.host;
console.log('outbound request ========================');
console.log(outboundData);
if (outboundData.method === 'OPTIONS') {
res.writeHead(200, {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Headers': outboundData.headers['access-control-request-headers'] || ''
});
return res.end();
}
https.request(outboundData, function (bungieRes) {
var initialHeaders = {
'access-control-allow-origin': '*',
'origin': outboundData.headers.origin
};
res.writeHead(
bungieRes.statusCode,
Object.keys(bungieRes.headers).reduce(copyHeaderFrom(bungieRes.headers), initialHeaders)
);
bungieRes.pipe(res);
}).end();
}).listen(process.argv[2], function () {
console.log('Bungie Proxy Server running on port %s', this.address().port);
});
I'm using node.js 0.6.18, and the following code makes node.js close the TCP connection between every two requests (verified with strace on Linux). How do I make node.js reuse the same TCP connection for multiple HTTP requests (i.e. keep-alive)? Please note that the webserver is capable of keep-alive, it works with other clients. The webserver returns a chunked HTTP response.
var http = require('http');
var cookie = 'FOO=bar';
function work() {
var options = {
host: '127.0.0.1',
port: 3333,
path: '/',
method: 'GET',
headers: {Cookie: cookie},
};
process.stderr.write('.')
var req = http.request(options, function(res) {
if (res.statusCode != 200) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
process.exit(1)
}
res.setEncoding('utf8');
res.on('data', function (chunk) {});
res.on('end', function () { work(); });
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
process.exit(1);
});
req.end();
}
work()
I was able to get this to work (verified with strace) by creating an http.Agent and setting its maxSockets property to 1. I don't know if this is the ideal way to do it; however, it does meet the requirements. One thing that I did notice is that what the docs claimed about http.Agent behavior did not accurately describe how it worked in practice. Code below:
var http = require('http');
var cookie = 'FOO=bar';
var agent = new http.Agent;
agent.maxSockets = 1;
function work() {
var options = {
host: '127.0.0.1',
port: 3000,
path: '/',
method: 'GET',
headers: {Cookie: cookie},
agent: agent
};
process.stderr.write('.')
var req = http.request(options, function(res) {
if (res.statusCode != 200) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
process.exit(1)
}
res.setEncoding('utf8');
res.on('data', function (chunk) {});
res.on('end', function () { work(); });
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
process.exit(1);
});
req.end();
}
work()
EDIT: I should add that I did my testing with node.js v0.8.7
you can just set:
http.globalAgent.keepAlive = true