I'm using node.js 0.6.18, and the following code makes node.js close the TCP connection between every two requests (verified with strace on Linux). How do I make node.js reuse the same TCP connection for multiple HTTP requests (i.e. keep-alive)? Please note that the webserver is capable of keep-alive, it works with other clients. The webserver returns a chunked HTTP response.
var http = require('http');
var cookie = 'FOO=bar';
function work() {
var options = {
host: '127.0.0.1',
port: 3333,
path: '/',
method: 'GET',
headers: {Cookie: cookie},
};
process.stderr.write('.')
var req = http.request(options, function(res) {
if (res.statusCode != 200) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
process.exit(1)
}
res.setEncoding('utf8');
res.on('data', function (chunk) {});
res.on('end', function () { work(); });
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
process.exit(1);
});
req.end();
}
work()
I was able to get this to work (verified with strace) by creating an http.Agent and setting its maxSockets property to 1. I don't know if this is the ideal way to do it; however, it does meet the requirements. One thing that I did notice is that what the docs claimed about http.Agent behavior did not accurately describe how it worked in practice. Code below:
var http = require('http');
var cookie = 'FOO=bar';
var agent = new http.Agent;
agent.maxSockets = 1;
function work() {
var options = {
host: '127.0.0.1',
port: 3000,
path: '/',
method: 'GET',
headers: {Cookie: cookie},
agent: agent
};
process.stderr.write('.')
var req = http.request(options, function(res) {
if (res.statusCode != 200) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
process.exit(1)
}
res.setEncoding('utf8');
res.on('data', function (chunk) {});
res.on('end', function () { work(); });
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
process.exit(1);
});
req.end();
}
work()
EDIT: I should add that I did my testing with node.js v0.8.7
you can just set:
http.globalAgent.keepAlive = true
Related
I need to put some delays between each axios POST call so POST calls are in sequence -- wait one finishes before issue the next POST.
The delay code I put in seems to delay -- console log shows "Delaying" and pauses there for some seconds, but on the server side, POST calls are still concurrent.
import * as Axios from "axios";
delay(delayTime:number) {
var tNow = Date.now();
var dateDiff = 0;
do {
dateDiff = Date.now() - tNow;
} while (dateDiff < delayTime); //milliseconds
}
// the code below is in a for loop
let axiosConfig = {
url: myurl,
method: ‘POST’,
timeout: 5,
headers: {
'Authorization' : AuthStr,
'Accept' : 'application/json',
'Content-Type' : 'application/json'
},
data: objContent
}
console.log(">>>>>>>>>>>>>>>>>>> Delaying”);
delay(10000);
let request = Axios.create().request(axiosConfig).
catch((rejection:any) => {
// some code
});
In order to wait one HTTP request finishes before issue the next request, you need to queue the HTTP request (queue async operation). The steps are:
When HTTP request is needed, add it to the queue.
Check whether there are any elements in the queue. If any, pick up one and execute it.
After that HTTP request is finished, goto step 2.
Example code would look like below:
const axios = require('axios');
let queue = [];
function sendRequest(callback) {
axios.get('http://example.com')
.then(function() {
callback();
}).catch(function () {
callback();
});
}
function addRequestToQueue() {
let id = (Math.random()*100000).toFixed(0);
if (queue.length === 0) {
sendRequest(function() {
queue.splice(queue.indexOf(id), 1);
consumeQueue();
});
}
queue.push(id);
}
function consumeQueue() {
if (!queue.length) {
return;
}
let id = queue[0];
sendRequest(function() {
queue.splice(queue.indexOf(id), 1);
consumeQueue();
});
}
addRequestToQueue();
addRequestToQueue();
addRequestToQueue();
addRequestToQueue();
addRequestToQueue();
I attempting to use ASP and vb.net to run a node.js file that returns a port number from Bungie's API. The issue I am having is that I can use this application locally, but not if I were to publish and deploy the code. The way I do this in Visual Studio is open the Package Manager Console, type in "node proxy.js," and copy the returned value (proxy.js is the node file I'm executing). Then I'd take that port number and add it into my code.
Obviously, this isn't the ideal way to execute this file. So, my question is: is there a way I can execute this file from the code behind in VB.NET? Or, is there a way that I can take my node file and execute it in VB? Here is the proxy.js file I'm working with:
var https = require('https');
var http = require('http');
var BUNGIE = {
host: 'www.bungie.net',
port: 443
};
function copyHeaderFrom(source) {
return function (target, k) {
if (typeof target[k.toLowerCase()] === 'undefined') {
target[k] = source[k];
}
return target;
};
}
http.createServer(function (req, res) {
var outboundData = {
method: req.method,
host: BUNGIE.host,
port: BUNGIE.port,
path: req.url,
headers: req.headers
};
console.log('req.headers.cookie:', req.headers.cookie);
outboundData.headers.host = BUNGIE.host;
console.log('outbound request ========================');
console.log(outboundData);
if (outboundData.method === 'OPTIONS') {
res.writeHead(200, {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Headers': outboundData.headers['access-control-request-headers'] || ''
});
return res.end();
}
https.request(outboundData, function (bungieRes) {
var initialHeaders = {
'access-control-allow-origin': '*',
'origin': outboundData.headers.origin
};
res.writeHead(
bungieRes.statusCode,
Object.keys(bungieRes.headers).reduce(copyHeaderFrom(bungieRes.headers), initialHeaders)
);
bungieRes.pipe(res);
}).end();
}).listen(process.argv[2], function () {
console.log('Bungie Proxy Server running on port %s', this.address().port);
});
I am using a CasperJS script as a web service, accessing it from a node server. What I did not manage to do is to make Casper be 'multithread'. If I make two simultaneously requests to Casper from postman the result will be something scrambled between both requests for one response, and the second will be null. I saw that PhantomJS has a page principle, but I did not find anything similar for Casper.
Can i call Casper's web service with multiple requests at the same time and get correct/coherent responses?
Is there some configuration needed for the web server to allow me to do this?
Should the request be done in a 'special manner'? Are there any caveats regarding this that i should be aware of?
If it can only function sequentially, would starting multiple servers on the same machine but different ports solve the issue?
Here is the casper web service i am talking about. When I make a request like
locahost:1338/?query=name.name it will crawl for that query on the
specified url. My problem comes when I make 2 parallel requests with different queries.
//includes web server modules
"use strict";
var port = 1338;
var server = require('webserver').create();
var url = 'url to scrap';
//start web server
var service = server.listen(port, function(request, response) {
var arr1 = [];
var arr2 = [];
var arr3 = [];
var casper = require("casper").create({
verbose: true,
logLevel: 'error',
pageSettings: {
loadImages: false,
loadPlugins: false,
userAgent: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36'
},
clientScripts: ["vendor/jquery-1.12.1.js"]
});
casper.start(url, function() {
}, function(){
console.log(url + " not found");
return;
});
casper.waitForSelector('.cssClass', function() {
}, function(){
console.log("not found");
return;
});
casper.then(function() {
var query = getQuery(request.url);
casper.sendKeys('.cssClass', query);
casper.click('.cssClass');
casper.waitForSelector('.cssClass', function(){
arr1 = this.evaluate(function(){
var nodeList = document.querySelectorAll(".cssClass");
return Array.prototype.map.call(nodeList, function(node){
return node.textContent;
});
});
}, function(){
console.log("not found");
return;
});
casper.then(function(){
if(names.length > 0)
{
casper.waitForSelector('.cssClass', function(){
arr2 = this.evaluate(function(){
var nodeList = document.querySelectorAll(".cssClass");
return Array.prototype.map.call(nodeList, function(node){
return node.textContent;
});
});
console.log("found");
}, function(){
console.log("not found");
return;
});
casper.waitForSelector('.cssClass', function(){
arr3 = this.evaluate(function(){
var nodeList = document.querySelectorAll(".cssClass");
return Array.prototype.map.call(nodeList, function(node){
return node.src;
});
});
console.log("found");
}, function(){
console.log("not found");
return;
});
}
});
});
casper.run(function() {
response.statusCode = 200;
response.write(JSON.stringify({p1: arr1, p2: arr2, p3: arr3}));
response.close();
});
});
console.log('\nServer running at http://localhost:' + port+'/');
Our web application is consuming a REST API that requires Basic Authentication.
I want to write the browser-sync gulp task like this:
gulp.task('browser-sync', function() {
browserSync({
server: {
baseDir: './'
},
authenticate: 'authentication-token-here-djfhjsdfjsgdf'
});
});
How might this be configured?
You can do it easily with the middleware option that processes each request:
gulp.task('browser-sync', function() {
browserSync({
server: {
baseDir: './'
},
middleware: [
function(req, res, next) {
const user = 'user';
const pass = 'pass';
let authorized = false;
// See if authorization exist in the request and matches username/password
if (req.headers.authorization) {
const credentials = new Buffer(req.headers.authorization.replace('Basic ', ''), 'base64').toString().split(/:(.*)/)
if (credentials[0] === user && credentials[1] === pass) {
authorized = true;
}
}
if (authorized) {
// Proceed to fulfill the request
next();
} else {
// Authorization doesn't exist / doesn't match, send authorization request in the response header
res.writeHead(401, {'WWW-Authenticate': 'Basic realm="Authenticate"'})
res.end();
}
}
]
});
});
I am testing an XML-RPC set up in node.js and would like to test the server receiving calls and responding as well as the client making calls to the server and receiving a response in the same node session. If I run http.createServer and http.request with the same host and port, I get:
Error: ECONNREFUSED, Connection refused
at Socket._onConnect (net.js:600:18)
at IOWatcher.onWritable [as callback] (net.js:186:12)
Test code that will generate the errror:
var http = require('http')
var options = {
host: 'localhost'
, port: 8000
}
// Set up server and start listening
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'})
res.end('success')
}).listen(options.port, options.host)
// Client call
// Gets Error: ECONNREFUSED, Connection refused
var clientRequest = http.request(options, function(res) {
res.on('data', function (chunk) {
console.log('Called')
})
})
clientRequest.write('')
clientRequest.end()
While the code above will work if separated into two files and run as separate node instances, is there a way to get the above to run on the same node instance?
As it was mentioned above your http-server may not be running at the time you make a request. Using setTimeout is completely wrong. Use the callback parameter in listen method instead:
var http = require('http')
var options = {
host: 'localhost'
, port: 8000
}
// Set up server and start listening
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'})
res.end('success')
}).listen(options.port, options.host, function() {
// Client call
// No error, server is listening
var clientRequest = http.request(options, function(res) {
res.on('data', function (chunk) {
console.log('Called')
})
})
clientRequest.write('')
clientRequest.end()
});
Your HTTP server probably isn't fully loaded and operational at the time when you are doing your request to it. Try wrap your client request with setTimeout to give your server a time to set up, for example like this:
var http = require('http')
var options = {
host: 'localhost'
, port: 8000
}
// Set up server and start listening
http.createServer(function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'})
res.end('success')
}).listen(options.port, options.host)
setTimeout(function() {
// Client call
// Shouldn't get error
var clientRequest = http.request(options, function(res) {
res.on('data', function (chunk) {
console.log('Called')
})
})
clientRequest.write('')
clientRequest.end()
}, 5000);