Which one of the following node.js HTTP proxy implementations is more performant?
The first implementation is:
var http = require('http');
http.createServer(function(request, response) {
var proxy = http.createClient(80, "google.com")
var proxy_request = proxy.request(request.method, request.url, request.headers);
proxy_request.addListener('response', function (proxy_response) {
proxy_response.addListener('data', function(chunk) {
response.write(chunk, 'binary');
});
proxy_response.addListener('end', function() {
response.end();
});
response.writeHead(proxy_response.statusCode, proxy_response.headers);
});
request.addListener('data', function(chunk) {
proxy_request.write(chunk, 'binary');
});
request.addListener('end', function() {
proxy_request.end();
});
}).listen(8080);
The second one uses stream.pipe() and it's like:
var http = require('http');
http.createServer(function(request, response) {
var proxy = http.createClient(80, "google.com");
var proxy_request = proxy.request(request.method, request.url, request.headers);
proxy_request.on('response', function (proxy_response) {
proxy_response.pipe(response);
response.writeHead(proxy_response.statusCode, proxy_response.headers);
});
request.pipe(proxy_request);
}).listen(8080);
The first one might blow up your process if the file is big and the clients connection is slow or if an uploaded file is big and the servers upload bandwidth is small. Use pipe, it's designed for this kind of stuff.
Also, use an existing module from npm for this:
many features and used in production at nodejitsu: http-proxy
fast: bouncy
Related
I am working on the meteor application ,which has iron router in application .I required to access http request and response object in router ,for accessing the request data.
To access the request and response objects, all you need to do is tell Iron Router that this route is a server only route by passing an options object with { where: 'server' } like so:
Router.route('/hello', function () {
var req = this.request;
var res = this.response;
res.end('hello from the server\n');
}, { where: 'server' });
Or if you want to use different functions for different Methods:
Router.route('/hi', { where: 'server' })
.get(function () {
var req = this.request;
var res = this.response;
res.end('hello from the server\n');
})
.post(function () {
var req = this.request;
var res = this.response;
doSomething(req.body)
res.end('done\n');
});
});
See more in the docs: http://iron-meteor.github.io/iron-router/
I discoverd recently chunked response.
I agree that most of the time we want to work on a full response.
But what if I want to work on a chunked response.
How would i do this with the $http service??
You can define a function with the angularjs promise $q wrapping the XMLHttpRequest.
var chunkedRequestWithPromise = function () {
var deferred = $q.defer();
var xhr = new XMLHttpRequest()
xhr.open("GET", 'https://yoururl.com/chunked', true)
xhr.onprogress = function () {
deferred.notify(xhr.responseText);
}
xhr.onreadystatechange = function (oEvent) {
if (xhr.readyState === 4) {
if (xhr.status === 200) {
deferred.resolve('success');
} else {
deferred.reject(xhr.statusText);
}
}
};
xhr.send();
return deferred.promise;
};
and use it:
chunkedRequestWithPromise().then(successFn,errorFn,notifyFn);
I've found a way to do something that looks correct to me
http://www.igvita.com/2011/08/26/server-sent-event-notifications-with-html5/
Of course there is also the websocket api, but it seams heavy for my uses
Another possibility would be the write another http service which would give control over the http response
I'm planning to do a series of HTTP requests in NodeJS though Tor.
Tor uses SOCKS5 so I went out and searched for a way to proxify HTTP requests in NodeJS.
I'm planning to the the default http.request() function to do the work. However, I can't seem to find a way to use a proxy with that. Someone suggested that I could do this:
var http = require("http");
var options = {
host: "localhost",
port: 9050,
path: "http://check.torproject.org",
method: 'GET',
headers: {
Host: "http://check.torproject.org",
}
};
var req = http.request(options, function(res) {
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
But it didn't work.
So, any suggestions?
I've just published two modules that should help you do this: socks5-http-client and socks5-https-client.
Just use those instead of the default http module. The API is the same. For example:
require('socks5-http-client').request(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
I know I'm answering an old question but there is a better solution available for this question, about how to use sock4 & sock5 proxy in Node.js. For the sake of simplicity, I will be using request-promise module however you can also use bare request module.
Requrement: socks-proxy-agent, request-promise
Example:
async function main() {
var proxy = "socks4://1.2.3.4:35618"
var agent = new SocksProxyAgent(proxy);
var options = {
uri: 'http://targetUrl',
agent: agent,
headers: {
'User-Agent': 'Request-Promise'
}
}
try {
var responce = await rp(options)
} catch(err) {
console.log(err)
}
console.log(responce) }
Not a complete answer, but you may want to keep your eye on these two modules.
https://github.com/Ayms/node-Tor
Support is being added into: https://github.com/Ayms/node-bot.
I sent him an email asking when he expected this to be complete, will update this post soon with that information.
I had the same problem and used polipo as proxy between node and TOR
node (request) - polilp httproxy:8123 - polipo - tor (socks5:9050).
For mac (osx with brew) it worked like this:
brew install polipo tor
tor # start top
polipo socksParentProxy=localhost:9050 # start polipo
Working example with request
var request = require('request');
var options = {'url':'https://check.torproject.org/', 'proxy':'http://localhost:8123'}
request(options,
function (error, response, body) {
if (error){
console.log(error);
return;
}
var usingTor = (body.indexOf('Congratulations. This browser is configured to use Tor.') !== -1);
expect(usingTor).to.equal(true);
});
If you're on *nix machine, you can use tsocks. It will "socksify" the whole process so you can use it even for anything which does not support proxies at all. This article has some great examples
Basically it's as easy as doing tsocks node myscript.js. I am not sure if it works with tsocks npm start but you could give it a try (npm starts your code as a subprocess)
All you need is some setup first (put server = 127.0.0.1 to etc/tsocks.conf)
Yo should try with polipo, that work for me;
http://ccm.net/faq/805-installing-an-easy-http-proxy-cache-polipo
I am using node-http-proxy. However, in addition to relaying HTTP requests, I also need to listen to the incoming and outgoing data.
Intercepting the response data is where I'm struggling. Node's ServerResponse object (and more generically the WritableStream interface) doesn't broadcast a 'data' event. http-proxy seems to create it's own internal request, which produces a ClientResponse object (which does broadcast the 'data' event) however this object is not exposed publically outside the proxy.
Any ideas how to solve this without monkey-patching node-http-proxy or creating a wrapper around the response object?
Related issue in issues of node-http-proxy on Github seems to imply this is not possible. For future attempts by others, here is how I hacked the issue:
you'll quickly find out that the proxy is only calling writeHead(), write() and end() methods of the res object
since res is already an EventEmitter, you can start emitting new custom events
listen for these new events to assemble the response data and then use it
var eventifyResponse = function(res) {
var methods = ['writeHead', 'write', 'end'];
methods.forEach(function(method){
var oldMethod = res[method]; // remember original method
res[method] = function() { // replace with a wrapper
oldMethod.apply(this, arguments); // call original method
arguments = Array.prototype.slice.call(arguments, 0);
arguments.unshift("method_" + method);
this.emit.apply(this, arguments); // broadcast the event
};
});
};
res = eventifyResponse(res), outputData = '';
res.on('method_writeHead', function(statusCode, headers) { saveHeaders(); });
res.on('method_write', function(data) { outputData += data; });
res.on('method_end', function(data) { use_data(outputData + data); });
proxy.proxyRequest(req, res, options)
This is a simple proxy server sniffing the traffic and writing it to console:
var http = require('http'),
httpProxy = require('http-proxy');
//
// Create a proxy server with custom application logic
//
var proxy = httpProxy.createProxyServer({});
// assign events
proxy.on('proxyRes', function (proxyRes, req, res) {
// collect response data
var proxyResData='';
proxyRes.on('data', function (chunk) {
proxyResData +=chunk;
});
proxyRes.on('end',function () {
var snifferData =
{
request:{
data:req.body,
headers:req.headers,
url:req.url,
method:req.method},
response:{
data:proxyResData,
headers:proxyRes.headers,
statusCode:proxyRes.statusCode}
};
console.log(snifferData);
});
// console.log('RAW Response from the target', JSON.stringify(proxyRes.headers, true, 2));
});
proxy.on('proxyReq', function(proxyReq, req, res, options) {
// collect request data
req.body='';
req.on('data', function (chunk) {
req.body +=chunk;
});
req.on('end', function () {
});
});
proxy.on('error',
function(err)
{
console.error(err);
});
// run the proxy server
var server = http.createServer(function(req, res) {
// every time a request comes proxy it:
proxy.web(req, res, {
target: 'http://localhost:4444'
});
});
console.log("listening on port 5556")
server.listen(5556);
I tried your hack but it didn't work for me. My use case is simple: I want to log the in- and outgoing traffic from an Android app to our staging server which is secured by basic auth.
https://github.com/greim/hoxy/
was the solution for me. My node-http-proxy always returned 500 (while the direct request to stage did not). Maybe the authorization headers would not be forwarded correctly or whatever.
Hoxy worked fine right from the start.
npm install hoxy [-g]
hoxy --port=<local-port> --stage=<your stage host>:<port>
As rules for logging I specified:
request: $aurl.log()
request: #log-headers()
request: $method.log()
request: $request-body.log()
response: $url.log()
response: $status-code.log()
response: $response-body.log()
Beware, this prints any binary content.
I'm trying to create a static file server in nodejs more as an exercise to understand node than as a perfect server. I'm well aware of projects like Connect and node-static and fully intend to use those libraries for more production-ready code, but I also like to understand the basics of what I'm working with. With that in mind, I've coded up a small server.js:
var http = require('http'),
url = require('url'),
path = require('path'),
fs = require('fs');
var mimeTypes = {
"html": "text/html",
"jpeg": "image/jpeg",
"jpg": "image/jpeg",
"png": "image/png",
"js": "text/javascript",
"css": "text/css"};
http.createServer(function(req, res) {
var uri = url.parse(req.url).pathname;
var filename = path.join(process.cwd(), uri);
path.exists(filename, function(exists) {
if(!exists) {
console.log("not exists: " + filename);
res.writeHead(200, {'Content-Type': 'text/plain'});
res.write('404 Not Found\n');
res.end();
}
var mimeType = mimeTypes[path.extname(filename).split(".")[1]];
res.writeHead(200, mimeType);
var fileStream = fs.createReadStream(filename);
fileStream.pipe(res);
}); //end path.exists
}).listen(1337);
My question is twofold
Is this the "right" way to go about creating and streaming basic html etc in node or is there a better/more elegant/more robust method ?
Is the .pipe() in node basically just doing the following?
.
var fileStream = fs.createReadStream(filename);
fileStream.on('data', function (data) {
res.write(data);
});
fileStream.on('end', function() {
res.end();
});
Thanks everyone!
Less is more
Just go command prompt first on your project and use
$ npm install express
Then write your app.js code like so:
var express = require('express'),
app = express(),
port = process.env.PORT || 4000;
app.use(express.static(__dirname + '/public'));
app.listen(port);
You would then create a "public" folder where you place your files. I tried it the harder way first but you have to worry about mime types which is just having to map stuff which is time consuming and then worry about response types, etc. etc. etc.... no thank you.
Your basic server looks good, except:
There is a return statement missing.
res.write('404 Not Found\n');
res.end();
return; // <- Don't forget to return here !!
And:
res.writeHead(200, mimeType);
should be:
res.writeHead(200, {'Content-Type':mimeType});
Yes pipe() does basically that, it also pauses/resumes the source stream (in case the receiver is slower).
Here is the source code of the pipe() function: https://github.com/joyent/node/blob/master/lib/stream.js
I like understanding what's going on under the hood as well.
I noticed a few things in your code that you probably want to clean up:
It crashes when filename points to a directory, because exists is true and it tries to read a file stream. I used fs.lstatSync to determine directory existence.
It isn't using the HTTP response codes correctly (200, 404, etc)
While MimeType is being determined (from the file extension), it isn't being set correctly in res.writeHead (as stewe pointed out)
To handle special characters, you probably want to unescape the uri
It blindly follows symlinks (could be a security concern)
Given this, some of the apache options (FollowSymLinks, ShowIndexes, etc) start to make more sense. I've update the code for your simple file server as follows:
var http = require('http'),
url = require('url'),
path = require('path'),
fs = require('fs');
var mimeTypes = {
"html": "text/html",
"jpeg": "image/jpeg",
"jpg": "image/jpeg",
"png": "image/png",
"js": "text/javascript",
"css": "text/css"};
http.createServer(function(req, res) {
var uri = url.parse(req.url).pathname;
var filename = path.join(process.cwd(), unescape(uri));
var stats;
try {
stats = fs.lstatSync(filename); // throws if path doesn't exist
} catch (e) {
res.writeHead(404, {'Content-Type': 'text/plain'});
res.write('404 Not Found\n');
res.end();
return;
}
if (stats.isFile()) {
// path exists, is a file
var mimeType = mimeTypes[path.extname(filename).split(".").reverse()[0]];
res.writeHead(200, {'Content-Type': mimeType} );
var fileStream = fs.createReadStream(filename);
fileStream.pipe(res);
} else if (stats.isDirectory()) {
// path exists, is a directory
res.writeHead(200, {'Content-Type': 'text/plain'});
res.write('Index of '+uri+'\n');
res.write('TODO, show index?\n');
res.end();
} else {
// Symbolic link, other?
// TODO: follow symlinks? security?
res.writeHead(500, {'Content-Type': 'text/plain'});
res.write('500 Internal server error\n');
res.end();
}
}).listen(1337);
var http = require('http')
var fs = require('fs')
var server = http.createServer(function (req, res) {
res.writeHead(200, { 'content-type': 'text/plain' })
fs.createReadStream(process.argv[3]).pipe(res)
})
server.listen(Number(process.argv[2]))
How about this pattern, which avoids checking separately that the file exists
var fileStream = fs.createReadStream(filename);
fileStream.on('error', function (error) {
response.writeHead(404, { "Content-Type": "text/plain"});
response.end("file not found");
});
fileStream.on('open', function() {
var mimeType = mimeTypes[path.extname(filename).split(".")[1]];
response.writeHead(200, {'Content-Type': mimeType});
});
fileStream.on('end', function() {
console.log('sent file ' + filename);
});
fileStream.pipe(response);
I made a httpServer function with extra features for general usage based on #Jeff Ward answer
custtom dir
index.html returns if req === dir
Usage:
httpServer(dir).listen(port);
https://github.com/kenokabe/ConciseStaticHttpServer
Thanks.
the st module makes serving static files easy. Here is an extract of README.md:
var mount = st({ path: __dirname + '/static', url: '/static' })
http.createServer(function(req, res) {
var stHandled = mount(req, res);
if (stHandled)
return
else
res.end('this is not a static file')
}).listen(1338)
#JasonSebring answer pointed me in the right direction, however his code is outdated. Here is how you do it with the newest connect version.
var connect = require('connect'),
serveStatic = require('serve-static'),
serveIndex = require('serve-index');
var app = connect()
.use(serveStatic('public'))
.use(serveIndex('public', {'icons': true, 'view': 'details'}))
.listen(3000);
In connect GitHub Repository there are other middlewares you can use.