I'm planning to do a series of HTTP requests in NodeJS though Tor.
Tor uses SOCKS5 so I went out and searched for a way to proxify HTTP requests in NodeJS.
I'm planning to the the default http.request() function to do the work. However, I can't seem to find a way to use a proxy with that. Someone suggested that I could do this:
var http = require("http");
var options = {
host: "localhost",
port: 9050,
path: "http://check.torproject.org",
method: 'GET',
headers: {
Host: "http://check.torproject.org",
}
};
var req = http.request(options, function(res) {
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
But it didn't work.
So, any suggestions?
I've just published two modules that should help you do this: socks5-http-client and socks5-https-client.
Just use those instead of the default http module. The API is the same. For example:
require('socks5-http-client').request(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
I know I'm answering an old question but there is a better solution available for this question, about how to use sock4 & sock5 proxy in Node.js. For the sake of simplicity, I will be using request-promise module however you can also use bare request module.
Requrement: socks-proxy-agent, request-promise
Example:
async function main() {
var proxy = "socks4://1.2.3.4:35618"
var agent = new SocksProxyAgent(proxy);
var options = {
uri: 'http://targetUrl',
agent: agent,
headers: {
'User-Agent': 'Request-Promise'
}
}
try {
var responce = await rp(options)
} catch(err) {
console.log(err)
}
console.log(responce) }
Not a complete answer, but you may want to keep your eye on these two modules.
https://github.com/Ayms/node-Tor
Support is being added into: https://github.com/Ayms/node-bot.
I sent him an email asking when he expected this to be complete, will update this post soon with that information.
I had the same problem and used polipo as proxy between node and TOR
node (request) - polilp httproxy:8123 - polipo - tor (socks5:9050).
For mac (osx with brew) it worked like this:
brew install polipo tor
tor # start top
polipo socksParentProxy=localhost:9050 # start polipo
Working example with request
var request = require('request');
var options = {'url':'https://check.torproject.org/', 'proxy':'http://localhost:8123'}
request(options,
function (error, response, body) {
if (error){
console.log(error);
return;
}
var usingTor = (body.indexOf('Congratulations. This browser is configured to use Tor.') !== -1);
expect(usingTor).to.equal(true);
});
If you're on *nix machine, you can use tsocks. It will "socksify" the whole process so you can use it even for anything which does not support proxies at all. This article has some great examples
Basically it's as easy as doing tsocks node myscript.js. I am not sure if it works with tsocks npm start but you could give it a try (npm starts your code as a subprocess)
All you need is some setup first (put server = 127.0.0.1 to etc/tsocks.conf)
Yo should try with polipo, that work for me;
http://ccm.net/faq/805-installing-an-easy-http-proxy-cache-polipo
Related
I'm working with Meteor and trying to establish NTLM authentication. I'm using the HTTP package to go through the NTLM "handshake" with the remote server. However, I'm running into an issue that I think is caused by the connection not being kept alive. I can't make it all the way through the handshake, because the remote server expects me to reuse the same HTTP connection but I'm making a new one every time (I'm not sure how note to). Below is some of what I'm using to do this. I'd appreciate any help here. Is there any alternative package to HTTP that has a keep alive option? Can I somehow reuse the same connection with the default HTTP package? Thanks.
var msg1 = Ntlm.createMessage1(hostname);
HTTP.get(url, {
headers: {
"Authorization": "NTLM " + msg1.toBase64()
}
}, function (error, result) {
if (result != null) {
var response = result.headers["www-authenticate"];
var msg2 = Ntlm.decodeMessage2(response);
//here I should respond to msg2 on same kept-alive connection, but I'm not, so it's failing
var msg3 = Ntlm.createMessage3(msg2, hostname);
HTTP.get(url, {
headers: {
"Authorization": "NTLM " + msg3.toBase64()
}
}, function (error, req) {
if (req != null) {
if (req.statusCode == 200) {
Ntlm.authorized = true; //success
} else {
//error, what I'm getting now
//"401 - Unauthorized: Access is denied due to invalid credentials"
}
} else Ntlm.error(error);
});
} else Ntlm.error(error);
});
I don't think you can keep alive a connection with the default meteor http package. However, there are extensions and replacements to this default package. One of them that seems to support setting keepalive to true is this called http-more. There is more info about this package "requests" support here: https://github.com/request/request#requestoptions-callback
I have a server side route in my Meteor app where I can get, for instance, the client's IP:
this.route('foo', {
where: 'server',
path: '/bar',
action: function () {
var ip = context.request.connection.remoteAddress;
}
});
How can I access the referer field? Do I need to use Meteor Headers?
You can directly access the connect request object, which has the headers:
this.request.headers['referer']
Like this:
Router.map(function () {
this.route('foo', {
where: 'server',
path: '/bar',
action: function () {
console.log("All headers:", this.request.headers);
console.log("Referer:", this.request.headers['referer']);
}
})
});
I just had the same problem and after some more digging I figured out if the source page was secure (HTTPS) the browser will not send the referer.
As well if the source document is on the local computer a referer is not available. That fooled me for a while as I simply created a test.html on the Desktop for testing...
So the code in my route for blocking direct requests to this route from other websites looks like this now:
action: function () {
var self = this;
var host = self.request.headers.host;
var referer = self.request.headers.referer;
var regex = new RegExp("^http(?:s)?://" + host);
if(typeof self.request.headers.referer !== "undefined" && !regex.test(referer)) {
Meteor.log.error("Access blocked from " + referer);
self.response.statusCode = 403;
self.response.end(Handlebars.templates['403']());
return;
}
}
If the referer is set, it has to match the current hostname.
you can do document.referrer to get that information
alternatively you can use History API:
history.go(-1);
I am using node-http-proxy. However, in addition to relaying HTTP requests, I also need to listen to the incoming and outgoing data.
Intercepting the response data is where I'm struggling. Node's ServerResponse object (and more generically the WritableStream interface) doesn't broadcast a 'data' event. http-proxy seems to create it's own internal request, which produces a ClientResponse object (which does broadcast the 'data' event) however this object is not exposed publically outside the proxy.
Any ideas how to solve this without monkey-patching node-http-proxy or creating a wrapper around the response object?
Related issue in issues of node-http-proxy on Github seems to imply this is not possible. For future attempts by others, here is how I hacked the issue:
you'll quickly find out that the proxy is only calling writeHead(), write() and end() methods of the res object
since res is already an EventEmitter, you can start emitting new custom events
listen for these new events to assemble the response data and then use it
var eventifyResponse = function(res) {
var methods = ['writeHead', 'write', 'end'];
methods.forEach(function(method){
var oldMethod = res[method]; // remember original method
res[method] = function() { // replace with a wrapper
oldMethod.apply(this, arguments); // call original method
arguments = Array.prototype.slice.call(arguments, 0);
arguments.unshift("method_" + method);
this.emit.apply(this, arguments); // broadcast the event
};
});
};
res = eventifyResponse(res), outputData = '';
res.on('method_writeHead', function(statusCode, headers) { saveHeaders(); });
res.on('method_write', function(data) { outputData += data; });
res.on('method_end', function(data) { use_data(outputData + data); });
proxy.proxyRequest(req, res, options)
This is a simple proxy server sniffing the traffic and writing it to console:
var http = require('http'),
httpProxy = require('http-proxy');
//
// Create a proxy server with custom application logic
//
var proxy = httpProxy.createProxyServer({});
// assign events
proxy.on('proxyRes', function (proxyRes, req, res) {
// collect response data
var proxyResData='';
proxyRes.on('data', function (chunk) {
proxyResData +=chunk;
});
proxyRes.on('end',function () {
var snifferData =
{
request:{
data:req.body,
headers:req.headers,
url:req.url,
method:req.method},
response:{
data:proxyResData,
headers:proxyRes.headers,
statusCode:proxyRes.statusCode}
};
console.log(snifferData);
});
// console.log('RAW Response from the target', JSON.stringify(proxyRes.headers, true, 2));
});
proxy.on('proxyReq', function(proxyReq, req, res, options) {
// collect request data
req.body='';
req.on('data', function (chunk) {
req.body +=chunk;
});
req.on('end', function () {
});
});
proxy.on('error',
function(err)
{
console.error(err);
});
// run the proxy server
var server = http.createServer(function(req, res) {
// every time a request comes proxy it:
proxy.web(req, res, {
target: 'http://localhost:4444'
});
});
console.log("listening on port 5556")
server.listen(5556);
I tried your hack but it didn't work for me. My use case is simple: I want to log the in- and outgoing traffic from an Android app to our staging server which is secured by basic auth.
https://github.com/greim/hoxy/
was the solution for me. My node-http-proxy always returned 500 (while the direct request to stage did not). Maybe the authorization headers would not be forwarded correctly or whatever.
Hoxy worked fine right from the start.
npm install hoxy [-g]
hoxy --port=<local-port> --stage=<your stage host>:<port>
As rules for logging I specified:
request: $aurl.log()
request: #log-headers()
request: $method.log()
request: $request-body.log()
response: $url.log()
response: $status-code.log()
response: $response-body.log()
Beware, this prints any binary content.
This error happens whenever the node process make a http request to get user's information from a web API.
The scenario is :
I'm running a TCP server using node, and when it get "login" request from a client, it will send a http GET request to another web API to retrieve the user's information.
While users increasing, the node process starts to throw the "ETIMEOUT" error when retrieving user's info. And once if the error happened, all the request after that will throw the same error.
I've tried to perform the same request with wget but everything is fine, so I think maybe it's not a network problem.
And strangely, after increasing the open file limit to 10,0000 using ulimit -n, it goes well until the next level user increment.
The fetch function is here:
fetchUserInfo = function(callback) {
var http = require('http');
var opt = {
agent: false,
host: 'www.someapi.net',
port: 80,
path: '/userInfo.php'
}
var body = '';
var req = http.request(opt, function(res) {
res.setEncoding('utf8');
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function() {
if(callback) {
callback(body);
}
});
});
req.on('error', function(e) {
sys.log("User info fetch error: " + e.message);
if(callback) {
callback();
}
});
req.end();
}
My environment is Debian GNU/Linux 6.0 with node v0.4.10.
Which one of the following node.js HTTP proxy implementations is more performant?
The first implementation is:
var http = require('http');
http.createServer(function(request, response) {
var proxy = http.createClient(80, "google.com")
var proxy_request = proxy.request(request.method, request.url, request.headers);
proxy_request.addListener('response', function (proxy_response) {
proxy_response.addListener('data', function(chunk) {
response.write(chunk, 'binary');
});
proxy_response.addListener('end', function() {
response.end();
});
response.writeHead(proxy_response.statusCode, proxy_response.headers);
});
request.addListener('data', function(chunk) {
proxy_request.write(chunk, 'binary');
});
request.addListener('end', function() {
proxy_request.end();
});
}).listen(8080);
The second one uses stream.pipe() and it's like:
var http = require('http');
http.createServer(function(request, response) {
var proxy = http.createClient(80, "google.com");
var proxy_request = proxy.request(request.method, request.url, request.headers);
proxy_request.on('response', function (proxy_response) {
proxy_response.pipe(response);
response.writeHead(proxy_response.statusCode, proxy_response.headers);
});
request.pipe(proxy_request);
}).listen(8080);
The first one might blow up your process if the file is big and the clients connection is slow or if an uploaded file is big and the servers upload bandwidth is small. Use pipe, it's designed for this kind of stuff.
Also, use an existing module from npm for this:
many features and used in production at nodejitsu: http-proxy
fast: bouncy