Async processing in response modifying proxy with node-http-proxy

[ashish...@gmail.com]

Hi,

I am trying to write a proxy server in nodejs using node-http-proxy module by nodejitsu. Since I am new to nodejs so I have tried to use the modifyResponse-middleware.js example from [1] as a starting point.

var fs = require('fs'),
    util = require('util'),
    colors = require('colors'),
    http = require('http'),
    connect = require('connect'),
    httpProxy = require('http-proxy'),
    request = require('request'),
    Promise = require('es6-promise').Promise,
    crypto = require('crypto');

function get_sha1(data) {
    var shasum = crypto.createHash('sha1');
    shasum.update(data);
    return shasum.digest('hex');
}

function process_data(url, data) {
    // Asynchronously process data and write final data to a file on filesystem  at 'data/test-out' + tag + '.html'
    // It returns back a promise that can perform operations with final data when it is made available.
    // ...
}

//
// Basic Connect App
//
connect.createServer(
    function (req, res, next) {
        var _write = res.write, _writeHead = res.writeHead, url = '';
       
        url = req.url + '-at-'+ req.client._httpMessage.connection._idleStart;
        tag = get_sha1(url);
       
        res.writeHead = function (statusCode, reason, headers) {
            res.removeHeader('Content-Length');
            if (headers) {
                delete headers['content-length'];
            }
            _writeHead.apply(res, arguments);
        };

        res.write = function (data) {
            var promise = process_data(url, data);
            promise.then(function (result) {
                console.log(result);
                // Read ndata from file created by async promise fulfillment
                ndata = fs.readFileSync('data/test-out' + tag + '.html');
                _write.call(res, ndata);
            }, function (err) {
                console.log(err);
                _write.call(res, data);
            });
        };
        next();
    },
    function (req, res) {
        proxy.web(req, res);
    }
).listen(8013);

//
// Http Proxy Server
//
var proxy = httpProxy.createProxyServer({
    target: 'https://www.google.com'
});

proxy.on('error', function (e) {
    util.puts('Error occured while serving request'.red);
    console.log(e);
});

util.puts('Proxy Server'.blue + ' started '.green.bold + 'on port '.blue + '8013'.yellow);

In the example as highlighted I have a function call to process_data that returns a promise. This promise asynchronously make available the new data from the data got from proxied server upon fulfilment. But the problem that I am facing is that before the promise is fulfilled or failed the response write method completes and the client receives an empty response.

Kindly let me know how I can make this async processing work here in context of the nodejs connect based server.

Thanks.

Ashish Sharma

--

[1]: https://github.com/nodejitsu/node-http-proxy/blob/master/examples/middleware/modifyResponse-middleware.js

Sent from Windows Mail

[Ashish Sharma]

Found a solution to my problem in  tamper.js at  https://github.com/fgnass/tamper .

However, this library also made synchronous edit to body of intercepted http communication. So as a workaround it was extended to give support on asynch edits.

Line 59  body = tamper(body || '', req, this.headers) modified to body = tamper(body || '', req, this.headers, function () { // Do res.end() here})

--

PS: If you are facing a similar problem, please contact at github.com/kartaa .

[Floby]

Looks like a classic case of transform stream

var url = assembleTargetUrl(req).
request(url).pipe(myTransform()).pipe(res) ;

function myTranform() {
return es.map(function (data, callback )) {
process_data(data, callback)
}
}