proxyReq can't safely modify headers when used with keep alive (ERR_HTTP_HEADERS_SENT)

Created by: nickwesselman

The use of the socket event to drive proxyReq and allow header modification appears to be problematic when used with keepAlive and maxSockets on an HTTP agent. When a socket is immediately available to a request, the event fires in the ClientRequest constructor, so the request headers can be modified. But with keepAlive enabled on the agent, a socket may not be immediately available. In that case the request continues on and begins buffering its output, so by the time socket is fired, the request header has been 'sent'.

I have run into this while using chimurai/http-proxy-middleware (chimurai/http-proxy-middleware#472) but I was able to reproduce with a basic proxy example that has an agent limited to 5 sockets, and a k6 test which ramps up beyond that. Likely there needs to be a better extension point for initializing the request options, since node doesn't appear to provide an alternative hook for before the request starts buffering... unless I'm missing something here.

Basic Proxy

var http = require('http'),
    httpProxy = require('http-proxy');

var proxy = httpProxy.createProxyServer({
  target:'http://localhost:9003',
  agent: new http.Agent({
    keepAlive: true,
    maxSockets: 5,
    maxFreeSockets: 5
  })
}).listen(3000);

proxy.on('proxyReq', function(proxyReq, req, res, options) {
    proxyReq.setHeader('TEST', req.headers['k6-vu'] || "(none)");
});

http.createServer(function (req, res) {
  res.writeHead(200, { 'Content-Type': 'text/plain' });
  res.write('request successfully proxied to: ' + req.url + '\n' + JSON.stringify(req.headers, true, 2));
  res.end();
}).listen(9003);

console.log("Proxy listening on http://localhost:3000");

k6 Test

import http from 'k6/http';
import { Counter } from "k6/metrics";

export let options = {
    discardResponseBodies: false,
    scenarios: {
        headRequest: {
            executor: 'ramping-vus',
            exec: 'basicRequest',
            startVUs: 5,
            stages: [
                { duration: '5s', target: 10 }
            ]
        }
    },
    thresholds: {
        "pageError": [{threshold: "count<1", abortOnFail: true}],
    }
};

const pageError = new Counter("pageError");

export function basicRequest() {
    const params = {
        headers: {
            'K6-VU': __VU
        }
    };
    var response = http.request('GET', 'http://localhost:3000/', null, params);
    if (response.error || response.status !== 200) {
        pageError.add(1);
    }
}

Output

_http_outgoing.js:518
    throw new ERR_HTTP_HEADERS_SENT('set');
    ^

Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
    at ClientRequest.setHeader (_http_outgoing.js:518:11)
    at ProxyServer.<anonymous> (C:\dev\nwe-jss-test\jss13.0.1-sc9.3.0\keepalive-middleware-test\basic-proxy.js:14:14)
    at ProxyServer.emit (C:\dev\nwe-jss-test\jss13.0.1-sc9.3.0\keepalive-middleware-test\node_modules\eventemitter3\index.js:184:35)
    at ClientRequest.<anonymous> (C:\dev\nwe-jss-test\jss13.0.1-sc9.3.0\keepalive-middleware-test\node_modules\http-proxy\lib\http-proxy\passes\web-incoming.js:133:16)
    at ClientRequest.emit (events.js:327:22)
    at tickOnSocket (_http_client.js:703:7)
    at onSocketNT (_http_client.js:744:5)
    at processTicksAndRejections (internal/process/task_queues.js:84:21) {
  code: 'ERR_HTTP_HEADERS_SENT'
}