Added support for tracking downloaded bytes (#94)

README.md

@@ -790,7 +790,8 @@ request({url: 'https://www.google.com', verbose: true}, function (error, respons
       },
       "response": {
         "statusCode": 200,
-        "httpVersion": "1.1"
+        "httpVersion": "1.1".
+        "downloadedBytes": 1234,
       },
       "timingStart": 1552908287924,
       "timingStartTimer": 805.690674,

lib/helpers.js

@@ -3,6 +3,7 @@
 var jsonSafeStringify = require('json-stringify-safe')
 var crypto = require('crypto')
 var Buffer = require('safe-buffer').Buffer
+var { Transform } = require('stream')
 
 var defer = typeof setImmediate === 'undefined'
   ? process.nextTick
@@ -58,10 +59,28 @@ function version () {
   }
 }
 
+class SizeTrackerStream extends Transform {
+  constructor (options) {
+    super(options)
+    this.size = 0
+  }
+
+  _transform (chunk, encoding, callback) {
+    this.size += chunk.length
+    this.push(chunk)
+    callback()
+  }
+
+  _flush (callback) {
+    callback()
+  }
+}
+
 exports.safeStringify = safeStringify
 exports.md5 = md5
 exports.isReadStream = isReadStream
 exports.toBase64 = toBase64
 exports.copy = copy
 exports.version = version
 exports.defer = defer
+exports.SizeTrackerStream = SizeTrackerStream

lib/http2/request.js

@@ -135,14 +135,15 @@ class Http2Request extends EventEmitter {
     }
 
     this.stream.setEncoding(encoding)
+    return this
   }
 
   write (chunk) {
     if (!this[kHeadersFlushed]) {
       this._flushHeaders()
     }
 
-    this.stream.write(chunk)
+    return this.stream.write(chunk)
   }
 
   _flushHeaders () {
@@ -167,6 +168,8 @@ class Http2Request extends EventEmitter {
       this._flushHeaders()
     }
     this.stream.pipe(dest)
+
+    return dest
   }
 
   on (eventName, listener) {
@@ -183,20 +186,26 @@ class Http2Request extends EventEmitter {
       this._flushHeaders()
     }
     this.stream.destroy()
+
+    return this
   }
 
   end () {
     if (!this[kHeadersFlushed]) {
       this._flushHeaders()
     }
     this.stream.end()
+
+    return this
   }
 
   setTimeout (timeout, cb) {
     if (!this[kHeadersFlushed]) {
       this._flushHeaders()
     }
     this.stream.setTimeout(timeout, cb)
+
+    return this
   }
 
   removeHeader (headerKey) {
@@ -209,6 +218,8 @@ class Http2Request extends EventEmitter {
     }
 
     delete this.requestHeaders[headerKey]
+
+    return this
   }
 
   setHeader (headerKey, headerValue) {
@@ -221,6 +232,8 @@ class Http2Request extends EventEmitter {
     }
 
     this.requestHeaders[headerKey] = headerValue
+
+    return this
   }
 }
 
@@ -291,22 +304,27 @@ class ResponseProxy extends EventEmitter {
 
   pause () {
     this.reqStream.pause()
+    return this
   }
 
   resume () {
     this.reqStream.resume()
+    return this
   }
 
   pipe (dest) {
     this.reqStream.pipe(dest)
+    return dest
   }
 
   setEncoding (encoding) {
     this.reqStream.setEncoding(encoding)
+    return this
   }
 
   destroy () {
     this.reqStream.destroy()
+    return this
   }
 }
 

request.js

@@ -43,6 +43,7 @@ var toBase64 = helpers.toBase64
 var defer = helpers.defer
 var copy = helpers.copy
 var version = helpers.version
+var SizeTrackerStream = helpers.SizeTrackerStream
 var globalCookieJar = cookies.jar()
 
 var globalPool = {}
@@ -1375,6 +1376,8 @@ Request.prototype.onRequestResponse = function (response) {
     }
 
     var responseContent
+    var downloadSizeTracker = new SizeTrackerStream()
+
     if ((self.gzip || self.brotli) && !noBody(response.statusCode)) {
       var contentEncoding = response.headers['content-encoding'] || 'identity'
       contentEncoding = contentEncoding.trim().toLowerCase()
@@ -1390,23 +1393,23 @@ Request.prototype.onRequestResponse = function (response) {
 
       if (self.gzip && contentEncoding === 'gzip') {
         responseContent = zlib.createGunzip(zlibOptions)
-        response.pipe(responseContent)
+        response.pipe(downloadSizeTracker).pipe(responseContent)
       } else if (self.gzip && contentEncoding === 'deflate') {
         responseContent = inflate.createInflate(zlibOptions)
-        response.pipe(responseContent)
+        response.pipe(downloadSizeTracker).pipe(responseContent)
       } else if (self.brotli && contentEncoding === 'br') {
         responseContent = brotli.createBrotliDecompress()
-        response.pipe(responseContent)
+        response.pipe(downloadSizeTracker).pipe(responseContent)
       } else {
         // Since previous versions didn't check for Content-Encoding header,
         // ignore any invalid values to preserve backwards-compatibility
         if (contentEncoding !== 'identity') {
           debug('ignoring unrecognized Content-Encoding ' + contentEncoding)
         }
-        responseContent = response
+        responseContent = response.pipe(downloadSizeTracker)
       }
     } else {
-      responseContent = response
+      responseContent = response.pipe(downloadSizeTracker)
     }
 
     if (self.encoding) {
@@ -1436,9 +1439,9 @@ Request.prototype.onRequestResponse = function (response) {
     // results in some other characters.
     // For example: If the server intentionally responds with `ð\x9F\x98\x8A` as status message
     // but if the statusMessageEncoding option is set to `utf8`, then it would get converted to '😊'.
-    var statusMessage = String(responseContent.statusMessage)
+    var statusMessage = String(response.statusMessage)
     if (self.statusMessageEncoding && /[^\w\s-']/.test(statusMessage)) {
-      responseContent.statusMessage = Buffer.from(statusMessage, 'latin1').toString(self.statusMessageEncoding)
+      response.statusMessage = Buffer.from(statusMessage, 'latin1').toString(self.statusMessageEncoding)
     }
 
     if (self._paused) {
@@ -1483,6 +1486,7 @@ Request.prototype.onRequestResponse = function (response) {
       self.emit('data', chunk)
     })
     responseContent.once('end', function (chunk) {
+      self._reqResInfo.response.downloadedBytes = downloadSizeTracker.size
       self.emit('end', chunk)
     })
     responseContent.on('error', function (error) {

tests/server.js

@@ -111,8 +111,6 @@ exports.createPostValidator = function (text, reqContentType) {
       resp.writeHead(200, { 'content-type': 'text/plain' })
       resp.write(r)
       resp.end()
-      // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests.
-      req || req.stream || req.stream.session || req.stream.session.close || req.stream.session.close()
     })
   }
   return l
@@ -133,8 +131,6 @@ exports.createPostJSONValidator = function (value, reqContentType) {
       resp.writeHead(200, { 'content-type': 'application/json' })
       resp.write(r)
       resp.end()
-      // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests.
-      req || req.stream || req.stream.session || req.stream.session.close || req.stream.session.close()
     })
   }
   return l
@@ -145,9 +141,6 @@ exports.createGetResponse = function (text, contentType) {
     resp.writeHead(200, { 'content-type': contentType })
     resp.write(text)
     resp.end()
-
-    // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests.
-    req || req.stream || req.stream.session || req.stream.session.close || req.stream.session.close()
   }
   return l
 }
@@ -159,8 +152,6 @@ exports.createChunkResponse = function (chunks, contentType) {
       resp.write(chunk)
     })
     resp.end()
-    // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests.
-    req || req.stream || req.stream.session || req.stream.session.close || req.stream.session.close()
   }
   return l
 }

tests/test-agent.js

@@ -23,16 +23,16 @@ function httpAgent (t, options, req) {
   var r = (req || request)(options, function (_err, res, body) {
     t.ok(r.agent instanceof http.Agent, 'is http.Agent')
     t.equal(r.agent.options.keepAlive, true, 'is keepAlive')
-    t.equal(Object.keys(r.agent.sockets).length, 1, '1 socket name')
-
+    t.equal(Object.keys(r.agent.freeSockets).length, 1, '1 socket name')
     var name = (typeof r.agent.getName === 'function')
       ? r.agent.getName({port: s.port})
       : 'localhost:' + s.port // node 0.10-
-    t.equal(r.agent.sockets[name].length, 1, '1 open socket')
 
-    var socket = r.agent.sockets[name][0]
+    t.equal(r.agent.freeSockets[name].length, 1, '1 open socket')
+
+    var socket = r.agent.freeSockets[name][0]
     socket.on('close', function () {
-      t.equal(Object.keys(r.agent.sockets).length, 0, '0 open sockets')
+      t.equal(Object.keys(r.agent.freeSockets).length, 0, '0 open sockets')
       t.end()
     })
     socket.end()

tests/test-body-http2.js

@@ -5,8 +5,10 @@ var request = require('../index')
 var tape = require('tape')
 var path = require('path')
 var fs = require('fs')
+var destroyable = require('server-destroy')
 
 var s = server.createHttp2Server()
+destroyable(s)
 
 tape('setup', function (t) {
   s.listen(0, function () {
@@ -154,8 +156,6 @@ addTest('testPutMultipartPostambleCRLF', {
 tape('testBinaryFile', function (t) {
   s.on('/', function (req, res) {
     req.pipe(res)
-    // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests.
-    req.stream && req.stream.session && req.stream.session.close && req.stream.session.close()
   })
 
   request(
@@ -178,8 +178,6 @@ tape('testBinaryFile', function (t) {
 tape('typed array', function (t) {
   s.on('/', function (req, res) {
     req.pipe(res)
-    // Close the session if it's a HTTP/2 request. This is not representative of a true http/2 server that might keep the session open. But we need this to close the server in the tests.
-    req.stream && req.stream.session && req.stream.session.close && req.stream.session.close()
   })
 
   var data = new Uint8Array([1, 2, 3])
@@ -203,7 +201,7 @@ tape('typed array', function (t) {
 })
 
 tape('cleanup', function (t) {
-  s.close(function () {
+  s.destroy(function () {
     t.end()
   })
 })

tests/test-cookies-http2.js

@@ -2,13 +2,15 @@
 
 var request = require('../index')
 var tape = require('tape')
+var destroyable = require('server-destroy')
 var server = require('./server')
 
 var validUrl
 var malformedUrl
 var invalidUrl
 
 var s = server.createHttp2Server()
+destroyable(s)
 
 tape('setup', function (t) {
   s.listen(0, function () {
@@ -19,17 +21,14 @@ tape('setup', function (t) {
     s.on('/valid', (req, res) => {
       res.setHeader('set-cookie', 'foo=bar')
       res.end('okay')
-      res.stream.session.close()
     })
     s.on('/malformed', (req, res) => {
       res.setHeader('set-cookie', 'foo')
       res.end('okay')
-      res.stream.session.close()
     })
     s.on('/invalid', (req, res) => {
       res.setHeader('set-cookie', 'foo=bar; Domain=foo.com')
       res.end('okay')
-      res.stream.session.close()
     })
 
     t.end()
@@ -137,7 +136,7 @@ tape('custom store', function (t) {
 })
 
 tape('cleanup', function (t) {
-  s.close(function () {
+  s.destroy(function () {
     t.end()
   })
 })

tests/test-verbose-auto-http2.js

@@ -77,7 +77,7 @@ tape('HTTP: verbose=true', function (t) {
     t.deepEqual(Object.keys(debug[0].session), ['id', 'reused', 'data'])
     t.deepEqual(Object.keys(debug[0].session.data), ['addresses'])
     t.equal(debug[0].session.reused, false)
-    t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion'])
+    t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes'])
 
     t.notEqual(debug[0].response.headers.length, 0)
     t.equal(debug[0].response.headers[0].key, 'Date')
@@ -116,7 +116,7 @@ tape('HTTP: redirect(HTTPS) + verbose=true', function (t) {
     t.deepEqual(Object.keys(debug[1].session.data), ['addresses', 'tls'])
     t.deepEqual(Object.keys(debug[1].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate'])
     t.equal(debug[1].session.reused, false)
-    t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion'])
+    t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes'])
 
     t.end()
   })
@@ -144,7 +144,7 @@ tape('HTTPS: verbose=true', function (t) {
     t.deepEqual(Object.keys(debug[0].session.data), ['addresses', 'tls'])
     t.deepEqual(Object.keys(debug[0].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate'])
     t.equal(debug[0].session.reused, true)
-    t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion'])
+    t.deepEqual(Object.keys(debug[0].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes'])
 
     t.end()
   })
@@ -179,7 +179,7 @@ tape('HTTPS: redirect(HTTP) + verbose=true', function (t) {
     t.deepEqual(Object.keys(debug[1].session), ['id', 'reused', 'data'])
     t.deepEqual(Object.keys(debug[1].session.data), ['addresses'])
     t.equal(debug[1].session.reused, false)
-    t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion'])
+    t.deepEqual(Object.keys(debug[1].response), ['statusCode', 'headers', 'httpVersion', 'downloadedBytes'])
 
     t.end()
   })