From 1b0495e34d13fe72587bec8fbd752503275ab37d Mon Sep 17 00:00:00 2001 From: Alexander O'Mara Date: Tue, 9 May 2017 20:56:11 -0400 Subject: [PATCH 1/2] zlib: expose amount of data read for compression/decompression Added bytesRead property to Zlib engines and an option to expose the engine to convenience method callbacks. Fixes: https://github.com/nodejs/node/issues/8874 --- lib/zlib.js | 18 +++- test/parallel/test-zlib-bytes-read.js | 92 +++++++++++++++++++ .../parallel/test-zlib-convenience-methods.js | 70 +++++++++++++- 3 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 test/parallel/test-zlib-bytes-read.js diff --git a/lib/zlib.js b/lib/zlib.js index 00be56dffc44fb..1ec7d58987893d 100644 --- a/lib/zlib.js +++ b/lib/zlib.js @@ -77,6 +77,16 @@ function isInvalidStrategy(strategy) { // constants.Z_DEFAULT_STRATEGY (0) } +function responseData(engine, buffer) { + if (engine._opts.info) { + return { + buffer: buffer, + engine: engine + }; + } + return buffer; +} + function zlibBuffer(engine, buffer, callback) { // Streams do not support non-Buffer ArrayBufferViews yet. Convert it to a // Buffer without copying. @@ -121,7 +131,7 @@ function zlibBuffer(engine, buffer, callback) { buffers = []; engine.close(); - callback(err, buf); + callback(err, responseData(engine, buf)); } } @@ -134,7 +144,7 @@ function zlibBufferSync(engine, buffer) { var flushFlag = engine._finishFlushFlag; - return engine._processChunk(buffer, flushFlag); + return responseData(engine, engine._processChunk(buffer, flushFlag)); } function zlibOnError(message, errno) { @@ -168,6 +178,8 @@ class Zlib extends Transform { opts = opts || {}; super(opts); + this.bytesRead = 0; + this._opts = opts; this._chunkSize = opts.chunkSize || constants.Z_DEFAULT_CHUNK; @@ -408,6 +420,8 @@ class Zlib extends Transform { var have = availOutBefore - availOutAfter; assert(have >= 0, 'have should not go down'); + self.bytesRead += availInBefore - availInAfter; + if (have > 0) { var out = self._buffer.slice(self._offset, self._offset + have); self._offset += have; diff --git a/test/parallel/test-zlib-bytes-read.js b/test/parallel/test-zlib-bytes-read.js new file mode 100644 index 00000000000000..f07d736afacc98 --- /dev/null +++ b/test/parallel/test-zlib-bytes-read.js @@ -0,0 +1,92 @@ +'use strict'; +const common = require('../common'); +const assert = require('assert'); +const zlib = require('zlib'); + +const expectStr = 'abcdefghijklmnopqrstuvwxyz'.repeat(2); +const expectBuf = Buffer.from(expectStr); + +function createWriter(target, buffer) { + const writer = { size: 0 }; + const write = () => { + target.write(Buffer.from([buffer[writer.size++]])); + if (writer.size < buffer.length) { + setTimeout(write, 25); + } else { + target.end(); + } + }; + write(); + return writer; +} + +for (const method of [ + ['createGzip', 'createGunzip', false], + ['createGzip', 'createUnzip', false], + ['createDeflate', 'createInflate', true], + ['createDeflateRaw', 'createInflateRaw', true] +]) { + let compWriter; + let compData = new Buffer(0); + + const comp = zlib[method[0]](); + comp.on('data', function(d) { + compData = Buffer.concat([compData, d]); + assert.strictEqual(this.bytesRead, compWriter.size, + `Should get write size on ${method[0]} data.`); + }); + comp.on('end', common.mustCall(function() { + assert.strictEqual(this.bytesRead, compWriter.size, + `Should get write size on ${method[0]} end.`); + assert.strictEqual(this.bytesRead, expectStr.length, + `Should get data size on ${method[0]} end.`); + + { + let decompWriter; + let decompData = new Buffer(0); + + const decomp = zlib[method[1]](); + decomp.on('data', function(d) { + decompData = Buffer.concat([decompData, d]); + assert.strictEqual(this.bytesRead, decompWriter.size, + `Should get write size on ${method[0]}/` + + `${method[1]} data.`); + }); + decomp.on('end', common.mustCall(function() { + assert.strictEqual(this.bytesRead, compData.length, + `Should get compressed size on ${method[0]}/` + + `${method[1]} end.`); + assert.strictEqual(decompData.toString(), expectStr, + `Should get original string on ${method[0]}/` + + `${method[1]} end.`); + })); + decompWriter = createWriter(decomp, compData); + } + + // Some methods should allow extra data after the compressed data + if (method[2]) { + const compDataExtra = Buffer.concat([compData, new Buffer('extra')]); + + let decompWriter; + let decompData = new Buffer(0); + + const decomp = zlib[method[1]](); + decomp.on('data', function(d) { + decompData = Buffer.concat([decompData, d]); + assert.strictEqual(this.bytesRead, decompWriter.size, + `Should get write size on ${method[0]}/` + + `${method[1]} data.`); + }); + decomp.on('end', common.mustCall(function() { + assert.strictEqual(this.bytesRead, compData.length, + `Should get compressed size on ${method[0]}/` + + `${method[1]} end.`); + assert.strictEqual(decompData.toString(), expectStr, + `Should get original string on ${method[0]}/` + + `${method[1]} end.`); + })); + decompWriter = createWriter(decomp, compDataExtra); + } + })); + compWriter = createWriter(comp, expectBuf); +} diff --git a/test/parallel/test-zlib-convenience-methods.js b/test/parallel/test-zlib-convenience-methods.js index 5612575d8e7db4..bbecfb60a8d6f8 100644 --- a/test/parallel/test-zlib-convenience-methods.js +++ b/test/parallel/test-zlib-convenience-methods.js @@ -36,6 +36,10 @@ const opts = { chunkSize: 1024, }; +const optsInfo = { + info: true +}; + for (const [type, expect] of [ ['string', expectStr], ['Buffer', expectBuf], @@ -44,10 +48,10 @@ for (const [type, expect] of [ ) ]) { for (const method of [ - ['gzip', 'gunzip'], - ['gzip', 'unzip'], - ['deflate', 'inflate'], - ['deflateRaw', 'inflateRaw'], + ['gzip', 'gunzip', false], + ['gzip', 'unzip', false], + ['deflate', 'inflate', true], + ['deflateRaw', 'inflateRaw', true], ]) { zlib[method[0]](expect, opts, common.mustCall((err, result) => { zlib[method[1]](result, opts, common.mustCall((err, result) => { @@ -65,6 +69,35 @@ for (const [type, expect] of [ })); })); + zlib[method[0]](expect, optsInfo, common.mustCall((err, result) => { + assert.strictEqual(result.engine.bytesRead, expectStr.length, + `Should get input size after ${method[0]} ` + + `${type} with info option.`); + + const compressed = result.buffer; + zlib[method[1]](compressed, optsInfo, common.mustCall((err, result) => { + assert.strictEqual(result.buffer.toString(), expectStr, + `Should get original string after ${method[0]}/` + + `${method[1]} ${type} with info option.`); + assert.strictEqual(result.engine.bytesRead, compressed.length, + `Should get compressed size after ${method[0]}/` + + `${method[1]} ${type} with info option.`); + })); + + // Some methods should allow extra data after the compressed data + if (method[2]) { + const extra = Buffer.concat([compressed, new Buffer('extra')]); + zlib[method[1]](extra, optsInfo, common.mustCall((err, result) => { + assert.strictEqual(result.buffer.toString(), expectStr, + `Should get original string after ${method[0]}/` + + `${method[1]} ${type} with extra data.`); + assert.strictEqual(result.engine.bytesRead, compressed.length, + `Should get compressed size after ${method[0]}/` + + `${method[1]} ${type} with info option.`); + })); + } + })); + { const compressed = zlib[`${method[0]}Sync`](expect, opts); const decompressed = zlib[`${method[1]}Sync`](compressed, opts); @@ -81,5 +114,34 @@ for (const [type, expect] of [ `Should get original string after ${method[0]}Sync/` + `${method[1]}Sync ${type} without options.`); } + + + { + const compressed = zlib[`${method[0]}Sync`](expect, optsInfo); + assert.strictEqual(compressed.engine.bytesRead, expectStr.length, + `Should get input size after ${method[0]} ` + + `${type} with info option.`); + const decompressed = zlib[`${method[1]}Sync`](compressed.buffer, + optsInfo); + assert.strictEqual(decompressed.buffer.toString(), expectStr, + `Should get original string after ${method[0]}Sync/` + + `${method[1]}Sync ${type} without options.`); + assert.strictEqual(decompressed.engine.bytesRead, + compressed.buffer.length, + `Should get input size after ${method[0]}/` + + `${method[1]} ${type} with info option.`); + + if (method[2]) { + const extra = Buffer.concat([compressed.buffer, new Buffer('extra')]); + const decompressed = zlib[`${method[1]}Sync`](extra, optsInfo); + assert.strictEqual(decompressed.buffer.toString(), expectStr, + `Should get original string after ${method[0]}/` + + `${method[1]} ${type} with extra data.`); + assert.strictEqual(decompressed.engine.bytesRead, + compressed.buffer.length, + `Should get compressed size after ${method[0]}/` + + `${method[1]} ${type} with info option.`); + } + } } } From d1c87c16b0cf2ab90a2dff2a82960ef93124d85c Mon Sep 17 00:00:00 2001 From: Alexander O'Mara Date: Mon, 15 May 2017 19:46:48 -0400 Subject: [PATCH 2/2] zlib: simplified responseData return object --- lib/zlib.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/zlib.js b/lib/zlib.js index 1ec7d58987893d..4772ef9e5e34a6 100644 --- a/lib/zlib.js +++ b/lib/zlib.js @@ -79,10 +79,7 @@ function isInvalidStrategy(strategy) { function responseData(engine, buffer) { if (engine._opts.info) { - return { - buffer: buffer, - engine: engine - }; + return { buffer, engine }; } return buffer; }