zlib: expose amount of data read for compression/decompression

Added bytesRead property to Zlib engines and an option to expose the engine to convenience method callbacks. Fixes: nodejs#8874
AlexanderOMara · May 10, 2017 · 1b0495e · 1b0495e
1 parent faf6654
commit 1b0495e
Show file tree

Hide file tree

Showing 3 changed files with 174 additions and 6 deletions.
diff --git a/lib/zlib.js b/lib/zlib.js
@@ -77,6 +77,16 @@ function isInvalidStrategy(strategy) {
   //         constants.Z_DEFAULT_STRATEGY (0)
 }
 
+function responseData(engine, buffer) {
+  if (engine._opts.info) {
+    return {
+      buffer: buffer,
+      engine: engine
+    };
+  }
+  return buffer;
+}
+
 function zlibBuffer(engine, buffer, callback) {
   // Streams do not support non-Buffer ArrayBufferViews yet. Convert it to a
   // Buffer without copying.
@@ -121,7 +131,7 @@ function zlibBuffer(engine, buffer, callback) {
 
     buffers = [];
     engine.close();
-    callback(err, buf);
+    callback(err, responseData(engine, buf));
   }
 }
 
@@ -134,7 +144,7 @@ function zlibBufferSync(engine, buffer) {
 
   var flushFlag = engine._finishFlushFlag;
 
-  return engine._processChunk(buffer, flushFlag);
+  return responseData(engine, engine._processChunk(buffer, flushFlag));
 }
 
 function zlibOnError(message, errno) {
@@ -168,6 +178,8 @@ class Zlib extends Transform {
     opts = opts || {};
     super(opts);
 
+    this.bytesRead = 0;
+
     this._opts = opts;
     this._chunkSize = opts.chunkSize || constants.Z_DEFAULT_CHUNK;
 
@@ -408,6 +420,8 @@ class Zlib extends Transform {
       var have = availOutBefore - availOutAfter;
       assert(have >= 0, 'have should not go down');
 
+      self.bytesRead += availInBefore - availInAfter;
+
       if (have > 0) {
         var out = self._buffer.slice(self._offset, self._offset + have);
         self._offset += have;

diff --git a/test/parallel/test-zlib-bytes-read.js b/test/parallel/test-zlib-bytes-read.js
@@ -0,0 +1,92 @@
+'use strict';
+const common = require('../common');
+const assert = require('assert');
+const zlib = require('zlib');
+
+const expectStr = 'abcdefghijklmnopqrstuvwxyz'.repeat(2);
+const expectBuf = Buffer.from(expectStr);
+
+function createWriter(target, buffer) {
+  const writer = { size: 0 };
+  const write = () => {
+    target.write(Buffer.from([buffer[writer.size++]]));
+    if (writer.size < buffer.length) {
+      setTimeout(write, 25);
+    } else {
+      target.end();
+    }
+  };
+  write();
+  return writer;
+}
+
+for (const method of [
+  ['createGzip', 'createGunzip', false],
+  ['createGzip', 'createUnzip', false],
+  ['createDeflate', 'createInflate', true],
+  ['createDeflateRaw', 'createInflateRaw', true]
+]) {
+  let compWriter;
+  let compData = new Buffer(0);
+
+  const comp = zlib[method[0]]();
+  comp.on('data', function(d) {
+    compData = Buffer.concat([compData, d]);
+    assert.strictEqual(this.bytesRead, compWriter.size,
+                       `Should get write size on ${method[0]} data.`);
+  });
+  comp.on('end', common.mustCall(function() {
+    assert.strictEqual(this.bytesRead, compWriter.size,
+                       `Should get write size on ${method[0]} end.`);
+    assert.strictEqual(this.bytesRead, expectStr.length,
+                       `Should get data size on ${method[0]} end.`);
+
+    {
+      let decompWriter;
+      let decompData = new Buffer(0);
+
+      const decomp = zlib[method[1]]();
+      decomp.on('data', function(d) {
+        decompData = Buffer.concat([decompData, d]);
+        assert.strictEqual(this.bytesRead, decompWriter.size,
+                           `Should get write size on ${method[0]}/` +
+                           `${method[1]} data.`);
+      });
+      decomp.on('end', common.mustCall(function() {
+        assert.strictEqual(this.bytesRead, compData.length,
+                           `Should get compressed size on ${method[0]}/` +
+                           `${method[1]} end.`);
+        assert.strictEqual(decompData.toString(), expectStr,
+                           `Should get original string on ${method[0]}/` +
+                           `${method[1]} end.`);
+      }));
+      decompWriter = createWriter(decomp, compData);
+    }
+
+    // Some methods should allow extra data after the compressed data
+    if (method[2]) {
+      const compDataExtra = Buffer.concat([compData, new Buffer('extra')]);
+
+      let decompWriter;
+      let decompData = new Buffer(0);
+
+      const decomp = zlib[method[1]]();
+      decomp.on('data', function(d) {
+        decompData = Buffer.concat([decompData, d]);
+        assert.strictEqual(this.bytesRead, decompWriter.size,
+                           `Should get write size on ${method[0]}/` +
+                           `${method[1]} data.`);
+      });
+      decomp.on('end', common.mustCall(function() {
+        assert.strictEqual(this.bytesRead, compData.length,
+                           `Should get compressed size on ${method[0]}/` +
+                           `${method[1]} end.`);
+        assert.strictEqual(decompData.toString(), expectStr,
+                           `Should get original string on ${method[0]}/` +
+                           `${method[1]} end.`);
+      }));
+      decompWriter = createWriter(decomp, compDataExtra);
+    }
+  }));
+  compWriter = createWriter(comp, expectBuf);
+}
diff --git a/test/parallel/test-zlib-convenience-methods.js b/test/parallel/test-zlib-convenience-methods.js
@@ -36,6 +36,10 @@ const opts = {
   chunkSize: 1024,
 };
 
+const optsInfo = {
+  info: true
+};
+
 for (const [type, expect] of [
   ['string', expectStr],
   ['Buffer', expectBuf],
@@ -44,10 +48,10 @@ for (const [type, expect] of [
   )
 ]) {
   for (const method of [
-    ['gzip', 'gunzip'],
-    ['gzip', 'unzip'],
-    ['deflate', 'inflate'],
-    ['deflateRaw', 'inflateRaw'],
+    ['gzip', 'gunzip', false],
+    ['gzip', 'unzip', false],
+    ['deflate', 'inflate', true],
+    ['deflateRaw', 'inflateRaw', true],
   ]) {
     zlib[method[0]](expect, opts, common.mustCall((err, result) => {
       zlib[method[1]](result, opts, common.mustCall((err, result) => {
@@ -65,6 +69,35 @@ for (const [type, expect] of [
       }));
     }));
 
+    zlib[method[0]](expect, optsInfo, common.mustCall((err, result) => {
+      assert.strictEqual(result.engine.bytesRead, expectStr.length,
+                         `Should get input size after ${method[0]} ` +
+                         `${type} with info option.`);
+
+      const compressed = result.buffer;
+      zlib[method[1]](compressed, optsInfo, common.mustCall((err, result) => {
+        assert.strictEqual(result.buffer.toString(), expectStr,
+                           `Should get original string after ${method[0]}/` +
+                           `${method[1]} ${type} with info option.`);
+        assert.strictEqual(result.engine.bytesRead, compressed.length,
+                           `Should get compressed size after ${method[0]}/` +
+                           `${method[1]} ${type} with info option.`);
+      }));
+
+      // Some methods should allow extra data after the compressed data
+      if (method[2]) {
+        const extra = Buffer.concat([compressed, new Buffer('extra')]);
+        zlib[method[1]](extra, optsInfo, common.mustCall((err, result) => {
+          assert.strictEqual(result.buffer.toString(), expectStr,
+                             `Should get original string after ${method[0]}/` +
+                             `${method[1]} ${type} with extra data.`);
+          assert.strictEqual(result.engine.bytesRead, compressed.length,
+                             `Should get compressed size after ${method[0]}/` +
+                             `${method[1]} ${type} with info option.`);
+        }));
+      }
+    }));
+
     {
       const compressed = zlib[`${method[0]}Sync`](expect, opts);
       const decompressed = zlib[`${method[1]}Sync`](compressed, opts);
@@ -81,5 +114,34 @@ for (const [type, expect] of [
                          `Should get original string after ${method[0]}Sync/` +
                          `${method[1]}Sync ${type} without options.`);
     }
+
+
+    {
+      const compressed = zlib[`${method[0]}Sync`](expect, optsInfo);
+      assert.strictEqual(compressed.engine.bytesRead, expectStr.length,
+                         `Should get input size after ${method[0]} ` +
+                         `${type} with info option.`);
+      const decompressed = zlib[`${method[1]}Sync`](compressed.buffer,
+                                                    optsInfo);
+      assert.strictEqual(decompressed.buffer.toString(), expectStr,
+                         `Should get original string after ${method[0]}Sync/` +
+                         `${method[1]}Sync ${type} without options.`);
+      assert.strictEqual(decompressed.engine.bytesRead,
+                         compressed.buffer.length,
+                         `Should get input size after ${method[0]}/` +
+                         `${method[1]} ${type} with info option.`);
+
+      if (method[2]) {
+        const extra = Buffer.concat([compressed.buffer, new Buffer('extra')]);
+        const decompressed = zlib[`${method[1]}Sync`](extra, optsInfo);
+        assert.strictEqual(decompressed.buffer.toString(), expectStr,
+                           `Should get original string after ${method[0]}/` +
+                           `${method[1]} ${type} with extra data.`);
+        assert.strictEqual(decompressed.engine.bytesRead,
+                           compressed.buffer.length,
+                           `Should get compressed size after ${method[0]}/` +
+                           `${method[1]} ${type} with info option.`);
+      }
+    }
   }
 }