From b17aa291e4021308b02df875231d39101500339a Mon Sep 17 00:00:00 2001 From: Nicholas Paun Date: Tue, 10 Sep 2024 16:26:41 -0700 Subject: [PATCH] zlib: Add support for info: true option --- src/node/internal/internal_zlib.ts | 285 ++++++++++++------ src/node/internal/internal_zlib_base.ts | 10 + src/node/internal/zlib.d.ts | 11 +- .../api/node/tests/zlib-nodejs-test.js | 76 +++-- 4 files changed, 239 insertions(+), 143 deletions(-) diff --git a/src/node/internal/internal_zlib.ts b/src/node/internal/internal_zlib.ts index 0f11b88c7e7..b16ed986e7e 100644 --- a/src/node/internal/internal_zlib.ts +++ b/src/node/internal/internal_zlib.ts @@ -6,14 +6,15 @@ import { default as zlibUtil, type ZlibOptions, - type CompressCallback, - type InternalCompressCallback, type BrotliOptions, } from 'node-internal:zlib'; import { Buffer } from 'node-internal:internal_buffer'; import { validateUint32 } from 'node-internal:validators'; import { ERR_INVALID_ARG_TYPE } from 'node-internal:internal_errors'; -import { Zlib, Brotli } from 'node-internal:internal_zlib_base'; +import { Zlib, Brotli, type ZlibBase } from 'node-internal:internal_zlib_base'; + +type ZlibResult = Buffer | { buffer: Buffer; engine: ZlibBase }; +type CompressCallback = (err: Error | null, result?: ZlibResult) => void; const { CONST_DEFLATE, @@ -35,99 +36,148 @@ export function crc32( return zlibUtil.crc32(data, value); } +function processChunk( + engine: ZlibBase, + data: ArrayBufferView | string +): ZlibResult { + return { + engine, + // TODO(soon): What is the proper way to deal with ArrayBufferView to Buffer typing issues? + buffer: engine._processChunk( + typeof data === 'string' ? Buffer.from(data) : (data as Buffer), + engine._finishFlushFlag + ), + }; +} + +function zlibSyncImpl( + data: ArrayBufferView | string, + options: ZlibOptions, + mode: ZlibMode +): ZlibResult { + if (!options.info) { + // Fast path, where we send the data directly to C++ + return Buffer.from(zlibUtil.zlibSync(data, options, mode)); + } + + // Else, use the Engine class in sync mode + return processChunk(new CLASS_BY_MODE[mode](options), data); +} + export function inflateSync( data: ArrayBufferView | string, options: ZlibOptions = {} -): Buffer { - return Buffer.from(zlibUtil.zlibSync(data, options, zlibUtil.CONST_INFLATE)); +): ZlibResult { + return zlibSyncImpl(data, options, CONST_INFLATE); } export function deflateSync( data: ArrayBufferView | string, options: ZlibOptions = {} -): Buffer { - return Buffer.from(zlibUtil.zlibSync(data, options, zlibUtil.CONST_DEFLATE)); +): ZlibResult { + return zlibSyncImpl(data, options, CONST_DEFLATE); } export function gunzipSync( data: ArrayBufferView | string, options: ZlibOptions = {} -): Buffer { - return Buffer.from(zlibUtil.zlibSync(data, options, zlibUtil.CONST_GUNZIP)); +): ZlibResult { + return zlibSyncImpl(data, options, CONST_GUNZIP); } export function gzipSync( data: ArrayBufferView | string, options: ZlibOptions = {} -): Buffer { - return Buffer.from(zlibUtil.zlibSync(data, options, zlibUtil.CONST_GZIP)); +): ZlibResult { + return zlibSyncImpl(data, options, CONST_GZIP); } export function inflateRawSync( data: ArrayBufferView | string, options: ZlibOptions = {} -): Buffer { - return Buffer.from( - zlibUtil.zlibSync(data, options, zlibUtil.CONST_INFLATERAW) - ); +): ZlibResult { + return zlibSyncImpl(data, options, CONST_INFLATERAW); } export function deflateRawSync( data: ArrayBufferView | string, options: ZlibOptions = {} -): Buffer { - return Buffer.from( - zlibUtil.zlibSync(data, options, zlibUtil.CONST_DEFLATERAW) - ); +): ZlibResult { + return zlibSyncImpl(data, options, CONST_DEFLATERAW); } export function unzipSync( data: ArrayBufferView | string, options: ZlibOptions = {} -): Buffer { - return Buffer.from(zlibUtil.zlibSync(data, options, zlibUtil.CONST_UNZIP)); +): ZlibResult { + return zlibSyncImpl(data, options, CONST_UNZIP); } export function brotliDecompressSync( data: ArrayBufferView | string, options: BrotliOptions = {} -): Buffer { - return Buffer.from(zlibUtil.brotliDecompressSync(data, options)); +): ZlibResult { + if (!options.info) { + // Fast path, where we send the data directly to C++ + return Buffer.from(zlibUtil.brotliDecompressSync(data, options)); + } + + // Else, use the Engine class in sync mode + return processChunk(new BrotliDecompress(options), data); } export function brotliCompressSync( data: ArrayBufferView | string, options: BrotliOptions = {} -): Buffer { - return Buffer.from(zlibUtil.brotliCompressSync(data, options)); +): ZlibResult { + if (!options.info) { + // Fast path, where we send the data directly to C++ + return Buffer.from(zlibUtil.brotliCompressSync(data, options)); + } + + // Else, use the Engine class in sync mode + return processChunk(new BrotliCompress(options), data); } function normalizeArgs( - optionsOrCallback: ZlibOptions | CompressCallback, - callbackOrUndefined?: CompressCallback + options: ZlibOptions | CompressCallback, + callback?: CompressCallback ): [ZlibOptions, CompressCallback] { - if (typeof optionsOrCallback === 'function') { - return [{}, optionsOrCallback]; - } else if (typeof callbackOrUndefined === 'function') { - return [optionsOrCallback, callbackOrUndefined]; + if (typeof options === 'function') { + return [{}, options]; + } else if (typeof callback === 'function') { + return [options, callback]; } - throw new ERR_INVALID_ARG_TYPE('callback', 'Function', callbackOrUndefined); + throw new ERR_INVALID_ARG_TYPE('callback', 'Function', callback); } -function wrapCallback(callback: CompressCallback): InternalCompressCallback { - return (res: Error | ArrayBuffer) => { +function processChunkCaptureError( + engine: ZlibBase, + data: ArrayBufferView | string, + cb: CompressCallback +): void { + try { + const res = processChunk(engine, data); queueMicrotask(() => { - if (res instanceof Error) { - callback(res); - } else { - callback(null, Buffer.from(res)); - } + cb(null, res); }); - }; + } catch (err: unknown) { + if (err instanceof Error) { + queueMicrotask(() => { + cb(err); + }); + return; + } + + const unreachable = new Error('Unreachable'); + unreachable.cause = err; + throw unreachable; + } } -export function inflate( +function zlibImpl( + mode: ZlibMode, data: ArrayBufferView | string, optionsOrCallback: ZlibOptions | CompressCallback, callbackOrUndefined?: CompressCallback @@ -136,89 +186,79 @@ export function inflate( optionsOrCallback, callbackOrUndefined ); - zlibUtil.zlib(data, options, zlibUtil.CONST_INFLATE, wrapCallback(callback)); + + if (!options.info) { + // Fast path + zlibUtil.zlib(data, options, mode, (res) => { + queueMicrotask(() => { + if (res instanceof Error) { + callback(res); + } else { + callback(null, Buffer.from(res)); + } + }); + }); + + return; + } + + processChunkCaptureError(new CLASS_BY_MODE[mode](options), data, callback); +} + +export function inflate( + data: ArrayBufferView | string, + options: ZlibOptions | CompressCallback, + callback?: CompressCallback +): void { + zlibImpl(CONST_INFLATE, data, options, callback); } export function unzip( data: ArrayBufferView | string, - optionsOrCallback: ZlibOptions | CompressCallback, - callbackOrUndefined?: CompressCallback + options: ZlibOptions | CompressCallback, + callback?: CompressCallback ): void { - const [options, callback] = normalizeArgs( - optionsOrCallback, - callbackOrUndefined - ); - zlibUtil.zlib(data, options, zlibUtil.CONST_UNZIP, wrapCallback(callback)); + zlibImpl(CONST_UNZIP, data, options, callback); } export function inflateRaw( data: ArrayBufferView | string, - optionsOrCallback: ZlibOptions | CompressCallback, - callbackOrUndefined?: CompressCallback + options: ZlibOptions | CompressCallback, + callback?: CompressCallback ): void { - const [options, callback] = normalizeArgs( - optionsOrCallback, - callbackOrUndefined - ); - zlibUtil.zlib( - data, - options, - zlibUtil.CONST_INFLATERAW, - wrapCallback(callback) - ); + zlibImpl(CONST_INFLATERAW, data, options, callback); } export function gunzip( data: ArrayBufferView | string, - optionsOrCallback: ZlibOptions | CompressCallback, - callbackOrUndefined?: CompressCallback + options: ZlibOptions | CompressCallback, + callback?: CompressCallback ): void { - const [options, callback] = normalizeArgs( - optionsOrCallback, - callbackOrUndefined - ); - zlibUtil.zlib(data, options, zlibUtil.CONST_GUNZIP, wrapCallback(callback)); + zlibImpl(CONST_GUNZIP, data, options, callback); } export function deflate( data: ArrayBufferView | string, - optionsOrCallback: ZlibOptions | CompressCallback, - callbackOrUndefined?: CompressCallback + options: ZlibOptions | CompressCallback, + callback?: CompressCallback ): void { - const [options, callback] = normalizeArgs( - optionsOrCallback, - callbackOrUndefined - ); - zlibUtil.zlib(data, options, zlibUtil.CONST_DEFLATE, wrapCallback(callback)); + zlibImpl(CONST_DEFLATE, data, options, callback); } export function deflateRaw( data: ArrayBufferView | string, - optionsOrCallback: ZlibOptions | CompressCallback, - callbackOrUndefined?: CompressCallback + options: ZlibOptions | CompressCallback, + callback?: CompressCallback ): void { - const [options, callback] = normalizeArgs( - optionsOrCallback, - callbackOrUndefined - ); - zlibUtil.zlib( - data, - options, - zlibUtil.CONST_DEFLATERAW, - wrapCallback(callback) - ); + zlibImpl(CONST_DEFLATERAW, data, options, callback); } export function gzip( data: ArrayBufferView | string, - optionsOrCallback: ZlibOptions | CompressCallback, - callbackOrUndefined?: CompressCallback + options: ZlibOptions | CompressCallback, + callback?: CompressCallback ): void { - const [options, callback] = normalizeArgs( - optionsOrCallback, - callbackOrUndefined - ); - zlibUtil.zlib(data, options, zlibUtil.CONST_GZIP, wrapCallback(callback)); + zlibImpl(CONST_GZIP, data, options, callback); } export function brotliDecompress( @@ -230,7 +270,23 @@ export function brotliDecompress( optionsOrCallback, callbackOrUndefined ); - zlibUtil.brotliDecompress(data, options, wrapCallback(callback)); + + if (!options.info) { + // Fast path + zlibUtil.brotliDecompress(data, options, (res) => { + queueMicrotask(() => { + if (res instanceof Error) { + callback(res); + } else { + callback(null, Buffer.from(res)); + } + }); + }); + + return; + } + + processChunkCaptureError(new BrotliDecompress(options), data, callback); } export function brotliCompress( @@ -242,9 +298,24 @@ export function brotliCompress( optionsOrCallback, callbackOrUndefined ); - zlibUtil.brotliCompress(data, options, wrapCallback(callback)); -} + if (!options.info) { + // Fast path + zlibUtil.brotliCompress(data, options, (res) => { + queueMicrotask(() => { + if (res instanceof Error) { + callback(res); + } else { + callback(null, Buffer.from(res)); + } + }); + }); + + return; + } + + processChunkCaptureError(new BrotliCompress(options), data, callback); +} export class Gzip extends Zlib { public constructor(options: ZlibOptions) { super(options, CONST_GZIP); @@ -302,6 +373,16 @@ export class BrotliDecompress extends Brotli { } } +const CLASS_BY_MODE = { + [ZlibMode.DEFLATE]: Deflate, + [ZlibMode.INFLATE]: Inflate, + [ZlibMode.DEFLATERAW]: DeflateRaw, + [ZlibMode.INFLATERAW]: InflateRaw, + [ZlibMode.GZIP]: Gzip, + [ZlibMode.GUNZIP]: Gunzip, + [ZlibMode.UNZIP]: Unzip, +}; + export function createGzip(options: ZlibOptions): Gzip { return new Gzip(options); } @@ -339,3 +420,13 @@ export function createBrotliDecompress( ): BrotliDecompress { return new BrotliDecompress(options); } + +const enum ZlibMode { + DEFLATE = 1, + INFLATE = 2, + GZIP = 3, + GUNZIP = 4, + DEFLATERAW = 5, + INFLATERAW = 6, + UNZIP = 7, +} diff --git a/src/node/internal/internal_zlib_base.ts b/src/node/internal/internal_zlib_base.ts index 3e04bce47b8..dc546aa36e9 100644 --- a/src/node/internal/internal_zlib_base.ts +++ b/src/node/internal/internal_zlib_base.ts @@ -526,6 +526,16 @@ export class ZlibBase extends Transform { } // This function is left for backwards compatibility. + public _processChunk( + chunk: Buffer, + flushFlag: number, + cb?: undefined + ): Buffer; + public _processChunk( + chunk: Buffer, + flushFlag: number, + cb: () => void + ): undefined; public _processChunk( chunk: Buffer, flushFlag: number, diff --git a/src/node/internal/zlib.d.ts b/src/node/internal/zlib.d.ts index d04e7cfb594..879ac9b4a9e 100644 --- a/src/node/internal/zlib.d.ts +++ b/src/node/internal/zlib.d.ts @@ -1,14 +1,9 @@ import { owner_symbol, type Zlib } from 'node-internal:internal_zlib_base'; -export function crc32(data: ArrayBufferView, value: number): number; - -export type CompressCallback = ( - err: Error | null, - buffer?: ArrayBuffer -) => void; -export type InternalCompressCallback = (res: Error | ArrayBuffer) => void; +type InternalCompressCallback = (res: Error | ArrayBuffer) => void; export function crc32(data: ArrayBufferView | string, value: number): number; + export function zlibSync( data: ArrayBufferView | string, options: ZlibOptions, @@ -178,6 +173,8 @@ export interface BrotliOptions { } | undefined; maxOutputLength?: number | undefined; + // Not specified in NodeJS docs but the tests expect it + info?: boolean | undefined; } type ErrorHandler = (errno: number, code: string, message: string) => void; diff --git a/src/workerd/api/node/tests/zlib-nodejs-test.js b/src/workerd/api/node/tests/zlib-nodejs-test.js index 2dfbe075ded..719a5fd1049 100644 --- a/src/workerd/api/node/tests/zlib-nodejs-test.js +++ b/src/workerd/api/node/tests/zlib-nodejs-test.js @@ -2005,26 +2005,25 @@ export const convenienceMethods = { await promise; } - // TODO(soon): Enable this test - // { - // const { promise, resolve } = Promise.withResolvers(); - // zlib[method[0]](expect, optsInfo, (err, result) => { - // assert.ifError(err); - // - // const compressed = result.buffer; - // zlib[method[1]](compressed, optsInfo, (err, result) => { - // assert.ifError(err); - // assert.strictEqual( - // result.buffer.toString(), - // expectStr, - // `Should get original string after ${method[0]}/` + - // `${method[1]} ${type} with info option.` - // ); - // resolve(); - // }); - // }); - // await promise; - // } + { + const { promise, resolve } = Promise.withResolvers(); + zlib[method[0]](expect, optsInfo, (err, result) => { + assert.ifError(err); + + const compressed = result.buffer; + zlib[method[1]](compressed, optsInfo, (err, result) => { + assert.ifError(err); + assert.strictEqual( + result.buffer.toString(), + expectStr, + `Should get original string after ${method[0]}/` + + `${method[1]} ${type} with info option.` + ); + resolve(); + }); + }); + await promise; + } { const compressed = zlib[`${method[0]}Sync`](expect, opts); @@ -2048,25 +2047,24 @@ export const convenienceMethods = { ); } - // TODO(soon): Enable this test - // { - // const compressed = zlib[`${method[0]}Sync`](expect, optsInfo); - // const decompressed = zlib[`${method[1]}Sync`]( - // compressed.buffer, - // optsInfo - // ); - // assert.strictEqual( - // decompressed.buffer.toString(), - // expectStr, - // `Should get original string after ${method[0]}Sync/` + - // `${method[1]}Sync ${type} without options.` - // ); - // assert.ok( - // decompressed.engine instanceof zlib[method[3]], - // `Should get engine ${method[3]} after ${method[0]} ` + - // `${type} with info option.` - // ); - // } + { + const compressed = zlib[`${method[0]}Sync`](expect, optsInfo); + const decompressed = zlib[`${method[1]}Sync`]( + compressed.buffer, + optsInfo + ); + assert.strictEqual( + decompressed.buffer.toString(), + expectStr, + `Should get original string after ${method[0]}Sync/` + + `${method[1]}Sync ${type} without options.` + ); + assert.ok( + decompressed.engine instanceof zlib[method[3]], + `Should get engine ${method[3]} after ${method[0]} ` + + `${type} with info option.` + ); + } } }