diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 6e6b8e9..b8eb895 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -74,7 +74,7 @@ jobs: run: yarn - name: Run tests - run: yarn test + run: yarn test:only - name: Submit coverage data to codecov uses: codecov/codecov-action@v2 diff --git a/lib/getHashDigest.js b/lib/getHashDigest.js index e51d3d0..505d6a3 100644 --- a/lib/getHashDigest.js +++ b/lib/getHashDigest.js @@ -45,37 +45,54 @@ function encodeBufferToBase(buffer, base) { let crypto = undefined; let createXXHash64 = undefined; let createMd4 = undefined; +let BatchedHash = undefined; +let BulkUpdateDecorator = undefined; -function getHashDigest(buffer, hashType, digestType, maxLength) { - hashType = hashType || "xxhash64"; +function getHashDigest(buffer, algorithm, digestType, maxLength) { + algorithm = algorithm || "xxhash64"; maxLength = maxLength || 9999; let hash; - if (hashType === "xxhash64") { + if (algorithm === "xxhash64") { if (createXXHash64 === undefined) { createXXHash64 = require("./hash/xxhash64"); + + if (BatchedHash === undefined) { + BatchedHash = require("./hash/BatchedHash"); + } } - hash = createXXHash64(); - } else if (hashType === "md4") { + hash = new BatchedHash(createXXHash64()); + } else if (algorithm === "md4") { if (createMd4 === undefined) { createMd4 = require("./hash/md4"); } - hash = createMd4(); - } else if (hashType === "native-md4") { + hash = new BatchedHash(createMd4()); + } else if (algorithm === "native-md4") { if (typeof crypto === "undefined") { crypto = require("crypto"); + + if (BulkUpdateDecorator === undefined) { + BulkUpdateDecorator = require("./hash/BulkUpdateDecorator"); + } } - hash = crypto.createHash("md4"); + hash = new BulkUpdateDecorator(() => crypto.createHash("md4"), "md4"); } else { if (typeof crypto === "undefined") { crypto = require("crypto"); + + if (BulkUpdateDecorator === undefined) { + BulkUpdateDecorator = require("./hash/BulkUpdateDecorator"); + } } - hash = crypto.createHash(hashType); + hash = new BulkUpdateDecorator( + () => crypto.createHash(algorithm), + algorithm + ); } hash.update(buffer); @@ -87,8 +104,7 @@ function getHashDigest(buffer, hashType, digestType, maxLength) { digestType === "base49" || digestType === "base52" || digestType === "base58" || - digestType === "base62" || - digestType === "base64" + digestType === "base62" ) { return encodeBufferToBase(hash.digest(), digestType.substr(4)).substr( 0, diff --git a/lib/hash/BatchedHash.js b/lib/hash/BatchedHash.js new file mode 100644 index 0000000..694ad4f --- /dev/null +++ b/lib/hash/BatchedHash.js @@ -0,0 +1,64 @@ +const MAX_SHORT_STRING = require("./wasm-hash").MAX_SHORT_STRING; + +class BatchedHash { + constructor(hash) { + this.string = undefined; + this.encoding = undefined; + this.hash = hash; + } + + /** + * Update hash {@link https://nodejs.org/api/crypto.html#crypto_hash_update_data_inputencoding} + * @param {string|Buffer} data data + * @param {string=} inputEncoding data encoding + * @returns {this} updated hash + */ + update(data, inputEncoding) { + if (this.string !== undefined) { + if ( + typeof data === "string" && + inputEncoding === this.encoding && + this.string.length + data.length < MAX_SHORT_STRING + ) { + this.string += data; + + return this; + } + + this.hash.update(this.string, this.encoding); + this.string = undefined; + } + + if (typeof data === "string") { + if ( + data.length < MAX_SHORT_STRING && + // base64 encoding is not valid since it may contain padding chars + (!inputEncoding || !inputEncoding.startsWith("ba")) + ) { + this.string = data; + this.encoding = inputEncoding; + } else { + this.hash.update(data, inputEncoding); + } + } else { + this.hash.update(data); + } + + return this; + } + + /** + * Calculates the digest {@link https://nodejs.org/api/crypto.html#crypto_hash_digest_encoding} + * @param {string=} encoding encoding of the return value + * @returns {string|Buffer} digest + */ + digest(encoding) { + if (this.string !== undefined) { + this.hash.update(this.string, this.encoding); + } + + return this.hash.digest(encoding); + } +} + +module.exports = BatchedHash; diff --git a/lib/hash/BulkUpdateDecorator.js b/lib/hash/BulkUpdateDecorator.js new file mode 100644 index 0000000..f3f7c73 --- /dev/null +++ b/lib/hash/BulkUpdateDecorator.js @@ -0,0 +1,107 @@ +const BULK_SIZE = 2000; + +// We are using an object instead of a Map as this will stay static during the runtime +// so access to it can be optimized by v8 +const digestCaches = {}; + +class BulkUpdateDecorator { + /** + * @param {Hash | function(): Hash} hashOrFactory function to create a hash + * @param {string=} hashKey key for caching + */ + constructor(hashOrFactory, hashKey) { + this.hashKey = hashKey; + + if (typeof hashOrFactory === "function") { + this.hashFactory = hashOrFactory; + this.hash = undefined; + } else { + this.hashFactory = undefined; + this.hash = hashOrFactory; + } + + this.buffer = ""; + } + + /** + * Update hash {@link https://nodejs.org/api/crypto.html#crypto_hash_update_data_inputencoding} + * @param {string|Buffer} data data + * @param {string=} inputEncoding data encoding + * @returns {this} updated hash + */ + update(data, inputEncoding) { + if ( + inputEncoding !== undefined || + typeof data !== "string" || + data.length > BULK_SIZE + ) { + if (this.hash === undefined) { + this.hash = this.hashFactory(); + } + + if (this.buffer.length > 0) { + this.hash.update(this.buffer); + this.buffer = ""; + } + + this.hash.update(data, inputEncoding); + } else { + this.buffer += data; + + if (this.buffer.length > BULK_SIZE) { + if (this.hash === undefined) { + this.hash = this.hashFactory(); + } + + this.hash.update(this.buffer); + this.buffer = ""; + } + } + + return this; + } + + /** + * Calculates the digest {@link https://nodejs.org/api/crypto.html#crypto_hash_digest_encoding} + * @param {string=} encoding encoding of the return value + * @returns {string|Buffer} digest + */ + digest(encoding) { + let digestCache; + + const buffer = this.buffer; + + if (this.hash === undefined) { + // short data for hash, we can use caching + const cacheKey = `${this.hashKey}-${encoding}`; + + digestCache = digestCaches[cacheKey]; + + if (digestCache === undefined) { + digestCache = digestCaches[cacheKey] = new Map(); + } + + const cacheEntry = digestCache.get(buffer); + + if (cacheEntry !== undefined) { + return cacheEntry; + } + + this.hash = this.hashFactory(); + } + + if (buffer.length > 0) { + this.hash.update(buffer); + } + + const digestResult = this.hash.digest(encoding); + + if (digestCache !== undefined) { + digestCache.set(buffer, digestResult); + } + + return digestResult; + } +} + +module.exports = BulkUpdateDecorator; diff --git a/package.json b/package.json index 9c19a92..c0f3b4b 100644 --- a/package.json +++ b/package.json @@ -7,10 +7,11 @@ "big.js": "^6.1.1" }, "scripts": { - "lint": "prettier --list-different . && eslint lib test", + "lint": "prettier --list-different . && eslint .", "pretest": "yarn lint", "test": "jest", - "test:ci": "jest --coverage", + "test:only": "jest --coverage", + "test:ci": "yarn test:only", "release": "yarn test && standard-version" }, "license": "MIT", diff --git a/test/getHashDigest.test.js b/test/getHashDigest.test.js index 6611b1d..cf426b3 100644 --- a/test/getHashDigest.test.js +++ b/test/getHashDigest.test.js @@ -4,6 +4,27 @@ const loaderUtils = require("../"); describe("getHashDigest()", () => { [ + ["test string", "xxhash64", "hex", undefined, "e9e2c351e3c6b198"], + ["test string", "xxhash64", "base64", undefined, "6eLDUePGsZg="], + ["test string", "xxhash64", "base52", undefined, "byfYGDmnmyUr"], + ["abc\\0♥", "xxhash64", "hex", undefined, "4b9a34297dc03d20"], + ["abc\\0💩", "xxhash64", "hex", undefined, "86733ec125b93904"], + ["abc\\0💩", "xxhash64", "base64", undefined, "hnM+wSW5OQQ="], + ["abc\\0♥", "xxhash64", "base64", undefined, "S5o0KX3APSA="], + ["abc\\0💩", "xxhash64", "base52", undefined, "cfByjQcJZIU"], + ["abc\\0♥", "xxhash64", "base52", undefined, "qdLyAQjLlod"], + + ["test string", "md4", "hex", 4, "2e06"], + ["test string", "md4", "base64", undefined, "Lgbt1PFiMmjFpRcw2KCyrw=="], + ["test string", "md4", "base52", undefined, "egWqIKxsDHdZTteemJqXfuo"], + ["abc\\0♥", "md4", "hex", undefined, "46b9627fecf49b80eaf01c01d86ae9fd"], + ["abc\\0💩", "md4", "hex", undefined, "45aa5b332f8e562aaf0106ad6fc1d78f"], + ["abc\\0💩", "md4", "base64", undefined, "RapbMy+OViqvAQatb8HXjw=="], + ["abc\\0♥", "md4", "base64", undefined, "Rrlif+z0m4Dq8BwB2Grp/Q=="], + ["abc\\0💩", "md4", "base52", undefined, "dtXZENFEkYHXGxOkJbevPoD"], + ["abc\\0♥", "md4", "base52", undefined, "fYFFcfXRGsVweukHKlPayHs"], + + ["test string", "md5", "hex", 4, "6f8d"], [ "test string", "md5", @@ -11,29 +32,29 @@ describe("getHashDigest()", () => { undefined, "6f8db599de986fab7a21625b7916589c", ], - ["test string", "md5", "base64", undefined, "2sm1pVmS8xuGJLCdWpJoRL"], - // ["test string", "md5", "base64url", undefined, "b421md6Yb6t6IWJbeRZYnA"], - ["test string", "xxhash64", "hex", undefined, "e9e2c351e3c6b198"], - ["test string", "xxhash64", "base64", undefined, "9yNNKdhM-bF"], - ["test string", "xxhash64", "base52", undefined, "byfYGDmnmyUr"], - // ["test string", "xxhash64", "base64url", undefined, "6eLDUePGsZg"], - ["test string", "md4", "hex", 4, "2e06"], - ["test string", "md5", "hex", 4, "6f8d"], ["test string", "md5", "base52", undefined, "dJnldHSAutqUacjgfBQGLQx"], + ["test string", "md5", "base64", undefined, "b421md6Yb6t6IWJbeRZYnA=="], ["test string", "md5", "base26", 6, "bhtsgu"], + ["abc\\0♥", "md5", "hex", undefined, "2e897b64f8050e66aff98d38f7a012c5"], + ["abc\\0💩", "md5", "hex", undefined, "63ad5b3d675c5890e0c01ed339ba0187"], + ["abc\\0💩", "md5", "base64", undefined, "Y61bPWdcWJDgwB7TOboBhw=="], + ["abc\\0♥", "md5", "base64", undefined, "Lol7ZPgFDmav+Y0496ASxQ=="], + ["abc\\0💩", "md5", "base52", undefined, "djhVWGHaUKUxqxEhcTnOfBx"], + ["abc\\0♥", "md5", "base52", undefined, "eHeasSeRyOnorzxUJpayzJc"], + [ "test string", "sha512", "base64", undefined, - "2IS-kbfIPnVflXb9CzgoNESGCkvkb0urMmucPD9z8q6HuYz8RShY1-tzSUpm5-Ivx_u4H1MEzPgAhyhaZ7RKog", + "EObWR69EYkRC84jCwUp4f/ixfmFluD12fsBHdo2MvLcaGjIm58x4Frx5wEJ9lKnaaIxBo5kse/Xk18w+C+XbrA==", ], [ "test string", - "md5", + "sha512", "hex", undefined, - "6f8db599de986fab7a21625b7916589c", + "10e6d647af44624442f388c2c14a787ff8b17e6165b83d767ec047768d8cbcb71a1a3226e7cc7816bc79c0427d94a9da688c41a3992c7bf5e4d7cc3e0be5dbac", ], ].forEach((test) => { it( diff --git a/test/interpolateName.test.js b/test/interpolateName.test.js index 5e3464a..d84bf8e 100644 --- a/test/interpolateName.test.js +++ b/test/interpolateName.test.js @@ -50,13 +50,13 @@ describe("interpolateName()", () => { "/app/img/image.png", "[sha512:hash:base64:7].[ext]", "test content", - "2BKDTjl.png", + "DL9MrvO.png", ], [ "/app/img/image.png", "[sha512:contenthash:base64:7].[ext]", "test content", - "2BKDTjl.png", + "DL9MrvO.png", ], [ "/app/dir/file.png", @@ -104,19 +104,19 @@ describe("interpolateName()", () => { "/lib/components/modal/modal.css", "[name].[md4:hash:base64:20].[ext]", "test content", - "modal.1kNSGJ6n9ibMUEckC1Cp.css", + "modal.ppiZgUkxKA4vUnIZrWrH.css", ], [ "/lib/components/modal/modal.css", "[name].[md5:hash:base64:20].[ext]", "test content", - "modal.1n8osQznuT8jOAwdzg_n.css", + "modal.lHP90NiApDwht3eNNIch.css", ], [ "/lib/components/modal/modal.css", "[name].[md5:contenthash:base64:20].[ext]", "test content", - "modal.1n8osQznuT8jOAwdzg_n.css", + "modal.lHP90NiApDwht3eNNIch.css", ], // Should not interpret without `hash` or `contenthash` [ @@ -259,7 +259,7 @@ describe("interpolateName()", () => { ], [ [{}, "[hash:base64]", { content: "test string" }], - "9yNNKdhM-bF", + "6eLDUePGsZg=", "should interpolate [hash] token with options", ], [