From fa22d9ebe410419a4267ed219773b1e38eb12b20 Mon Sep 17 00:00:00 2001 From: Etienne Rossignon Date: Tue, 1 Jun 2021 08:27:21 +0200 Subject: [PATCH] fix issue #1192 & only compress VFS if --compress is set --- lib/producer.ts | 69 +++++++++++++++---- prelude/bootstrap.js | 18 +++-- test/test-10-pnpm/main.js | 4 ++ test/test-11-pnpm/main.js | 4 ++ test/test-11-pnpm/package.json | 2 +- test/test-1191/main.js | 24 ++++++- .../main.js | 4 +- .../test_with_new_fs_promises.js | 10 +-- test/test.js | 3 + 9 files changed, 110 insertions(+), 28 deletions(-) diff --git a/lib/producer.ts b/lib/producer.ts index 16e3edc4b..0cd3a9a62 100644 --- a/lib/producer.ts +++ b/lib/producer.ts @@ -253,31 +253,67 @@ interface ProducerOptions { doCompress: CompressType; } +/** + * instead of creating a vfs dicionnary with actual path as key + * we use a compression mechanism that can reduce significantly + * the memory footprint of the vfs in the code. + * + * without vfs compression: + * + * vfs = { + * "/folder1/folder2/file1.js": {}; + * "/folder1/folder2/folder3/file2.js": {}; + * "/folder1/folder2/folder3/file3.js": {}; + * } + * + * with compression : + * + * fileDictionary = { + * "folder1": "1", + * "folder2": "2", + * "file1": "3", + * "folder3": "4", + * "file2": "5", + * "file3": "6", + * } + * vfs = { + * "/1/2/3": {}; + * "/1/2/4/5": {}; + * "/1/2/4/6": {}; + * } + * + * note: the key is computed in base36 for further compression. + */ const fileDictionary: { [key: string]: string } = {}; let counter = 0; -function replace(k: string) { - let existingKey = fileDictionary[k]; +function getOrCreateHash(fileOrFolderName: string) { + let existingKey = fileDictionary[fileOrFolderName]; if (!existingKey) { const newkey = counter; counter += 1; existingKey = newkey.toString(36); - fileDictionary[k] = existingKey; + fileDictionary[fileOrFolderName] = existingKey; } return existingKey; } -const separator = '$'; - -function makeKey(filename: string, slash: string): string { - const a = filename.split(slash).map(replace).join(separator); - return a; +const separator = '/'; + +function makeKey( + doCompression: CompressType, + fullpath: string, + slash: string +): string { + if (doCompression === CompressType.None) return fullpath; + return fullpath.split(slash).map(getOrCreateHash).join(separator); } + export default function producer({ backpack, bakes, slash, target, symLinks, - doCompress + doCompress, }: ProducerOptions) { return new Promise((resolve, reject) => { if (!Buffer.alloc) { @@ -296,7 +332,7 @@ export default function producer({ for (const stripe of stripes) { let { snap } = stripe; snap = snapshotify(snap, slash); - const vfsKey = makeKey(snap, slash); + const vfsKey = makeKey(doCompress, snap, slash); if (!vfs[vfsKey]) vfs[vfsKey] = {}; } @@ -305,8 +341,8 @@ export default function producer({ for (const [key, value] of Object.entries(symLinks)) { const k = snapshotify(key, slash); const v = snapshotify(value, slash); - const vfsKey = makeKey(k, slash); - snapshotSymLinks[vfsKey] = makeKey(v, slash); + const vfsKey = makeKey(doCompress, k, slash); + snapshotSymLinks[vfsKey] = makeKey(doCompress, v, slash); } let meter: streamMeter.StreamMeter; @@ -357,7 +393,7 @@ export default function producer({ const { store } = prevStripe; let { snap } = prevStripe; snap = snapshotify(snap, slash); - const vfsKey = makeKey(snap, slash); + const vfsKey = makeKey(doCompress, snap, slash); vfs[vfsKey][store] = [track, meter.bytes]; track += meter.bytes; } @@ -384,7 +420,12 @@ export default function producer({ return cb(null, intoStream(Buffer.alloc(0))); } - cb(null, pipeMayCompressToNewMeter(intoStream(buffer || Buffer.from('')))); + cb( + null, + pipeMayCompressToNewMeter( + intoStream(buffer || Buffer.from('')) + ) + ); } ); } diff --git a/prelude/bootstrap.js b/prelude/bootstrap.js index 2890be6c4..3f135b895 100644 --- a/prelude/bootstrap.js +++ b/prelude/bootstrap.js @@ -188,8 +188,8 @@ console.log(translateNth(["", "rw"], 0, "d:\\snapshot\\countly\\plugins-ext\\")) console.log(translateNth(["", "a+"], 0, "d:\\snapshot\\countly\\plugins-ext\\1234")); */ const dictRev = {}; -const separator = '$'; -let maxKey = Object.keys(DICT).reduce((p, c) => { +const separator = '/'; +let maxKey = Object.values(DICT).reduce((p, c) => { const cc = parseInt(c, 36); return cc > p ? cc : p; }, 0); @@ -206,6 +206,9 @@ function replace(k) { } function makeKey(filename, slash) { + if (!DOCOMPRESS) { + return filename; + } const a = filename.split(slash).map(replace).join(separator); return a || filename; } @@ -214,13 +217,20 @@ Object.entries(DICT).forEach(([k, v]) => { }); function toOriginal(fShort) { + if (!DOCOMPRESS) { + return fShort; + } return fShort - .split('$') + .split(separator) .map((x) => dictRev[x]) .join(path.sep); } const symlinksEntries = Object.entries(SYMLINKS); + +// separator for substitution depends on platform; +const sepsep = DOCOMPRESS ? separator : path.sep; + function normalizePathAndFollowLink(f) { f = normalizePath(f); f = makeKey(f, path.sep); @@ -228,7 +238,7 @@ function normalizePathAndFollowLink(f) { while (needToSubstitute) { needToSubstitute = false; for (const [k, v] of symlinksEntries) { - if (f.startsWith(`${k}${separator}`) || f === k) { + if (f.startsWith(`${k}${sepsep}`) || f === k) { f = f.replace(k, v); needToSubstitute = true; break; diff --git a/test/test-10-pnpm/main.js b/test/test-10-pnpm/main.js index 248a32008..2369a4984 100644 --- a/test/test-10-pnpm/main.js +++ b/test/test-10-pnpm/main.js @@ -10,6 +10,10 @@ const utils = require('../utils.js'); // ignore this test if nodejs <= 10 , as recent version of PNPM do not support nodejs=10 const MAJOR_VERSION = parseInt(process.version.match(/v([0-9]+)/)[1], 10); if (MAJOR_VERSION < 12) { + console.log( + 'skiping test as it requires nodejs >= 12 and got', + MAJOR_VERSION + ); return; } diff --git a/test/test-11-pnpm/main.js b/test/test-11-pnpm/main.js index 8e70b5b26..0baa7f646 100644 --- a/test/test-11-pnpm/main.js +++ b/test/test-11-pnpm/main.js @@ -10,6 +10,10 @@ const utils = require('../utils.js'); // ignore this test if nodejs <= 10 , as recent version of PNPM do not support nodejs=10 const MAJOR_VERSION = parseInt(process.version.match(/v([0-9]+)/)[1], 10); if (MAJOR_VERSION < 12) { + console.log( + 'skiping test as it requires nodejs >= 12 and got', + MAJOR_VERSION + ); return; } diff --git a/test/test-11-pnpm/package.json b/test/test-11-pnpm/package.json index a4f207706..06d566980 100644 --- a/test/test-11-pnpm/package.json +++ b/test/test-11-pnpm/package.json @@ -11,6 +11,6 @@ "author": "", "license": "ISC", "dependencies": { - "bonjour": "*" + "bonjour": "3.5.0" } } diff --git a/test/test-1191/main.js b/test/test-1191/main.js index ad298e331..b15babb7f 100644 --- a/test/test-1191/main.js +++ b/test/test-1191/main.js @@ -32,11 +32,27 @@ const logRef = utils.spawn.sync('node', [path.join(__dirname, input)], { cwd: __dirname, expect: 0, }); + if (logRef.replace(/\r|\n/g, '') !== '42') { console.log(`expecting 42 but got ${logRef}`); process.exit(1); } +function doTestWithCompression() { + console.log('doTestWithCompression'); + utils.pkg.sync( + ['--compress', 'Brotli', '--target', target, '--output', output1, input], + { + // expect: 0, + } + ); + const log = utils.spawn.sync(path.join(__dirname, output1), [], { + cwd: __dirname, + // expect: 0, + stdio: ['inherit', 'pipe', 'pipe'], + }); + return log; +} function doTestNoCompression() { console.log('doTestNoCompression'); utils.pkg.sync(['--target', target, '--output', output2, input], { @@ -49,7 +65,6 @@ function doTestNoCompression() { }); return log; } - const logNoCompression = doTestNoCompression(); if (logNoCompression.stderr !== '') { console.log('NO COMPRESSION: expecting no error'); @@ -57,6 +72,13 @@ if (logNoCompression.stderr !== '') { process.exit(1); } +const logWithCompression = doTestWithCompression(); +if (logWithCompression.stderr !== '') { + console.log('NO COMPRESSION: expecting no error'); + console.log('but got =', logWithCompression.stderr); + process.exit(1); +} + // now with compress utils.vacuum.sync(output1); diff --git a/test/test-12-compression-various-file-access/main.js b/test/test-12-compression-various-file-access/main.js index 6e6b45fc6..ac80d6080 100644 --- a/test/test-12-compression-various-file-access/main.js +++ b/test/test-12-compression-various-file-access/main.js @@ -68,7 +68,8 @@ async function runTest(input) { console.log(' GZIPed:'); console.log(logGZip); - process.exit(1); + console.log('------------ ERROR'); + process.exit(1000); } utils.vacuum.sync(output); utils.vacuum.sync('gzip_' + output); @@ -83,3 +84,4 @@ console.log(' now testing with fs.promises'); const input2 = 'test_with_new_fs_promises.js'; runTest(input2); console.log('Done'); +process.exit(0); diff --git a/test/test-12-compression-various-file-access/test_with_new_fs_promises.js b/test/test-12-compression-various-file-access/test_with_new_fs_promises.js index 5f6d7d002..831e5f7c9 100644 --- a/test/test-12-compression-various-file-access/test_with_new_fs_promises.js +++ b/test/test-12-compression-various-file-access/test_with_new_fs_promises.js @@ -3,8 +3,8 @@ const path = require('path'); let fs_promises; -// ignore this test if nodejs <= 10 , as recent version of PNPM do not support nodejs=10 const MAJOR_VERSION = parseInt(process.version.match(/v([0-9]+)/)[1], 10); + if (MAJOR_VERSION >= 14) { // only work with nodeJs >= 14.0 fs_promises = require('fs/promises'); @@ -26,16 +26,12 @@ async function withPromises() { if (MAJOR_VERSION >= 14) { // eslint-disable-line no-unused-vars - const { bytesRead } = await fd.read({ - buffer, - position: 10, - encoding: 'ascii', - }); + const { bytesRead } = await fd.read(buffer, 0, buffer.length, 0); if (process.env.DEBUG) { console.log('bytesRead = ', bytesRead); } } else { - await fd.read(buffer, 0, buffer.length, 10); + await fd.read(buffer, 0, buffer.length, 0); } console.log(buffer.toString()); } catch (err) { diff --git a/test/test.js b/test/test.js index 9ff18cf32..4eb6b3221 100644 --- a/test/test.js +++ b/test/test.js @@ -42,8 +42,11 @@ function joinAndForward(d) { const list = []; +console.log('FLAVOR = ', flavor); if (flavor === 'only-npm') { list.push(joinAndForward('test-79-npm/main.js')); + list.push(joinAndForward('test-1191/main.js')); + list.push(joinAndForward('test-1192/main.js')); } else { list.push(joinAndForward('**/main.js')); if (flavor === 'no-npm') {