fix issue vercel#1192 & only compress VFS if --compress is set

lib/producer.ts

@@ -253,31 +253,67 @@ interface ProducerOptions {
   doCompress: CompressType;
 }
 
+/**
+ * instead of creating a vfs dicionnary with actual path as key
+ * we use a compression mechanism that can reduce significantly
+ * the memory footprint of the vfs in the code.
+ *
+ * without vfs compression:
+ *
+ * vfs = {
+ *   "/folder1/folder2/file1.js": {};
+ *   "/folder1/folder2/folder3/file2.js": {};
+ *   "/folder1/folder2/folder3/file3.js": {};
+ * }
+ *
+ * with compression :
+ *
+ * fileDictionary = {
+ *    "folder1": "1",
+ *    "folder2": "2",
+ *    "file1": "3",
+ *    "folder3": "4",
+ *    "file2": "5",
+ *    "file3": "6",
+ * }
+ * vfs = {
+ *   "/1/2/3": {};
+ *   "/1/2/4/5": {};
+ *   "/1/2/4/6": {};
+ * }
+ *
+ * note: the key is computed in base36 for further compression.
+ */
 const fileDictionary: { [key: string]: string } = {};
 let counter = 0;
-function replace(k: string) {
-  let existingKey = fileDictionary[k];
+function getOrCreateHash(fileOrFolderName: string) {
+  let existingKey = fileDictionary[fileOrFolderName];
   if (!existingKey) {
     const newkey = counter;
     counter += 1;
     existingKey = newkey.toString(36);
-    fileDictionary[k] = existingKey;
+    fileDictionary[fileOrFolderName] = existingKey;
   }
   return existingKey;
 }
-const separator = '$';
-
-function makeKey(filename: string, slash: string): string {
-  const a = filename.split(slash).map(replace).join(separator);
-  return a;
+const separator = '/';
+
+function makeKey(
+  doCompression: CompressType,
+  fullpath: string,
+  slash: string
+): string {
+  if (doCompression === CompressType.None) return fullpath;
+  return fullpath.split(slash).map(getOrCreateHash).join(separator);
 }
+
 export default function producer({
   backpack,
   bakes,
   slash,
   target,
   symLinks,
-  doCompress
+  doCompress,
 }: ProducerOptions) {
   return new Promise<void>((resolve, reject) => {
     if (!Buffer.alloc) {
@@ -296,7 +332,7 @@ export default function producer({
     for (const stripe of stripes) {
       let { snap } = stripe;
       snap = snapshotify(snap, slash);
-      const vfsKey = makeKey(snap, slash);
+      const vfsKey = makeKey(doCompress, snap, slash);
       if (!vfs[vfsKey]) vfs[vfsKey] = {};
     }
 
@@ -305,8 +341,8 @@ export default function producer({
     for (const [key, value] of Object.entries(symLinks)) {
       const k = snapshotify(key, slash);
       const v = snapshotify(value, slash);
-      const vfsKey = makeKey(k, slash);
-      snapshotSymLinks[vfsKey] = makeKey(v, slash);
+      const vfsKey = makeKey(doCompress, k, slash);
+      snapshotSymLinks[vfsKey] = makeKey(doCompress, v, slash);
     }
 
     let meter: streamMeter.StreamMeter;
@@ -357,7 +393,7 @@ export default function producer({
           const { store } = prevStripe;
           let { snap } = prevStripe;
           snap = snapshotify(snap, slash);
-          const vfsKey = makeKey(snap, slash);
+          const vfsKey = makeKey(doCompress, snap, slash);
           vfs[vfsKey][store] = [track, meter.bytes];
           track += meter.bytes;
         }
@@ -384,7 +420,12 @@ export default function producer({
                     return cb(null, intoStream(Buffer.alloc(0)));
                   }
 
-                  cb(null, pipeMayCompressToNewMeter(intoStream(buffer || Buffer.from(''))));
+                  cb(
+                    null,
+                    pipeMayCompressToNewMeter(
+                      intoStream(buffer || Buffer.from(''))
+                    )
+                  );
                 }
               );
             }

prelude/bootstrap.js

@@ -188,8 +188,8 @@ console.log(translateNth(["", "rw"], 0, "d:\\snapshot\\countly\\plugins-ext\\"))
 console.log(translateNth(["", "a+"], 0, "d:\\snapshot\\countly\\plugins-ext\\1234"));
 */
 const dictRev = {};
-const separator = '$';
-let maxKey = Object.keys(DICT).reduce((p, c) => {
+const separator = '/';
+let maxKey = Object.values(DICT).reduce((p, c) => {
   const cc = parseInt(c, 36);
   return cc > p ? cc : p;
 }, 0);
@@ -206,6 +206,9 @@ function replace(k) {
 }
 
 function makeKey(filename, slash) {
+  if (!DOCOMPRESS) {
+    return filename;
+  }
   const a = filename.split(slash).map(replace).join(separator);
   return a || filename;
 }
@@ -214,21 +217,28 @@ Object.entries(DICT).forEach(([k, v]) => {
 });
 
 function toOriginal(fShort) {
+  if (!DOCOMPRESS) {
+    return fShort;
+  }
   return fShort
-    .split('$')
+    .split(separator)
     .map((x) => dictRev[x])
     .join(path.sep);
 }
 
 const symlinksEntries = Object.entries(SYMLINKS);
+
+// separator for substitution depends on platform;
+const sepsep = DOCOMPRESS ? separator : path.sep;
+
 function normalizePathAndFollowLink(f) {
   f = normalizePath(f);
   f = makeKey(f, path.sep);
   let needToSubstitute = true;
   while (needToSubstitute) {
     needToSubstitute = false;
     for (const [k, v] of symlinksEntries) {
-      if (f.startsWith(`${k}${separator}`) || f === k) {
+      if (f.startsWith(`${k}${sepsep}`) || f === k) {
         f = f.replace(k, v);
         needToSubstitute = true;
         break;

test/test-10-pnpm/main.js

@@ -10,6 +10,10 @@ const utils = require('../utils.js');
 // ignore this test if nodejs <= 10 , as recent version of PNPM do not support nodejs=10
 const MAJOR_VERSION = parseInt(process.version.match(/v([0-9]+)/)[1], 10);
 if (MAJOR_VERSION < 12) {
+  console.log(
+    'skiping test as it requires nodejs >= 12 and got',
+    MAJOR_VERSION
+  );
   return;
 }
 

test/test-11-pnpm/main.js

@@ -10,6 +10,10 @@ const utils = require('../utils.js');
 // ignore this test if nodejs <= 10 , as recent version of PNPM do not support nodejs=10
 const MAJOR_VERSION = parseInt(process.version.match(/v([0-9]+)/)[1], 10);
 if (MAJOR_VERSION < 12) {
+  console.log(
+    'skiping test as it requires nodejs >= 12 and got',
+    MAJOR_VERSION
+  );
   return;
 }
 

test/test-11-pnpm/package.json

@@ -11,6 +11,6 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "bonjour": "*"
+    "bonjour": "3.5.0"
   }
 }

test/test-1191/main.js

@@ -32,11 +32,27 @@ const logRef = utils.spawn.sync('node', [path.join(__dirname, input)], {
   cwd: __dirname,
   expect: 0,
 });
+
 if (logRef.replace(/\r|\n/g, '') !== '42') {
   console.log(`expecting 42 but got ${logRef}`);
   process.exit(1);
 }
 
+function doTestWithCompression() {
+  console.log('doTestWithCompression');
+  utils.pkg.sync(
+    ['--compress', 'Brotli', '--target', target, '--output', output1, input],
+    {
+      //   expect: 0,
+    }
+  );
+  const log = utils.spawn.sync(path.join(__dirname, output1), [], {
+    cwd: __dirname,
+    // expect: 0,
+    stdio: ['inherit', 'pipe', 'pipe'],
+  });
+  return log;
+}
 function doTestNoCompression() {
   console.log('doTestNoCompression');
   utils.pkg.sync(['--target', target, '--output', output2, input], {
@@ -49,14 +65,20 @@ function doTestNoCompression() {
   });
   return log;
 }
-
 const logNoCompression = doTestNoCompression();
 if (logNoCompression.stderr !== '') {
   console.log('NO COMPRESSION: expecting no error');
   console.log('but got =', logNoCompression.stderr);
   process.exit(1);
 }
 
+const logWithCompression = doTestWithCompression();
+if (logWithCompression.stderr !== '') {
+  console.log('NO COMPRESSION: expecting no error');
+  console.log('but got =', logWithCompression.stderr);
+  process.exit(1);
+}
+
 // now with compress
 
 utils.vacuum.sync(output1);

test/test-12-compression-various-file-access/main.js

@@ -68,7 +68,8 @@ async function runTest(input) {
     console.log(' GZIPed:');
     console.log(logGZip);
 
-    process.exit(1);
+    console.log('------------ ERROR');
+    process.exit(1000);
   }
   utils.vacuum.sync(output);
   utils.vacuum.sync('gzip_' + output);
@@ -83,3 +84,4 @@ console.log('  now testing with fs.promises');
 const input2 = 'test_with_new_fs_promises.js';
 runTest(input2);
 console.log('Done');
+process.exit(0);

test/test-12-compression-various-file-access/test_with_new_fs_promises.js

@@ -3,8 +3,8 @@
 const path = require('path');
 
 let fs_promises;
-// ignore this test if nodejs <= 10 , as recent version of PNPM do not support nodejs=10
 const MAJOR_VERSION = parseInt(process.version.match(/v([0-9]+)/)[1], 10);
+
 if (MAJOR_VERSION >= 14) {
   // only work with nodeJs >= 14.0
   fs_promises = require('fs/promises');
@@ -26,16 +26,12 @@ async function withPromises() {
 
     if (MAJOR_VERSION >= 14) {
       // eslint-disable-line no-unused-vars
-      const { bytesRead } = await fd.read({
-        buffer,
-        position: 10,
-        encoding: 'ascii',
-      });
+      const { bytesRead } = await fd.read(buffer, 0, buffer.length, 0);
       if (process.env.DEBUG) {
         console.log('bytesRead = ', bytesRead);
       }
     } else {
-      await fd.read(buffer, 0, buffer.length, 10);
+      await fd.read(buffer, 0, buffer.length, 0);
     }
     console.log(buffer.toString());
   } catch (err) {

test/test.js

@@ -42,8 +42,11 @@ function joinAndForward(d) {
 
 const list = [];
 
+console.log('FLAVOR = ', flavor);
 if (flavor === 'only-npm') {
   list.push(joinAndForward('test-79-npm/main.js'));
+  list.push(joinAndForward('test-1191/main.js'));
+  list.push(joinAndForward('test-1192/main.js'));
 } else {
   list.push(joinAndForward('**/main.js'));
   if (flavor === 'no-npm') {