fix issue vercel#1192 & only compress VFS if --compress is set

lib/producer.ts

@@ -227,8 +227,7 @@ function nativePrebuildInstall(target: Target, nodeFile: string) {
   }
 
   execSync(
-    `${prebuild} -t ${nodeVersion} --platform ${platform[target.platform]
-    } --arch ${target.arch}`,
+    `${prebuild} -t ${nodeVersion} --platform ${platform[target.platform]} --arch ${target.arch}`,
     { cwd: dir }
   );
   fs.copyFileSync(nodeFile, nativeFile);
@@ -246,31 +245,67 @@ interface ProducerOptions {
   doCompress: CompressType;
 }
 
+/**
+ * instead of creating a vfs dicionnary with actual path as key
+ * we use a compression mechanism that can reduce significantly
+ * the memory footprint of the vfs in the code.
+ *
+ * without vfs compression:
+ *
+ * vfs = {
+ *   "/folder1/folder2/file1.js": {};
+ *   "/folder1/folder2/folder3/file2.js": {};
+ *   "/folder1/folder2/folder3/file3.js": {};
+ * }
+ *
+ * with compression :
+ *
+ * fileDictionary = {
+ *    "folder1": "1",
+ *    "folder2": "2",
+ *    "file1": "3",
+ *    "folder3": "4",
+ *    "file2": "5",
+ *    "file3": "6",
+ * }
+ * vfs = {
+ *   "/1/2/3": {};
+ *   "/1/2/4/5": {};
+ *   "/1/2/4/6": {};
+ * }
+ *
+ * note: the key is computed in base36 for further compression.
+ */
 const fileDictionary: { [key: string]: string } = {};
 let counter = 0;
-function replace(k: string) {
-  let existingKey = fileDictionary[k];
+function getOrCreateHash(fileOrFolderName: string) {
+  let existingKey = fileDictionary[fileOrFolderName];
   if (!existingKey) {
     const newkey = counter;
     counter += 1;
     existingKey = newkey.toString(36);
-    fileDictionary[k] = existingKey;
+    fileDictionary[fileOrFolderName] = existingKey;
   }
   return existingKey;
 }
-const separator = '$';
-
-function makeKey(filename: string, slash: string): string {
-  const a = filename.split(slash).map(replace).join(separator);
-  return a;
+const separator = '/';
+
+function makeKey(
+  doCompression: CompressType,
+  fullpath: string,
+  slash: string
+): string {
+  if (doCompression === CompressType.None) return fullpath;
+  return fullpath.split(slash).map(getOrCreateHash).join(separator);
 }
+
 export default function producer({
   backpack,
   bakes,
   slash,
   target,
   symLinks,
-  doCompress
+  doCompress,
 }: ProducerOptions) {
   return new Promise<void>((resolve, reject) => {
     if (!Buffer.alloc) {
@@ -289,7 +324,7 @@ export default function producer({
     for (const stripe of stripes) {
       let { snap } = stripe;
       snap = snapshotify(snap, slash);
-      const vfsKey = makeKey(snap, slash);
+      const vfsKey = makeKey(doCompress, snap, slash);
       if (!vfs[vfsKey]) vfs[vfsKey] = {};
     }
 
@@ -298,8 +333,8 @@ export default function producer({
     for (const [key, value] of Object.entries(symLinks)) {
       const k = snapshotify(key, slash);
       const v = snapshotify(value, slash);
-      const vfsKey = makeKey(k, slash);
-      snapshotSymLinks[vfsKey] = makeKey(v, slash);
+      const vfsKey = makeKey(doCompress, k, slash);
+      snapshotSymLinks[vfsKey] = makeKey(doCompress, v, slash);
     }
 
     let meter: streamMeter.StreamMeter;
@@ -350,7 +385,7 @@ export default function producer({
           const { store } = prevStripe;
           let { snap } = prevStripe;
           snap = snapshotify(snap, slash);
-          const vfsKey = makeKey(snap, slash);
+          const vfsKey = makeKey(doCompress, snap, slash);
           vfs[vfsKey][store] = [track, meter.bytes];
           track += meter.bytes;
         }
@@ -377,7 +412,12 @@ export default function producer({
                     return cb(null, intoStream(Buffer.alloc(0)));
                   }
 
-                  cb(null, pipeMayCompressToNewMeter(intoStream(buffer || Buffer.from(''))));
+                  cb(
+                    null,
+                    pipeMayCompressToNewMeter(
+                      intoStream(buffer || Buffer.from(''))
+                    )
+                  );
                 }
               );
             }

prelude/bootstrap.js

@@ -137,7 +137,7 @@ function readdirMountpoints(filename) {
       return path.dirname(interior) === filename;
     })
     .map(({ interior, exterior }) => {
-      if (isRegExp(interior)) return path.replace(interior, exterior);
+      if (isRegExp(interior)) return filename.replace(interior, exterior);
       return path.basename(interior);
     });
 }
@@ -190,8 +190,8 @@ console.log(translateNth(["", "rw"], 0, "d:\\snapshot\\countly\\plugins-ext\\"))
 console.log(translateNth(["", "a+"], 0, "d:\\snapshot\\countly\\plugins-ext\\1234"));
 */
 const dictRev = {};
-const separator = '$';
-let maxKey = Object.keys(DICT).reduce((p, c) => {
+const separator = '/';
+let maxKey = Object.values(DICT).reduce((p, c) => {
   const cc = parseInt(c, 36);
   return cc > p ? cc : p;
 }, 0);
@@ -208,6 +208,9 @@ function replace(k) {
 }
 
 function makeKey(filename, slash) {
+  if (!DOCOMPRESS) {
+    return filename;
+  }
   const a = filename.split(slash).map(replace).join(separator);
   return a || filename;
 }
@@ -216,21 +219,28 @@ Object.entries(DICT).forEach(([k, v]) => {
 });
 
 function toOriginal(fShort) {
+  if (!DOCOMPRESS) {
+    return fShort;
+  }
   return fShort
-    .split('$')
+    .split(separator)
     .map((x) => dictRev[x])
     .join(sep);
 }
 
 const symlinksEntries = Object.entries(SYMLINKS);
+
+// separator for substitution depends on platform;
+const sepsep = DOCOMPRESS ? separator : sep;
+
 function normalizePathAndFollowLink(f) {
   f = normalizePath(f);
   f = makeKey(f, sep);
   let needToSubstitue = true;
   while (needToSubstitue) {
     needToSubstitue = false;
     for (const [k, v] of symlinksEntries) {
-      if (f.startsWith(`${k}${separator}`) || f === k) {
+      if (f.startsWith(`${k}${sepsep}`) || f === k) {
         f = f.replace(k, v);
         needToSubstitue = true;
         break;
@@ -483,7 +493,7 @@ function payloadFileSync(pointer) {
 
 (() => {
   process.pkg.path = {};
-  process.pkg.path.resolve = () => {
+  process.pkg.path.resolve = function resolve() {
     const args = cloneArgs(arguments);
     args.unshift(path.dirname(ENTRYPOINT));
     return path.resolve.apply(path, args); // eslint-disable-line prefer-spread

test/test-10-pnpm/main.js

@@ -10,6 +10,10 @@ const utils = require('../utils.js');
 // ignore this test if nodejs <= 10 , as recent version of PNPM do not support nodejs=10
 const MAJOR_VERSION = parseInt(process.version.match(/v([0-9]+)/)[1], 10);
 if (MAJOR_VERSION < 12) {
+  console.log(
+    'skiping test as it requires nodejs >= 12 and got',
+    MAJOR_VERSION
+  );
   return;
 }
 

test/test-11-pnpm/main.js

@@ -10,6 +10,10 @@ const utils = require('../utils.js');
 // ignore this test if nodejs <= 10 , as recent version of PNPM do not support nodejs=10
 const MAJOR_VERSION = parseInt(process.version.match(/v([0-9]+)/)[1], 10);
 if (MAJOR_VERSION < 12) {
+  console.log(
+    'skiping test as it requires nodejs >= 12 and got',
+    MAJOR_VERSION
+  );
   return;
 }
 

test/test-11-pnpm/package.json

@@ -11,6 +11,6 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "bonjour": "*"
+    "bonjour": "3.5.0"
   }
 }

test/test-1135-issue/main.js

@@ -33,7 +33,6 @@ console.log('node version = ', version);
 const npmlog = utils.exec.sync('npm install');
 console.log('npm log :', npmlog);
 
-// verify that we have the .pnpm folder and a symlinks module in node_modules
 assert(fs.lstatSync(path.join(__dirname, 'node_modules/canvas')).isDirectory());
 assert(
   fs.lstatSync(path.join(__dirname, 'node_modules/canvas/build')).isDirectory()

test/test-1191/.gitignore

@@ -0,0 +1,3 @@
+output*
+test.sqlite
+package-lock*

test/test-1191/main.js

@@ -12,7 +12,8 @@ assert(__dirname === process.cwd());
 /* eslint-disable no-unused-vars */
 const target = process.argv[2] || 'host';
 const ext = process.platform === 'win32' ? '.exe' : '';
-const output = './output' + ext;
+const output1 = './output1' + ext;
+const output2 = './output2' + ext;
 const input = './index.js';
 
 // remove any possible left-over
@@ -21,40 +22,67 @@ utils.vacuum.sync('./node_modules');
 const version = utils.exec.sync('node --version');
 console.log('node version = ', version);
 
-// launch `npm install`
-const npmlog = utils.exec.sync('npm install');
-console.log('npm log :', npmlog);
-
-utils.pkg.sync(['--target', target, '--output', output, input], {
-  expect: 0,
-});
+// launch `yarn`
+const yarnlog = utils.exec.sync('yarn');
+console.log('yarn log :', yarnlog);
 
 // -----------------------------------------------------------------------
 // Execute programm outside pjg
 const logRef = utils.spawn.sync('node', [path.join(__dirname, input)], {
   cwd: __dirname,
   expect: 0,
 });
+
 if (logRef.replace(/\r|\n/g, '') !== '42') {
   console.log(`expecting 42 but got ${logRef}`);
   process.exit(1);
 }
-const log = utils.spawn.sync(path.join(__dirname, output), [], {
-  cwd: __dirname,
-  expect: 1,
-  stdio: ['inherit', 'pipe', 'pipe'],
-});
 
-if (
-  !log.stderr.match(
-    /was not included into executable at compilation stage. Please recompile adding it as asset or script/
-  )
-) {
-  console.log('expecting  warning message');
-  console.log('but got =', log.stderr);
+function doTestWithCompression() {
+  console.log('doTestWithCompression');
+  utils.pkg.sync(
+    ['--compress', 'Brotli', '--target', target, '--output', output1, input],
+    {
+      //   expect: 0,
+    }
+  );
+  const log = utils.spawn.sync(path.join(__dirname, output1), [], {
+    cwd: __dirname,
+    // expect: 0,
+    stdio: ['inherit', 'pipe', 'pipe'],
+  });
+  return log;
+}
+function doTestNoCompression() {
+  console.log('doTestNoCompression');
+  utils.pkg.sync(['--target', target, '--output', output2, input], {
+    //  expect: 0,
+  });
+  const log = utils.spawn.sync(path.join(__dirname, output2), [], {
+    cwd: __dirname,
+    expect: 0,
+    stdio: ['inherit', 'pipe', 'pipe'],
+  });
+  return log;
+}
+const logNoCompression = doTestNoCompression();
+if (logNoCompression.stderr !== '') {
+  console.log('NO COMPRESSION: expecting no error');
+  console.log('but got =', logNoCompression.stderr);
+  process.exit(1);
+}
+
+const logWithCompression = doTestWithCompression();
+if (logWithCompression.stderr !== '') {
+  console.log('NO COMPRESSION: expecting no error');
+  console.log('but got =', logWithCompression.stderr);
   process.exit(1);
 }
-utils.vacuum.sync(output);
+
+// now with compress
+
+utils.vacuum.sync(output1);
+utils.vacuum.sync(output2);
 utils.vacuum.sync('node_modules');
 utils.vacuum.sync('package-lock.json');
 utils.vacuum.sync('test.sqlite');

test/test-1192/.gitignore

@@ -0,0 +1 @@
+dist/