fix issue vercel#1192 & only compress VFS if --compress is set

lib/producer.ts

@@ -227,7 +227,8 @@ function nativePrebuildInstall(target: Target, nodeFile: string) {
   }
 
   execSync(
-    `${prebuild} -t ${nodeVersion} --platform ${platform[target.platform]
+    `${prebuild} -t ${nodeVersion} --platform ${
+      platform[target.platform]
     } --arch ${target.arch}`,
     { cwd: dir }
   );
@@ -246,31 +247,67 @@ interface ProducerOptions {
   doCompress: CompressType;
 }
 
+/**
+ * instead of creating a vfs dicionnary with actual path as key
+ * we use a compression mechanism that can reduce significantly
+ * the memory footprint of the vfs in the code.
+ *
+ * without vfs compression:
+ *
+ * vfs = {
+ *   "/folder1/folder2/file1.js": {};
+ *   "/folder1/folder2/folder3/file2.js": {};
+ *   "/folder1/folder2/folder3/file3.js": {};
+ * }
+ *
+ * with compression :
+ *
+ * fileDictionary = {
+ *    "folder1": "1",
+ *    "folder2": "2",
+ *    "file1": "3",
+ *    "folder3": "4",
+ *    "file2": "5",
+ *    "file3": "6",
+ * }
+ * vfs = {
+ *   "/1/2/3": {};
+ *   "/1/2/4/5": {};
+ *   "/1/2/4/6": {};
+ * }
+ *
+ * note: the key is computed in base36 for further compression.
+ */
 const fileDictionary: { [key: string]: string } = {};
 let counter = 0;
-function replace(k: string) {
-  let existingKey = fileDictionary[k];
+function getOrCreateHash(fileOrFolderName: string) {
+  let existingKey = fileDictionary[fileOrFolderName];
   if (!existingKey) {
     const newkey = counter;
     counter += 1;
     existingKey = newkey.toString(36);
-    fileDictionary[k] = existingKey;
+    fileDictionary[fileOrFolderName] = existingKey;
   }
   return existingKey;
 }
-const separator = '$';
-
-function makeKey(filename: string, slash: string): string {
-  const a = filename.split(slash).map(replace).join(separator);
-  return a;
+const separator = '/';
+
+function makeKey(
+  doCompression: CompressType,
+  fullpath: string,
+  slash: string
+): string {
+  if (doCompression === CompressType.None) return fullpath;
+  return fullpath.split(slash).map(getOrCreateHash).join(separator);
 }
+
 export default function producer({
   backpack,
   bakes,
   slash,
   target,
   symLinks,
-  doCompress
+  doCompress,
 }: ProducerOptions) {
   return new Promise<void>((resolve, reject) => {
     if (!Buffer.alloc) {
@@ -289,7 +326,7 @@ export default function producer({
     for (const stripe of stripes) {
       let { snap } = stripe;
       snap = snapshotify(snap, slash);
-      const vfsKey = makeKey(snap, slash);
+      const vfsKey = makeKey(doCompress, snap, slash);
       if (!vfs[vfsKey]) vfs[vfsKey] = {};
     }
 
@@ -298,8 +335,8 @@ export default function producer({
     for (const [key, value] of Object.entries(symLinks)) {
       const k = snapshotify(key, slash);
       const v = snapshotify(value, slash);
-      const vfsKey = makeKey(k, slash);
-      snapshotSymLinks[vfsKey] = makeKey(v, slash);
+      const vfsKey = makeKey(doCompress, k, slash);
+      snapshotSymLinks[vfsKey] = makeKey(doCompress, v, slash);
     }
 
     let meter: streamMeter.StreamMeter;
@@ -350,7 +387,7 @@ export default function producer({
           const { store } = prevStripe;
           let { snap } = prevStripe;
           snap = snapshotify(snap, slash);
-          const vfsKey = makeKey(snap, slash);
+          const vfsKey = makeKey(doCompress, snap, slash);
           vfs[vfsKey][store] = [track, meter.bytes];
           track += meter.bytes;
         }
@@ -377,7 +414,12 @@ export default function producer({
                     return cb(null, intoStream(Buffer.alloc(0)));
                   }
 
-                  cb(null, pipeMayCompressToNewMeter(intoStream(buffer || Buffer.from(''))));
+                  cb(
+                    null,
+                    pipeMayCompressToNewMeter(
+                      intoStream(buffer || Buffer.from(''))
+                    )
+                  );
                 }
               );
             }

prelude/bootstrap.js

@@ -137,7 +137,7 @@ function readdirMountpoints(filename) {
       return path.dirname(interior) === filename;
     })
     .map(({ interior, exterior }) => {
-      if (isRegExp(interior)) return path.replace(interior, exterior);
+      if (isRegExp(interior)) return filename.replace(interior, exterior);
       return path.basename(interior);
     });
 }
@@ -190,8 +190,8 @@ console.log(translateNth(["", "rw"], 0, "d:\\snapshot\\countly\\plugins-ext\\"))
 console.log(translateNth(["", "a+"], 0, "d:\\snapshot\\countly\\plugins-ext\\1234"));
 */
 const dictRev = {};
-const separator = '$';
-let maxKey = Object.keys(DICT).reduce((p, c) => {
+const separator = '/';
+let maxKey = Object.values(DICT).reduce((p, c) => {
   const cc = parseInt(c, 36);
   return cc > p ? cc : p;
 }, 0);
@@ -208,6 +208,9 @@ function replace(k) {
 }
 
 function makeKey(filename, slash) {
+  if (!DOCOMPRESS) {
+    return filename;
+  }
   const a = filename.split(slash).map(replace).join(separator);
   return a || filename;
 }
@@ -216,8 +219,11 @@ Object.entries(DICT).forEach(([k, v]) => {
 });
 
 function toOriginal(fShort) {
+  if (!DOCOMPRESS) {
+    return fShort;
+  }
   return fShort
-    .split('$')
+    .split(separator)
     .map((x) => dictRev[x])
     .join(sep);
 }
@@ -483,7 +489,7 @@ function payloadFileSync(pointer) {
 
 (() => {
   process.pkg.path = {};
-  process.pkg.path.resolve = () => {
+  process.pkg.path.resolve = function resolve() {
     const args = cloneArgs(arguments);
     args.unshift(path.dirname(ENTRYPOINT));
     return path.resolve.apply(path, args); // eslint-disable-line prefer-spread

test/test-1191/.gitignore

@@ -0,0 +1,3 @@
+output*
+test.sqlite
+package-lock*

test/test-1191/main.js

@@ -12,7 +12,8 @@ assert(__dirname === process.cwd());
 /* eslint-disable no-unused-vars */
 const target = process.argv[2] || 'host';
 const ext = process.platform === 'win32' ? '.exe' : '';
-const output = './output' + ext;
+const output1 = './output1' + ext;
+const output2 = './output2' + ext;
 const input = './index.js';
 
 // remove any possible left-over
@@ -25,36 +26,72 @@ console.log('node version = ', version);
 const npmlog = utils.exec.sync('npm install');
 console.log('npm log :', npmlog);
 
-utils.pkg.sync(['--target', target, '--output', output, input], {
-  expect: 0,
-});
-
 // -----------------------------------------------------------------------
 // Execute programm outside pjg
 const logRef = utils.spawn.sync('node', [path.join(__dirname, input)], {
   cwd: __dirname,
   expect: 0,
 });
+
 if (logRef.replace(/\r|\n/g, '') !== '42') {
   console.log(`expecting 42 but got ${logRef}`);
   process.exit(1);
 }
-const log = utils.spawn.sync(path.join(__dirname, output), [], {
-  cwd: __dirname,
-  expect: 1,
-  stdio: ['inherit', 'pipe', 'pipe'],
-});
 
-if (
-  !log.stderr.match(
-    /was not included into executable at compilation stage. Please recompile adding it as asset or script/
-  )
-) {
-  console.log('expecting  warning message');
-  console.log('but got =', log.stderr);
+function doTestWithCompression() {
+  console.log('doTestWithCompression');
+  utils.pkg.sync(
+    ['--compress', 'Brotli', '--target', target, '--output', output1, input],
+    {
+      expect: 0,
+    }
+  );
+  const log = utils.spawn.sync(path.join(__dirname, output1), [], {
+    cwd: __dirname,
+    expect: 0,
+    stdio: ['inherit', 'pipe', 'pipe'],
+  });
+  return log;
+}
+function doTestNoCompression() {
+  console.log('doTestNoCompression');
+  utils.pkg.sync(['--target', target, '--output', output2, input], {
+    expect: 0,
+  });
+  const log = utils.spawn.sync(path.join(__dirname, output2), [], {
+    cwd: __dirname,
+    expect: 0,
+    stdio: ['inherit', 'pipe', 'pipe'],
+  });
+  return log;
+}
+const logNoCompression = doTestNoCompression();
+if (logNoCompression.stderr !== '') {
+  console.log('NO COMPRESSION: expecting no error');
+  console.log('but got =', logNoCompression.stderr);
+  process.exit(1);
+}
+
+const logWithCompression = doTestWithCompression();
+if (logWithCompression.stderr !== '') {
+  console.log('NO COMPRESSION: expecting no error');
+  console.log('but got =', logWithCompression.stderr);
   process.exit(1);
 }
-utils.vacuum.sync(output);
+// if (
+//   !log.stderr.match(
+//     /was not included into executable at compilation stage. Please recompile adding it as asset or script/
+//   )
+// ) {
+//   console.log('expecting  warning message');
+//   console.log('but got =', log.stderr);
+//   process.exit(1);
+// }
+
+// now with compress
+
+utils.vacuum.sync(output1);
+utils.vacuum.sync(output2);
 utils.vacuum.sync('node_modules');
 utils.vacuum.sync('package-lock.json');
 utils.vacuum.sync('test.sqlite');

test/test-1192/.gitignore

@@ -0,0 +1 @@
+dist/

test/test-1192/main.js

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const path = require('path');
+const assert = require('assert');
+const utils = require('../utils.js');
+
+assert(!module.parent);
+assert(__dirname === process.cwd());
+
+/* eslint-disable no-unused-vars */
+const target = process.argv[2] || 'host';
+const ext = process.platform === 'win32' ? '.exe' : '';
+const cmd = process.platform === 'win32' ? '.cmd' : '';
+const output = './output' + ext;
+const input = './package.json';
+
+// remove any possible left-over
+utils.vacuum.sync('./node_modules');
+
+const version = utils.exec.sync('node --version');
+console.log('node version = ', version);
+
+// launch `npm install`
+const npmlog = utils.exec.sync('npm' + cmd + ' install');
+// console.log('npm log :', npmlog);
+
+utils.pkg.sync(['--target', target, '--output', output, input], {
+  expect: 0,
+});
+
+// -----------------------------------------------------------------------
+// Execute programm outside pjg
+const logRef = utils.spawn.sync(
+  'node',
+  [path.join(__dirname, 'src/index.js')],
+  {
+    cwd: __dirname,
+    expect: 0,
+    stdio: ['inherit', 'pipe', 'pipe'],
+  }
+);
+
+const log = utils.spawn.sync(path.join(__dirname, output), [], {
+  cwd: __dirname,
+  expect: 0,
+  stdio: ['inherit', 'pipe', 'pipe'],
+});
+
+if (logRef.stdout !== log.stdout) {
+  console.log('expecting', logRef.stdout);
+  console.log('but got =', log.stdout);
+  process.exit(1);
+}
+if (logRef.stderr !== log.stderr) {
+  console.log('expecting', logRef.stderr);
+  console.log('but got =', log.stderr);
+  process.exit(1);
+}
+
+utils.vacuum.sync(output);
+utils.vacuum.sync('node_modules');
+utils.vacuum.sync('package-lock.json');
+console.log('Done');

test/test-1192/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "pkg-express",
+  "version": "1.0.0",
+  "description": "Illustrates issue between pkg 5.2.X and express (works with with pkg pre 5.2.X).",
+  "repository": "",
+  "main": "src/index.js",
+  "bin": "src/index.js",
+  "license": "MIT",
+  "scripts": {
+    "postinstall": "npm run make",
+    "start": "node .",
+    "make": "pkg . -o ./dist/pkg-express.exe"
+  },
+  "dependencies": {
+    "express": "^4.17.1",
+    "pug": "^3.0.2"
+  },
+  "devDependencies": {
+    "pkg": "5.2.1"
+  },
+  "pkg": {
+    "assets": "./public/views/*.pug"
+  }
+}

test/test-1192/public/views/index.pug

@@ -0,0 +1,8 @@
+
+html
+  head
+    title Hello World!
+  body
+    p Hello World!
+
+