compress files in virtual file system (#885)

.eslintrc

@@ -1,4 +1,5 @@
 {
+  // note you must disable the base rule as it can report incorrect errors
   "extends": ["airbnb-base", "prettier"],
   "rules": {
     "no-bitwise": "off",
@@ -10,8 +11,10 @@
     "consistent-return": "off",
     "no-restricted-syntax": "off",
     "import/prefer-default-export": "off",
-    "camelcase": "off"
-  },
+    "camelcase": "off",
+    "no-shadow": "off",
+    "@typescript-eslint/no-shadow": ["error"]
+    },
   "overrides": [
     {
       "files": ["*.ts"],

README.md

@@ -192,6 +192,16 @@ option to `pkg`. First ensure your computer meets the
 requirements to compile original Node.js:
 [BUILDING.md](https://github.com/nodejs/node/blob/master/BUILDING.md)
 
+### Compression
+
+Pass `--compress Brotli` or `--compress GZip` to `pkg` to compress further the content of the files store in the exectable.
+
+This option can reduce the size of the embedded file system by up to 60%.
+
+The startup time of the application might be reduced slightly.
+
+`-C` can be used as a shortcut for `--compress `.
+
 ### Environment
 
 | Var            | Description                                                                               |

lib/compress_type.ts

@@ -0,0 +1,6 @@
+
+export enum CompressType {
+    None = 0,
+    GZip = 1,
+    Brotli = 2
+};

lib/help.ts

@@ -19,6 +19,7 @@ export default function help() {
     --public             speed up and disclose the sources of top-level project
     --public-packages    force specified packages to be considered public
     --no-bytecode        skip bytecode generation and include source files as plain js
+    -C, --compress       [default=None] compression algorithm = Brotli or GZip
 
   ${chalk.dim('Examples:')}
 
@@ -36,5 +37,11 @@ export default function help() {
     ${chalk.cyan('$ pkg --public-packages "packageA,packageB" index.js')}
   ${chalk.gray('–')} Consider all packages to be public
     ${chalk.cyan('$ pkg --public-packages "*" index.js')}
+  ${chalk.gray('–')} Bakes '--expose-gc' into executable
+    ${chalk.cyan('$ pkg --options expose-gc index.js')}
+  ${chalk.gray(
+    '–'
+  )} reduce size of the data packed inside the executable with GZip
+    ${chalk.cyan('$ pkg --compress GZip index.js')}
 `);
 }

lib/index.ts

@@ -1,6 +1,6 @@
 /* eslint-disable require-atomic-updates */
 
-import { mkdirp, readFile, remove, stat, readFileSync } from 'fs-extra';
+import { existsSync, mkdirp, readFile, remove, stat, readFileSync } from 'fs-extra';
 import { need, system } from 'pkg-fetch';
 import assert from 'assert';
 import minimist from 'minimist';
@@ -16,6 +16,7 @@ import refine from './refiner';
 import { shutdown } from './fabricator';
 import walk, { Marker, WalkerParams } from './walker';
 import { Target, NodeTarget, SymLinks } from './types';
+import { CompressType } from './compress_type';
 
 const { version } = JSON.parse(
   readFileSync(path.join(__dirname, '../package.json'), 'utf-8')
@@ -25,15 +26,6 @@ function isConfiguration(file: string) {
   return isPackageJson(file) || file.endsWith('.config.json');
 }
 
-async function exists(file: string) {
-  try {
-    await stat(file);
-    return true;
-  } catch (error) {
-    return false;
-  }
-}
-
 // http://www.openwall.com/lists/musl/2012/12/08/4
 
 const {
@@ -245,6 +237,8 @@ export async function exec(argv2: string[]) {
       't',
       'target',
       'targets',
+      'C',
+      'compress',
     ],
     default: { bytecode: true },
   });
@@ -272,7 +266,31 @@ export async function exec(argv2: string[]) {
 
   const forceBuild = argv.b || argv.build;
 
-  // _
+  // doCompress
+  const algo = argv.C || argv.compress || 'None';
+
+
+  let doCompress: CompressType = CompressType.None;
+  switch (algo.toLowerCase()) {
+    case 'brotli':
+    case 'br':
+      doCompress = CompressType.Brotli;
+      break;
+    case 'gzip':
+    case 'gz':
+      doCompress = CompressType.GZip;
+      break;
+    case 'none':
+      break;
+    default:
+      // eslint-disable-next-line no-console
+      throw wasReported(`Invalid compression algorithm ${algo} ( should be None, Brotli or Gzip)`);
+
+  }
+  if (doCompress !== CompressType.None) {
+    // eslint-disable-next-line no-console
+    console.log('compression: ', CompressType[doCompress]);
+  }
 
   if (!argv._.length) {
     throw wasReported('Entry file/directory is expected', [
@@ -288,14 +306,13 @@ export async function exec(argv2: string[]) {
 
   let input = path.resolve(argv._[0]);
 
-  if (!(await exists(input))) {
+  if (!existsSync(input)) {
     throw wasReported('Input file does not exist', [input]);
   }
 
   if ((await stat(input)).isDirectory()) {
     input = path.join(input, 'package.json');
-
-    if (!(await exists(input))) {
+    if (!existsSync(input)) {
       throw wasReported('Input file does not exist', [input]);
     }
   }
@@ -330,10 +347,10 @@ export async function exec(argv2: string[]) {
         }
       }
       inputBin = path.resolve(path.dirname(input), bin);
-      if (!(await exists(inputBin))) {
+      if (!existsSync(inputBin)) {
         throw wasReported(
           'Bin file does not exist (taken from package.json ' +
-            "'bin' property)",
+          "'bin' property)",
           [inputBin]
         );
       }
@@ -362,8 +379,7 @@ export async function exec(argv2: string[]) {
 
   if (config) {
     config = path.resolve(config);
-
-    if (!(await exists(config))) {
+    if (!existsSync(config)) {
       throw wasReported('Config file does not exist', [config]);
     }
 
@@ -603,7 +619,7 @@ export async function exec(argv2: string[]) {
   log.debug('Targets:', JSON.stringify(targets, null, 2));
 
   for (const target of targets) {
-    if (target.output && (await exists(target.output))) {
+    if (target.output && existsSync(target.output)) {
       if ((await stat(target.output)).isFile()) {
         await remove(target.output);
       } else {
@@ -615,14 +631,8 @@ export async function exec(argv2: string[]) {
       await mkdirp(path.dirname(target.output));
     }
 
-    await producer({
-      backpack,
-      bakes,
-      slash: target.platform === 'win' ? '\\' : '/',
-      target: target as Target,
-      symLinks,
-    });
-
+    const slash = target.platform === 'win' ? '\\' : '/';
+    await producer({ backpack, bakes, slash, target: target as Target, symLinks, doCompress });
     if (target.platform !== 'win' && target.output) {
       await plusx(target.output);
     }

lib/packer.ts

@@ -1,8 +1,8 @@
 /* eslint-disable complexity */
 
 import assert from 'assert';
-import fs from 'fs-extra';
-import path from 'path';
+import * as fs from 'fs-extra';
+import * as path from 'path';
 
 import {
   STORE_BLOB,
@@ -157,7 +157,7 @@ export default function packer({
     }
   }
   const prelude =
-    `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS) { 
+    `return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS, DICT, DOCOMPRESS) { 
         ${bootstrapText}${
       log.debugMode ? diagnosticText : ''
     }\n})(function (exports) {\n${commonText}\n},\n` +
@@ -166,6 +166,10 @@ export default function packer({
     `%DEFAULT_ENTRYPOINT%` +
     `\n,\n` +
     `%SYMLINKS%` +
+    '\n,\n' +
+    '%DICT%' +
+    '\n,\n' +
+    '%DOCOMPRESS%' +
     `\n);`;
 
   return { prelude, entrypoint, stripes };