module: refactor modules bootstrap

PR-URL: #29937
Reviewed-By: Myles Borins <myles.borins@gmail.com>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>

doc/api/esm.md

@@ -905,13 +905,13 @@ _isMain_ is **true** when resolving the Node.js application entry point.
 >    1. Throw a _Module Not Found_ error.
 > 1. If _pjson.exports_ is not **null** or **undefined**, then
 >    1. If _pjson.exports_ is a String or Array, then
->       1. Return _PACKAGE_EXPORTS_TARGET_RESOLVE(packageURL, pjson.exports,
->          "")_.
+>       1. Return **PACKAGE_EXPORTS_TARGET_RESOLVE**(_packageURL_,
+>          _pjson.exports_, "")_.
 >    1. If _pjson.exports is an Object, then
 >       1. If _pjson.exports_ contains a _"."_ property, then
 >          1. Let _mainExport_ be the _"."_ property in _pjson.exports_.
->          1. Return _PACKAGE_EXPORTS_TARGET_RESOLVE(packageURL, mainExport,
->             "")_.
+>          1. Return **PACKAGE_EXPORTS_TARGET_RESOLVE**(_packageURL_,
+>             _mainExport_, "")_.
 > 1. If _pjson.main_ is a String, then
 >    1. Let _resolvedMain_ be the URL resolution of _packageURL_, "/", and
 >       _pjson.main_.

lib/internal/bootstrap/loaders.js

@@ -221,7 +221,10 @@ NativeModule.prototype.compileForPublicLoader = function(needToSyncExports) {
   this.compile();
   if (needToSyncExports) {
     if (!this.exportKeys) {
-      this.exportKeys = Object.keys(this.exports);
+      // When using --expose-internals, we do not want to reflect the named
+      // exports from core modules as this can trigger unnecessary getters.
+      const internal = this.id.startsWith('internal/');
+      this.exportKeys = internal ? [] : Object.keys(this.exports);
     }
     this.getESMFacade();
     this.syncExports();

lib/internal/bootstrap/pre_execution.js

@@ -5,6 +5,7 @@ const { Object, SafeWeakMap } = primordials;
 const { getOptionValue } = require('internal/options');
 const { Buffer } = require('buffer');
 const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes;
+const path = require('path');
 
 function prepareMainThreadExecution(expandArgv1 = false) {
   // Patch the process object with legacy properties and normalizations
@@ -404,7 +405,6 @@ function initializeESMLoader() {
         'The ESM module loader is experimental.',
         'ExperimentalWarning', undefined);
     }
-
     const {
       setImportModuleDynamicallyCallback,
       setInitializeImportMetaObjectCallback
@@ -414,14 +414,6 @@ function initializeESMLoader() {
     // track of for different ESM modules.
     setInitializeImportMetaObjectCallback(esm.initializeImportMetaObject);
     setImportModuleDynamicallyCallback(esm.importModuleDynamicallyCallback);
-    const userLoader = getOptionValue('--experimental-loader');
-    // If --experimental-loader is specified, create a loader with user hooks.
-    // Otherwise create the default loader.
-    if (userLoader) {
-      const { emitExperimentalWarning } = require('internal/util');
-      emitExperimentalWarning('--experimental-loader');
-    }
-    esm.initializeLoader(process.cwd(), userLoader);
   }
 }
 
@@ -446,11 +438,70 @@ function loadPreloadModules() {
   }
 }
 
+function resolveMainPath(main) {
+  const { toRealPath, Module: CJSModule } =
+      require('internal/modules/cjs/loader');
+
+  // Note extension resolution for the main entry point can be deprecated in a
+  // future major.
+  let mainPath = CJSModule._findPath(path.resolve(main), null, true);
+  if (!mainPath)
+    return;
+
+  const preserveSymlinksMain = getOptionValue('--preserve-symlinks-main');
+  if (!preserveSymlinksMain)
+    mainPath = toRealPath(mainPath);
+
+  return mainPath;
+}
+
+function shouldUseESMLoader(mainPath) {
+  const experimentalModules = getOptionValue('--experimental-modules');
+  if (!experimentalModules)
+    return false;
+  const userLoader = getOptionValue('--experimental-loader');
+  if (userLoader)
+    return true;
+  // Determine the module format of the main
+  if (mainPath && mainPath.endsWith('.mjs'))
+    return true;
+  if (!mainPath || mainPath.endsWith('.cjs'))
+    return false;
+  const { readPackageScope } = require('internal/modules/cjs/loader');
+  const pkg = readPackageScope(mainPath);
+  return pkg && pkg.data.type === 'module';
+}
+
+function runMainESM(mainPath) {
+  const esmLoader = require('internal/process/esm_loader');
+  const { pathToFileURL } = require('internal/url');
+  const { hasUncaughtExceptionCaptureCallback } =
+      require('internal/process/execution');
+  return esmLoader.initializeLoader().then(() => {
+    const main = path.isAbsolute(mainPath) ?
+      pathToFileURL(mainPath).href : mainPath;
+    return esmLoader.ESMLoader.import(main).catch((e) => {
+      if (hasUncaughtExceptionCaptureCallback()) {
+        process._fatalException(e);
+        return;
+      }
+      internalBinding('errors').triggerUncaughtException(
+        e,
+        true /* fromPromise */
+      );
+    });
+  });
+}
+
+
 module.exports = {
   patchProcessObject,
+  resolveMainPath,
+  runMainESM,
   setupCoverageHooks,
   setupWarningHandler,
   setupDebugEnv,
+  shouldUseESMLoader,
   prepareMainThreadExecution,
   initializeDeprecations,
   initializeESMLoader,

lib/internal/main/run_main_module.js

@@ -10,8 +10,7 @@ const CJSModule = require('internal/modules/cjs/loader').Module;
 
 markBootstrapComplete();
 
-// Note: this actually tries to run the module as a ESM first if
-// --experimental-modules is on.
-// TODO(joyeecheung): can we move that logic to here? Note that this
-// is an undocumented method available via `require('module').runMain`
-CJSModule.runMain();
+// Note: this loads the module through the ESM loader if
+// --experimental-loader is provided or --experimental-modules is on
+// and the module is determined to be an ES module
+CJSModule.runMain(process.argv[1]);

lib/internal/main/worker_thread.js

@@ -135,8 +135,9 @@ port.on('message', (message) => {
       const { evalScript } = require('internal/process/execution');
       evalScript('[worker eval]', filename);
     } else {
-      process.argv[1] = filename; // script filename
-      require('module').runMain();
+      // script filename
+      const CJSModule = require('internal/modules/cjs/loader').Module;
+      CJSModule.runMain(process.argv[1] = filename);
     }
   } else if (message.type === STDIO_PAYLOAD) {
     const { stream, chunk, encoding } = message;

lib/internal/modules/cjs/loader.js

@@ -70,9 +70,14 @@ const {
   ERR_REQUIRE_ESM
 } = require('internal/errors').codes;
 const { validateString } = require('internal/validators');
+const {
+  resolveMainPath,
+  shouldUseESMLoader,
+  runMainESM
+} = require('internal/bootstrap/pre_execution');
 const pendingDeprecation = getOptionValue('--pending-deprecation');
 
-module.exports = { wrapSafe, Module };
+module.exports = { wrapSafe, Module, toRealPath, readPackageScope };
 
 let asyncESM, ModuleJob, ModuleWrap, kInstantiated;
 
@@ -898,6 +903,10 @@ Module.prototype.load = function(filename) {
   this.paths = Module._nodeModulePaths(path.dirname(filename));
 
   const extension = findLongestRegisteredExtension(filename);
+  // allow .mjs to be overridden
+  if (filename.endsWith('.mjs') && !Module._extensions['.mjs']) {
+    throw new ERR_REQUIRE_ESM(filename);
+  }
   Module._extensions[extension](this, filename);
   this.loaded = true;
 
@@ -911,14 +920,19 @@ Module.prototype.load = function(filename) {
     if (module !== undefined && module.module !== undefined) {
       if (module.module.getStatus() >= kInstantiated)
         module.module.setExport('default', exports);
-    } else { // preemptively cache
+    } else {
+      // Preemptively cache
+      // We use a function to defer promise creation for async hooks.
       ESMLoader.moduleMap.set(
         url,
-        new ModuleJob(ESMLoader, url, () =>
+        // Module job creation will start promises.
+        // We make it a function to lazily trigger those promises
+        // for async hooks compatibility.
+        () => new ModuleJob(ESMLoader, url, () =>
           new ModuleWrap(url, undefined, ['default'], function() {
             this.setExport('default', exports);
           })
-        )
+        , false /* isMain */, false /* inspectBrk */)
       );
     }
   }
@@ -947,15 +961,15 @@ Module.prototype.require = function(id) {
 let resolvedArgv;
 let hasPausedEntry = false;
 
-function wrapSafe(filename, content) {
+function wrapSafe(filename, content, cjsModuleInstance) {
   if (patched) {
     const wrapper = Module.wrap(content);
     return vm.runInThisContext(wrapper, {
       filename,
       lineOffset: 0,
       displayErrors: true,
       importModuleDynamically: experimentalModules ? async (specifier) => {
-        const loader = await asyncESM.loaderPromise;
+        const loader = asyncESM.ESMLoader;
         return loader.import(specifier, normalizeReferrerURL(filename));
       } : undefined,
     });
@@ -981,17 +995,16 @@ function wrapSafe(filename, content) {
       ]
     );
   } catch (err) {
-    if (experimentalModules) {
+    if (experimentalModules && process.mainModule === cjsModuleInstance)
       enrichCJSError(err);
-    }
     throw err;
   }
 
   if (experimentalModules) {
     const { callbackMap } = internalBinding('module_wrap');
     callbackMap.set(compiled.cacheKey, {
       importModuleDynamically: async (specifier) => {
-        const loader = await asyncESM.loaderPromise;
+        const loader = asyncESM.ESMLoader;
         return loader.import(specifier, normalizeReferrerURL(filename));
       }
     });
@@ -1014,7 +1027,7 @@ Module.prototype._compile = function(content, filename) {
   }
 
   maybeCacheSourceMap(filename, content, this);
-  const compiledWrapper = wrapSafe(filename, content);
+  const compiledWrapper = wrapSafe(filename, content, this);
 
   var inspectorWrapper = null;
   if (getOptionValue('--inspect-brk') && process._eval == null) {
@@ -1070,7 +1083,11 @@ Module._extensions['.js'] = function(module, filename) {
           'files in that package scope as ES modules.\nInstead rename ' +
           `${basename} to end in .cjs, change the requiring code to use ` +
           'import(), or remove "type": "module" from ' +
-          `${path.resolve(pkg.path, 'package.json')}.`
+          `${path.resolve(pkg.path, 'package.json')}.`,
+          undefined,
+          undefined,
+          undefined,
+          true
         );
         warnRequireESM = false;
       }
@@ -1113,26 +1130,16 @@ Module._extensions['.node'] = function(module, filename) {
   return process.dlopen(module, path.toNamespacedPath(filename));
 };
 
-Module._extensions['.mjs'] = function(module, filename) {
-  throw new ERR_REQUIRE_ESM(filename);
-};
-
 // Bootstrap main module.
-Module.runMain = function() {
-  // Load the main module--the command line argument.
-  if (experimentalModules) {
-    asyncESM.loaderPromise.then((loader) => {
-      return loader.import(pathToFileURL(process.argv[1]).href);
-    })
-    .catch((e) => {
-      internalBinding('errors').triggerUncaughtException(
-        e,
-        true /* fromPromise */
-      );
-    });
-    return;
+Module.runMain = function(main = process.argv[1]) {
+  const resolvedMain = resolveMainPath(main);
+  const useESMLoader = shouldUseESMLoader(resolvedMain);
+  module.exports.asyncRunMain = useESMLoader;
+  if (useESMLoader) {
+    runMainESM(resolvedMain || main);
+  } else {
+    Module._load(main, null, true);
   }
-  Module._load(process.argv[1], null, true);
 };
 
 function createRequireFromPath(filename) {
@@ -1238,7 +1245,7 @@ Module.Module = Module;
 
 // We have to load the esm things after module.exports!
 if (experimentalModules) {
-  asyncESM = require('internal/process/esm_loader');
   ModuleJob = require('internal/modules/esm/module_job');
+  asyncESM = require('internal/process/esm_loader');
   ({ ModuleWrap, kInstantiated } = internalBinding('module_wrap'));
 }

lib/internal/modules/esm/loader.js

@@ -22,6 +22,7 @@ const createDynamicModule = require(
   'internal/modules/esm/create_dynamic_module');
 const { translators } = require('internal/modules/esm/translators');
 const { ModuleWrap } = internalBinding('module_wrap');
+const { getOptionValue } = require('internal/options');
 
 const debug = require('internal/util/debuglog').debuglog('esm');
 
@@ -118,7 +119,7 @@ class Loader {
     url = pathToFileURL(`${process.cwd()}/[eval${++this.evalIndex}]`).href
   ) {
     const evalInstance = (url) => new ModuleWrap(url, undefined, source, 0, 0);
-    const job = new ModuleJob(this, url, evalInstance, false);
+    const job = new ModuleJob(this, url, evalInstance, false, false);
     this.moduleMap.set(url, job);
     const { module, result } = await job.run();
     return {
@@ -146,6 +147,9 @@ class Loader {
   async getModuleJob(specifier, parentURL) {
     const { url, format } = await this.resolve(specifier, parentURL);
     let job = this.moduleMap.get(url);
+    // CommonJS will set functions for lazy job evaluation.
+    if (typeof job === 'function')
+      this.moduleMap.set(url, job = job());
     if (job !== undefined)
       return job;
 
@@ -169,7 +173,10 @@ class Loader {
       loaderInstance = translators.get(format);
     }
 
-    job = new ModuleJob(this, url, loaderInstance, parentURL === undefined);
+    const inspectBrk = parentURL === undefined &&
+        format === 'module' && getOptionValue('--inspect-brk');
+    job = new ModuleJob(this, url, loaderInstance, parentURL === undefined,
+                        inspectBrk);
     this.moduleMap.set(url, job);
     return job;
   }

lib/internal/modules/esm/module_job.js

@@ -9,7 +9,6 @@ const {
 const { ModuleWrap } = internalBinding('module_wrap');
 
 const { decorateErrorStack } = require('internal/util');
-const { getOptionValue } = require('internal/options');
 const assert = require('internal/assert');
 const resolvedPromise = SafePromise.resolve();
 
@@ -22,9 +21,10 @@ let hasPausedEntry = false;
 class ModuleJob {
   // `loader` is the Loader instance used for loading dependencies.
   // `moduleProvider` is a function
-  constructor(loader, url, moduleProvider, isMain) {
+  constructor(loader, url, moduleProvider, isMain, inspectBrk) {
     this.loader = loader;
     this.isMain = isMain;
+    this.inspectBrk = inspectBrk;
 
     // This is a Promise<{ module, reflect }>, whose fields will be copied
     // onto `this` by `link()` below once it has been resolved.
@@ -83,12 +83,12 @@ class ModuleJob {
     };
     await addJobsToDependencyGraph(this);
     try {
-      if (!hasPausedEntry && this.isMain && getOptionValue('--inspect-brk')) {
+      if (!hasPausedEntry && this.inspectBrk) {
         hasPausedEntry = true;
         const initWrapper = internalBinding('inspector').callAndPauseOnStart;
         initWrapper(this.module.instantiate, this.module);
       } else {
-        this.module.instantiate();
+        this.module.instantiate(true);
       }
     } catch (e) {
       decorateErrorStack(e);

lib/internal/modules/esm/module_map.js

@@ -16,7 +16,8 @@ class ModuleMap extends SafeMap {
   }
   set(url, job) {
     validateString(url, 'url');
-    if (job instanceof ModuleJob !== true) {
+    if (job instanceof ModuleJob !== true &&
+        typeof job !== 'function') {
       throw new ERR_INVALID_ARG_TYPE('job', 'ModuleJob', job);
     }
     debug(`Storing ${url} in ModuleMap`);