module: resolve and instantiate loader pipeline hooks

.eslintrc.yaml

@@ -11,7 +11,7 @@ parserOptions:
   ecmaVersion: 2017
 
 overrides:
-  - files: ["doc/api/esm.md", "*.mjs"]
+  - files: ["doc/api/esm.md", "*.mjs", "test/es-module/test-esm-example-loader.js"]
     parserOptions:
       sourceType: module
 
@@ -117,6 +117,7 @@ rules:
   keyword-spacing: error
   linebreak-style: [error, unix]
   max-len: [error, {code: 80,
+                ignorePattern: "^\/\/ Flags:",
                 ignoreRegExpLiterals: true,
                 ignoreUrls: true,
                 tabWidth: 2}]

doc/api/esm.md

@@ -98,4 +98,111 @@ fs.readFile('./foo.txt', (err, body) => {
 });
 ```
 
+## Loader hooks
+
+<!-- type=misc -->
+
+To customize the default module resolution, loader hooks can optionally be
+provided via a `--loader ./loader-name.mjs` argument to Node.
+
+When hooks are used they only apply to ES module loading and not to any
+CommonJS modules loaded.
+
+### Resolve hook
+
+The resolve hook returns the resolved file URL and module format for a
+given module specifier and parent file URL:
+
+```js
+import url from 'url';
+
+export async function resolve(specifier, parentModuleURL, defaultResolver) {
+  return {
+    url: new URL(specifier, parentModuleURL).href,
+    format: 'esm'
+  };
+}
+```
+
+The default NodeJS ES module resolution function is provided as a third
+argument to the resolver for easy compatibility workflows.
+
+In addition to returning the resolved file URL value, the resolve hook also
+returns a `format` property specifying the module format of the resolved
+module. This can be one of `"esm"`, `"cjs"`, `"json"`, `"builtin"` or
+`"addon"`.
+
+For example a dummy loader to load JavaScript restricted to browser resolution
+rules with only JS file extension and Node builtin modules support could
+be written:
+
+```js
+import url from 'url';
+import path from 'path';
+import process from 'process';
+
+const builtins = new Set(
+  Object.keys(process.binding('natives')).filter((str) =>
+    /^(?!(?:internal|node|v8)\/)/.test(str))
+);
+const JS_EXTENSIONS = new Set(['.js', '.mjs']);
+
+export function resolve(specifier, parentModuleURL/*, defaultResolve */) {
+  if (builtins.has(specifier)) {
+    return {
+      url: specifier,
+      format: 'builtin'
+    };
+  }
+  if (/^\.{0,2}[/]/.test(specifier) !== true && !specifier.startsWith('file:')) {
+    // For node_modules support:
+    // return defaultResolve(specifier, parentModuleURL);
+    throw new Error(
+      `imports must begin with '/', './', or '../'; '${specifier}' does not`);
+  }
+  const resolved = new url.URL(specifier, parentModuleURL);
+  const ext = path.extname(resolved.pathname);
+  if (!JS_EXTENSIONS.has(ext)) {
+    throw new Error(
+      `Cannot load file with non-JavaScript file extension ${ext}.`);
+  }
+  return {
+    url: resolved.href,
+    format: 'esm'
+  };
+}
+```
+
+With this loader, running:
+
+```
+NODE_OPTIONS='--experimental-modules --loader ./custom-loader.mjs' node x.js
+```
+
+would load the module `x.js` as an ES module with relative resolution support
+(with `node_modules` loading skipped in this example).
+
+### Dynamic instantiate hook
+
+To create a custom dynamic module that doesn't correspond to one of the
+existing `format` interpretations, the `dynamicInstantiate` hook can be used.
+This hook is called only for modules that return `format: "dynamic"` from
+the `resolve` hook.
+
+```js
+export async function dynamicInstantiate(url) {
+  return {
+    exports: ['customExportName'],
+    execute: (exports) => {
+      // get and set functions provided for pre-allocated export names
+      exports.customExportName.set('value');
+    }
+  };
+}
+```
+
+With the list of module exports provided upfront, the `execute` function will
+then be called at the exact point of module evalutation order for that module
+in the import tree.
+
 [Node.js EP for ES Modules]: https://github.com/nodejs/node-eps/blob/master/002-es-modules.md

lib/internal/errors.js

@@ -328,6 +328,8 @@ E('ERR_TRANSFORM_WITH_LENGTH_0',
 E('ERR_UNESCAPED_CHARACTERS',
   (name) => `${name} contains unescaped characters`);
 E('ERR_UNKNOWN_ENCODING', 'Unknown encoding: %s');
+E('ERR_UNKNOWN_FILE_EXTENSION', 'Unknown file extension: %s');
+E('ERR_UNKNOWN_MODULE_FORMAT', 'Unknown module format: %s');
 E('ERR_UNKNOWN_SIGNAL', 'Unknown signal: %s');
 E('ERR_UNKNOWN_STDIN_TYPE', 'Unknown stdin file type');
 E('ERR_UNKNOWN_STREAM_TYPE', 'Unknown stream file type');

lib/internal/loader/Loader.js

@@ -1,20 +1,21 @@
 'use strict';
 
-const { URL } = require('url');
 const { getURLFromFilePath } = require('internal/url');
 
 const {
-  getNamespaceOfModuleWrap
+  getNamespaceOfModuleWrap,
+  createDynamicModule
 } = require('internal/loader/ModuleWrap');
 
 const ModuleMap = require('internal/loader/ModuleMap');
 const ModuleJob = require('internal/loader/ModuleJob');
-const resolveRequestUrl = require('internal/loader/resolveRequestUrl');
+const ModuleRequest = require('internal/loader/ModuleRequest');
 const errors = require('internal/errors');
+const debug = require('util').debuglog('esm');
 
 function getBase() {
   try {
-    return getURLFromFilePath(`${process.cwd()}/`);
+    return getURLFromFilePath(`${process.cwd()}/`).href;
   } catch (e) {
     e.stack;
     // If the current working directory no longer exists.
@@ -28,45 +29,75 @@ function getBase() {
 class Loader {
   constructor(base = getBase()) {
     this.moduleMap = new ModuleMap();
-    if (typeof base !== 'undefined' && base instanceof URL !== true) {
-      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'URL');
+    if (typeof base !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'string');
     }
     this.base = base;
+    this.resolver = ModuleRequest.resolve.bind(null);
+    this.dynamicInstantiate = undefined;
   }
 
-  async resolve(specifier) {
-    const request = resolveRequestUrl(this.base, specifier);
-    if (request.url.protocol !== 'file:') {
-      throw new errors.Error('ERR_INVALID_PROTOCOL',
-                             request.url.protocol, 'file:');
-    }
-    return request.url;
+  hook({ resolve = ModuleRequest.resolve, dynamicInstantiate }) {
+    this.resolver = resolve.bind(null);
+    this.dynamicInstantiate = dynamicInstantiate;
   }
 
-  async getModuleJob(dependentJob, specifier) {
-    if (!this.moduleMap.has(dependentJob.url)) {
-      throw new errors.Error('ERR_MISSING_MODULE', dependentJob.url);
+  async resolve(specifier, parentURL = this.base) {
+    if (typeof parentURL !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE',
+                                 'parentURL', 'string');
+    }
+    const { url, format } = await this.resolver(specifier, parentURL,
+                                                ModuleRequest.resolve);
+
+    if (typeof format !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'format',
+                                 ['esm', 'cjs', 'builtin', 'addon', 'json']);
+    }
+    if (typeof url !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string');
+    }
+
+    if (format === 'builtin') {
+      return { url: `node:${url}`, format };
     }
-    const request = await resolveRequestUrl(dependentJob.url, specifier);
-    const url = `${request.url}`;
-    if (this.moduleMap.has(url)) {
-      return this.moduleMap.get(url);
+
+    if (format !== 'dynamic') {
+      if (!ModuleRequest.loaders.has(format)) {
+        throw new errors.Error('ERR_UNKNOWN_MODULE_FORMAT', format);
+      }
+      if (!url.startsWith('file:')) {
+        throw new errors.Error('ERR_INVALID_PROTOCOL', url, 'file:');
+      }
     }
-    const dependencyJob = new ModuleJob(this, request);
-    this.moduleMap.set(url, dependencyJob);
-    return dependencyJob;
+
+    return { url, format };
   }
 
-  async import(specifier) {
-    const request = await resolveRequestUrl(this.base, specifier);
-    const url = `${request.url}`;
-    let job;
-    if (this.moduleMap.has(url)) {
-      job = this.moduleMap.get(url);
-    } else {
-      job = new ModuleJob(this, request);
+  async getModuleJob(specifier, parentURL = this.base) {
+    const { url, format } = await this.resolve(specifier, parentURL);
+    let job = this.moduleMap.get(url);
+    if (job === undefined) {
+      let loaderInstance;
+      if (format === 'dynamic') {
+        loaderInstance = async (url) => {
+          const { exports, execute } = await this.dynamicInstantiate(url);
+          return createDynamicModule(exports, url, (reflect) => {
+            debug(`Loading custom loader ${url}`);
+            execute(reflect.exports);
+          });
+        };
+      } else {
+        loaderInstance = ModuleRequest.loaders.get(format);
+      }
+      job = new ModuleJob(this, url, loaderInstance);
       this.moduleMap.set(url, job);
     }
+    return job;
+  }
+
+  async import(specifier, parentURL = this.base) {
+    const job = await this.getModuleJob(specifier, parentURL);
     const module = await job.run();
     return getNamespaceOfModuleWrap(module);
   }

lib/internal/loader/ModuleJob.js

@@ -2,51 +2,40 @@
 
 const { SafeSet, SafePromise } = require('internal/safe_globals');
 const resolvedPromise = SafePromise.resolve();
-const resolvedArrayPromise = SafePromise.resolve([]);
-const { ModuleWrap } = require('internal/loader/ModuleWrap');
 
-const NOOP = () => { /* No-op */ };
 class ModuleJob {
   /**
    * @param {module: ModuleWrap?, compiled: Promise} moduleProvider
    */
-  constructor(loader, moduleProvider, url) {
-    this.url = `${moduleProvider.url}`;
-    this.moduleProvider = moduleProvider;
+  constructor(loader, url, moduleProvider) {
     this.loader = loader;
     this.error = null;
     this.hadError = false;
 
-    if (moduleProvider instanceof ModuleWrap !== true) {
-      // linked == promise for dependency jobs, with module populated,
-      // module wrapper linked
-      this.modulePromise = this.moduleProvider.createModule();
-      this.module = undefined;
-      const linked = async () => {
-        const dependencyJobs = [];
-        this.module = await this.modulePromise;
-        this.module.link(async (dependencySpecifier) => {
-          const dependencyJobPromise =
-            this.loader.getModuleJob(this, dependencySpecifier);
-          dependencyJobs.push(dependencyJobPromise);
-          const dependencyJob = await dependencyJobPromise;
-          return dependencyJob.modulePromise;
-        });
-        return SafePromise.all(dependencyJobs);
-      };
-      this.linked = linked();
+    // linked == promise for dependency jobs, with module populated,
+    // module wrapper linked
+    this.moduleProvider = moduleProvider;
+    this.modulePromise = this.moduleProvider(url);
+    this.module = undefined;
+    this.reflect = undefined;
+    const linked = async () => {
+      const dependencyJobs = [];
+      ({ module: this.module,
+         reflect: this.reflect } = await this.modulePromise);
+      this.module.link(async (dependencySpecifier) => {
+        const dependencyJobPromise =
+            this.loader.getModuleJob(dependencySpecifier, url);
+        dependencyJobs.push(dependencyJobPromise);
+        const dependencyJob = await dependencyJobPromise;
+        return (await dependencyJob.modulePromise).module;
+      });
+      return SafePromise.all(dependencyJobs);
+    };
+    this.linked = linked();
 
-      // instantiated == deep dependency jobs wrappers instantiated,
-      //module wrapper instantiated
-      this.instantiated = undefined;
-    } else {
-      const getModuleProvider = async () => moduleProvider;
-      this.modulePromise = getModuleProvider();
-      this.moduleProvider = { finish: NOOP };
-      this.module = moduleProvider;
-      this.linked = resolvedArrayPromise;
-      this.instantiated = this.modulePromise;
-    }
+    // instantiated == deep dependency jobs wrappers instantiated,
+    // module wrapper instantiated
+    this.instantiated = undefined;
   }
 
   instantiate() {