From b0b7943e8fe40b30eccb716e8a58239efcfe6627 Mon Sep 17 00:00:00 2001 From: Bradley Farias Date: Fri, 2 Jul 2021 15:57:01 -0500 Subject: [PATCH] esm: working mock test PR-URL: https://github.com/nodejs/node/pull/39240 Reviewed-By: James M Snell Reviewed-By: Geoffrey Booth --- doc/api/esm.md | 34 +++ .../modules/esm/initialize_import_meta.js | 32 +++ lib/internal/modules/esm/loader.js | 71 ++++- lib/internal/modules/esm/translators.js | 28 +- test/es-module/test-esm-loader-mock.mjs | 45 ++++ .../es-module-loaders/loader-side-effect.mjs | 6 +- .../es-module-loaders/mock-loader.mjs | 244 ++++++++++++++++++ test/parallel/test-bootstrap-modules.js | 1 + 8 files changed, 425 insertions(+), 36 deletions(-) create mode 100644 lib/internal/modules/esm/initialize_import_meta.js create mode 100644 test/es-module/test-esm-loader-mock.mjs create mode 100644 test/fixtures/es-module-loaders/mock-loader.mjs diff --git a/doc/api/esm.md b/doc/api/esm.md index 48fd7cd9631fbe..db6d01eea31a12 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -827,6 +827,9 @@ its own `require` using `module.createRequire()`. ```js /** + * @param {{ + port: MessagePort, + }} utilities Things that preload code might find useful * @returns {string} Code to run before application startup */ export function globalPreload() { @@ -843,6 +846,35 @@ const require = createRequire(cwd() + '/'); } ``` +In order to allow communication between the application and the loader, another +argument is provided to the preload code: `port`. This is available as a +parameter to the loader hook and inside of the source text returned by the hook. +Some care must be taken in order to properly call [`port.ref()`][] and +[`port.unref()`][] to prevent a process from being in a state where it won't +close normally. + +```js +/** + * This example has the application context send a message to the loader + * and sends the message back to the application context + * @param {{ + port: MessagePort, + }} utilities Things that preload code might find useful + * @returns {string} Code to run before application startup + */ +export function globalPreload({ port }) { + port.onmessage = (evt) => { + port.postMessage(evt.data); + }; + return `\ + port.postMessage('console.log("I went to the Loader and back");'); + port.onmessage = (evt) => { + eval(evt.data); + }; + `; +} +``` + ### Examples The various loader hooks can be used together to accomplish wide-ranging @@ -1417,6 +1449,8 @@ success! [`module.createRequire()`]: module.md#modulecreaterequirefilename [`module.syncBuiltinESMExports()`]: module.md#modulesyncbuiltinesmexports [`package.json`]: packages.md#nodejs-packagejson-field-definitions +[`port.ref()`]: https://nodejs.org/dist/latest-v17.x/docs/api/worker_threads.html#portref +[`port.unref()`]: https://nodejs.org/dist/latest-v17.x/docs/api/worker_threads.html#portunref [`process.dlopen`]: process.md#processdlopenmodule-filename-flags [`string`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String [`util.TextDecoder`]: util.md#class-utiltextdecoder diff --git a/lib/internal/modules/esm/initialize_import_meta.js b/lib/internal/modules/esm/initialize_import_meta.js new file mode 100644 index 00000000000000..322b4c59be1561 --- /dev/null +++ b/lib/internal/modules/esm/initialize_import_meta.js @@ -0,0 +1,32 @@ +'use strict'; + +const { getOptionValue } = require('internal/options'); +const experimentalImportMetaResolve = +getOptionValue('--experimental-import-meta-resolve'); +const { PromisePrototypeThen, PromiseReject } = primordials; +const asyncESM = require('internal/process/esm_loader'); + +function createImportMetaResolve(defaultParentUrl) { + return async function resolve(specifier, parentUrl = defaultParentUrl) { + return PromisePrototypeThen( + asyncESM.esmLoader.resolve(specifier, parentUrl), + ({ url }) => url, + (error) => ( + error.code === 'ERR_UNSUPPORTED_DIR_IMPORT' ? + error.url : PromiseReject(error)) + ); + }; +} + +function initializeImportMeta(meta, context) { + const url = context.url; + + // Alphabetical + if (experimentalImportMetaResolve) + meta.resolve = createImportMetaResolve(url); + meta.url = url; +} + +module.exports = { + initializeImportMeta +}; diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index 3c135d3601b3cc..91f570297be341 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -19,6 +19,7 @@ const { SafeWeakMap, globalThis, } = primordials; +const { MessageChannel } = require('internal/worker/io'); const { ERR_INVALID_ARG_TYPE, @@ -40,6 +41,9 @@ const { defaultResolve, DEFAULT_CONDITIONS, } = require('internal/modules/esm/resolve'); +const { + initializeImportMeta +} = require('internal/modules/esm/initialize_import_meta'); const { defaultLoad } = require('internal/modules/esm/load'); const { translators } = require( 'internal/modules/esm/translators'); @@ -77,6 +81,8 @@ class ESMLoader { defaultResolve, ]; + #importMetaInitializer = initializeImportMeta; + /** * Map of already-loaded CJS modules to use */ @@ -409,7 +415,18 @@ class ESMLoader { if (!count) return; for (let i = 0; i < count; i++) { - const preload = this.#globalPreloaders[i](); + const channel = new MessageChannel(); + const { + port1: insidePreload, + port2: insideLoader, + } = channel; + + insidePreload.unref(); + insideLoader.unref(); + + const preload = this.#globalPreloaders[i]({ + port: insideLoader + }); if (preload == null) return; @@ -423,22 +440,60 @@ class ESMLoader { const { compileFunction } = require('vm'); const preloadInit = compileFunction( preload, - ['getBuiltin'], + ['getBuiltin', 'port', 'setImportMetaCallback'], { filename: '', } ); const { NativeModule } = require('internal/bootstrap/loaders'); - - FunctionPrototypeCall(preloadInit, globalThis, (builtinName) => { - if (NativeModule.canBeRequiredByUsers(builtinName)) { - return require(builtinName); + // We only allow replacing the importMetaInitializer during preload, + // after preload is finished, we disable the ability to replace it + // + // This exposes accidentally setting the initializer too late by + // throwing an error. + let finished = false; + let replacedImportMetaInitializer = false; + let next = this.#importMetaInitializer; + try { + // Calls the compiled preload source text gotten from the hook + // Since the parameters are named we use positional parameters + // see compileFunction above to cross reference the names + FunctionPrototypeCall( + preloadInit, + globalThis, + // Param getBuiltin + (builtinName) => { + if (NativeModule.canBeRequiredByUsers(builtinName)) { + return require(builtinName); + } + throw new ERR_INVALID_ARG_VALUE('builtinName', builtinName); + }, + // Param port + insidePreload, + // Param setImportMetaCallback + (fn) => { + if (finished || typeof fn !== 'function') { + throw new ERR_INVALID_ARG_TYPE('fn', fn); + } + replacedImportMetaInitializer = true; + const parent = next; + next = (meta, context) => { + return fn(meta, context, parent); + }; + }); + } finally { + finished = true; + if (replacedImportMetaInitializer) { + this.#importMetaInitializer = next; } - throw new ERR_INVALID_ARG_VALUE('builtinName', builtinName); - }); + } } } + importMetaInitialize(meta, context) { + this.#importMetaInitializer(meta, context); + } + /** * Resolve the location of the module. * diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index fdeaba0549ae9b..297ac95e662670 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -8,8 +8,6 @@ const { ObjectGetPrototypeOf, ObjectPrototypeHasOwnProperty, ObjectKeys, - PromisePrototypeThen, - PromiseReject, SafeArrayIterator, SafeMap, SafeSet, @@ -52,9 +50,6 @@ const { const { maybeCacheSourceMap } = require('internal/source_map/source_map_cache'); const moduleWrap = internalBinding('module_wrap'); const { ModuleWrap } = moduleWrap; -const { getOptionValue } = require('internal/options'); -const experimentalImportMetaResolve = - getOptionValue('--experimental-import-meta-resolve'); const asyncESM = require('internal/process/esm_loader'); const { emitWarningSync } = require('internal/process/warning'); const { TextDecoder } = require('internal/encoding'); @@ -111,25 +106,6 @@ async function importModuleDynamically(specifier, { url }, assertions) { return asyncESM.esmLoader.import(specifier, url, assertions); } -function createImportMetaResolve(defaultParentUrl) { - return async function resolve(specifier, parentUrl = defaultParentUrl) { - return PromisePrototypeThen( - asyncESM.esmLoader.resolve(specifier, parentUrl), - ({ url }) => url, - (error) => ( - error.code === 'ERR_UNSUPPORTED_DIR_IMPORT' ? - error.url : PromiseReject(error)) - ); - }; -} - -function initializeImportMeta(meta, { url }) { - // Alphabetical - if (experimentalImportMetaResolve) - meta.resolve = createImportMetaResolve(url); - meta.url = url; -} - // Strategy for loading a standard JavaScript module. translators.set('module', async function moduleStrategy(url, source, isMain) { assertBufferSource(source, true, 'load'); @@ -138,7 +114,9 @@ translators.set('module', async function moduleStrategy(url, source, isMain) { debug(`Translating StandardModule ${url}`); const module = new ModuleWrap(url, undefined, source, 0, 0); moduleWrap.callbackMap.set(module, { - initializeImportMeta, + initializeImportMeta: (meta, wrap) => this.importMetaInitialize(meta, { + url: wrap.url + }), importModuleDynamically, }); return module; diff --git a/test/es-module/test-esm-loader-mock.mjs b/test/es-module/test-esm-loader-mock.mjs new file mode 100644 index 00000000000000..2783bf694d239a --- /dev/null +++ b/test/es-module/test-esm-loader-mock.mjs @@ -0,0 +1,45 @@ +// Flags: --loader ./test/fixtures/es-module-loaders/mock-loader.mjs +import '../common/index.mjs'; +import assert from 'assert/strict'; + +// This is provided by test/fixtures/es-module-loaders/mock-loader.mjs +import mock from 'node:mock'; + +mock('node:events', { + EventEmitter: 'This is mocked!' +}); + +// This resolves to node:events +// It is intercepted by mock-loader and doesn't return the normal value +assert.deepStrictEqual(await import('events'), Object.defineProperty({ + __proto__: null, + EventEmitter: 'This is mocked!' +}, Symbol.toStringTag, { + enumerable: false, + value: 'Module' +})); + +const mutator = mock('node:events', { + EventEmitter: 'This is mocked v2!' +}); + +// It is intercepted by mock-loader and doesn't return the normal value. +// This is resolved separately from the import above since the specifiers +// are different. +const mockedV2 = await import('node:events'); +assert.deepStrictEqual(mockedV2, Object.defineProperty({ + __proto__: null, + EventEmitter: 'This is mocked v2!' +}, Symbol.toStringTag, { + enumerable: false, + value: 'Module' +})); + +mutator.EventEmitter = 'This is mocked v3!'; +assert.deepStrictEqual(mockedV2, Object.defineProperty({ + __proto__: null, + EventEmitter: 'This is mocked v3!' +}, Symbol.toStringTag, { + enumerable: false, + value: 'Module' +})); diff --git a/test/fixtures/es-module-loaders/loader-side-effect.mjs b/test/fixtures/es-module-loaders/loader-side-effect.mjs index 5c80724fbb95f6..e91cdea0527881 100644 --- a/test/fixtures/es-module-loaders/loader-side-effect.mjs +++ b/test/fixtures/es-module-loaders/loader-side-effect.mjs @@ -1,5 +1,5 @@ // Arrow function so it closes over the this-value of the preload scope. -const globalPreload = () => { +const globalPreloadSrc = () => { /* global getBuiltin */ const assert = getBuiltin('assert'); const vm = getBuiltin('vm'); @@ -24,9 +24,9 @@ const implicitGlobalConst = 42 * 42; globalThis.explicitGlobalProperty = 42 * 42 * 42; } -export function getGlobalPreloadCode() { +export function globalPreload() { return `\ -(${globalPreload.toString()})(); +(${globalPreloadSrc.toString()})(); `; } diff --git a/test/fixtures/es-module-loaders/mock-loader.mjs b/test/fixtures/es-module-loaders/mock-loader.mjs new file mode 100644 index 00000000000000..4187137b105616 --- /dev/null +++ b/test/fixtures/es-module-loaders/mock-loader.mjs @@ -0,0 +1,244 @@ +import { receiveMessageOnPort } from 'node:worker_threads'; +const mockedModuleExports = new Map(); +let currentMockVersion = 0; + +// This loader causes a new module `node:mock` to become available as a way to +// swap module resolution results for mocking purposes. It uses this instead +// of import.meta so that CommonJS can still use the functionality. +// +// It does so by allowing non-mocked modules to live in normal URL cache +// locations but creates 'mock-facade:' URL cache location for every time a +// module location is mocked. Since a single URL can be mocked multiple +// times but it cannot be removed from the cache, `mock-facade:` URLs have a +// form of mock-facade:$VERSION:$REPLACING_URL with the parameters being URL +// percent encoded every time a module is resolved. So if a module for +// 'file:///app.js' is mocked it might look like +// 'mock-facade:12:file%3A%2F%2F%2Fapp.js'. This encoding is done to prevent +// problems like mocking URLs with special URL characters like '#' or '?' from +// accidentally being picked up as part of the 'mock-facade:' URL containing +// the mocked URL. +// +// NOTE: due to ESM spec, once a specifier has been resolved in a source text +// it cannot be changed. So things like the following DO NOT WORK: +// +// ```mjs +// import mock from 'node:mock'; +// mock('file:///app.js', {x:1}); +// const namespace1 = await import('file:///app.js'); +// namespace1.x; // 1 +// mock('file:///app.js', {x:2}); +// const namespace2 = await import('file:///app.js'); +// namespace2.x; // STILL 1, because this source text already set the specifier +// // for 'file:///app.js', a different specifier that resolves +// // to that could still get a new namespace though +// assert(namespace1 === namespace2); +// ``` + +/** + * FIXME: this is a hack to workaround loaders being + * single threaded for now, just ensures that the MessagePort drains + */ +function doDrainPort() { + let msg; + while (msg = receiveMessageOnPort(preloadPort)) { + onPreloadPortMessage(msg.message); + } +} + +/** + * @param param0 message from the application context + */ +function onPreloadPortMessage({ + mockVersion, resolved, exports +}) { + currentMockVersion = mockVersion; + mockedModuleExports.set(resolved, exports); +} +let preloadPort; +export function globalPreload({port}) { + // Save the communication port to the application context to send messages + // to it later + preloadPort = port; + // Every time the application context sends a message over the port + port.on('message', onPreloadPortMessage); + // This prevents the port that the Loader/application talk over + // from keeping the process alive, without this, an application would be kept + // alive just because a loader is waiting for messages + port.unref(); + + const insideAppContext = (getBuiltin, port, setImportMetaCallback) => { + /** + * This is the Map that saves *all* the mocked URL -> replacement Module + * mappings + * @type {Map} + */ + let mockedModules = new Map(); + let mockVersion = 0; + /** + * This is the value that is placed into the `node:mock` default export + * + * @example + * ```mjs + * import mock from 'node:mock'; + * const mutator = mock('file:///app.js', {x:1}); + * const namespace = await import('file:///app.js'); + * namespace.x; // 1; + * mutator.x = 2; + * namespace.x; // 2; + * ``` + * + * @param {string} resolved an absolute URL HREF string + * @param {object} replacementProperties an object to pick properties from + * to act as a module namespace + * @returns {object} a mutator object that can update the module namespace + * since we can't do something like old Object.observe + */ + const doMock = (resolved, replacementProperties) => { + let exportNames = Object.keys(replacementProperties); + let namespace = Object.create(null); + /** + * @type {Array<(name: string)=>void>} functions to call whenever an + * export name is updated + */ + let listeners = []; + for (const name of exportNames) { + let currentValueForPropertyName = replacementProperties[name]; + Object.defineProperty(namespace, name, { + enumerable: true, + get() { + return currentValueForPropertyName; + }, + set(v) { + currentValueForPropertyName = v; + for (let fn of listeners) { + try { + fn(name); + } catch { + } + } + } + }); + } + mockedModules.set(resolved, { + namespace, + listeners + }); + mockVersion++; + // Inform the loader that the `resolved` URL should now use the specific + // `mockVersion` and has export names of `exportNames` + // + // This allows the loader to generate a fake module for that version + // and names the next time it resolves a specifier to equal `resolved` + port.postMessage({ mockVersion, resolved, exports: exportNames }); + return namespace; + } + // Sets the import.meta properties up + // has the normal chaining workflow with `defaultImportMetaInitializer` + setImportMetaCallback((meta, context, defaultImportMetaInitializer) => { + /** + * 'node:mock' creates its default export by plucking off of import.meta + * and must do so in order to get the communications channel from inside + * preloadCode + */ + if (context.url === 'node:mock') { + meta.doMock = doMock; + return; + } + /** + * Fake modules created by `node:mock` get their meta.mock utility set + * to the corresponding value keyed off `mockedModules` and use this + * to setup their exports/listeners properly + */ + if (context.url.startsWith('mock-facade:')) { + let [proto, version, encodedTargetURL] = context.url.split(':'); + let decodedTargetURL = decodeURIComponent(encodedTargetURL); + if (mockedModules.has(decodedTargetURL)) { + meta.mock = mockedModules.get(decodedTargetURL); + return; + } + } + /** + * Ensure we still get things like `import.meta.url` + */ + defaultImportMetaInitializer(meta, context); + }); + }; + return `(${insideAppContext})(getBuiltin, port, setImportMetaCallback)` +} + + +// Rewrites node: loading to mock-facade: so that it can be intercepted +export function resolve(specifier, context, defaultResolve) { + if (specifier === 'node:mock') { + return { + url: specifier + }; + } + doDrainPort(); + const def = defaultResolve(specifier, context); + if (context.parentURL?.startsWith('mock-facade:')) { + // Do nothing, let it get the "real" module + } else if (mockedModuleExports.has(def.url)) { + return { + url: `mock-facade:${currentMockVersion}:${encodeURIComponent(def.url)}` + }; + }; + return { + url: `${def.url}` + }; +} + +export function load(url, context, defaultLoad) { + doDrainPort(); + if (url === 'node:mock') { + /** + * Simply grab the import.meta.doMock to establish the communication + * channel with preloadCode + */ + return { + source: 'export default import.meta.doMock', + format: 'module' + }; + } + /** + * Mocked fake module, not going to be handled in default way so it + * generates the source text, then short circuits + */ + if (url.startsWith('mock-facade:')) { + let [proto, version, encodedTargetURL] = url.split(':'); + let ret = generateModule(mockedModuleExports.get( + decodeURIComponent(encodedTargetURL) + )); + return { + source: ret, + format: 'module' + }; + } + return defaultLoad(url, context); +} + +/** + * + * @param {Array} exports name of the exports of the module + * @returns {string} + */ +function generateModule(exports) { + let body = [ + 'export {};', + 'let mapping = {__proto__: null};' + ]; + for (const [i, name] of Object.entries(exports)) { + let key = JSON.stringify(name); + body.push(`var _${i} = import.meta.mock.namespace[${key}];`); + body.push(`Object.defineProperty(mapping, ${key}, { enumerable: true, set(v) {_${i} = v;}, get() {return _${i};} });`); + body.push(`export {_${i} as ${name}};`); + } + body.push(`import.meta.mock.listeners.push(${ + () => { + for (var k in mapping) { + mapping[k] = import.meta.mock.namespace[k]; + } + } + });`); + return body.join('\n'); +} diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index b8101388a9c64d..19ba89a290b6d7 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -77,6 +77,7 @@ const expectedModules = new Set([ 'NativeModule internal/modules/esm/module_job', 'NativeModule internal/modules/esm/module_map', 'NativeModule internal/modules/esm/resolve', + 'NativeModule internal/modules/esm/initialize_import_meta', 'NativeModule internal/modules/esm/translators', 'NativeModule internal/process/esm_loader', 'NativeModule internal/options',