From d9b8fae6d313c58028173868c14aea95998887aa Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Sat, 15 Jul 2023 11:50:11 +0200 Subject: [PATCH] esm: add back `globalPreload` tests and fix failing ones --- lib/internal/main/eval_string.js | 2 +- lib/internal/modules/esm/hooks.js | 71 +++-- lib/internal/modules/esm/loader.js | 13 + lib/internal/process/esm_loader.js | 2 + test/es-module/test-esm-loader-hooks.mjs | 123 ++++++++- test/es-module/test-esm-loader-mock.mjs | 45 ++++ .../es-module-loaders/mock-loader.mjs | 249 ++++++++++++++++++ 7 files changed, 474 insertions(+), 31 deletions(-) create mode 100644 test/es-module/test-esm-loader-mock.mjs create mode 100644 test/fixtures/es-module-loaders/mock-loader.mjs diff --git a/lib/internal/main/eval_string.js b/lib/internal/main/eval_string.js index ec6a2d51af5450..dc59a2ce4f7709 100644 --- a/lib/internal/main/eval_string.js +++ b/lib/internal/main/eval_string.js @@ -24,7 +24,7 @@ markBootstrapComplete(); const source = getOptionValue('--eval'); const print = getOptionValue('--print'); -const loadESM = getOptionValue('--import').length > 0; +const loadESM = getOptionValue('--import').length > 0 || getOptionValue('--experimental-loader').length > 0; if (getOptionValue('--input-type') === 'module') evalModule(source, print); else { diff --git a/lib/internal/modules/esm/hooks.js b/lib/internal/modules/esm/hooks.js index cef2897bd68967..fa34e4b39a5950 100644 --- a/lib/internal/modules/esm/hooks.js +++ b/lib/internal/modules/esm/hooks.js @@ -10,6 +10,7 @@ const { Promise, SafeSet, StringPrototypeSlice, + StringPrototypeStartsWith, StringPrototypeToUpperCase, globalThis, } = primordials; @@ -30,6 +31,7 @@ const { ERR_INVALID_RETURN_PROPERTY_VALUE, ERR_INVALID_RETURN_VALUE, ERR_LOADER_CHAIN_INCOMPLETE, + ERR_UNKNOWN_BUILTIN_MODULE, ERR_WORKER_UNSERIALIZABLE_ERROR, } = require('internal/errors').codes; const { exitCodes: { kUnfinishedTopLevelAwait } } = internalBinding('errors'); @@ -521,14 +523,14 @@ class HooksProxy { this.#worker.on('exit', process.exit); } - #waitForWorker() { + waitForWorker() { if (!this.#isReady) { const { kIsOnline } = require('internal/worker'); if (!this.#worker[kIsOnline]) { debug('wait for signal from worker'); AtomicsWait(this.#lock, WORKER_TO_MAIN_THREAD_NOTIFICATION, 0); const response = this.#worker.receiveMessageSync(); - if (response.message.status === 'exit') { return; } + if (response == null || response.message.status === 'exit') { return; } const { preloadScripts } = this.#unwrapMessage(response); this.#executePreloadScripts(preloadScripts); } @@ -538,7 +540,7 @@ class HooksProxy { } async makeAsyncRequest(method, ...args) { - this.#waitForWorker(); + this.waitForWorker(); MessageChannel ??= require('internal/worker/io').MessageChannel; const asyncCommChannel = new MessageChannel(); @@ -578,7 +580,7 @@ class HooksProxy { } makeSyncRequest(method, ...args) { - this.#waitForWorker(); + this.waitForWorker(); // Pass work to the worker. debug('post sync message to worker', { method, args }); @@ -620,35 +622,66 @@ class HooksProxy { } } + #importMetaInitializer = require('internal/modules/esm/initialize_import_meta').initializeImportMeta; + + importMetaInitialize(meta, context, loader) { + this.#importMetaInitializer(meta, context, loader); + } + #executePreloadScripts(preloadScripts) { for (let i = 0; i < preloadScripts.length; i++) { const { code, port } = preloadScripts[i]; const { compileFunction } = require('vm'); const preloadInit = compileFunction( code, - ['getBuiltin', 'port'], + ['getBuiltin', 'port', 'setImportMetaCallback'], { filename: '', }, ); + let finished = false; + let replacedImportMetaInitializer = false; + let next = this.#importMetaInitializer; const { BuiltinModule } = require('internal/bootstrap/realm'); // Calls the compiled preload source text gotten from the hook // Since the parameters are named we use positional parameters // see compileFunction above to cross reference the names - FunctionPrototypeCall( - preloadInit, - globalThis, - // Param getBuiltin - (builtinName) => { - if (BuiltinModule.canBeRequiredByUsers(builtinName) && - BuiltinModule.canBeRequiredWithoutScheme(builtinName)) { - return require(builtinName); - } - throw new ERR_INVALID_ARG_VALUE('builtinName', builtinName); - }, - // Param port - port, - ); + try { + FunctionPrototypeCall( + preloadInit, + globalThis, + // Param getBuiltin + (builtinName) => { + if (StringPrototypeStartsWith(builtinName, 'node:')) { + builtinName = StringPrototypeSlice(builtinName, 5); + } else if (!BuiltinModule.canBeRequiredWithoutScheme(builtinName)) { + throw new ERR_UNKNOWN_BUILTIN_MODULE(builtinName); + } + if (BuiltinModule.canBeRequiredByUsers(builtinName)) { + return require(builtinName); + } + throw new ERR_UNKNOWN_BUILTIN_MODULE(builtinName); + }, + // Param port + port, + // setImportMetaCallback + (fn) => { + if (finished || typeof fn !== 'function') { + throw new ERR_INVALID_ARG_TYPE('fn', fn); + } + replacedImportMetaInitializer = true; + const parent = next; + next = (meta, context) => { + return fn(meta, context, parent); + }; + }, + ); + } finally { + finished = true; + if (replacedImportMetaInitializer) { + this.#importMetaInitializer = next; + } + } } } } diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index b73ba2eb3c8154..9dd032b4227fa2 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -277,6 +277,11 @@ class DefaultModuleLoader { meta = importMetaInitializer(meta, context, this); return meta; } + + /** + * No-op when no hooks have been supplied. + */ + forceLoadHooks() {} } ObjectSetPrototypeOf(DefaultModuleLoader.prototype, null); @@ -349,6 +354,14 @@ class CustomizedModuleLoader extends DefaultModuleLoader { return result; } + + importMetaInitialize(meta, context) { + hooksProxy.importMetaInitialize(meta, context, this); + } + + forceLoadHooks() { + hooksProxy.waitForWorker(); + } } diff --git a/lib/internal/process/esm_loader.js b/lib/internal/process/esm_loader.js index 9a84ed944e87c4..e735101ab18c47 100644 --- a/lib/internal/process/esm_loader.js +++ b/lib/internal/process/esm_loader.js @@ -36,6 +36,8 @@ module.exports = { parentURL, kEmptyObject, )); + } else { + esmLoader.forceLoadHooks(); } await callback(esmLoader); } catch (err) { diff --git a/test/es-module/test-esm-loader-hooks.mjs b/test/es-module/test-esm-loader-hooks.mjs index 4cd1e7297c08d7..7c83f65f71a807 100644 --- a/test/es-module/test-esm-loader-hooks.mjs +++ b/test/es-module/test-esm-loader-hooks.mjs @@ -1,6 +1,7 @@ import { spawnPromisified } from '../common/index.mjs'; import * as fixtures from '../common/fixtures.mjs'; import assert from 'node:assert'; +import os from 'node:os'; import { execPath } from 'node:process'; import { describe, it } from 'node:test'; @@ -422,18 +423,118 @@ describe('Loader hooks', { concurrency: true }, () => { }); }); - it('should handle globalPreload returning undefined', async () => { - const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ - '--no-warnings', - '--experimental-loader', - 'data:text/javascript,export function globalPreload(){}', - fixtures.path('empty.js'), - ]); + describe('globalPreload', () => { + it('should handle globalPreload returning undefined', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + 'data:text/javascript,export function globalPreload(){}', + fixtures.path('empty.js'), + ]); - assert.strictEqual(stderr, ''); - assert.strictEqual(stdout, ''); - assert.strictEqual(code, 0); - assert.strictEqual(signal, null); + assert.strictEqual(stderr, ''); + assert.strictEqual(stdout, ''); + assert.strictEqual(code, 0); + assert.strictEqual(signal, null); + }); + + it('should handle loading node:test', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + 'data:text/javascript,export function globalPreload(){return `getBuiltin("node:test")()`}', + fixtures.path('empty.js'), + ]); + + assert.strictEqual(stderr, ''); + assert.match(stdout, /\n# pass 1\r?\n/); + assert.strictEqual(code, 0); + assert.strictEqual(signal, null); + }); + + it('should handle loading node:os with node: prefix', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + 'data:text/javascript,export function globalPreload(){return `console.log(getBuiltin("node:os").arch())`}', + fixtures.path('empty.js'), + ]); + + assert.strictEqual(stderr, ''); + assert.strictEqual(stdout.trim(), os.arch()); + assert.strictEqual(code, 0); + assert.strictEqual(signal, null); + }); + + it('should handle loading node:os without node: prefix', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + 'data:text/javascript,export function globalPreload(){return `console.log(getBuiltin("os").arch())`}', + fixtures.path('empty.js'), + ]); + + assert.strictEqual(stderr, ''); + assert.strictEqual(stdout.trim(), os.arch()); + assert.strictEqual(code, 0); + assert.strictEqual(signal, null); + }); + + it('should throw when loading node:test without node: prefix', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + 'data:text/javascript,export function globalPreload(){return `getBuiltin("test")()`}', + fixtures.path('empty.js'), + ]); + + assert.match(stderr, /ERR_UNKNOWN_BUILTIN_MODULE/); + assert.strictEqual(stdout, ''); + assert.strictEqual(code, 1); + assert.strictEqual(signal, null); + }); + + it('should register globals set from globalPreload', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + 'data:text/javascript,export function globalPreload(){return "this.myGlobal=4"}', + '--print', 'myGlobal', + ]); + + assert.strictEqual(stderr, ''); + assert.strictEqual(stdout.trim(), '4'); + assert.strictEqual(code, 0); + assert.strictEqual(signal, null); + }); + + it('should log console.log calls returned from globalPreload', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + 'data:text/javascript,export function globalPreload(){return `console.log("Hello from globalPreload")`}', + fixtures.path('empty.js'), + ]); + + assert.strictEqual(stderr, ''); + assert.strictEqual(stdout.trim(), 'Hello from globalPreload'); + assert.strictEqual(code, 0); + assert.strictEqual(signal, null); + }); + + it('should crash if globalPreload returns code that throws', async () => { + const { code, signal, stdout, stderr } = await spawnPromisified(execPath, [ + '--no-warnings', + '--experimental-loader', + 'data:text/javascript,export function globalPreload(){return `throw new Error("error from globalPreload")`}', + fixtures.path('empty.js'), + ]); + + assert.match(stderr, /error from globalPreload/); + assert.strictEqual(stdout, ''); + assert.strictEqual(code, 1); + assert.strictEqual(signal, null); + }); }); it('should be fine to call `process.removeAllListeners("beforeExit")` from the main thread', async () => { diff --git a/test/es-module/test-esm-loader-mock.mjs b/test/es-module/test-esm-loader-mock.mjs new file mode 100644 index 00000000000000..2783bf694d239a --- /dev/null +++ b/test/es-module/test-esm-loader-mock.mjs @@ -0,0 +1,45 @@ +// Flags: --loader ./test/fixtures/es-module-loaders/mock-loader.mjs +import '../common/index.mjs'; +import assert from 'assert/strict'; + +// This is provided by test/fixtures/es-module-loaders/mock-loader.mjs +import mock from 'node:mock'; + +mock('node:events', { + EventEmitter: 'This is mocked!' +}); + +// This resolves to node:events +// It is intercepted by mock-loader and doesn't return the normal value +assert.deepStrictEqual(await import('events'), Object.defineProperty({ + __proto__: null, + EventEmitter: 'This is mocked!' +}, Symbol.toStringTag, { + enumerable: false, + value: 'Module' +})); + +const mutator = mock('node:events', { + EventEmitter: 'This is mocked v2!' +}); + +// It is intercepted by mock-loader and doesn't return the normal value. +// This is resolved separately from the import above since the specifiers +// are different. +const mockedV2 = await import('node:events'); +assert.deepStrictEqual(mockedV2, Object.defineProperty({ + __proto__: null, + EventEmitter: 'This is mocked v2!' +}, Symbol.toStringTag, { + enumerable: false, + value: 'Module' +})); + +mutator.EventEmitter = 'This is mocked v3!'; +assert.deepStrictEqual(mockedV2, Object.defineProperty({ + __proto__: null, + EventEmitter: 'This is mocked v3!' +}, Symbol.toStringTag, { + enumerable: false, + value: 'Module' +})); diff --git a/test/fixtures/es-module-loaders/mock-loader.mjs b/test/fixtures/es-module-loaders/mock-loader.mjs new file mode 100644 index 00000000000000..062be39603e851 --- /dev/null +++ b/test/fixtures/es-module-loaders/mock-loader.mjs @@ -0,0 +1,249 @@ +import { receiveMessageOnPort } from 'node:worker_threads'; +const mockedModuleExports = new Map(); +let currentMockVersion = 0; + +// This loader causes a new module `node:mock` to become available as a way to +// swap module resolution results for mocking purposes. It uses this instead +// of import.meta so that CommonJS can still use the functionality. +// +// It does so by allowing non-mocked modules to live in normal URL cache +// locations but creates 'mock-facade:' URL cache location for every time a +// module location is mocked. Since a single URL can be mocked multiple +// times but it cannot be removed from the cache, `mock-facade:` URLs have a +// form of mock-facade:$VERSION:$REPLACING_URL with the parameters being URL +// percent encoded every time a module is resolved. So if a module for +// 'file:///app.js' is mocked it might look like +// 'mock-facade:12:file%3A%2F%2F%2Fapp.js'. This encoding is done to prevent +// problems like mocking URLs with special URL characters like '#' or '?' from +// accidentally being picked up as part of the 'mock-facade:' URL containing +// the mocked URL. +// +// NOTE: due to ESM spec, once a specifier has been resolved in a source text +// it cannot be changed. So things like the following DO NOT WORK: +// +// ```mjs +// import mock from 'node:mock'; +// mock('file:///app.js', {x:1}); +// const namespace1 = await import('file:///app.js'); +// namespace1.x; // 1 +// mock('file:///app.js', {x:2}); +// const namespace2 = await import('file:///app.js'); +// namespace2.x; // STILL 1, because this source text already set the specifier +// // for 'file:///app.js', a different specifier that resolves +// // to that could still get a new namespace though +// assert(namespace1 === namespace2); +// ``` + +/** + * FIXME: this is a hack to workaround loaders being + * single threaded for now, just ensures that the MessagePort drains + */ +function doDrainPort() { + let msg; + while (msg = receiveMessageOnPort(preloadPort)) { + onPreloadPortMessage(msg.message); + } +} + +/** + * @param param0 message from the application context + */ +function onPreloadPortMessage({ + mockVersion, resolved, exports +}) { + currentMockVersion = mockVersion; + mockedModuleExports.set(resolved, exports); +} +let preloadPort; +export function globalPreload({port}) { + // Save the communication port to the application context to send messages + // to it later + preloadPort = port; + // Every time the application context sends a message over the port + port.on('message', onPreloadPortMessage); + // This prevents the port that the Loader/application talk over + // from keeping the process alive, without this, an application would be kept + // alive just because a loader is waiting for messages + port.unref(); + + const insideAppContext = (getBuiltin, port, setImportMetaCallback) => { + /** + * This is the Map that saves *all* the mocked URL -> replacement Module + * mappings + * @type {Map} + */ + let mockedModules = new Map(); + let mockVersion = 0; + /** + * This is the value that is placed into the `node:mock` default export + * + * @example + * ```mjs + * import mock from 'node:mock'; + * const mutator = mock('file:///app.js', {x:1}); + * const namespace = await import('file:///app.js'); + * namespace.x; // 1; + * mutator.x = 2; + * namespace.x; // 2; + * ``` + * + * @param {string} resolved an absolute URL HREF string + * @param {object} replacementProperties an object to pick properties from + * to act as a module namespace + * @returns {object} a mutator object that can update the module namespace + * since we can't do something like old Object.observe + */ + const doMock = (resolved, replacementProperties) => { + let exportNames = Object.keys(replacementProperties); + let namespace = Object.create(null); + /** + * @type {Array<(name: string)=>void>} functions to call whenever an + * export name is updated + */ + let listeners = []; + for (const name of exportNames) { + let currentValueForPropertyName = replacementProperties[name]; + Object.defineProperty(namespace, name, { + enumerable: true, + get() { + return currentValueForPropertyName; + }, + set(v) { + currentValueForPropertyName = v; + for (let fn of listeners) { + try { + fn(name); + } catch { + } + } + } + }); + } + mockedModules.set(resolved, { + namespace, + listeners + }); + mockVersion++; + // Inform the loader that the `resolved` URL should now use the specific + // `mockVersion` and has export names of `exportNames` + // + // This allows the loader to generate a fake module for that version + // and names the next time it resolves a specifier to equal `resolved` + port.postMessage({ mockVersion, resolved, exports: exportNames }); + return namespace; + } + // Sets the import.meta properties up + // has the normal chaining workflow with `defaultImportMetaInitializer` + setImportMetaCallback((meta, context, defaultImportMetaInitializer) => { + /** + * 'node:mock' creates its default export by plucking off of import.meta + * and must do so in order to get the communications channel from inside + * preloadCode + */ + if (context.url === 'node:mock') { + meta.doMock = doMock; + return; + } + /** + * Fake modules created by `node:mock` get their meta.mock utility set + * to the corresponding value keyed off `mockedModules` and use this + * to setup their exports/listeners properly + */ + if (context.url.startsWith('mock-facade:')) { + let [proto, version, encodedTargetURL] = context.url.split(':'); + let decodedTargetURL = decodeURIComponent(encodedTargetURL); + if (mockedModules.has(decodedTargetURL)) { + meta.mock = mockedModules.get(decodedTargetURL); + return; + } + } + /** + * Ensure we still get things like `import.meta.url` + */ + defaultImportMetaInitializer(meta, context); + }); + }; + return `(${insideAppContext})(getBuiltin, port, setImportMetaCallback)` +} + + +// Rewrites node: loading to mock-facade: so that it can be intercepted +export async function resolve(specifier, context, defaultResolve) { + if (specifier === 'node:mock') { + return { + shortCircuit: true, + url: specifier + }; + } + doDrainPort(); + const def = await defaultResolve(specifier, context); + if (context.parentURL?.startsWith('mock-facade:')) { + // Do nothing, let it get the "real" module + } else if (mockedModuleExports.has(def.url)) { + return { + shortCircuit: true, + url: `mock-facade:${currentMockVersion}:${encodeURIComponent(def.url)}` + }; + }; + return { + shortCircuit: true, + url: def.url, + }; +} + +export async function load(url, context, defaultLoad) { + doDrainPort(); + if (url === 'node:mock') { + /** + * Simply grab the import.meta.doMock to establish the communication + * channel with preloadCode + */ + return { + shortCircuit: true, + source: 'export default import.meta.doMock', + format: 'module' + }; + } + /** + * Mocked fake module, not going to be handled in default way so it + * generates the source text, then short circuits + */ + if (url.startsWith('mock-facade:')) { + let [proto, version, encodedTargetURL] = url.split(':'); + let ret = generateModule(mockedModuleExports.get( + decodeURIComponent(encodedTargetURL) + )); + return { + shortCircuit: true, + source: ret, + format: 'module' + }; + } + return defaultLoad(url, context); +} + +/** + * + * @param {Array} exports name of the exports of the module + * @returns {string} + */ +function generateModule(exports) { + let body = [ + 'export {};', + 'let mapping = {__proto__: null};' + ]; + for (const [i, name] of Object.entries(exports)) { + let key = JSON.stringify(name); + body.push(`var _${i} = import.meta.mock.namespace[${key}];`); + body.push(`Object.defineProperty(mapping, ${key}, { enumerable: true, set(v) {_${i} = v;}, get() {return _${i};} });`); + body.push(`export {_${i} as ${name}};`); + } + body.push(`import.meta.mock.listeners.push(${ + () => { + for (var k in mapping) { + mapping[k] = import.meta.mock.namespace[k]; + } + } + });`); + return body.join('\n'); +}