-
-
Notifications
You must be signed in to change notification settings - Fork 6.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: support WebAssembly (Wasm) imports in ESM modules #13505
Changes from 8 commits
95ef976
645db88
9985179
5d73f9a
bc6d38c
734bbc5
612d480
dcff11e
e524e40
08c8906
66c8312
6586a24
01f8d78
27d73b6
2aea674
79e0f40
0d4945e
26db3fd
1e6d75b
d4e9b73
ad32edd
397bcd1
976ef61
66a2312
7188086
26e7022
607fc0f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
{ | ||
"type": "module", | ||
"name": "native-esm-wasm", | ||
"jest": { | ||
"testEnvironment": "node", | ||
"transform": {} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
/** | ||
* Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. | ||
* | ||
* This source code is licensed under the MIT license found in the | ||
* LICENSE file in the root directory of this source tree. | ||
*/ | ||
|
||
// the point here is that it's the node core module | ||
// eslint-disable-next-line no-restricted-imports | ||
import {readFileSync} from 'fs'; | ||
// The file was generated by wasm-pack | ||
import {getAnswer} from '../42.wasm'; | ||
|
||
const wasmFileBuffer = readFileSync('42.wasm'); | ||
|
||
test('supports native wasm imports', () => { | ||
expect(getAnswer()).toBe(42); | ||
}); | ||
|
||
test('supports imports from "data:application/wasm" URI with base64 encoding', async () => { | ||
const importedWasmModule = await import( | ||
`data:application/wasm;base64,${wasmFileBuffer.toString('base64')}` | ||
); | ||
expect(importedWasmModule.getAnswer()).toBe(42); | ||
}); | ||
|
||
test('imports from "data:text/wasm" URI without explicit encoding fail', async () => { | ||
await expect(() => | ||
import(`data:application/wasm,${wasmFileBuffer.toString('base64')}`), | ||
).rejects.toThrow('Missing data URI encoding'); | ||
}); | ||
|
||
test('imports from "data:text/wasm" URI with invalid encoding fail', async () => { | ||
await expect(() => | ||
import('data:application/wasm;charset=utf-8,oops'), | ||
).rejects.toThrow('Invalid data URI encoding: charset=utf-8'); | ||
}); |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -398,9 +398,12 @@ export default class Runtime { | |
|
||
// unstable as it should be replaced by https://github.com/nodejs/modules/issues/393, and we don't want people to use it | ||
unstable_shouldLoadAsEsm(path: string): boolean { | ||
return Resolver.unstable_shouldLoadAsEsm( | ||
path, | ||
this._config.extensionsToTreatAsEsm, | ||
return ( | ||
path.endsWith('wasm') || | ||
Resolver.unstable_shouldLoadAsEsm( | ||
path, | ||
this._config.extensionsToTreatAsEsm, | ||
) | ||
); | ||
} | ||
|
||
|
@@ -441,6 +444,19 @@ export default class Runtime { | |
'Promise initialization should be sync - please report this bug to Jest!', | ||
); | ||
|
||
if (modulePath.endsWith('wasm')) { | ||
const wasm = this._importWasmModule( | ||
fs.readFileSync(modulePath), | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we have a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done in 0d4945e. I had to change the type for |
||
modulePath, | ||
context, | ||
); | ||
|
||
this._esmoduleRegistry.set(cacheKey, wasm); | ||
|
||
transformResolve(); | ||
return wasm; | ||
} | ||
|
||
if (this._resolver.isCoreModule(modulePath)) { | ||
const core = this._importCoreModule(modulePath, context); | ||
this._esmoduleRegistry.set(cacheKey, core); | ||
|
@@ -567,56 +583,67 @@ export default class Runtime { | |
} | ||
|
||
const mime = match.groups.mime; | ||
if (mime === 'application/wasm') { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Quite a few lines below are marked as changed because of extra indent. Hiding whitespace in diff viewer makes it a bit easier to see the real difference. |
||
throw new Error('WASM is currently not supported'); | ||
} | ||
|
||
const encoding = match.groups.encoding; | ||
let code = match.groups.code; | ||
if (!encoding || encoding === 'charset=utf-8') { | ||
code = decodeURIComponent(code); | ||
} else if (encoding === 'base64') { | ||
code = Buffer.from(code, 'base64').toString(); | ||
} else { | ||
throw new Error(`Invalid data URI encoding: ${encoding}`); | ||
} | ||
|
||
let module; | ||
if (mime === 'application/json') { | ||
module = new SyntheticModule( | ||
['default'], | ||
function () { | ||
const obj = JSON.parse(code); | ||
// @ts-expect-error: TS doesn't know what `this` is | ||
this.setExport('default', obj); | ||
}, | ||
{context, identifier: specifier}, | ||
|
||
if (mime === 'application/wasm') { | ||
if (!encoding) { | ||
throw new Error('Missing data URI encoding'); | ||
} | ||
if (encoding !== 'base64') { | ||
throw new Error(`Invalid data URI encoding: ${encoding}`); | ||
} | ||
module = await this._importWasmModule( | ||
Buffer.from(match.groups.code, 'base64'), | ||
specifier, | ||
context, | ||
); | ||
} else { | ||
module = new SourceTextModule(code, { | ||
context, | ||
identifier: specifier, | ||
importModuleDynamically: async ( | ||
specifier: string, | ||
referencingModule: VMModule, | ||
) => { | ||
invariant( | ||
runtimeSupportsVmModules, | ||
'You need to run with a version of node that supports ES Modules in the VM API. See https://jestjs.io/docs/ecmascript-modules', | ||
); | ||
const module = await this.resolveModule( | ||
specifier, | ||
referencingModule.identifier, | ||
referencingModule.context, | ||
); | ||
let code = match.groups.code; | ||
if (!encoding || encoding === 'charset=utf-8') { | ||
code = decodeURIComponent(code); | ||
} else if (encoding === 'base64') { | ||
code = Buffer.from(code, 'base64').toString(); | ||
} else { | ||
throw new Error(`Invalid data URI encoding: ${encoding}`); | ||
} | ||
|
||
return this.linkAndEvaluateModule(module); | ||
}, | ||
initializeImportMeta(meta: ImportMeta) { | ||
// no `jest` here as it's not loaded in a file | ||
meta.url = specifier; | ||
}, | ||
}); | ||
if (mime === 'application/json') { | ||
module = new SyntheticModule( | ||
['default'], | ||
function () { | ||
const obj = JSON.parse(code); | ||
// @ts-expect-error: TS doesn't know what `this` is | ||
this.setExport('default', obj); | ||
}, | ||
{context, identifier: specifier}, | ||
); | ||
} else { | ||
module = new SourceTextModule(code, { | ||
context, | ||
identifier: specifier, | ||
importModuleDynamically: async ( | ||
specifier: string, | ||
referencingModule: VMModule, | ||
) => { | ||
invariant( | ||
runtimeSupportsVmModules, | ||
'You need to run with a version of node that supports ES Modules in the VM API. See https://jestjs.io/docs/ecmascript-modules', | ||
); | ||
const module = await this.resolveModule( | ||
specifier, | ||
referencingModule.identifier, | ||
referencingModule.context, | ||
); | ||
|
||
return this.linkAndEvaluateModule(module); | ||
}, | ||
initializeImportMeta(meta: ImportMeta) { | ||
// no `jest` here as it's not loaded in a file | ||
meta.url = specifier; | ||
}, | ||
}); | ||
} | ||
} | ||
|
||
this._esmoduleRegistry.set(specifier, module); | ||
|
@@ -1640,6 +1667,50 @@ export default class Runtime { | |
return evaluateSyntheticModule(module); | ||
} | ||
|
||
private async _importWasmModule( | ||
source: Buffer, | ||
identifier: string, | ||
context: VMContext, | ||
) { | ||
const wasmModule = await WebAssembly.compile(source); | ||
|
||
const exports = WebAssembly.Module.exports(wasmModule); | ||
const imports = WebAssembly.Module.imports(wasmModule); | ||
|
||
const moduleLookup: Record<string, VMModule> = {}; | ||
for (const {module} of imports) { | ||
if (moduleLookup[module] === undefined) { | ||
moduleLookup[module] = await this.linkAndEvaluateModule( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should these be put into There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I’ve replaced There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmmm I should have kept
This happens when Wasm imports refer back to a javascript file that calls it. E.g.:
which is then mistakenly passed to to I can try submiting a follow-up PR with a fix in a few days. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks! There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
await this.resolveModule(module, identifier, context), | ||
); | ||
} | ||
} | ||
|
||
const syntheticModule = new SyntheticModule( | ||
exports.map(({name}) => name), | ||
function () { | ||
const importsObject: WebAssembly.Imports = {}; | ||
for (const {module, name} of imports) { | ||
if (!importsObject[module]) { | ||
importsObject[module] = {}; | ||
} | ||
importsObject[module][name] = moduleLookup[module].namespace[name]; | ||
} | ||
const wasmInstance = new WebAssembly.Instance( | ||
wasmModule, | ||
importsObject, | ||
); | ||
for (const {name} of exports) { | ||
// @ts-expect-error: TS doesn't know what `this` is | ||
this.setExport(name, wasmInstance.exports[name]); | ||
} | ||
}, | ||
{context, identifier}, | ||
); | ||
|
||
return syntheticModule; | ||
} | ||
|
||
private _getMockedNativeModule(): typeof nativeModule.Module { | ||
if (this._moduleImplementation) { | ||
return this._moduleImplementation; | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
File source: https://github.com/hasharchives/wasm-ts-esm-in-node-jest-and-nextjs
Happy to replace this wasm with some canonical example file, but I don’t know any. Ideally, this file would export a function with arguments (mine is just
getAnswer = () => 42
).There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Maybe something from https://github.com/mdn/webassembly-examples?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
that seems reasonable 👍
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done in 79e0f40, using
add(i32, i32)
from mdn/webassembly-examples.