From ce358c42f48557d9c37793f926bb0e9ac1d059b6 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Fri, 10 Mar 2023 15:12:42 -0800 Subject: [PATCH] Move wasm module preload plugin out of library_browser.js. NFC Also test the preload plugin system under node. I was working previously but not covered by any of the test we run in CI. I'm hoping to unify this preloading system with the one planned as part of #18552. --- .../source/docs/api_reference/preamble.js.rst | 2 - src/library_browser.js | 60 +++---------------- src/library_dylink.js | 46 +++++++++++++- src/library_fs.js | 24 ++++++-- src/library_wasmfs.js | 21 +++++-- src/preamble.js | 4 ++ test/test_other.py | 42 +++++++++++++ 7 files changed, 132 insertions(+), 67 deletions(-) diff --git a/site/source/docs/api_reference/preamble.js.rst b/site/source/docs/api_reference/preamble.js.rst index 6f5e5102ec2fe..d96c0130f3de6 100644 --- a/site/source/docs/api_reference/preamble.js.rst +++ b/site/source/docs/api_reference/preamble.js.rst @@ -437,8 +437,6 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t function SAFE_HEAP_LOAD(dest, bytes, isFloat, unsigned) function SAFE_FT_MASK(value, mask) function CHECK_OVERFLOW(value, bits, ignore, sig) - Module["preloadedImages"] - Module["preloadedAudios"] .. PRIVATE NOTES (not rendered) : diff --git a/src/library_browser.js b/src/library_browser.js index d0f394efdd77d..93a10b8b33554 100644 --- a/src/library_browser.js +++ b/src/library_browser.js @@ -25,9 +25,6 @@ var LibraryBrowser = { Module["resumeMainLoop"] = function Module_resumeMainLoop() { Browser.mainLoop.resume() }; Module["getUserMedia"] = function Module_getUserMedia() { Browser.getUserMedia() }; Module["createContext"] = function Module_createContext(canvas, useWebGL, setInModule, webGLContextAttributes) { return Browser.createContext(canvas, useWebGL, setInModule, webGLContextAttributes) }; -#if MAIN_MODULE - var preloadedWasm = {}; -#endif var preloadedImages = {}; var preloadedAudios = {};`, @@ -101,8 +98,6 @@ var LibraryBrowser = { workers: [], init: function() { - if (!Module["preloadPlugins"]) Module["preloadPlugins"] = []; // needs to exist even in workers - if (Browser.initted) return; Browser.initted = true; @@ -116,12 +111,15 @@ var LibraryBrowser = { Browser.BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : (!Browser.hasBlobConstructor ? err("warning: no BlobBuilder") : null)); Browser.URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : undefined; if (!Module.noImageDecoding && typeof Browser.URLObject == 'undefined') { - err("warning: Browser does not support creating object URLs. Built-in browser image decoding will not be available."); +#if ENVIRONMENT_MAY_BE_NODE + if (!ENVIRONMENT_IS_NODE) +#endif + err("warning: Browser does not support creating object URLs. Built-in browser image decoding will not be available."); Module.noImageDecoding = true; } // Support for plugins that can process preloaded files. You can add more of these to - // your app by creating and appending to Module.preloadPlugins. + // your app by creating and appending to preloadPlugins. // // Each plugin is asked if it can handle a file based on the file's name. If it can, // it is given the file's raw data. When it is done, it calls a callback with the file's @@ -172,7 +170,7 @@ var LibraryBrowser = { }; img.src = url; }; - Module['preloadPlugins'].push(imagePlugin); + preloadPlugins.push(imagePlugin); var audioPlugin = {}; audioPlugin['canHandle'] = function audioPlugin_canHandle(name) { @@ -243,34 +241,7 @@ var LibraryBrowser = { return fail(); } }; - Module['preloadPlugins'].push(audioPlugin); - -#if MAIN_MODULE - // Use string keys here to avoid minification since the plugin consumer - // also uses string keys. - var wasmPlugin = { - 'promiseChainEnd': Promise.resolve(), - 'canHandle': function(name) { - return !Module.noWasmDecoding && name.endsWith('.so') - }, - 'handle': function(byteArray, name, onload, onerror) { - // loadWebAssemblyModule can not load modules out-of-order, so rather - // than just running the promises in parallel, this makes a chain of - // promises to run in series. - wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then( - () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then( - (module) => { - preloadedWasm[name] = module; - onload(); - }, - (err) => { - console.warn("Couldn't instantiate wasm: " + name + " '" + err + "'"); - onerror(); - }); - } - }; - Module['preloadPlugins'].push(wasmPlugin); -#endif // MAIN_MODULE + preloadPlugins.push(audioPlugin); // Canvas event setup @@ -313,23 +284,6 @@ var LibraryBrowser = { } }, - // Tries to handle an input byteArray using preload plugins. Returns true if - // it was handled. - handledByPreloadPlugin: function(byteArray, fullname, finish, onerror) { - // Ensure plugins are ready. - Browser.init(); - - var handled = false; - Module['preloadPlugins'].forEach(function(plugin) { - if (handled) return; - if (plugin['canHandle'](fullname)) { - plugin['handle'](byteArray, fullname, finish, onerror); - handled = true; - } - }); - return handled; - }, - createContext: function(/** @type {HTMLCanvasElement} */ canvas, useWebGL, setInModule, webGLContextAttributes) { if (useWebGL && Module.ctx && canvas == Module.canvas) return Module.ctx; // no need to recreate GL context if it's already been created for this canvas. diff --git a/src/library_dylink.js b/src/library_dylink.js index cdbc16844bad6..eb3e833f92b5e 100644 --- a/src/library_dylink.js +++ b/src/library_dylink.js @@ -10,6 +10,38 @@ var dlopenMissingError = "'To use dlopen, you need enable dynamic linking, see h var LibraryDylink = { #if RELOCATABLE +#if FORCE_FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM + $registerWasmPlugin: function() { + // Use string keys here to avoid minification since the plugin consumer + // also uses string keys. + var wasmPlugin = { + 'promiseChainEnd': Promise.resolve(), + 'canHandle': function(name) { + return !Module.noWasmDecoding && name.endsWith('.so') + }, + 'handle': function(byteArray, name, onload, onerror) { + // loadWebAssemblyModule can not load modules out-of-order, so rather + // than just running the promises in parallel, this makes a chain of + // promises to run in series. + wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then( + () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then( + (module) => { + preloadedWasm[name] = module; + onload(); + }, + (err) => { + console.warn("Couldn't instantiate wasm: " + name + " '" + err + "'"); + onerror(); + }); + } + }; + preloadPlugins.push(wasmPlugin); + }, + + $preloadedWasm__deps: ['$registerWasmPlugin'], + $preloadedWasm__postset: `registerWasmPlugin();`, + $preloadedWasm: {}, +#endif $isSymbolDefined: function(symName) { // Ignore 'stub' symbols that are auto-generated as part of the original @@ -887,7 +919,12 @@ var LibraryDylink = { // If a library was already loaded, it is not loaded a second time. However // flags.global and flags.nodelete are handled every time a load request is made. // Once a library becomes "global" or "nodelete", it cannot be removed or unloaded. - $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule', '$isInternalSym', '$mergeLibSymbols', '$newDSO'], + $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule', + '$isInternalSym', '$mergeLibSymbols', '$newDSO', +#if FORCE_FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM + '$preloadedWasm' +#endif + ], $loadDynamicLibrary__docs: ` /** * @param {number=} handle @@ -957,11 +994,16 @@ var LibraryDylink = { // libName -> exports function getExports() { +#if FORCE_FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM // lookup preloaded cache first - if (typeof preloadedWasm != 'undefined' && preloadedWasm[libName]) { + if (preloadedWasm[libName]) { +#if DYLINK_DEBUG + err('using preloaded module for: ' + libName); +#endif var libModule = preloadedWasm[libName]; return flags.loadAsync ? Promise.resolve(libModule) : libModule; } +#endif // module not preloaded - load lib data and create new module from it if (flags.loadAsync) { diff --git a/src/library_fs.js b/src/library_fs.js index 05d307bf7dad0..6672eba2c1845 100644 --- a/src/library_fs.js +++ b/src/library_fs.js @@ -86,10 +86,6 @@ Object.defineProperties(FSNode.prototype, { }); FS.FSNode = FSNode; FS.staticInit();` + -#if USE_CLOSURE_COMPILER - // Declare variable for Closure, FS.createPreloadedFile() below calls Browser.handledByPreloadPlugin() - '/**@suppress {duplicate, undefinedVars}*/var Browser;' + -#endif // Get module methods from settings '{{{ EXPORTED_RUNTIME_METHODS.filter(function(func) { return func.substr(0, 3) === 'FS_' }).map(function(func){return 'Module["' + func + '"] = FS.' + func.substr(3) + ";"}).reduce(function(str, func){return str + func;}, '') }}}'; }, @@ -1862,6 +1858,24 @@ FS.staticInit();` + node.stream_ops = stream_ops; return node; }, + + // Tries to handle an input byteArray using preload plugins. Returns true if + // it was handled. + handledByPreloadPlugin: function(byteArray, fullname, finish, onerror) { + // Ensure plugins are ready. + if (typeof browser != 'undefined') Browser.init(); + + var handled = false; + preloadPlugins.forEach(function(plugin) { + if (handled) return; + if (plugin['canHandle'](fullname)) { + plugin['handle'](byteArray, fullname, finish, onerror); + handled = true; + } + }); + return handled; + }, + // Preloads a file asynchronously. You can call this before run, for example in // preRun. run will be delayed until this file arrives and is set up. // If you call it after run(), you may want to pause the main loop until it @@ -1888,7 +1902,7 @@ FS.staticInit();` + if (onload) onload(); removeRunDependency(dep); } - if (Browser.handledByPreloadPlugin(byteArray, fullname, finish, () => { + if (FS.handledByPreloadPlugin(byteArray, fullname, finish, () => { if (onerror) onerror(); removeRunDependency(dep); })) { diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js index 68bf1405d9eb2..b837e71a0cd8a 100644 --- a/src/library_wasmfs.js +++ b/src/library_wasmfs.js @@ -7,10 +7,6 @@ mergeInto(LibraryManager.library, { $wasmFSPreloadedFiles: [], $wasmFSPreloadedDirs: [], -#if USE_CLOSURE_COMPILER - // Declare variable for Closure, FS.createPreloadedFile() below calls Browser.handledByPreloadPlugin() - $FS__postset: '/**@suppress {duplicate, undefinedVars}*/var Browser;', -#endif $FS__deps: [ '$wasmFSPreloadedFiles', '$wasmFSPreloadedDirs', @@ -23,6 +19,21 @@ mergeInto(LibraryManager.library, { ], $FS : { // TODO: Clean up the following functions - currently copied from library_fs.js directly. + handledByPreloadPlugin: function(byteArray, fullname, finish, onerror) { + // Ensure plugins are ready. + if (typeof browser != 'undefined') Browser.init(); + + var handled = false; + preloadPlugins.forEach(function(plugin) { + if (handled) return; + if (plugin['canHandle'](fullname)) { + plugin['handle'](byteArray, fullname, finish, onerror); + handled = true; + } + }); + return handled; + }, + createPreloadedFile: (parent, name, url, canRead, canWrite, onload, onerror, dontCreateFile, canOwn, preFinish) => { // TODO: use WasmFS code to resolve and join the path here? var fullname = name ? parent + '/' + name : parent; @@ -37,7 +48,7 @@ mergeInto(LibraryManager.library, { removeRunDependency(dep); } #if !MINIMAL_RUNTIME - if (Browser.handledByPreloadPlugin(byteArray, fullname, finish, () => { + if (FS.handledByPreloadPlugin(byteArray, fullname, finish, () => { if (onerror) onerror(); removeRunDependency(dep); })) { diff --git a/src/preamble.js b/src/preamble.js index 69b09341662ec..790ce290b23df 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -19,6 +19,10 @@ Module.realPrint = out; out = err = () => {}; #endif +#if FORCE_FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM +{{{ makeModuleReceiveWithVar('preloadPlugins', undefined, '[]') }}} +#endif + #if RELOCATABLE {{{ makeModuleReceiveWithVar('dynamicLibraries', undefined, '[]', true) }}} #endif diff --git a/test/test_other.py b/test/test_other.py index 6d13645dc8d08..5763d68cf2077 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -13228,3 +13228,45 @@ def test_pthreads_flag(self): # See https://github.com/llvm/llvm-project/commit/c800391fb974cdaaa62bd74435f76408c2e5ceae err = self.expect_fail([EMCC, '-pthreads', '-c', test_file('hello_world.c')]) self.assertContained('emcc: error: unrecognized command-line option `-pthreads`; did you mean `-pthread`?', err) + + def test_preload_module(self): + # TODO(sbc): This test is copyied from test_browser.py. Perhaps find a better way to + # share code between them. + create_file('library.c', r''' + #include + int library_func() { + return 42; + } + ''') + self.run_process([EMCC, 'library.c', '-sSIDE_MODULE', '-o', 'library.so']) + create_file('main.c', r''' + #include + #include + #include + #include + int main() { +#ifdef PRELOAD + int found = EM_ASM_INT( + return preloadedWasm['/library.so'] !== undefined; + ); + assert(found); +#endif + // Remove the shared library from the host filesystem. + // When not preloaded, which should cause the dlopen to fail. + EM_ASM(require('fs').unlinkSync('library.so')); + void *lib_handle = dlopen("/library.so", RTLD_NOW); + assert(lib_handle); + typedef int (*voidfunc)(); + voidfunc x = (voidfunc)dlsym(lib_handle, "library_func"); + assert(x); + assert(x() == 42); + printf("done\n"); + return 0; + } + ''') + self.do_runf('main.c', 'done\n', emcc_args=['-sMAIN_MODULE=2', '-DPRELOAD', '--preload-file', '.@/', '--use-preload-plugins']) + + # Run the test again but without preloading. In this case the removal of the shared + # library file should cause the dlopen to fail. + self.run_process([EMCC, 'library.c', '-sSIDE_MODULE', '-o', 'library.so']) + self.do_runf('main.c', 'Error in loading dynamic library', emcc_args=['-sMAIN_MODULE=2'], assert_returncode=NON_ZERO)