Move wasm module preload plugin out of library_browser.js. NFC

Also test the preload plugin system under node. It was working previously but not covered by any of the test we run in CI. I'm hoping to unify this preloading system with the one planned as part of #18552.
emscripten-core · May 3, 2023 · 85780b3 · 85780b3
1 parent b72453c
commit 85780b3
Show file tree

Hide file tree

Showing 14 changed files with 126 additions and 73 deletions.
diff --git a/src/compiler.js b/src/compiler.js
@@ -64,10 +64,15 @@ Object.assign(global, settings);
 
 global.symbolsOnly = symbolsOnlyArg != -1;
 
+if (!ALL_INCOMING_MODULE_JS_API.length) {
+  ALL_INCOMING_MODULE_JS_API = INCOMING_MODULE_JS_API
+}
+
 EXPORTED_FUNCTIONS = new Set(EXPORTED_FUNCTIONS);
 WASM_EXPORTS = new Set(WASM_EXPORTS);
 SIDE_MODULE_EXPORTS = new Set(SIDE_MODULE_EXPORTS);
 INCOMING_MODULE_JS_API = new Set(INCOMING_MODULE_JS_API);
+ALL_INCOMING_MODULE_JS_API = new Set(ALL_INCOMING_MODULE_JS_API);
 WEAK_IMPORTS = new Set(WEAK_IMPORTS);
 if (symbolsOnly) {
   INCLUDE_FULL_LIBRARY = 1;

diff --git a/src/library_browser.js b/src/library_browser.js
@@ -12,6 +12,10 @@ var LibraryBrowser = {
     '$safeSetTimeout',
     '$warnOnce',
     'emscripten_set_main_loop_timing',
+    '$preloadPlugins',
+#if MAIN_MODULE
+    '$preloadedWasm',
+#endif
   ],
   $Browser__postset: `
     // exports
@@ -25,9 +29,6 @@ var LibraryBrowser = {
     Module["resumeMainLoop"] = function Module_resumeMainLoop() { Browser.mainLoop.resume() };
     Module["getUserMedia"] = function Module_getUserMedia() { Browser.getUserMedia() };
     Module["createContext"] = function Module_createContext(canvas, useWebGL, setInModule, webGLContextAttributes) { return Browser.createContext(canvas, useWebGL, setInModule, webGLContextAttributes) };
-#if MAIN_MODULE
-    var preloadedWasm = {};
-#endif
     var preloadedImages = {};
     var preloadedAudios = {};`,
 
@@ -101,18 +102,19 @@ var LibraryBrowser = {
     workers: [],
 
     init: function() {
-      if (!Module["preloadPlugins"]) Module["preloadPlugins"] = []; // needs to exist even in workers
-
       if (Browser.initted) return;
       Browser.initted = true;
       Browser.URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : undefined;
       if (!Module.noImageDecoding && typeof Browser.URLObject == 'undefined') {
-        err("warning: Browser does not support creating object URLs. Built-in browser image decoding will not be available.");
+#if ENVIRONMENT_MAY_BE_NODE
+        if (!ENVIRONMENT_IS_NODE)
+#endif
+          err("warning: Browser does not support creating object URLs. Built-in browser image decoding will not be available.");
         Module.noImageDecoding = true;
       }
 
       // Support for plugins that can process preloaded files. You can add more of these to
-      // your app by creating and appending to Module.preloadPlugins.
+      // your app by creating and appending to preloadPlugins.
       //
       // Each plugin is asked if it can handle a file based on the file's name. If it can,
       // it is given the file's raw data. When it is done, it calls a callback with the file's
@@ -151,7 +153,7 @@ var LibraryBrowser = {
         };
         img.src = url;
       };
-      Module['preloadPlugins'].push(imagePlugin);
+      preloadPlugins.push(imagePlugin);
 
       var audioPlugin = {};
       audioPlugin['canHandle'] = function audioPlugin_canHandle(name) {
@@ -214,34 +216,7 @@ var LibraryBrowser = {
           finish(audio); // try to use it even though it is not necessarily ready to play
         }, 10000);
       };
-      Module['preloadPlugins'].push(audioPlugin);
-
-#if MAIN_MODULE
-      // Use string keys here to avoid minification since the plugin consumer
-      // also uses string keys.
-      var wasmPlugin = {
-        'promiseChainEnd': Promise.resolve(),
-        'canHandle': function(name) {
-          return !Module.noWasmDecoding && name.endsWith('.so')
-        },
-        'handle': function(byteArray, name, onload, onerror) {
-          // loadWebAssemblyModule can not load modules out-of-order, so rather
-          // than just running the promises in parallel, this makes a chain of
-          // promises to run in series.
-          wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then(
-            () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then(
-              (module) => {
-                preloadedWasm[name] = module;
-                onload();
-              },
-              (err) => {
-                console.warn("Couldn't instantiate wasm: " + name + " '" + err + "'");
-                onerror();
-              });
-        }
-      };
-      Module['preloadPlugins'].push(wasmPlugin);
-#endif // MAIN_MODULE
+      preloadPlugins.push(audioPlugin);
 
       // Canvas event setup
 
@@ -284,23 +259,6 @@ var LibraryBrowser = {
       }
     },
 
-    // Tries to handle an input byteArray using preload plugins. Returns true if
-    // it was handled.
-    handledByPreloadPlugin: function(byteArray, fullname, finish, onerror) {
-      // Ensure plugins are ready.
-      Browser.init();
-
-      var handled = false;
-      Module['preloadPlugins'].forEach((plugin) => {
-        if (handled) return;
-        if (plugin['canHandle'](fullname)) {
-          plugin['handle'](byteArray, fullname, finish, onerror);
-          handled = true;
-        }
-      });
-      return handled;
-    },
-
     createContext: function(/** @type {HTMLCanvasElement} */ canvas, useWebGL, setInModule, webGLContextAttributes) {
       if (useWebGL && Module.ctx && canvas == Module.canvas) return Module.ctx; // no need to recreate GL context if it's already been created for this canvas.
 

diff --git a/src/library_dylink.js b/src/library_dylink.js
@@ -10,6 +10,39 @@ var dlopenMissingError = "'To use dlopen, you need enable dynamic linking, see h
 
 var LibraryDylink = {
 #if RELOCATABLE
+  $registerWasmPlugin__deps: ['$preloadPlugins'],
+  $registerWasmPlugin: function() {
+    // Use string keys here to avoid minification since the plugin consumer
+    // also uses string keys.
+    var wasmPlugin = {
+      'promiseChainEnd': Promise.resolve(),
+      'canHandle': function(name) {
+        return !Module.noWasmDecoding && name.endsWith('.so')
+      },
+      'handle': function(byteArray, name, onload, onerror) {
+        // loadWebAssemblyModule can not load modules out-of-order, so rather
+        // than just running the promises in parallel, this makes a chain of
+        // promises to run in series.
+        wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then(
+          () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then(
+            (module) => {
+              preloadedWasm[name] = module;
+              onload();
+            },
+            (error) => {
+              err('failed to instantiate wasm: ' + name + ': ' + error);
+              onerror();
+            });
+      }
+    };
+    preloadPlugins.push(wasmPlugin);
+  },
+
+  $preloadedWasm__deps: ['$registerWasmPlugin'],
+  $preloadedWasm__postset: `
+    registerWasmPlugin();
+    `,
+  $preloadedWasm: {},
 
   $isSymbolDefined: function(symName) {
     // Ignore 'stub' symbols that are auto-generated as part of the original
@@ -884,7 +917,9 @@ var LibraryDylink = {
   // If a library was already loaded, it is not loaded a second time. However
   // flags.global and flags.nodelete are handled every time a load request is made.
   // Once a library becomes "global" or "nodelete", it cannot be removed or unloaded.
-  $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule', '$isInternalSym', '$mergeLibSymbols', '$newDSO', '$asyncLoad'],
+  $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule',
+                              '$isInternalSym', '$mergeLibSymbols', '$newDSO',
+                              '$asyncLoad', '$preloadedWasm'],
   $loadDynamicLibrary__docs: `
     /**
      * @param {number=} handle
@@ -955,7 +990,10 @@ var LibraryDylink = {
     // libName -> exports
     function getExports() {
       // lookup preloaded cache first
-      if (typeof preloadedWasm != 'undefined' && preloadedWasm[libName]) {
+      if (preloadedWasm[libName]) {
+#if DYLINK_DEBUG
+        err('using preloaded module for: ' + libName);
+#endif
         var libModule = preloadedWasm[libName];
         return flags.loadAsync ? Promise.resolve(libModule) : libModule;
       }

diff --git a/src/library_fs.js b/src/library_fs.js
@@ -93,10 +93,6 @@ Object.defineProperties(FSNode.prototype, {
 FS.FSNode = FSNode;
 FS.createPreloadedFile = FS_createPreloadedFile;
 FS.staticInit();` +
-#if USE_CLOSURE_COMPILER
-           // Declare variable for Closure, FS.createPreloadedFile() below calls Browser.handledByPreloadPlugin()
-           '/**@suppress {duplicate, undefinedVars}*/var Browser;' +
-#endif
            // Get module methods from settings
            '{{{ EXPORTED_RUNTIME_METHODS.filter(function(func) { return func.substr(0, 3) === 'FS_' }).map(function(func){return 'Module["' + func + '"] = FS.' + func.substr(3) + ";"}).reduce(function(str, func){return str + func;}, '') }}}';
   },

diff --git a/src/library_fs_shared.js b/src/library_fs_shared.js
@@ -5,6 +5,29 @@
  */
 
 mergeInto(LibraryManager.library, {
+  $preloadPlugins: "{{{ makeModuleReceiveExpr('preloadPlugins', '[]') }}}",
+
+#if !MINIMAL_RUNTIME
+  // Tries to handle an input byteArray using preload plugins. Returns true if
+  // it was handled.
+  $FS_handledByPreloadPlugin__internal: true,
+  $FS_handledByPreloadPlugin__deps: ['$preloadPlugins'],
+  $FS_handledByPreloadPlugin: function(byteArray, fullname, finish, onerror) {
+    // Ensure plugins are ready.
+    if (typeof Browser != 'undefined') Browser.init();
+
+    var handled = false;
+    preloadPlugins.forEach(function(plugin) {
+      if (handled) return;
+      if (plugin['canHandle'](fullname)) {
+        plugin['handle'](byteArray, fullname, finish, onerror);
+        handled = true;
+      }
+    });
+    return handled;
+  },
+#endif
+
   // Preloads a file asynchronously. You can call this before run, for example in
   // preRun. run will be delayed until this file arrives and is set up.
   // If you call it after run(), you may want to pause the main loop until it
@@ -17,7 +40,11 @@ mergeInto(LibraryManager.library, {
   // You can also call this with a typed array instead of a url. It will then
   // do preloading for the Image/Audio part, as if the typed array were the
   // result of an XHR that you did manually.
-  $FS_createPreloadedFile__deps: ['$asyncLoad'],
+  $FS_createPreloadedFile__deps: ['$asyncLoad',
+#if !MINIMAL_RUNTIME
+    '$FS_handledByPreloadPlugin',
+#endif
+  ],
   $FS_createPreloadedFile: function(parent, name, url, canRead, canWrite, onload, onerror, dontCreateFile, canOwn, preFinish) {
 #if WASMFS
     // TODO: use WasmFS code to resolve and join the path here?
@@ -38,7 +65,7 @@ mergeInto(LibraryManager.library, {
         removeRunDependency(dep);
       }
 #if !MINIMAL_RUNTIME
-      if (Browser.handledByPreloadPlugin(byteArray, fullname, finish, () => {
+      if (FS_handledByPreloadPlugin(byteArray, fullname, finish, () => {
         if (onerror) onerror();
         removeRunDependency(dep);
       })) {
@@ -76,4 +103,5 @@ mergeInto(LibraryManager.library, {
     if (canWrite) mode |= {{{ cDefs.S_IWUGO }}};
     return mode;
   },
+
 });
diff --git a/src/library_lz4.js b/src/library_lz4.js
@@ -6,7 +6,7 @@
 
 #if LZ4
 mergeInto(LibraryManager.library, {
-  $LZ4__deps: ['$FS'],
+  $LZ4__deps: ['$FS', '$preloadPlugins'],
   $LZ4: {
     DIR_MODE: {{{ cDefs.S_IFDIR }}} | 511 /* 0777 */,
     FILE_MODE: {{{ cDefs.S_IFREG }}} | 511 /* 0777 */,
@@ -52,7 +52,7 @@ mergeInto(LibraryManager.library, {
         pack['metadata'].files.forEach(function(file) {
           var handled = false;
           var fullname = file.filename;
-          Module['preloadPlugins'].forEach(function(plugin) {
+          preloadPlugins.forEach(function(plugin) {
             if (handled) return;
             if (plugin['canHandle'](fullname)) {
               var dep = getUniqueRunDependency('fp ' + fullname);

diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js
@@ -7,11 +7,7 @@
 mergeInto(LibraryManager.library, {
   $wasmFSPreloadedFiles: [],
   $wasmFSPreloadedDirs: [],
-  // Declare variable for Closure, FS.createPreloadedFile() below calls Browser.handledByPreloadPlugin()
   $FS__postset: `
-#if USE_CLOSURE_COMPILER
-/**@suppress {duplicate, undefinedVars}*/var Browser;
-#endif
 FS.createPreloadedFile = FS_createPreloadedFile;
 `,
   $FS__deps: [

diff --git a/src/parseTools.js b/src/parseTools.js
@@ -732,7 +732,7 @@ function makeRemovedModuleAPIAssert(moduleName, localName) {
 function checkReceiving(name) {
   // ALL_INCOMING_MODULE_JS_API contains all valid incoming module API symbols
   // so calling makeModuleReceive* with a symbol not in this list is an error
-  assert(ALL_INCOMING_MODULE_JS_API.includes(name));
+  assert(ALL_INCOMING_MODULE_JS_API.has(name));
 }
 
 // Make code to receive a value on the incoming Module object.

diff --git a/test/other/metadce/test_metadce_hello_O0.jssize b/test/other/metadce/test_metadce_hello_O0.jssize
@@ -1 +1 @@
-23827
+23842
diff --git a/test/other/metadce/test_metadce_hello_dylink.jssize b/test/other/metadce/test_metadce_hello_dylink.jssize
@@ -1 +1 @@
-27887
+28153
diff --git a/test/other/metadce/test_metadce_minimal_O0.jssize b/test/other/metadce/test_metadce_minimal_O0.jssize
@@ -1 +1 @@
-20076
+20091
diff --git a/test/other/test_unoptimized_code_size.js.size b/test/other/test_unoptimized_code_size.js.size
@@ -1 +1 @@
-59626
+59646
diff --git a/test/other/test_unoptimized_code_size_strict.js.size b/test/other/test_unoptimized_code_size_strict.js.size
@@ -1 +1 @@
-58568
+58588
diff --git a/test/test_other.py b/test/test_other.py
@@ -13390,3 +13390,35 @@ def test_node_pthreads_err_out(self):
   def test_windows_batch_file_dp0_expansion_bug(self):
     create_file('build_with_quotes.bat',  f'@"emcc" {test_file("hello_world.c")}')
     self.run_process(['build_with_quotes.bat'])
+
+  def test_preload_module(self):
+    # TODO(sbc): This test is copyied from test_browser.py.  Perhaps find a better way to
+    # share code between them.
+    create_file('library.c', r'''
+      #include <stdio.h>
+      int library_func() {
+        return 42;
+      }
+    ''')
+    self.run_process([EMCC, 'library.c', '-sSIDE_MODULE', '-o', 'library.so'])
+    create_file('main.c', r'''
+      #include <assert.h>
+      #include <dlfcn.h>
+      #include <stdio.h>
+      #include <emscripten.h>
+      int main() {
+        int found = EM_ASM_INT(
+          return preloadedWasm['/library.so'] !== undefined;
+        );
+        assert(found);
+        void *lib_handle = dlopen("/library.so", RTLD_NOW);
+        assert(lib_handle);
+        typedef int (*voidfunc)();
+        voidfunc x = (voidfunc)dlsym(lib_handle, "library_func");
+        assert(x);
+        assert(x() == 42);
+        printf("done\n");
+        return 0;
+      }
+    ''')
+    self.do_runf('main.c', 'done\n', emcc_args=['-sMAIN_MODULE=2', '--preload-file', '.@/', '--use-preload-plugins'])