Move wasm module preload plugin out of library_browser.js. NFC

Also test the preload plugin system under node. I was working previously but not covered by any of the test we run in CI. I'm hoping to unify this preloading system with the one planned as part of #18552.
emscripten-core · Mar 15, 2023 · ce358c4 · ce358c4
1 parent 92a8464
commit ce358c4
Show file tree

Hide file tree

Showing 7 changed files with 132 additions and 67 deletions.
diff --git a/site/source/docs/api_reference/preamble.js.rst b/site/source/docs/api_reference/preamble.js.rst
@@ -437,8 +437,6 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t
   function SAFE_HEAP_LOAD(dest, bytes, isFloat, unsigned)
   function SAFE_FT_MASK(value, mask)
   function CHECK_OVERFLOW(value, bits, ignore, sig)
-  Module["preloadedImages"]
-  Module["preloadedAudios"]
 
 
 .. PRIVATE NOTES (not rendered) :

diff --git a/src/library_browser.js b/src/library_browser.js
@@ -25,9 +25,6 @@ var LibraryBrowser = {
     Module["resumeMainLoop"] = function Module_resumeMainLoop() { Browser.mainLoop.resume() };
     Module["getUserMedia"] = function Module_getUserMedia() { Browser.getUserMedia() };
     Module["createContext"] = function Module_createContext(canvas, useWebGL, setInModule, webGLContextAttributes) { return Browser.createContext(canvas, useWebGL, setInModule, webGLContextAttributes) };
-#if MAIN_MODULE
-    var preloadedWasm = {};
-#endif
     var preloadedImages = {};
     var preloadedAudios = {};`,
 
@@ -101,8 +98,6 @@ var LibraryBrowser = {
     workers: [],
 
     init: function() {
-      if (!Module["preloadPlugins"]) Module["preloadPlugins"] = []; // needs to exist even in workers
-
       if (Browser.initted) return;
       Browser.initted = true;
 
@@ -116,12 +111,15 @@ var LibraryBrowser = {
       Browser.BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : (!Browser.hasBlobConstructor ? err("warning: no BlobBuilder") : null));
       Browser.URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : undefined;
       if (!Module.noImageDecoding && typeof Browser.URLObject == 'undefined') {
-        err("warning: Browser does not support creating object URLs. Built-in browser image decoding will not be available.");
+#if ENVIRONMENT_MAY_BE_NODE
+        if (!ENVIRONMENT_IS_NODE)
+#endif
+          err("warning: Browser does not support creating object URLs. Built-in browser image decoding will not be available.");
         Module.noImageDecoding = true;
       }
 
       // Support for plugins that can process preloaded files. You can add more of these to
-      // your app by creating and appending to Module.preloadPlugins.
+      // your app by creating and appending to preloadPlugins.
       //
       // Each plugin is asked if it can handle a file based on the file's name. If it can,
       // it is given the file's raw data. When it is done, it calls a callback with the file's
@@ -172,7 +170,7 @@ var LibraryBrowser = {
         };
         img.src = url;
       };
-      Module['preloadPlugins'].push(imagePlugin);
+      preloadPlugins.push(imagePlugin);
 
       var audioPlugin = {};
       audioPlugin['canHandle'] = function audioPlugin_canHandle(name) {
@@ -243,34 +241,7 @@ var LibraryBrowser = {
           return fail();
         }
       };
-      Module['preloadPlugins'].push(audioPlugin);
-
-#if MAIN_MODULE
-      // Use string keys here to avoid minification since the plugin consumer
-      // also uses string keys.
-      var wasmPlugin = {
-        'promiseChainEnd': Promise.resolve(),
-        'canHandle': function(name) {
-          return !Module.noWasmDecoding && name.endsWith('.so')
-        },
-        'handle': function(byteArray, name, onload, onerror) {
-          // loadWebAssemblyModule can not load modules out-of-order, so rather
-          // than just running the promises in parallel, this makes a chain of
-          // promises to run in series.
-          wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then(
-            () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then(
-              (module) => {
-                preloadedWasm[name] = module;
-                onload();
-              },
-              (err) => {
-                console.warn("Couldn't instantiate wasm: " + name + " '" + err + "'");
-                onerror();
-              });
-        }
-      };
-      Module['preloadPlugins'].push(wasmPlugin);
-#endif // MAIN_MODULE
+      preloadPlugins.push(audioPlugin);
 
       // Canvas event setup
 
@@ -313,23 +284,6 @@ var LibraryBrowser = {
       }
     },
 
-    // Tries to handle an input byteArray using preload plugins. Returns true if
-    // it was handled.
-    handledByPreloadPlugin: function(byteArray, fullname, finish, onerror) {
-      // Ensure plugins are ready.
-      Browser.init();
-
-      var handled = false;
-      Module['preloadPlugins'].forEach(function(plugin) {
-        if (handled) return;
-        if (plugin['canHandle'](fullname)) {
-          plugin['handle'](byteArray, fullname, finish, onerror);
-          handled = true;
-        }
-      });
-      return handled;
-    },
-
     createContext: function(/** @type {HTMLCanvasElement} */ canvas, useWebGL, setInModule, webGLContextAttributes) {
       if (useWebGL && Module.ctx && canvas == Module.canvas) return Module.ctx; // no need to recreate GL context if it's already been created for this canvas.
 

diff --git a/src/library_dylink.js b/src/library_dylink.js
@@ -10,6 +10,38 @@ var dlopenMissingError = "'To use dlopen, you need enable dynamic linking, see h
 
 var LibraryDylink = {
 #if RELOCATABLE
+#if FORCE_FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM
+  $registerWasmPlugin: function() {
+    // Use string keys here to avoid minification since the plugin consumer
+    // also uses string keys.
+    var wasmPlugin = {
+      'promiseChainEnd': Promise.resolve(),
+      'canHandle': function(name) {
+        return !Module.noWasmDecoding && name.endsWith('.so')
+      },
+      'handle': function(byteArray, name, onload, onerror) {
+        // loadWebAssemblyModule can not load modules out-of-order, so rather
+        // than just running the promises in parallel, this makes a chain of
+        // promises to run in series.
+        wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then(
+          () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then(
+            (module) => {
+              preloadedWasm[name] = module;
+              onload();
+            },
+            (err) => {
+              console.warn("Couldn't instantiate wasm: " + name + " '" + err + "'");
+              onerror();
+            });
+      }
+    };
+    preloadPlugins.push(wasmPlugin);
+  },
+
+  $preloadedWasm__deps: ['$registerWasmPlugin'],
+  $preloadedWasm__postset: `registerWasmPlugin();`,
+  $preloadedWasm: {},
+#endif
 
   $isSymbolDefined: function(symName) {
     // Ignore 'stub' symbols that are auto-generated as part of the original
@@ -887,7 +919,12 @@ var LibraryDylink = {
   // If a library was already loaded, it is not loaded a second time. However
   // flags.global and flags.nodelete are handled every time a load request is made.
   // Once a library becomes "global" or "nodelete", it cannot be removed or unloaded.
-  $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule', '$isInternalSym', '$mergeLibSymbols', '$newDSO'],
+  $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule',
+                              '$isInternalSym', '$mergeLibSymbols', '$newDSO',
+#if FORCE_FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM
+                              '$preloadedWasm'
+#endif
+  ],
   $loadDynamicLibrary__docs: `
     /**
      * @param {number=} handle
@@ -957,11 +994,16 @@ var LibraryDylink = {
 
     // libName -> exports
     function getExports() {
+#if FORCE_FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM
       // lookup preloaded cache first
-      if (typeof preloadedWasm != 'undefined' && preloadedWasm[libName]) {
+      if (preloadedWasm[libName]) {
+#if DYLINK_DEBUG
+        err('using preloaded module for: ' + libName);
+#endif
         var libModule = preloadedWasm[libName];
         return flags.loadAsync ? Promise.resolve(libModule) : libModule;
       }
+#endif
 
       // module not preloaded - load lib data and create new module from it
       if (flags.loadAsync) {

diff --git a/src/library_fs.js b/src/library_fs.js
@@ -86,10 +86,6 @@ Object.defineProperties(FSNode.prototype, {
 });
 FS.FSNode = FSNode;
 FS.staticInit();` +
-#if USE_CLOSURE_COMPILER
-           // Declare variable for Closure, FS.createPreloadedFile() below calls Browser.handledByPreloadPlugin()
-           '/**@suppress {duplicate, undefinedVars}*/var Browser;' +
-#endif
            // Get module methods from settings
            '{{{ EXPORTED_RUNTIME_METHODS.filter(function(func) { return func.substr(0, 3) === 'FS_' }).map(function(func){return 'Module["' + func + '"] = FS.' + func.substr(3) + ";"}).reduce(function(str, func){return str + func;}, '') }}}';
   },
@@ -1862,6 +1858,24 @@ FS.staticInit();` +
       node.stream_ops = stream_ops;
       return node;
     },
+
+    // Tries to handle an input byteArray using preload plugins. Returns true if
+    // it was handled.
+    handledByPreloadPlugin: function(byteArray, fullname, finish, onerror) {
+      // Ensure plugins are ready.
+      if (typeof browser != 'undefined') Browser.init();
+
+      var handled = false;
+      preloadPlugins.forEach(function(plugin) {
+        if (handled) return;
+        if (plugin['canHandle'](fullname)) {
+          plugin['handle'](byteArray, fullname, finish, onerror);
+          handled = true;
+        }
+      });
+      return handled;
+    },
+
     // Preloads a file asynchronously. You can call this before run, for example in
     // preRun. run will be delayed until this file arrives and is set up.
     // If you call it after run(), you may want to pause the main loop until it
@@ -1888,7 +1902,7 @@ FS.staticInit();` +
           if (onload) onload();
           removeRunDependency(dep);
         }
-        if (Browser.handledByPreloadPlugin(byteArray, fullname, finish, () => {
+        if (FS.handledByPreloadPlugin(byteArray, fullname, finish, () => {
           if (onerror) onerror();
           removeRunDependency(dep);
         })) {

diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js
@@ -7,10 +7,6 @@
 mergeInto(LibraryManager.library, {
   $wasmFSPreloadedFiles: [],
   $wasmFSPreloadedDirs: [],
-#if USE_CLOSURE_COMPILER
-  // Declare variable for Closure, FS.createPreloadedFile() below calls Browser.handledByPreloadPlugin()
-  $FS__postset: '/**@suppress {duplicate, undefinedVars}*/var Browser;',
-#endif
   $FS__deps: [
     '$wasmFSPreloadedFiles',
     '$wasmFSPreloadedDirs',
@@ -23,6 +19,21 @@ mergeInto(LibraryManager.library, {
   ],
   $FS : {
     // TODO: Clean up the following functions - currently copied from library_fs.js directly.
+    handledByPreloadPlugin: function(byteArray, fullname, finish, onerror) {
+      // Ensure plugins are ready.
+      if (typeof browser != 'undefined') Browser.init();
+
+      var handled = false;
+      preloadPlugins.forEach(function(plugin) {
+        if (handled) return;
+        if (plugin['canHandle'](fullname)) {
+          plugin['handle'](byteArray, fullname, finish, onerror);
+          handled = true;
+        }
+      });
+      return handled;
+    },
+
     createPreloadedFile: (parent, name, url, canRead, canWrite, onload, onerror, dontCreateFile, canOwn, preFinish) => {
       // TODO: use WasmFS code to resolve and join the path here?
       var fullname = name ? parent + '/' + name : parent;
@@ -37,7 +48,7 @@ mergeInto(LibraryManager.library, {
           removeRunDependency(dep);
         }
 #if !MINIMAL_RUNTIME
-        if (Browser.handledByPreloadPlugin(byteArray, fullname, finish, () => {
+        if (FS.handledByPreloadPlugin(byteArray, fullname, finish, () => {
           if (onerror) onerror();
           removeRunDependency(dep);
         })) {

diff --git a/src/preamble.js b/src/preamble.js
@@ -19,6 +19,10 @@ Module.realPrint = out;
 out = err = () => {};
 #endif
 
+#if FORCE_FILESYSTEM && SYSCALLS_REQUIRE_FILESYSTEM
+{{{ makeModuleReceiveWithVar('preloadPlugins', undefined, '[]') }}}
+#endif
+
 #if RELOCATABLE
 {{{ makeModuleReceiveWithVar('dynamicLibraries', undefined, '[]', true) }}}
 #endif

diff --git a/test/test_other.py b/test/test_other.py
@@ -13228,3 +13228,45 @@ def test_pthreads_flag(self):
     # See https://github.com/llvm/llvm-project/commit/c800391fb974cdaaa62bd74435f76408c2e5ceae
     err = self.expect_fail([EMCC, '-pthreads', '-c', test_file('hello_world.c')])
     self.assertContained('emcc: error: unrecognized command-line option `-pthreads`; did you mean `-pthread`?', err)
+
+  def test_preload_module(self):
+    # TODO(sbc): This test is copyied from test_browser.py.  Perhaps find a better way to
+    # share code between them.
+    create_file('library.c', r'''
+      #include <stdio.h>
+      int library_func() {
+        return 42;
+      }
+    ''')
+    self.run_process([EMCC, 'library.c', '-sSIDE_MODULE', '-o', 'library.so'])
+    create_file('main.c', r'''
+      #include <assert.h>
+      #include <dlfcn.h>
+      #include <stdio.h>
+      #include <emscripten.h>
+      int main() {
+#ifdef PRELOAD
+        int found = EM_ASM_INT(
+          return preloadedWasm['/library.so'] !== undefined;
+        );
+        assert(found);
+#endif
+        // Remove the shared library from the host filesystem.
+        // When not preloaded, which should cause the dlopen to fail.
+        EM_ASM(require('fs').unlinkSync('library.so'));
+        void *lib_handle = dlopen("/library.so", RTLD_NOW);
+        assert(lib_handle);
+        typedef int (*voidfunc)();
+        voidfunc x = (voidfunc)dlsym(lib_handle, "library_func");
+        assert(x);
+        assert(x() == 42);
+        printf("done\n");
+        return 0;
+      }
+    ''')
+    self.do_runf('main.c', 'done\n', emcc_args=['-sMAIN_MODULE=2', '-DPRELOAD', '--preload-file', '.@/', '--use-preload-plugins'])
+
+    # Run the test again but without preloading.  In this case the removal of the shared
+    # library file should cause the dlopen to fail.
+    self.run_process([EMCC, 'library.c', '-sSIDE_MODULE', '-o', 'library.so'])
+    self.do_runf('main.c', 'Error in loading dynamic library', emcc_args=['-sMAIN_MODULE=2'], assert_returncode=NON_ZERO)