diff --git a/src/library_dylink.js b/src/library_dylink.js index 63e98421e1dcc..d7676cdc2036e 100644 --- a/src/library_dylink.js +++ b/src/library_dylink.js @@ -25,7 +25,7 @@ var LibraryDylink = { // than just running the promises in parallel, this makes a chain of // promises to run in series. wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then( - () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then( + () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true}, name)).then( (exports) => { #if DYLINK_DEBUG dbg(`registering preloadedWasm: ${name}`); @@ -618,6 +618,7 @@ var LibraryDylink = { // promise that resolves to its exports if the loadAsync flag is set. $loadWebAssemblyModule__docs: ` /** + * @param {string=} libName * @param {Object=} localScope * @param {number=} handle */`, @@ -629,7 +630,10 @@ var LibraryDylink = { '$currentModuleWeakSymbols', '$alignMemory', '$zeroMemory', '$updateTableMap', ], - $loadWebAssemblyModule: function(binary, flags, localScope, handle) { + $loadWebAssemblyModule: function(binary, flags, libName, localScope, handle) { +#if DYLINK_DEBUG + dbg(`loadWebAssemblyModule: ${libName}`); +#endif var metadata = getDylinkMetadata(binary); currentModuleWeakSymbols = metadata.weakImports; #if ASSERTIONS @@ -752,10 +756,20 @@ var LibraryDylink = { '{{{ WASI_MODULE_NAME }}}': proxy, }; - function postInstantiation(instance) { + function postInstantiation(module, instance) { #if ASSERTIONS // the table should be unchanged assert(wasmTable === originalTable); +#endif +#if PTHREADS + if (!ENVIRONMENT_IS_PTHREAD && libName) { +#if DYLINK_DEBUG + dbg(`registering sharedModules: ${libName}`) +#endif + // cache all loaded modules in `sharedModules`, which gets passed + // to new workers when they are created. + sharedModules[libName] = module; + } #endif // add new entries to functionsInTableMap updateTableMap(tableBase, metadata.tableSize); @@ -844,16 +858,16 @@ var LibraryDylink = { if (flags.loadAsync) { if (binary instanceof WebAssembly.Module) { var instance = new WebAssembly.Instance(binary, info); - return Promise.resolve(postInstantiation(instance)); + return Promise.resolve(postInstantiation(binary, instance)); } return WebAssembly.instantiate(binary, info).then( - (result) => postInstantiation(result.instance) + (result) => postInstantiation(result.module, result.instance) ); } var module = binary instanceof WebAssembly.Module ? binary : new WebAssembly.Module(binary); var instance = new WebAssembly.Instance(module, info); - return postInstantiation(instance); + return postInstantiation(module, instance); } // now load needed libraries and the module itself. @@ -968,6 +982,16 @@ var LibraryDylink = { // libName -> libData function loadLibData() { +#if PTHREADS + var sharedMod = sharedModules[libName]; +#if DYLINK_DEBUG + dbg(`checking sharedModules: ${libName}: ${sharedMod ? 'found' : 'not found'}`); +#endif + if (sharedMod) { + return flags.loadAsync ? Promise.resolve(sharedMod) : sharedMod; + } +#endif + // for wasm, we can use fetch for async, but for fs mode we can only imitate it if (handle) { var data = {{{ makeGetValue('handle', C_STRUCTS.dso.file_data, '*') }}}; @@ -1007,10 +1031,10 @@ var LibraryDylink = { // module not preloaded - load lib data and create new module from it if (flags.loadAsync) { - return loadLibData().then((libData) => loadWebAssemblyModule(libData, flags, localScope, handle)); + return loadLibData().then((libData) => loadWebAssemblyModule(libData, flags, libName, localScope, handle)); } - return loadWebAssemblyModule(loadLibData(), flags, localScope, handle); + return loadWebAssemblyModule(loadLibData(), flags, libName, localScope, handle); } // module for lib is loaded - update the dso & global namespace diff --git a/src/library_pthread.js b/src/library_pthread.js index ab30f07a0409a..218d4c502aa77 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -410,7 +410,9 @@ var LibraryPThread = { 'wasmOffsetConverter': wasmOffsetConverter, #endif #if MAIN_MODULE - 'dynamicLibraries': Module['dynamicLibraries'], + // Share all modules that have been loaded so far. New workers + // won't start running threads until these are all loaded. + 'sharedModules': sharedModules, #endif #if ASSERTIONS 'workerID': worker.workerID, @@ -431,6 +433,7 @@ var LibraryPThread = { ) { return onMaybeReady(); } + let pthreadPoolReady = Promise.all(PThread.unusedWorkers.map(PThread.loadWasmModuleToWorker)); #if PTHREAD_POOL_DELAY_LOAD // PTHREAD_POOL_DELAY_LOAD means we want to proceed synchronously without diff --git a/src/postamble.js b/src/postamble.js index 712165158661e..6ec0203723863 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -131,6 +131,12 @@ function stackCheckInit() { } #endif +#if MAIN_MODULE && PTHREADS +// Map of modules to be shared with new threads. This gets populated by the +// main thread and shared with all new workers. +var sharedModules = Module['sharedModules'] || []; +#endif + #if MAIN_READS_PARAMS function run(args = arguments_) { #else diff --git a/src/worker.js b/src/worker.js index 7d653fb0505f5..14b78fde7a191 100644 --- a/src/worker.js +++ b/src/worker.js @@ -151,7 +151,10 @@ function handleMessage(e) { #endif // MINIMAL_RUNTIME #if MAIN_MODULE - Module['dynamicLibraries'] = e.data.dynamicLibraries; + Module['sharedModules'] = e.data.sharedModules; +#if RUNTIME_DEBUG + dbg(`received ${Object.keys(e.data.sharedModules).length} shared modules: ${Object.keys(e.data.sharedModules)}`); +#endif #endif // Use `const` here to ensure that the variable is scoped only to diff --git a/test/other/metadce/test_metadce_hello_dylink.jssize b/test/other/metadce/test_metadce_hello_dylink.jssize index d7c41b7d32c0c..62b62ad8bcd45 100644 --- a/test/other/metadce/test_metadce_hello_dylink.jssize +++ b/test/other/metadce/test_metadce_hello_dylink.jssize @@ -1 +1 @@ -15023 +15032 diff --git a/test/test_core.py b/test/test_core.py index 4022a5d558428..0a12125c2a05a 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -9474,34 +9474,40 @@ def test_pthread_dylink_main_module_1(self): self.do_runf(test_file('hello_world.c')) @needs_dylink - @node_pthreads - def test_Module_dynamicLibraries_pthreads(self): + @parameterized({ + '': ([],), + 'pthreads': (['-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME', '-pthread', '-Wno-experimental'],) + }) + def test_Module_dynamicLibraries(self, args): # test that Module.dynamicLibraries works with pthreads - self.emcc_args += ['-pthread', '-Wno-experimental'] + self.emcc_args += args self.emcc_args += ['--pre-js', 'pre.js'] - self.set_setting('PROXY_TO_PTHREAD') - self.set_setting('EXIT_RUNTIME') # This test is for setting dynamicLibraries at runtime so we don't # want emscripten loading `liblib.so` automatically (which it would # do without this setting. self.set_setting('NO_AUTOLOAD_DYLIBS') create_file('pre.js', ''' - if (typeof importScripts == 'undefined') { // !ENVIRONMENT_IS_WORKER - // Load liblib.so by default on non-workers - Module['dynamicLibraries'] = ['liblib.so']; - } else { - // Verify whether the main thread passes Module.dynamicLibraries to the worker - assert(Module['dynamicLibraries'].includes('liblib.so')); - } + Module['dynamicLibraries'] = ['liblib.so']; ''') + if args: + self.setup_node_pthreads() + create_file('post.js', ''' + if (ENVIRONMENT_IS_PTHREAD) { + err('sharedModules: ' + Object.keys(sharedModules)); + assert('liblib.so' in sharedModules); + assert(sharedModules['liblib.so'] instanceof WebAssembly.Module); + } + ''') + self.emcc_args += ['--post-js', 'post.js'] + self.dylink_test( r''' #include <stdio.h> int side(); int main() { - printf("result is %d", side()); + printf("result is %d\n", side()); return 0; } ''', diff --git a/test/test_other.py b/test/test_other.py index 9c149a68d1d19..669f776d79a94 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -13473,12 +13473,20 @@ def test_preload_module(self, args): struct stat statbuf; assert(stat("/library.so", &statbuf) == 0); - // Check that it was preloaded + // Check that it was preloaded. + // The preloading actually only happens on the main thread where the filesystem + // lives. On worker threads the module object is shared via preloadedModules. if (emscripten_is_main_runtime_thread()) { int found = EM_ASM_INT( return preloadedWasm['/library.so'] !== undefined; ); assert(found); + } else { + int found = EM_ASM_INT( + err(sharedModules); + return sharedModules['/library.so'] !== undefined; + ); + assert(found); } void *lib_handle = dlopen("/library.so", RTLD_NOW); assert(lib_handle);