diff --git a/src/library_dylink.js b/src/library_dylink.js index 4ae16887f8b07..b48ba3abc0895 100644 --- a/src/library_dylink.js +++ b/src/library_dylink.js @@ -25,7 +25,7 @@ var LibraryDylink = { // than just running the promises in parallel, this makes a chain of // promises to run in series. wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then( - () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then( + () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true}, name)).then( (exports) => { #if DYLINK_DEBUG dbg(`registering preloadedWasm: ${name}`); @@ -319,15 +319,20 @@ var LibraryDylink = { }, #else // MAIN_MODULE != 0 // dynamic linker/loader (a-la ld.so on ELF systems) - $LDSO__deps: ['$newDSO'], + $LDSO__deps: ['$newDSO', '$loadDylibs'], $LDSO: { // name -> dso [refcount, name, module, global]; Used by dlopen loadedLibsByName: {}, // handle -> dso; Used by dlsym loadedLibsByHandle: {}, - init: () => newDSO('__main__', {{{ cDefs.RTLD_DEFAULT }}}, wasmImports), + init: () => { + newDSO('__main__', {{{ cDefs.RTLD_DEFAULT }}}, wasmImports); + loadDylibs(); + }, }, + $LDSO__postset: 'addOnPreRun(LDSO.init);', + $dlSetError__internal: true, $dlSetError__deps: ['__dl_seterr', '$stringToUTF8OnStack', '$withStackSave'], $dlSetError: function(msg) { @@ -607,6 +612,7 @@ var LibraryDylink = { // promise that resolves to its exports if the loadAsync flag is set. $loadWebAssemblyModule__docs: ` /** + * @param {string=} libName * @param {Object=} localScope * @param {number=} handle */`, @@ -621,7 +627,10 @@ var LibraryDylink = { '$createInvokeFunction', #endif ], - $loadWebAssemblyModule: function(binary, flags, localScope, handle) { + $loadWebAssemblyModule: function(binary, flags, libName, localScope, handle) { +#if DYLINK_DEBUG + dbg(`loadWebAssemblyModule: ${libName}`); +#endif var metadata = getDylinkMetadata(binary); currentModuleWeakSymbols = metadata.weakImports; #if ASSERTIONS @@ -744,10 +753,20 @@ var LibraryDylink = { '{{{ WASI_MODULE_NAME }}}': proxy, }; - function postInstantiation(instance) { + function postInstantiation(module, instance) { #if ASSERTIONS // the table should be unchanged assert(wasmTable === originalTable); +#endif +#if PTHREADS + if (!ENVIRONMENT_IS_PTHREAD && libName) { +#if DYLINK_DEBUG + dbg(`registering sharedModules: ${libName}`) +#endif + // cache all loaded modules in `sharedModules`, which gets passed + // to new workers when they are created. + sharedModules[libName] = module; + } #endif // add new entries to functionsInTableMap updateTableMap(tableBase, metadata.tableSize); @@ -838,16 +857,16 @@ var LibraryDylink = { if (flags.loadAsync) { if (binary instanceof WebAssembly.Module) { var instance = new WebAssembly.Instance(binary, info); - return Promise.resolve(postInstantiation(instance)); + return Promise.resolve(postInstantiation(binary, instance)); } return WebAssembly.instantiate(binary, info).then( - (result) => postInstantiation(result.instance) + (result) => postInstantiation(result.module, result.instance) ); } var module = binary instanceof WebAssembly.Module ? binary : new WebAssembly.Module(binary); var instance = new WebAssembly.Instance(module, info); - return postInstantiation(instance); + return postInstantiation(module, instance); } // now load needed libraries and the module itself. @@ -964,6 +983,16 @@ var LibraryDylink = { // libName -> libData function loadLibData() { +#if PTHREADS + var sharedMod = sharedModules[libName]; +#if DYLINK_DEBUG + dbg(`checking sharedModules: ${libName}: ${sharedMod ? 'found' : 'not found'}`); +#endif + if (sharedMod) { + return flags.loadAsync ? Promise.resolve(sharedMod) : sharedMod; + } +#endif + // for wasm, we can use fetch for async, but for fs mode we can only imitate it if (handle) { var data = {{{ makeGetValue('handle', C_STRUCTS.dso.file_data, '*') }}}; @@ -1003,10 +1032,10 @@ var LibraryDylink = { // module not preloaded - load lib data and create new module from it if (flags.loadAsync) { - return loadLibData().then((libData) => loadWebAssemblyModule(libData, flags, localScope, handle)); + return loadLibData().then((libData) => loadWebAssemblyModule(libData, flags, libName, localScope, handle)); } - return loadWebAssemblyModule(loadLibData(), flags, localScope, handle); + return loadWebAssemblyModule(loadLibData(), flags, libName, localScope, handle); } // module for lib is loaded - update the dso & global namespace diff --git a/src/library_pthread.js b/src/library_pthread.js index 3ead63051ca96..fa7b254d0cea3 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -105,6 +105,14 @@ var LibraryPThread = { PThread.allocateUnusedWorker(); } #endif +#if !MINIMAL_RUNTIME + // MINIMAL_RUNTIME takes care of calling loadWasmModuleToAllWorkers + // in postamble_minimal.js + addOnPreRun(() => { + addRunDependency('loading-workers') + PThread.loadWasmModuleToAllWorkers(() => removeRunDependency('loading-workers')); + }); +#endif #if MAIN_MODULE PThread.outstandingPromises = {}; // Finished threads are threads that have finished running but we not yet @@ -402,7 +410,9 @@ var LibraryPThread = { 'wasmOffsetConverter': wasmOffsetConverter, #endif #if MAIN_MODULE - 'dynamicLibraries': Module['dynamicLibraries'], + // Shared all modules that have been loaded so far. New workers + // won't start running threads until these are all loaded. + 'sharedModules': sharedModules, #endif #if ASSERTIONS 'workerID': worker.workerID, @@ -423,6 +433,7 @@ var LibraryPThread = { ) { return onMaybeReady(); } + let pthreadPoolReady = Promise.all(PThread.unusedWorkers.map(PThread.loadWasmModuleToWorker)); #if PTHREAD_POOL_DELAY_LOAD // PTHREAD_POOL_DELAY_LOAD means we want to proceed synchronously without diff --git a/src/parseTools.js b/src/parseTools.js index 167767e0f2915..12393c071ae21 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -746,7 +746,7 @@ function makeRemovedModuleAPIAssert(moduleName, localName) { function checkReceiving(name) { // ALL_INCOMING_MODULE_JS_API contains all valid incoming module API symbols // so calling makeModuleReceive* with a symbol not in this list is an error - assert(ALL_INCOMING_MODULE_JS_API.has(name)); + assert(ALL_INCOMING_MODULE_JS_API.has(name), `${name} is not part of INCOMING_MODULE_JS_API`); } // Make code to receive a value on the incoming Module object. diff --git a/src/postamble.js b/src/postamble.js index 91e91f003d977..e9cd85f8df640 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -131,10 +131,15 @@ function stackCheckInit() { } #endif -#if RELOCATABLE +#if RELOCATABLE && PTHREADS +// Map of modules to be shared with new threads. This gets populated by the +// main thread an shared with all new workers. +var sharedModules = Module['sharedModules'] ||[]; var dylibsLoaded = false; #if '$LDSO' in addedLibraryItems -LDSO.init(); +if (ENVIRONMENT_IS_PTHREAD) { + LDSO.init(); +} #endif #endif @@ -158,15 +163,11 @@ function run() { stackCheckInit(); #endif -#if RELOCATABLE - if (!dylibsLoaded) { - // Loading of dynamic libraries needs to happen on each thread, so we can't - // use the normal __ATPRERUN__ mechanism. -#if MAIN_MODULE +#if MAIN_MODULE && PTHREADS + if (!dylibsLoaded && ENVIRONMENT_IS_PTHREAD) { + // Loading of dynamic libraries needs to happen on each thread, so we can't + // use the normal __ATPRERUN__ mechanism. loadDylibs(); -#else - reportUndefinedSymbols(); -#endif dylibsLoaded = true; // Loading dylibs can add run dependencies. @@ -177,6 +178,8 @@ function run() { return; } } +#elif RELOCATABLE && !MAIN_MODULE + reportUndefinedSymbols(); #endif #if WASM_WORKERS diff --git a/src/preamble.js b/src/preamble.js index 51783e5a5f386..085c68483063b 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1047,13 +1047,7 @@ function createWasm() { // We now have the Wasm module loaded up, keep a reference to the compiled module so we can post it to the workers. wasmModule = module; #endif - -#if PTHREADS - PThread.loadWasmModuleToAllWorkers(() => removeRunDependency('wasm-instantiate')); -#else // singlethreaded build: removeRunDependency('wasm-instantiate'); -#endif // ~PTHREADS - return exports; } // wait for the pthread pool (if any) diff --git a/src/worker.js b/src/worker.js index 4c8fe3cebc4f7..14b78fde7a191 100644 --- a/src/worker.js +++ b/src/worker.js @@ -81,7 +81,7 @@ var out = () => { throw 'out() is not defined in worker.js.'; } #endif var err = threadPrintErr; self.alert = threadAlert; -#if RUNTIME_DEBUG +#if ASSERTIONS || RUNTIME_DEBUG var dbg = threadPrintErr; #endif @@ -151,7 +151,10 @@ function handleMessage(e) { #endif // MINIMAL_RUNTIME #if MAIN_MODULE - Module['dynamicLibraries'] = e.data.dynamicLibraries; + Module['sharedModules'] = e.data.sharedModules; +#if RUNTIME_DEBUG + dbg(`received ${Object.keys(e.data.sharedModules).length} shared modules: ${Object.keys(e.data.sharedModules)}`); +#endif #endif // Use `const` here to ensure that the variable is scoped only to diff --git a/test/test_core.py b/test/test_core.py index 7abd9ccf7653b..218d175fa4c34 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -9474,34 +9474,39 @@ def test_pthread_dylink_main_module_1(self): self.do_runf(test_file('hello_world.c')) @needs_dylink - @node_pthreads - def test_Module_dynamicLibraries_pthreads(self): + @parameterized({ + '': ([],), + 'pthreads': (['-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME', '-pthread', '-Wno-experimental'],) + }) + def test_Module_dynamicLibraries(self, args): # test that Module.dynamicLibraries works with pthreads - self.emcc_args += ['-pthread', '-Wno-experimental'] + self.emcc_args += args self.emcc_args += ['--pre-js', 'pre.js'] - self.set_setting('PROXY_TO_PTHREAD') - self.set_setting('EXIT_RUNTIME') # This test is for setting dynamicLibraries at runtime so we don't # want emscripten loading `liblib.so` automatically (which it would # do without this setting. self.set_setting('NO_AUTOLOAD_DYLIBS') create_file('pre.js', ''' - if (typeof importScripts == 'undefined') { // !ENVIRONMENT_IS_WORKER - // Load liblib.so by default on non-workers - Module['dynamicLibraries'] = ['liblib.so']; - } else { - // Verify whether the main thread passes Module.dynamicLibraries to the worker - assert(Module['dynamicLibraries'].includes('liblib.so')); - } + Module['dynamicLibraries'] = ['liblib.so']; ''') + if args: + self.setup_node_pthreads() + create_file('post.js', ''' + if (ENVIRONMENT_IS_PTHREAD) { + err('sharedModules: ' + Object.keys(sharedModules)); + assert('liblib.so' in sharedModules); + } + ''') + self.emcc_args += ['--post-js', 'post.js'] + self.dylink_test( r''' #include int side(); int main() { - printf("result is %d", side()); + printf("result is %d\n", side()); return 0; } ''', diff --git a/test/test_other.py b/test/test_other.py index 2781163ff71f6..8301db486181d 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -2331,7 +2331,8 @@ def test_undefined_symbols(self, action): print('checking "%s" %s' % (args, value)) extra = ['-s', action + '_ON_UNDEFINED_SYMBOLS=%d' % value] if action else [] proc = self.run_process([EMXX, '-sUSE_SDL', 'main.cpp'] + extra + args, stderr=PIPE, check=False) - print(proc.stderr) + if common.EMTEST_VERBOSE: + print(proc.stderr) if value or action is None: # The default is that we error in undefined symbols self.assertContained('undefined symbol: something', proc.stderr) @@ -13469,12 +13470,20 @@ def test_preload_module(self, args): struct stat statbuf; assert(stat("/library.so", &statbuf) == 0); - // Check that it was preloaded + // Check that it was preloaded. + // The preloading actually only happens on the main thread where the filesystem + // lives. On worker threads the module object is shared via preloadedModules. if (emscripten_is_main_runtime_thread()) { int found = EM_ASM_INT( return preloadedWasm['/library.so'] !== undefined; ); assert(found); + } else { + int found = EM_ASM_INT( + err(sharedModules); + return sharedModules['/library.so'] !== undefined; + ); + assert(found); } void *lib_handle = dlopen("/library.so", RTLD_NOW); assert(lib_handle);