From e045661d41e1e6b46624261ec1f3fab79cf167a8 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Tue, 16 May 2023 17:35:03 -0700 Subject: [PATCH] Share side modules with worker threads via postMessage Any side modules that are loaded at the time of worker creation are shared with the worker via postMessage. As a followup we should extend this to modules that are loaded after the worker is created but before the pthread runs (for example when a module is loaded while a worker is unused). --- emcc.py | 3 + src/library_dylink.js | 132 +++++++++++------- src/library_pthread.js | 13 +- src/parseTools.js | 2 +- src/postamble.js | 30 +--- src/preamble.js | 15 +- src/worker.js | 7 +- .../metadce/test_metadce_hello_dylink.jssize | 2 +- .../test_metadce_minimal_pthreads.jssize | 2 +- test/other/test_unoptimized_code_size.js.size | 2 +- ...t_unoptimized_code_size_no_asserts.js.size | 2 +- .../test_unoptimized_code_size_strict.js.size | 2 +- test/test_core.py | 31 ++-- test/test_other.py | 13 +- 14 files changed, 150 insertions(+), 106 deletions(-) diff --git a/emcc.py b/emcc.py index 1e9f7753ec547..71f5a0a4a1357 100755 --- a/emcc.py +++ b/emcc.py @@ -2076,7 +2076,10 @@ def phase_linker_setup(options, state, newargs): assert not settings.SIDE_MODULE if settings.MAIN_MODULE == 1: settings.INCLUDE_FULL_LIBRARY = 1 + # Called from preamble.js once the main module is instantiated. settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$loadDylibs'] + if settings.STACK_OVERFLOW_CHECK == 2: + settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$setDylinkStackLimits'] settings.REQUIRED_EXPORTS += ['malloc'] if settings.MAIN_MODULE == 1 or settings.SIDE_MODULE == 1: diff --git a/src/library_dylink.js b/src/library_dylink.js index 7c1daba178e73..15f07e432a825 100644 --- a/src/library_dylink.js +++ b/src/library_dylink.js @@ -25,7 +25,7 @@ var LibraryDylink = { // than just running the promises in parallel, this makes a chain of // promises to run in series. wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then( - () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true})).then( + () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true}, name)).then( (exports) => { #if DYLINK_DEBUG dbg(`registering preloadedWasm: ${name}`); @@ -66,13 +66,47 @@ var LibraryDylink = { return true; }, + // Dynamic version of shared.py:make_invoke. This is needed for invokes + // that originate from side modules since these are not known at JS + // generation time. +#if !DISABLE_EXCEPTION_CATCHING || SUPPORT_LONGJMP == 'emscripten' + $createInvokeFunction__internal: true, + $createInvokeFunction__deps: ['$dynCall', 'setThrew'], + $createInvokeFunction: function(sig) { + return function() { + var sp = stackSave(); + try { + return dynCall(sig, arguments[0], Array.prototype.slice.call(arguments, 1)); + } catch(e) { + stackRestore(sp); + // Create a try-catch guard that rethrows the Emscripten EH exception. +#if EXCEPTION_STACK_TRACES + // Exceptions thrown from C++ and longjmps will be an instance of + // EmscriptenEH. + if (!(e instanceof EmscriptenEH)) throw e; +#else + // Exceptions thrown from C++ will be a pointer (number) and longjmp + // will throw the number Infinity. Use the compact and fast "e !== e+0" + // test to check if e was not a Number. + if (e !== e+0) throw e; +#endif + _setThrew(1, 0); + } + } + }, +#endif + // Resolve a global symbol by name. This is used during module loading to // resolve imports, and by `dlsym` when used with `RTLD_DEFAULT`. // Returns both the resolved symbol (i.e. a function or a global) along with // the canonical name of the symbol (in some cases is modify the symbol as // part of the loop process, so that actual symbol looked up has a different // name). - $resolveGlobalSymbol__deps: ['$isSymbolDefined'], + $resolveGlobalSymbol__deps: ['$isSymbolDefined', +#if !DISABLE_EXCEPTION_CATCHING || SUPPORT_LONGJMP == 'emscripten' + '$createInvokeFunction', +#endif + ], $resolveGlobalSymbol__internal: true, $resolveGlobalSymbol: function(symName, direct = false) { var sym; @@ -325,7 +359,12 @@ var LibraryDylink = { loadedLibsByName: {}, // handle -> dso; Used by dlsym loadedLibsByHandle: {}, - init: () => newDSO('__main__', {{{ cDefs.RTLD_DEFAULT }}}, wasmImports), + init: () => { +#if ASSERTIONS + assert(wasmImports); +#endif + newDSO('__main__', {{{ cDefs.RTLD_DEFAULT }}}, wasmImports); + }, }, $dlSetError__internal: true, @@ -340,36 +379,6 @@ var LibraryDylink = { }); }, - // Dynamic version of shared.py:make_invoke. This is needed for invokes - // that originate from side modules since these are not known at JS - // generation time. -#if !DISABLE_EXCEPTION_CATCHING || SUPPORT_LONGJMP == 'emscripten' - $createInvokeFunction__internal: true, - $createInvokeFunction__deps: ['$dynCall', 'setThrew'], - $createInvokeFunction: function(sig) { - return function() { - var sp = stackSave(); - try { - return dynCall(sig, arguments[0], Array.prototype.slice.call(arguments, 1)); - } catch(e) { - stackRestore(sp); - // Create a try-catch guard that rethrows the Emscripten EH exception. -#if EXCEPTION_STACK_TRACES - // Exceptions thrown from C++ and longjmps will be an instance of - // EmscriptenEH. - if (!(e instanceof EmscriptenEH)) throw e; -#else - // Exceptions thrown from C++ will be a pointer (number) and longjmp - // will throw the number Infinity. Use the compact and fast "e !== e+0" - // test to check if e was not a Number. - if (e !== e+0) throw e; -#endif - _setThrew(1, 0); - } - } - }, -#endif - // We support some amount of allocation during startup in the case of // dynamic linking, which needs to allocate memory for dynamic libraries that // are loaded. That has to happen before the main program can start to run, @@ -607,6 +616,7 @@ var LibraryDylink = { // promise that resolves to its exports if the loadAsync flag is set. $loadWebAssemblyModule__docs: ` /** + * @param {string=} libName * @param {Object=} localScope * @param {number=} handle */`, @@ -617,11 +627,11 @@ var LibraryDylink = { '$alignMemory', '$zeroMemory', '$currentModuleWeakSymbols', '$alignMemory', '$zeroMemory', '$updateTableMap', -#if !DISABLE_EXCEPTION_CATCHING || SUPPORT_LONGJMP == 'emscripten' - '$createInvokeFunction', -#endif ], - $loadWebAssemblyModule: function(binary, flags, localScope, handle) { + $loadWebAssemblyModule: function(binary, flags, libName, localScope, handle) { +#if DYLINK_DEBUG + dbg(`loadWebAssemblyModule: ${libName}`); +#endif var metadata = getDylinkMetadata(binary); currentModuleWeakSymbols = metadata.weakImports; #if ASSERTIONS @@ -744,10 +754,20 @@ var LibraryDylink = { '{{{ WASI_MODULE_NAME }}}': proxy, }; - function postInstantiation(instance) { + function postInstantiation(module, instance) { #if ASSERTIONS // the table should be unchanged assert(wasmTable === originalTable); +#endif +#if PTHREADS + if (!ENVIRONMENT_IS_PTHREAD && libName) { +#if DYLINK_DEBUG + dbg(`registering sharedModules: ${libName}`) +#endif + // cache all loaded modules in `sharedModules`, which gets passed + // to new workers when they are created. + sharedModules[libName] = module; + } #endif // add new entries to functionsInTableMap updateTableMap(tableBase, metadata.tableSize); @@ -760,12 +780,11 @@ var LibraryDylink = { } #if STACK_OVERFLOW_CHECK >= 2 if (moduleExports['__set_stack_limits']) { -#if PTHREADS // When we are on an uninitialized pthread we delay calling // __set_stack_limits until $setDylinkStackLimits. - if (!ENVIRONMENT_IS_PTHREAD || runtimeInitialized) -#endif - moduleExports['__set_stack_limits']({{{ to64('_emscripten_stack_get_base()') }}}, {{{ to64('_emscripten_stack_get_end()') }}}); + if (runtimeInitialized) { + moduleExports['__set_stack_limits']({{{ to64('_emscripten_stack_get_base()') }}}, {{{ to64('_emscripten_stack_get_end()') }}}); + } } #endif @@ -838,16 +857,16 @@ var LibraryDylink = { if (flags.loadAsync) { if (binary instanceof WebAssembly.Module) { var instance = new WebAssembly.Instance(binary, info); - return Promise.resolve(postInstantiation(instance)); + return Promise.resolve(postInstantiation(binary, instance)); } return WebAssembly.instantiate(binary, info).then( - (result) => postInstantiation(result.instance) + (result) => postInstantiation(result.module, result.instance) ); } var module = binary instanceof WebAssembly.Module ? binary : new WebAssembly.Module(binary); var instance = new WebAssembly.Instance(module, info); - return postInstantiation(instance); + return postInstantiation(module, instance); } // now load needed libraries and the module itself. @@ -863,7 +882,7 @@ var LibraryDylink = { return loadModule(); }, -#if STACK_OVERFLOW_CHECK >= 2 && PTHREADS +#if STACK_OVERFLOW_CHECK >= 2 // With PTHREADS we load libraries before we are running a pthread and // therefore before we have a stack. Instead we delay calling // `__set_stack_limits` until we start running a thread. We also need to call @@ -964,6 +983,16 @@ var LibraryDylink = { // libName -> libData function loadLibData() { +#if PTHREADS + var sharedMod = sharedModules[libName]; +#if DYLINK_DEBUG + dbg(`checking sharedModules: ${libName}: ${sharedMod ? 'found' : 'not found'}`); +#endif + if (sharedMod) { + return flags.loadAsync ? Promise.resolve(sharedMod) : sharedMod; + } +#endif + // for wasm, we can use fetch for async, but for fs mode we can only imitate it if (handle) { var data = {{{ makeGetValue('handle', C_STRUCTS.dso.file_data, '*') }}}; @@ -1003,10 +1032,10 @@ var LibraryDylink = { // module not preloaded - load lib data and create new module from it if (flags.loadAsync) { - return loadLibData().then((libData) => loadWebAssemblyModule(libData, flags, localScope, handle)); + return loadLibData().then((libData) => loadWebAssemblyModule(libData, flags, libName, localScope, handle)); } - return loadWebAssemblyModule(loadLibData(), flags, localScope, handle); + return loadWebAssemblyModule(loadLibData(), flags, libName, localScope, handle); } // module for lib is loaded - update the dso & global namespace @@ -1039,9 +1068,6 @@ var LibraryDylink = { $loadDylibs__internal: true, $loadDylibs__deps: ['$loadDynamicLibrary', '$reportUndefinedSymbols'], $loadDylibs: function() { -#if DYLINK_DEBUG - dbg(`loadDylibs: ${dynamicLibraries}`); -#endif if (!dynamicLibraries.length) { #if DYLINK_DEBUG dbg('loadDylibs: no libraries to preload'); @@ -1050,6 +1076,10 @@ var LibraryDylink = { return; } +#if DYLINK_DEBUG + dbg(`loadDylibs: ${dynamicLibraries}`); +#endif + // Load binaries asynchronously addRunDependency('loadDylibs'); dynamicLibraries.reduce((chain, lib) => { diff --git a/src/library_pthread.js b/src/library_pthread.js index 3ead63051ca96..fa7b254d0cea3 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -105,6 +105,14 @@ var LibraryPThread = { PThread.allocateUnusedWorker(); } #endif +#if !MINIMAL_RUNTIME + // MINIMAL_RUNTIME takes care of calling loadWasmModuleToAllWorkers + // in postamble_minimal.js + addOnPreRun(() => { + addRunDependency('loading-workers') + PThread.loadWasmModuleToAllWorkers(() => removeRunDependency('loading-workers')); + }); +#endif #if MAIN_MODULE PThread.outstandingPromises = {}; // Finished threads are threads that have finished running but we not yet @@ -402,7 +410,9 @@ var LibraryPThread = { 'wasmOffsetConverter': wasmOffsetConverter, #endif #if MAIN_MODULE - 'dynamicLibraries': Module['dynamicLibraries'], + // Shared all modules that have been loaded so far. New workers + // won't start running threads until these are all loaded. + 'sharedModules': sharedModules, #endif #if ASSERTIONS 'workerID': worker.workerID, @@ -423,6 +433,7 @@ var LibraryPThread = { ) { return onMaybeReady(); } + let pthreadPoolReady = Promise.all(PThread.unusedWorkers.map(PThread.loadWasmModuleToWorker)); #if PTHREAD_POOL_DELAY_LOAD // PTHREAD_POOL_DELAY_LOAD means we want to proceed synchronously without diff --git a/src/parseTools.js b/src/parseTools.js index 167767e0f2915..12393c071ae21 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -746,7 +746,7 @@ function makeRemovedModuleAPIAssert(moduleName, localName) { function checkReceiving(name) { // ALL_INCOMING_MODULE_JS_API contains all valid incoming module API symbols // so calling makeModuleReceive* with a symbol not in this list is an error - assert(ALL_INCOMING_MODULE_JS_API.has(name)); + assert(ALL_INCOMING_MODULE_JS_API.has(name), `${name} is not part of INCOMING_MODULE_JS_API`); } // Make code to receive a value on the incoming Module object. diff --git a/src/postamble.js b/src/postamble.js index 8e01e640c3516..d235e9da81d4b 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -131,11 +131,10 @@ function stackCheckInit() { } #endif -#if RELOCATABLE -var dylibsLoaded = false; -#if '$LDSO' in addedLibraryItems -LDSO.init(); -#endif +#if MAIN_MODULE && PTHREADS +// Map of modules to be shared with new threads. This gets populated by the +// main thread an shared with all new workers. +var sharedModules = Module['sharedModules'] ||[]; #endif #if MAIN_READS_PARAMS @@ -158,25 +157,8 @@ function run() { stackCheckInit(); #endif -#if RELOCATABLE - if (!dylibsLoaded) { - // Loading of dynamic libraries needs to happen on each thread, so we can't - // use the normal __ATPRERUN__ mechanism. -#if MAIN_MODULE - loadDylibs(); -#else - reportUndefinedSymbols(); -#endif - dylibsLoaded = true; - - // Loading dylibs can add run dependencies. - if (runDependencies > 0) { -#if RUNTIME_DEBUG - dbg('loadDylibs added run() dependencies, not running yet'); -#endif - return; - } - } +#if RELOCATABLE && !MAIN_MODULE + reportUndefinedSymbols(); #endif #if WASM_WORKERS diff --git a/src/preamble.js b/src/preamble.js index 70097ae03387c..0104eb60faa9f 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -234,9 +234,12 @@ function initRuntime() { #if STACK_OVERFLOW_CHECK >= 2 #if RUNTIME_DEBUG - dbg('__set_stack_limits: ' + _emscripten_stack_get_base() + ', ' + _emscripten_stack_get_end()); + dbg(`__set_stack_limits: ${ptrToString(_emscripten_stack_get_base())}, ${ptrToString(_emscripten_stack_get_end())}`); #endif ___set_stack_limits(_emscripten_stack_get_base(), _emscripten_stack_get_end()); +#if MAIN_MODULE + setDylinkStackLimits(_emscripten_stack_get_base(), _emscripten_stack_get_end()); +#endif #endif #if RELOCATABLE callRuntimeCallbacks(__RELOC_FUNCS__); @@ -980,6 +983,10 @@ function createWasm() { } #endif mergeLibSymbols(exports, 'main') +#if '$LDSO' in addedLibraryItems + LDSO.init(); +#endif + loadDylibs(); #endif #if MEMORY64 @@ -1047,13 +1054,7 @@ function createWasm() { // We now have the Wasm module loaded up, keep a reference to the compiled module so we can post it to the workers. wasmModule = module; #endif - -#if PTHREADS - PThread.loadWasmModuleToAllWorkers(() => removeRunDependency('wasm-instantiate')); -#else // singlethreaded build: removeRunDependency('wasm-instantiate'); -#endif // ~PTHREADS - return exports; } // wait for the pthread pool (if any) diff --git a/src/worker.js b/src/worker.js index 4c8fe3cebc4f7..14b78fde7a191 100644 --- a/src/worker.js +++ b/src/worker.js @@ -81,7 +81,7 @@ var out = () => { throw 'out() is not defined in worker.js.'; } #endif var err = threadPrintErr; self.alert = threadAlert; -#if RUNTIME_DEBUG +#if ASSERTIONS || RUNTIME_DEBUG var dbg = threadPrintErr; #endif @@ -151,7 +151,10 @@ function handleMessage(e) { #endif // MINIMAL_RUNTIME #if MAIN_MODULE - Module['dynamicLibraries'] = e.data.dynamicLibraries; + Module['sharedModules'] = e.data.sharedModules; +#if RUNTIME_DEBUG + dbg(`received ${Object.keys(e.data.sharedModules).length} shared modules: ${Object.keys(e.data.sharedModules)}`); +#endif #endif // Use `const` here to ensure that the variable is scoped only to diff --git a/test/other/metadce/test_metadce_hello_dylink.jssize b/test/other/metadce/test_metadce_hello_dylink.jssize index 5da2a4f7c562e..62b62ad8bcd45 100644 --- a/test/other/metadce/test_metadce_hello_dylink.jssize +++ b/test/other/metadce/test_metadce_hello_dylink.jssize @@ -1 +1 @@ -15060 +15032 diff --git a/test/other/metadce/test_metadce_minimal_pthreads.jssize b/test/other/metadce/test_metadce_minimal_pthreads.jssize index 13dac5d04b508..51cd2147d6059 100644 --- a/test/other/metadce/test_metadce_minimal_pthreads.jssize +++ b/test/other/metadce/test_metadce_minimal_pthreads.jssize @@ -1 +1 @@ -15415 +15456 diff --git a/test/other/test_unoptimized_code_size.js.size b/test/other/test_unoptimized_code_size.js.size index cc84a949cd741..068c130d5b5bd 100644 --- a/test/other/test_unoptimized_code_size.js.size +++ b/test/other/test_unoptimized_code_size.js.size @@ -1 +1 @@ -59630 +59629 diff --git a/test/other/test_unoptimized_code_size_no_asserts.js.size b/test/other/test_unoptimized_code_size_no_asserts.js.size index 5ff8d77a15fd1..2a7a3ea68f817 100644 --- a/test/other/test_unoptimized_code_size_no_asserts.js.size +++ b/test/other/test_unoptimized_code_size_no_asserts.js.size @@ -1 +1 @@ -33256 +33255 diff --git a/test/other/test_unoptimized_code_size_strict.js.size b/test/other/test_unoptimized_code_size_strict.js.size index 7b6c1d7f7e426..bf3a0db19d3b5 100644 --- a/test/other/test_unoptimized_code_size_strict.js.size +++ b/test/other/test_unoptimized_code_size_strict.js.size @@ -1 +1 @@ -58572 +58571 diff --git a/test/test_core.py b/test/test_core.py index 7abd9ccf7653b..218d175fa4c34 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -9474,34 +9474,39 @@ def test_pthread_dylink_main_module_1(self): self.do_runf(test_file('hello_world.c')) @needs_dylink - @node_pthreads - def test_Module_dynamicLibraries_pthreads(self): + @parameterized({ + '': ([],), + 'pthreads': (['-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME', '-pthread', '-Wno-experimental'],) + }) + def test_Module_dynamicLibraries(self, args): # test that Module.dynamicLibraries works with pthreads - self.emcc_args += ['-pthread', '-Wno-experimental'] + self.emcc_args += args self.emcc_args += ['--pre-js', 'pre.js'] - self.set_setting('PROXY_TO_PTHREAD') - self.set_setting('EXIT_RUNTIME') # This test is for setting dynamicLibraries at runtime so we don't # want emscripten loading `liblib.so` automatically (which it would # do without this setting. self.set_setting('NO_AUTOLOAD_DYLIBS') create_file('pre.js', ''' - if (typeof importScripts == 'undefined') { // !ENVIRONMENT_IS_WORKER - // Load liblib.so by default on non-workers - Module['dynamicLibraries'] = ['liblib.so']; - } else { - // Verify whether the main thread passes Module.dynamicLibraries to the worker - assert(Module['dynamicLibraries'].includes('liblib.so')); - } + Module['dynamicLibraries'] = ['liblib.so']; ''') + if args: + self.setup_node_pthreads() + create_file('post.js', ''' + if (ENVIRONMENT_IS_PTHREAD) { + err('sharedModules: ' + Object.keys(sharedModules)); + assert('liblib.so' in sharedModules); + } + ''') + self.emcc_args += ['--post-js', 'post.js'] + self.dylink_test( r''' #include int side(); int main() { - printf("result is %d", side()); + printf("result is %d\n", side()); return 0; } ''', diff --git a/test/test_other.py b/test/test_other.py index 3cbe043529f50..640ff3a11fdf5 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -2331,7 +2331,8 @@ def test_undefined_symbols(self, action): print('checking "%s" %s' % (args, value)) extra = ['-s', action + '_ON_UNDEFINED_SYMBOLS=%d' % value] if action else [] proc = self.run_process([EMXX, '-sUSE_SDL', 'main.cpp'] + extra + args, stderr=PIPE, check=False) - print(proc.stderr) + if common.EMTEST_VERBOSE: + print(proc.stderr) if value or action is None: # The default is that we error in undefined symbols self.assertContained('undefined symbol: something', proc.stderr) @@ -13470,12 +13471,20 @@ def test_preload_module(self, args): struct stat statbuf; assert(stat("/library.so", &statbuf) == 0); - // Check that it was preloaded + // Check that it was preloaded. + // The preloading actually only happens on the main thread where the filesystem + // lives. On worker threads the module object is shared via preloadedModules. if (emscripten_is_main_runtime_thread()) { int found = EM_ASM_INT( return preloadedWasm['/library.so'] !== undefined; ); assert(found); + } else { + int found = EM_ASM_INT( + err(sharedModules); + return sharedModules['/library.so'] !== undefined; + ); + assert(found); } void *lib_handle = dlopen("/library.so", RTLD_NOW); assert(lib_handle);