Fix for loading wasm files under Node.js and in browser when files we…

ChangeLog.markdown

@@ -10,6 +10,10 @@ Not all changes are documented here. In particular, new features, user-oriented
 Current Trunk
 -------------
 
+ - Fix `Module.locateFile` to resolve relative paths to *.wasm, *.mem and other files relatively to JavaScript file rather than current working directory
+ - Add second argument `prefix` to `Module.locateFile` function that contains path to JavaScript file where files are loaded from by default
+ - Remove `Module.*PrefixURL` APIs (use `Module.locateFile` instead)
+
 v1.38.8: 07/06/2018
 -------------------
  - Fix a regression in 1.38.7 with binaryen no longer bundling binaryen.js (which emscripten doesn't need, that's just for handwritten JS users, but emscripten did check for its prescence).

emcc.py

@@ -2598,7 +2598,10 @@ def modularize():
 %(src)s
 
   return %(EXPORT_NAME)s;
-}%(instantiate)s;
+};
+%(EXPORT_NAME)s = %(EXPORT_NAME)s.bind({
+  _currentScript: typeof document !== 'undefined' ? document.currentScript : undefined
+})%(instantiate)s;
 ''' % {
     'EXPORT_NAME': shared.Settings.EXPORT_NAME,
     'src': src,
@@ -2704,11 +2707,7 @@ def generate_html(target, options, js_target, target_basename,
       script.un_src()
       script.inline = ('''
           var memoryInitializer = '%s';
-          if (typeof Module['locateFile'] === 'function') {
-            memoryInitializer = Module['locateFile'](memoryInitializer);
-          } else if (Module['memoryInitializerPrefixURL']) {
-            memoryInitializer = Module['memoryInitializerPrefixURL'] + memoryInitializer;
-          }
+          memoryInitializer = Module['locateFile'] ? Module['locateFile'](memoryInitializer, '') : memoryInitializer;
           Module['memoryInitializerRequestURL'] = memoryInitializer;
           var meminitXHR = Module['memoryInitializerRequest'] = new XMLHttpRequest();
           meminitXHR.open('GET', memoryInitializer, true);

site/source/docs/api_reference/module.rst

@@ -51,15 +51,9 @@ The following ``Module`` attributes affect code execution. Set them to customize
 
 	The commandline arguments. The value of ``arguments`` contains the values returned if compiled code checks ``argc`` and ``argv``.
 
-
-.. js:attribute:: Module.filePackagePrefixURL
-
-	This is the "prefix" URL for a preloaded data file that is hosted separately from its JavaScript and HTML files (it includes the full path up to, but not including, the data file). See :ref:`packaging-files-data-file-location` for more information.
-
-
 .. js:attribute:: Module.locateFile
 
-	If set, this method will be called when the runtime needs to load a file, such as a ``.wasm`` WebAssembly file, ``.mem`` memory init file, or a file generated by the file packager. The function receives the URL, and should return the actual URL. This lets you host file packages or the ``.mem`` file etc. on a different location than the current directory (which is the default expectation), for example if you want to host them on a CDN. Note that ``locateFile`` is sort of a generalization of ``Module.filePackagePrefixURL``.
+	If set, this method will be called when the runtime needs to load a file, such as a ``.wasm`` WebAssembly file, ``.mem`` memory init file, or a file generated by the file packager. The function receives the relative path to the file as configured in build process and a ``prefix`` (path to JavaScript file), and should return the actual URL. This lets you host file packages or the ``.mem`` file etc. on a different location than the directory of JavaScript file (which is the default expectation), for example if you want to host them on a CDN. NOTE: ``prefix`` might be empty string for memory initializer file is loaded in parallel with JS or in packager, so be careful with those.
 
 .. js:attribute:: Module.logReadFiles
 

site/source/docs/porting/files/packaging_files.rst

@@ -71,7 +71,7 @@ Changing the data file location
 
 By default, the **.data** file containing all the preloaded files is loaded from the same URL as your **.js** file. In some cases it may be useful to have the data file in a different location from the other files — for example if your **.html** and **.js** change a lot you may want to keep the data file on a fast CDN somewhere else.
 
-This model is supported by changing the :js:attr:`Module.filePackagePrefixURL` to be the URL where the data file is stored (this is a prefix, so should include the full path before the data's file name). The attribute must be specified in a ``<script>`` element before the one that loads the data file.
+This model is supported by specifying :js:attr:`Module.locateFile` function to return URL where the data file is stored. The function must be specified in a ``<script>`` element before the one that loads the data file.
 
 
 .. _packaging-files-packaged-file-location:

site/source/docs/porting/pthreads.rst

@@ -46,7 +46,7 @@ The Emscripten implementation for the pthreads API should follow the POSIX stand
 
 - Note that the function emscripten_num_logical_cores() will always return the value of navigator.hardwareConcurrency, i.e. the number of logical cores on the system, even when shared memory is not supported. This means that it is possible for emscripten_num_logical_cores() to return a value greater than 1, while at the same time emscripten_has_threading_support() can return false. The return value of emscripten_has_threading_support() denotes whether the browser has shared memory support available.
 
-Also note that when compiling code that uses pthreads, an additional JavaScript file `pthread-main.js` is generated alongside the output .js file. That file must be deployed with the rest of the generated code files. By default, `pthread-main.js` will be loaded relative to the main HTML page URL. If it is desirable to load the file from a different location e.g. in a CDN environment, then one can define the `Module.locateFile(filename)` function in the main HTML `Module` object to return the URL of the target location of the `pthread-main.js` entry point. If this function is not defined in `Module`, then the relative location specified by `Module.pthreadMainPrefixURL + '/pthread-main.js'` will be used instead. If this is prefix URL is not specified either, then the default location relative to the main HTML file is used.
+Also note that when compiling code that uses pthreads, an additional JavaScript file `pthread-main.js` is generated alongside the output .js file. That file must be deployed with the rest of the generated code files. By default, `pthread-main.js` will be loaded relative to the main HTML page URL. If it is desirable to load the file from a different location e.g. in a CDN environment, then one can define the `Module.locateFile(filename)` function in the main HTML `Module` object to return the URL of the target location of the `pthread-main.js` entry point. If this function is not defined in `Module`, then the default location relative to the main HTML file is used.
 
 Running code and tests
 ======================

src/Fetch.js

@@ -111,10 +111,9 @@ var Fetch = {
 
       var fetchJs = 'fetch-worker.js';
       // Allow HTML module to configure the location where the 'pthread-main.js' file will be loaded from,
-      // either via Module.locateFile() function, or via Module.pthreadMainPrefixURL string. If neither
-      // of these are passed, then the default URL 'pthread-main.js' relative to the main html file is loaded.
-      if (typeof Module['locateFile'] === 'function') fetchJs = Module['locateFile'](fetchJs);
-      else if (Module['pthreadMainPrefixURL']) fetchJs = Module['pthreadMainPrefixURL'] + fetchJs;
+      // via Module.locateFile() function. If not specified, then the default URL 'pthread-main.js' relative
+      // to the main html file is loaded.
+      fetchJs = locateFile(fetchJs);
       Fetch.worker = new Worker(fetchJs);
       Fetch.worker.onmessage = function(e) {
         out('fetch-worker sent a message: ' + e.filename + ':' + e.lineno + ': ' + e.message);

src/library_debugger_toolkit.js

@@ -445,12 +445,7 @@ var CyberDWARFHeapPrinter = function(cdFileLocation) {
   }
 
   function initialize_debugger(cb) {
-    var cdFile;
-    if (typeof Module['locateFile'] === 'function') {
-      cdFileLocation = Module['locateFile'](cdFileLocation);
-    } else if (Module['cdInitializerPrefixURL']) {
-      cdFileLocation = Module['cdInitializerPrefixURL'] + cdFileLocation;
-    }
+    cdFileLocation = locateFile(cdFileLocation);
     if (ENVIRONMENT_IS_NODE || ENVIRONMENT_IS_SHELL) {
       var data = Module['read'](cdFileLocation);
       install_cyberdwarf(data);

src/library_pthread.js

@@ -254,10 +254,9 @@ var LibraryPThread = {
       var numWorkersLoaded = 0;
       var pthreadMainJs = 'pthread-main.js';
       // Allow HTML module to configure the location where the 'pthread-main.js' file will be loaded from,
-      // either via Module.locateFile() function, or via Module.pthreadMainPrefixURL string. If neither
-      // of these are passed, then the default URL 'pthread-main.js' relative to the main html file is loaded.
-      if (typeof Module['locateFile'] === 'function') pthreadMainJs = Module['locateFile'](pthreadMainJs);
-      else if (Module['pthreadMainPrefixURL']) pthreadMainJs = Module['pthreadMainPrefixURL'] + pthreadMainJs;
+      // via Module.locateFile() function. If not specified, then the default URL 'pthread-main.js' relative
+      // to the main html file is loaded.
+      pthreadMainJs = locateFile(pthreadMainJs);
 
       for (var i = 0; i < numWorkers; ++i) {
         var worker = new Worker(pthreadMainJs);

src/postamble.js

@@ -40,11 +40,7 @@ if (memoryInitializer && !ENVIRONMENT_IS_PTHREAD) {
 if (memoryInitializer) {
 #endif
   if (!isDataURI(memoryInitializer)) {
-    if (typeof Module['locateFile'] === 'function') {
-      memoryInitializer = Module['locateFile'](memoryInitializer);
-    } else if (Module['memoryInitializerPrefixURL']) {
-      memoryInitializer = Module['memoryInitializerPrefixURL'] + memoryInitializer;
-    }
+    memoryInitializer = locateFile(memoryInitializer);
   }
   if (ENVIRONMENT_IS_NODE || ENVIRONMENT_IS_SHELL) {
     var data = Module['readBinary'](memoryInitializer);
@@ -88,8 +84,8 @@ if (memoryInitializer) {
             response = data.buffer;
           } else {
 #endif
-            // If you see this warning, the issue may be that you are using locateFile or memoryInitializerPrefixURL, and defining them in JS. That
-            // means that the HTML file doesn't know about them, and when it tries to create the mem init request early, does it to the wrong place.
+            // If you see this warning, the issue may be that you are using locateFile and defining it in JS. That
+            // means that the HTML file doesn't know about it, and when it tries to create the mem init request early, does it to the wrong place.
             // Look in your browser's devtools network console to see what's going on.
             console.warn('a problem seems to have happened with Module.memoryInitializerRequest, status: ' + request.status + ', retrying ' + memoryInitializer);
             doBrowserLoad();

src/preamble.js

@@ -2020,16 +2020,14 @@ function integrateWasmJS() {
   var wasmBinaryFile = '{{{ WASM_BINARY_FILE }}}';
   var asmjsCodeFile = '{{{ ASMJS_CODE_FILE }}}';
 
-  if (typeof Module['locateFile'] === 'function') {
   if (!isDataURI(wasmTextFile)) {
-      wasmTextFile = Module['locateFile'](wasmTextFile);
+    wasmTextFile = locateFile(wasmTextFile);
   }
   if (!isDataURI(wasmBinaryFile)) {
-      wasmBinaryFile = Module['locateFile'](wasmBinaryFile);
+    wasmBinaryFile = locateFile(wasmBinaryFile);
   }
   if (!isDataURI(asmjsCodeFile)) {
-      asmjsCodeFile = Module['locateFile'](asmjsCodeFile);
-    }
+    asmjsCodeFile = locateFile(asmjsCodeFile);
   }
 
   // utilities

src/shell.js

@@ -83,9 +83,26 @@ if (!ENVIRONMENT_IS_PTHREAD) PthreadWorkerInit = {};
 var currentScriptUrl = (typeof document !== 'undefined' && document.currentScript) ? document.currentScript.src : undefined;
 #endif
 
+#if ASSERTIONS
+assert(typeof Module['memoryInitializerPrefixURL'] === 'undefined', 'Module.memoryInitializerPrefixURL option was removed, use Module.locateFile instead');
+assert(typeof Module['pthreadMainPrefixURL'] === 'undefined', 'Module.pthreadMainPrefixURL option was removed, use Module.locateFile instead');
+assert(typeof Module['cdInitializerPrefixURL'] === 'undefined', 'Module.cdInitializerPrefixURL option was removed, use Module.locateFile instead');
+assert(typeof Module['filePackagePrefixURL'] === 'undefined', 'Module.filePackagePrefixURL option was removed, use Module.locateFile instead');
+#endif
+
+// `/` should be present at the end if `scriptDirectory` is not empty
+var scriptDirectory = '';
+function locateFile(path) {
+  if (Module['locateFile']) {
+    return Module['locateFile'](path, scriptDirectory);
+  } else {
+    return scriptDirectory + path;
+  }
+}
+
 #if ENVIRONMENT_MAY_BE_NODE
 if (ENVIRONMENT_IS_NODE) {
-
+  scriptDirectory = __dirname + '/';
 #if ENVIRONMENT
 #if ASSERTIONS
   if (!(typeof process === 'object' && typeof require === 'function')) throw new Error('not compiled for this environment (did you build to HTML and try to run it not on the web, or set ENVIRONMENT to something - like node - and run it someplace else - like on the web?)');
@@ -212,6 +229,15 @@ if (ENVIRONMENT_IS_SHELL) {
 #endif // ENVIRONMENT_MAY_BE_SHELL
 #if ENVIRONMENT_MAY_BE_WEB_OR_WORKER
 if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) {
+  if (ENVIRONMENT_IS_WEB) {
+    // We do this for `-s MODULARIZE=1` where at the time when this code runs document.currentScript might already be unavailable
+    var currentScript = this['_currentScript'] || document.currentScript;
+    if (currentScript.src.indexOf('blob:') !== 0) {
+      scriptDirectory = currentScript.src.split('/').slice(0, -1).join('/') + '/';
+    }
+  } else if (ENVIRONMENT_IS_WORKER) {
+    scriptDirectory = self.location.href.split('/').slice(0, -1).join('/') + '/';
+  }
 
 #if ENVIRONMENT
 #if ASSERTIONS

tests/test_browser.py

@@ -500,7 +500,7 @@ def test_custom_file_package_url(self):
     open(os.path.join(self.get_dir(), 'subdirr', 'data1.txt'), 'w').write('''1214141516171819''')
     # change the file package base dir to look in a "cdn". note that normally you would add this in your own custom html file etc., and not by
     # modifying the existing shell in this manner
-    open(self.in_dir('shell.html'), 'w').write(open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { filePackagePrefixURL: "cdn/", '))
+    open(self.in_dir('shell.html'), 'w').write(open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { locateFile: function (path, prefix) {if (path.endsWith(".wasm")) {return prefix + path;} else {return "cdn/" + path;}}, '))
     open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(self.with_report_result(r'''
       #include <stdio.h>
       #include <string.h>
@@ -555,7 +555,7 @@ def setup(assetLocalization):
               setTimeout(function() { window.close() }, 1000);
             }
             var Module = {
-              filePackagePrefixURL: "''' + assetLocalization + r'''",
+              locateFile: function (path, prefix) {if (path.endsWith(".wasm")) {return prefix + path;} else {return "''' + assetLocalization + r'''" + path;}},
               print: (function() {
                 var element = document.getElementById('output');
                 return function(text) { element.innerHTML += text.replace('\n', '<br>', 'g') + '<br>';};
@@ -588,8 +588,8 @@ def test():
     test()
 
 
-    # TODO: CORS, test using a full url for filePackagePrefixURL
-    #open(self.in_dir('shell.html'), 'w').write(open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { filePackagePrefixURL: "http:/localhost:8888/cdn/", '))
+    # TODO: CORS, test using a full url for locateFile
+    #open(self.in_dir('shell.html'), 'w').write(open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { locateFile: function (path) {return "http:/localhost:8888/cdn/" + path;}, '))
     #test()
 
   def test_sdl_swsurface(self):
@@ -3459,8 +3459,8 @@ def test_pthread_custom_pthread_main_url(self):
       }
     '''))
 
-    # Test that it is possible to define "Module.pthreadMainPrefixURL" string to locate where pthread-main.js will be loaded from.
-    open(self.in_dir('shell.html'), 'w').write(open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { pthreadMainPrefixURL: "cdn/", '))
+    # Test that it is possible to define "Module.locateFile" string to locate where pthread-main.js will be loaded from.
+    open(self.in_dir('shell.html'), 'w').write(open(path_from_root('src', 'shell.html')).read().replace('var Module = {', 'var Module = { locateFile: function (path, prefix) {if (path.endsWith(".wasm")) {return prefix + path;} else {return "cdn/" + path;}}, '))
     Popen([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--shell-file', 'shell.html', '-s', 'WASM=0', '-s', 'IN_TEST_HARNESS=1', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=1', '-o', 'test.html']).communicate()
     shutil.move('pthread-main.js', os.path.join('cdn', 'pthread-main.js'))
     self.run_browser('test.html', '', '/report_result?1')
@@ -3936,3 +3936,39 @@ def test_unicode_html_shell(self):
     open(self.in_dir('shell.html'), 'w').write(open(path_from_root('src', 'shell.html')).read().replace('Emscripten-Generated Code', 'Emscripten-Generated Emoji 😅'))
     subprocess.check_output([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--shell-file', 'shell.html', '-o', 'test.html'])
     self.run_browser('test.html', None, '/report_result?0')
+
+  # Tests that Emscripten-compiled applications can be run from a relative path in browser that is different than the address of the current page
+  def test_browser_run_from_different_directory(self):
+    src = open(path_from_root('tests', 'browser_test_hello_world.c')).read()
+    open('test.c', 'w').write(self.with_report_result(src))
+    Popen([PYTHON, EMCC, 'test.c', '-o', 'test.html', '-O3']).communicate()
+
+    if not os.path.exists('subdir'):
+      os.mkdir('subdir')
+    shutil.move('test.js', os.path.join('subdir', 'test.js'))
+    shutil.move('test.wasm', os.path.join('subdir', 'test.wasm'))
+    src = open('test.html').read()
+    # Make sure JS is loaded from subdirectory
+    open('test-subdir.html', 'w').write(src.replace('test.js', 'subdir/test.js'))
+
+    self.run_browser('test-subdir.html', None, '/report_result?0')
+
+  # Similar to `test_browser_run_from_different_directory`, but asynchronous because of `-s MODULARIZE=1`
+  def test_browser_run_from_different_directory_async(self):
+    src = open(path_from_root('tests', 'browser_test_hello_world.c')).read()
+    open('test.c', 'w').write(self.with_report_result(src))
+    # compile the code with the modularize feature and the preload-file option enabled
+    Popen([PYTHON, EMCC, 'test.c', '-o', 'test.js', '-s', 'MODULARIZE=1', '-O3']).communicate()
+    if not os.path.exists('subdir'):
+      os.mkdir('subdir')
+    shutil.move('test.js', os.path.join('subdir', 'test.js'))
+    shutil.move('test.wasm', os.path.join('subdir', 'test.wasm'))
+    # Make sure JS is loaded from subdirectory
+    open('test-subdir.html', 'w').write('''
+      <script src="subdir/test.js"></script>
+      <script>
+        Module();
+      </script>
+    ''')
+
+    self.run_browser('test-subdir.html', None, '/report_result?0')

tests/test_other.py

@@ -5102,7 +5102,7 @@ def do(name, moar_opts):
       do('normal', [])
       do('no_nuthin', ['-s', 'EXPORTED_RUNTIME_METHODS=[]'])
       print('  ', sizes)
-      assert abs(sizes['no_nuthin'] - sizes['normal']) < 10
+      assert abs(sizes['no_nuthin'] - sizes['normal']) < 15
       assert sizes['no_nuthin'] < absolute
     test(['-s', 'ASSERTIONS=0'], 95000) # we don't care about code size with assertions
     test(['-O1'], 80000)
@@ -8526,3 +8526,11 @@ def test_html_preprocess(self):
 T4:NO_EXIT_RUNTIME > 1
 T5:NO_EXIT_RUNTIME
 T6:(else) !NO_EXIT_RUNTIME""", output)
+
+  # Tests that Emscripten-compiled applications can be run from a relative path with node command line that is different than the current working directory.
+  def test_node_js_run_from_different_directory(self):
+      if not os.path.exists('subdir'):
+          os.mkdir('subdir')
+      run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'), '-o', os.path.join('subdir', 'a.js'), '-O3'])
+      ret = run_process(NODE_JS + [os.path.join('subdir', 'a.js')], stdout=PIPE).stdout
+      assert 'hello, world!' in ret