Add suppport for external ports

test/other/ports/external.py

@@ -0,0 +1,62 @@
+# Copyright 2024 The Emscripten Authors.  All rights reserved.
+# Emscripten is available under two separate licenses, the MIT license and the
+# University of Illinois/NCSA Open Source License.  Both these licenses can be
+# found in the LICENSE file.
+
+import os
+from typing import Dict, Optional
+
+OPTIONS = {
+  'value1': 'Value for define TEST_VALUE_1',
+  'value2': 'Value for define TEST_VALUE_2',
+  'dependency': 'A dependency'
+}
+
+# user options (from --use-port)
+opts: Dict[str, Optional[str]] = {
+  'value1': None,
+  'value2': None,
+  'dependency': None
+}
+
+deps = []
+
+
+def get_lib_name(settings):
+  return 'lib_external.a'
+
+
+def get(ports, settings, shared):
+  # for simplicity in testing, the source is in the same folder as the port and not fetched as a tarball
+  source_dir = os.path.dirname(os.path.abspath(__file__))
+
+  def create(final):
+    ports.install_headers(source_dir)
+    print(f'about to build {source_dir}')
+    ports.build_port(source_dir, final, 'external')
+
+  return [shared.cache.get_lib(get_lib_name(settings), create, what='port')]
+
+
+def clear(ports, settings, shared):
+  shared.cache.erase_lib(get_lib_name(settings))
+
+
+def process_args(ports):
+  args = ['-isystem', ports.get_include_dir('external')]
+  if opts['value1']:
+    args.append(f'-DTEST_VALUE_1={opts["value1"]}')
+  if opts['value2']:
+    args.append(f'-DTEST_VALUE_2={opts["value2"]}')
+  if opts['dependency']:
+    args.append(f'-DTEST_DEPENDENCY_{opts["dependency"].upper()}')
+  return args
+
+
+def process_dependencies(settings):
+  if opts['dependency']:
+    deps.append(opts['dependency'])
+
+
+def handle_options(options):
+  opts.update(options)

test/other/ports/my_port.c

@@ -0,0 +1,3 @@
+int my_port_fn(int value) {
+  return value;
+}

test/other/ports/my_port.h

@@ -0,0 +1 @@
+int my_port_fn(int);

test/other/ports/simple.py

@@ -0,0 +1,30 @@
+# Copyright 2024 The Emscripten Authors.  All rights reserved.
+# Emscripten is available under two separate licenses, the MIT license and the
+# University of Illinois/NCSA Open Source License.  Both these licenses can be
+# found in the LICENSE file.
+
+import os
+
+
+def get_lib_name(settings):
+  return 'lib_simple.a'
+
+
+def get(ports, settings, shared):
+  # for simplicity in testing, the source is in the same folder as the port and not fetched as a tarball
+  source_dir = os.path.dirname(os.path.abspath(__file__))
+
+  def create(final):
+    ports.install_headers(source_dir)
+    ports.build_port(source_dir, final, 'simple')
+
+  return [shared.cache.get_lib(get_lib_name(settings), create, what='port')]
+
+
+def clear(ports, settings, shared):
+  shared.cache.erase_lib(get_lib_name(settings))
+
+
+def process_args(ports):
+  args = ['-isystem', ports.get_include_dir('simple')]
+  return args

test/other/test_external_ports.c

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2024 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+#include <my_port.h>
+#include <assert.h>
+#include <stdio.h>
+
+#ifdef TEST_DEPENDENCY_SDL2
+#include <SDL2/SDL.h>
+#endif
+
+// TEST_VALUE_1 and TEST_VALUE_2 are defined via port options
+#ifndef TEST_VALUE_1
+#define TEST_VALUE_1 0
+#endif
+#ifndef TEST_VALUE_2
+#define TEST_VALUE_2 0
+#endif
+
+int main() {
+  assert(my_port_fn(99) == 99); // check that we can call a function from my_port.h
+  printf("value1=%d&value2=%d\n", TEST_VALUE_1, TEST_VALUE_2);
+#ifdef TEST_DEPENDENCY_SDL2
+  SDL_version version;
+  SDL_VERSION(&version);
+  printf("sdl2=%d\n", version.major);
+#endif
+  return 0;
+}

test/other/test_external_ports_simple.c

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2024 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+#include <my_port.h>
+#include <assert.h>
+
+int main() {
+  assert(my_port_fn(99) == 99); // check that we can call a function from my_port.h
+  return 0;
+}

test/test_other.py

@@ -2393,6 +2393,40 @@ def test_contrib_ports(self):
     # with a different contrib port when there is another one
     self.emcc(test_file('other/test_contrib_ports.cpp'), ['--use-port=contrib.glfw3'])
 
+  @crossplatform
+  def test_external_ports_simple(self):
+    if config.FROZEN_CACHE:
+      self.skipTest("test doesn't work with frozen cache")
+    simple_port_path = test_file("other/ports/simple.py")
+    self.emcc(test_file('other/test_external_ports_simple.c'), [f'--use-port={simple_port_path}'], output_filename='a.out.js')
+    self.run_js('a.out.js')
+
+  @crossplatform
+  def test_external_ports(self):
+    if config.FROZEN_CACHE:
+      self.skipTest("test doesn't work with frozen cache")
+    external_port_path = test_file("other/ports/external.py")
+    # testing no option
+    self.emcc(test_file('other/test_external_ports.c'), [f'--use-port={external_port_path}'], output_filename='a0.out.js')
+    output = self.run_js('a0.out.js')
+    self.assertContained('value1=0&value2=0\n', output)
+    # testing 1 option
+    self.emcc(test_file('other/test_external_ports.c'), [f'--use-port={external_port_path}:value1=12'], output_filename='a1.out.js')
+    output = self.run_js('a1.out.js')
+    self.assertContained('value1=12&value2=0\n', output)
+    self.emcc(test_file('other/test_external_ports.c'), [f'--use-port={external_port_path}:value1=12:value2=36'], output_filename='a2.out.js')
+    # testing 2 options
+    output = self.run_js('a2.out.js')
+    self.assertContained('value1=12&value2=36\n', output)
+    # testing dependency
+    self.emcc(test_file('other/test_external_ports.c'), [f'--use-port={external_port_path}:dependency=sdl2'], output_filename='a3.out.js')
+    output = self.run_js('a3.out.js')
+    self.assertContained('sdl2=2\n', output)
+    # testing invalid dependency
+    stderr = self.expect_fail([EMCC, test_file('other/test_external_ports.c'), f'--use-port={external_port_path}:dependency=invalid', '-o', 'a4.out.js'])
+    self.assertFalse(os.path.exists('a4.out.js'))
+    self.assertContained('Unknown dependency invalid for port external', stderr)
+
   def test_link_memcpy(self):
     # memcpy can show up *after* optimizations, so after our opportunity to link in libc, so it must be special-cased
     create_file('main.c', r'''

tools/ports/__init__.py

@@ -8,6 +8,8 @@
 import os
 import shutil
 import glob
+import importlib.util
+import sys
 from typing import Set
 from tools import cache
 from tools import config
@@ -33,8 +35,7 @@
 logger = logging.getLogger('ports')
 
 
-def load_port(name):
-  port = __import__(name, globals(), level=1, fromlist=[None])
+def init_port(name, port):
   ports.append(port)
   port.is_contrib = name.startswith('contrib.')
   port.name = name
@@ -63,6 +64,26 @@ def load_port(name):
       utils.exit_with_error('duplicate port variant: %s' % variant)
     port_variants[variant] = (port.name, extra_settings)
 
+  validate_port(port)
+
+
+def load_port_by_name(name):
+  port = __import__(name, globals(), level=1, fromlist=[None])
+  init_port(name, port)
+
+
+def load_port_by_path(path):
+  name = os.path.splitext(os.path.basename(path))[0]
+  if name in ports_by_name:
+    utils.exit_with_error(f'port path [{path}] is invalid: duplicate port name {name}')
+  module_name = f'tools.ports.{name}'
+  spec = importlib.util.spec_from_file_location(module_name, path)
+  port = importlib.util.module_from_spec(spec)
+  sys.modules[module_name] = port
+  spec.loader.exec_module(port)
+  init_port(name, port)
+  return name
+
 
 def validate_port(port):
   expected_attrs = ['get', 'clear', 'show']
@@ -74,30 +95,20 @@ def validate_port(port):
     assert hasattr(port, a), 'port %s is missing %s' % (port, a)
 
 
-def validate_ports():
-  for port in ports:
-    validate_port(port)
-    for dep in port.deps:
-      if dep not in ports_by_name:
-        utils.exit_with_error('unknown dependency in port: %s' % dep)
-
-
 @ToolchainProfiler.profile()
 def read_ports():
   for filename in os.listdir(ports_dir):
     if not filename.endswith('.py') or filename == '__init__.py':
       continue
     filename = os.path.splitext(filename)[0]
-    load_port(filename)
+    load_port_by_name(filename)
 
   contrib_dir = os.path.join(ports_dir, 'contrib')
   for filename in os.listdir(contrib_dir):
     if not filename.endswith('.py') or filename == '__init__.py':
       continue
     filename = os.path.splitext(filename)[0]
-    load_port('contrib.' + filename)
-
-  validate_ports()
+    load_port_by_name('contrib.' + filename)
 
 
 def get_all_files_under(dirname):
@@ -386,6 +397,8 @@ def resolve_dependencies(port_set, settings):
   def add_deps(node):
     node.process_dependencies(settings)
     for d in node.deps:
+      if d not in ports_by_name:
+        utils.exit_with_error(f'Unknown dependency {d} for port {node.name}')
       dep = ports_by_name[d]
       if dep not in port_set:
         port_set.add(dep)
@@ -400,11 +413,18 @@ def handle_use_port_error(arg, message):
 
 
 def handle_use_port_arg(settings, arg):
-  args = arg.split(':', 1)
-  name, options = args[0], None
-  if len(args) == 2:
-    options = args[1]
-  if name not in ports_by_name:
+  # Ignore ':' in first or second char of string since we could be dealing with a windows drive separator
+  pos = arg.find(':', 2)
+  if pos != -1:
+    name, options = arg[:pos], arg[pos + 1:]
+  else:
+    name, options = arg, None
+  if name.endswith('.py'):
+    port_file_path = name
+    if not os.path.isfile(port_file_path):
+      handle_use_port_error(arg, f'not a valid port path: {port_file_path}')
+    name = load_port_by_path(port_file_path)
+  elif name not in ports_by_name:
     handle_use_port_error(arg, f'invalid port name: {name}')
   ports_needed.add(name)
   if options: