fix: support cross compiling for wasm with make generator (#222)

* fix: support cross compiling for wasm with make generator

* fix: lint

* refactor for readability

Co-authored-by: Christian Clauss <cclauss@me.com>

* replace separator in make generator on Windows

* snake_case

* found more place to replace sep

* lint

* replace sep in compiler path

* fix sed unterminated `s' command error on Windows

* path includes `\` so replace the ended `\` only

* replace `\` with `/` in depfile on win

* lint

* fix: trailing `\` in raw string

* revert: flavor can be set via `-f make-linux` so no need to change the mac params

* fix: also do not use raw string in windows branch due to trailing `\` break editor highlight

* fix: respect user specified AR_target environment variable

* feat: detect wasm flavor

* lint: Too many return statements

* fix get compiler predefines on windows

* GetCrossCompilerPredefines always return dict

* do not broad exceptions

* test: GetCrossCompilerPredefines

* fix lint

* refactor: do not put so many lines in try block

* fix: finally block should wait until subprocess terminate

* suggestion change

---------

Co-authored-by: Christian Clauss <cclauss@me.com>

pylib/gyp/common.py

@@ -422,17 +422,58 @@ def EnsureDirExists(path):
     except OSError:
         pass
 
+def GetCrossCompilerPredefines():  # -> dict
+    cmd = []
+    if CC := os.environ.get("CC_target") or os.environ.get("CC"):
+        cmd += CC.split(" ")
+        if CFLAGS := os.environ.get("CFLAGS"):
+            cmd += CFLAGS.split(" ")
+    elif CXX := os.environ.get("CXX_target") or os.environ.get("CXX"):
+        cmd += CXX.split(" ")
+        if CXXFLAGS := os.environ.get("CXXFLAGS"):
+            cmd += CXXFLAGS.split(" ")
+    else:
+        return {}
 
-def GetFlavor(params):
+    if sys.platform == "win32":
+        fd, input = tempfile.mkstemp(suffix=".c")
+        try:
+            os.close(fd)
+            out = subprocess.Popen(
+                [*cmd, "-dM", "-E", "-x", "c", input],
+                shell=True,
+                stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+            )
+            stdout = out.communicate()[0]
+        finally:
+            os.unlink(input)
+    else:
+        input = "/dev/null"
+        out = subprocess.Popen(
+            [*cmd, "-dM", "-E", "-x", "c", input],
+            shell=False,
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+        )
+        stdout = out.communicate()[0]
+
+    defines = {}
+    lines = stdout.decode("utf-8").replace("\r\n", "\n").split("\n")
+    for line in lines:
+        if not line:
+            continue
+        define_directive, key, *value = line.split(" ")
+        assert define_directive == "#define"
+        defines[key] = " ".join(value)
+    return defines
+
+def GetFlavorByPlatform():
     """Returns |params.flavor| if it's set, the system's default flavor else."""
     flavors = {
         "cygwin": "win",
         "win32": "win",
         "darwin": "mac",
     }
 
-    if "flavor" in params:
-        return params["flavor"]
     if sys.platform in flavors:
         return flavors[sys.platform]
     if sys.platform.startswith("sunos"):
@@ -452,6 +493,18 @@ def GetFlavor(params):
 
     return "linux"
 
+def GetFlavor(params):
+    if "flavor" in params:
+        return params["flavor"]
+
+    defines = GetCrossCompilerPredefines()
+    if "__EMSCRIPTEN__" in defines:
+        return "emscripten"
+    if "__wasm__" in defines:
+        return "wasi" if "__wasi__" in defines else "wasm"
+
+    return GetFlavorByPlatform()
+
 
 def CopyTool(flavor, out_path, generator_flags={}):
     """Finds (flock|mac|win)_tool.gyp in the gyp directory and copies it

pylib/gyp/common_test.py

@@ -9,7 +9,9 @@
 import gyp.common
 import unittest
 import sys
-
+import os
+import subprocess
+from unittest.mock import patch, MagicMock
 
 class TestTopologicallySorted(unittest.TestCase):
     def test_Valid(self):
@@ -73,6 +75,62 @@ def test_platform_default(self):
     def test_param(self):
         self.assertFlavor("foobar", "linux2", {"flavor": "foobar"})
 
+    class MockCommunicate:
+        def __init__(self, stdout):
+            self.stdout = stdout
+
+        def decode(self, encoding):
+            return self.stdout
+
+    @patch("os.close")
+    @patch("os.unlink")
+    @patch("tempfile.mkstemp")
+    def test_GetCrossCompilerPredefines(self, mock_mkstemp, mock_unlink, mock_close):
+        mock_close.return_value = None
+        mock_unlink.return_value = None
+        mock_mkstemp.return_value = (0, "temp.c")
+
+        def mock_run(env, defines_stdout):
+            with patch("subprocess.Popen") as mock_popen:
+                mock_process = MagicMock()
+                mock_process.communicate.return_value = (
+                    TestGetFlavor.MockCommunicate(defines_stdout), None)
+                mock_process.stdout = MagicMock()
+                mock_popen.return_value = mock_process
+                expected_input = "temp.c" if sys.platform == "win32" else "/dev/null"
+                with patch.dict(os.environ, env):
+                    defines = gyp.common.GetCrossCompilerPredefines()
+                    flavor = gyp.common.GetFlavor({})
+                if env.get("CC_target"):
+                    mock_popen.assert_called_with(
+                        [env["CC_target"], "-dM", "-E", "-x", "c", expected_input],
+                        shell=sys.platform == "win32",
+                        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                return [defines, flavor]
+
+        [defines1, _] = mock_run({}, "")
+        self.assertDictEqual({}, defines1)
+
+        [defines2, flavor2] = mock_run(
+            { "CC_target": "/opt/wasi-sdk/bin/clang" },
+            "#define __wasm__ 1\n#define __wasi__ 1\n"
+        )
+        self.assertDictEqual({ "__wasm__": "1", "__wasi__": "1" }, defines2)
+        self.assertEqual("wasi", flavor2)
+
+        [defines3, flavor3] = mock_run(
+            { "CC_target": "/opt/wasi-sdk/bin/clang" },
+            "#define __wasm__ 1\n"
+        )
+        self.assertDictEqual({ "__wasm__": "1" }, defines3)
+        self.assertEqual("wasm", flavor3)
+
+        [defines4, flavor4] = mock_run(
+            { "CC_target": "/emsdk/upstream/emscripten/emcc" },
+            "#define __EMSCRIPTEN__ 1\n"
+        )
+        self.assertDictEqual({ "__EMSCRIPTEN__": "1" }, defines4)
+        self.assertEqual("emscripten", flavor4)
 
 if __name__ == "__main__":
     unittest.main()

pylib/gyp/generator/make.py

@@ -25,6 +25,7 @@
 import os
 import re
 import subprocess
+import sys
 import gyp
 import gyp.common
 import gyp.xcode_emulation
@@ -378,7 +379,7 @@ def CalculateGeneratorInputInfo(params):
 CXXFLAGS.target ?= $(CPPFLAGS) $(CXXFLAGS)
 LINK.target ?= %(LINK.target)s
 LDFLAGS.target ?= $(LDFLAGS)
-AR.target ?= $(AR)
+AR.target ?= %(AR.target)s
 PLI.target ?= %(PLI.target)s
 
 # C++ apps need to be linked with g++.
@@ -442,13 +443,21 @@ def CalculateGeneratorInputInfo(params):
 define fixup_dep
 # The depfile may not exist if the input file didn't have any #includes.
 touch $(depfile).raw
-# Fixup path as in (1).
-sed -e "s|^$(notdir $@)|$@|" $(depfile).raw >> $(depfile)
+# Fixup path as in (1).""" +
+    (r"""
+sed -e "s|^$(notdir $@)|$@|" -re 's/\\\\([^$$])/\/\1/g' $(depfile).raw >> $(depfile)"""
+    if sys.platform == 'win32' else r"""
+sed -e "s|^$(notdir $@)|$@|" $(depfile).raw >> $(depfile)""") +
+    r"""
 # Add extra rules as in (2).
 # We remove slashes and replace spaces with new lines;
 # remove blank lines;
-# delete the first line and append a colon to the remaining lines.
-sed -e 's|\\||' -e 'y| |\n|' $(depfile).raw |\
+# delete the first line and append a colon to the remaining lines.""" +
+    ("""
+sed -e 's/\\\\\\\\$$//' -e 's/\\\\\\\\/\\//g' -e 'y| |\\n|' $(depfile).raw |\\"""
+    if sys.platform == 'win32' else """
+sed -e 's|\\\\||' -e 'y| |\\n|' $(depfile).raw |\\""") +
+    r"""
   grep -v '^$$'                             |\
   sed -e 1d -e 's|$$|:|'                     \
     >> $(depfile)
@@ -724,6 +733,10 @@ def QuoteIfNecessary(string):
         string = '"' + string.replace('"', '\\"') + '"'
     return string
 
+def replace_sep(string):
+    if sys.platform == 'win32':
+        string = string.replace('\\\\', '/').replace('\\', '/')
+    return string
 
 def StringToMakefileVariable(string):
     """Convert a string to a value that is acceptable as a make variable name."""
@@ -859,7 +872,7 @@ def Write(
             self.output = self.ComputeMacBundleOutput(spec)
             self.output_binary = self.ComputeMacBundleBinaryOutput(spec)
         else:
-            self.output = self.output_binary = self.ComputeOutput(spec)
+            self.output = self.output_binary = replace_sep(self.ComputeOutput(spec))
 
         self.is_standalone_static_library = bool(
             spec.get("standalone_static_library", 0)
@@ -985,7 +998,7 @@ def WriteSubMake(self, output_filename, makefile_path, targets, build_dir):
         # sub-project dir (see test/subdirectory/gyptest-subdir-all.py).
         self.WriteLn(
             "export builddir_name ?= %s"
-            % os.path.join(os.path.dirname(output_filename), build_dir)
+            % replace_sep(os.path.join(os.path.dirname(output_filename), build_dir))
         )
         self.WriteLn(".PHONY: all")
         self.WriteLn("all:")
@@ -2063,7 +2076,7 @@ def WriteList(self, value_list, variable=None, prefix="", quoter=QuoteIfNecessar
         """
         values = ""
         if value_list:
-            value_list = [quoter(prefix + value) for value in value_list]
+            value_list = [replace_sep(quoter(prefix + value)) for value in value_list]
             values = " \\\n\t" + " \\\n\t".join(value_list)
         self.fp.write(f"{variable} :={values}\n\n")
 
@@ -2369,10 +2382,12 @@ def WriteAutoRegenerationRule(params, root_makefile, makefile_name, build_files)
         "\t$(call do_cmd,regen_makefile)\n\n"
         % {
             "makefile_name": makefile_name,
-            "deps": " ".join(SourceifyAndQuoteSpaces(bf) for bf in build_files),
-            "cmd": gyp.common.EncodePOSIXShellList(
-                [gyp_binary, "-fmake"] + gyp.RegenerateFlags(options) + build_files_args
+            "deps": replace_sep(
+                " ".join(SourceifyAndQuoteSpaces(bf) for bf in build_files)
             ),
+            "cmd": replace_sep(gyp.common.EncodePOSIXShellList(
+                [gyp_binary, "-fmake"] + gyp.RegenerateFlags(options) + build_files_args
+            )),
         }
     )
 
@@ -2435,33 +2450,52 @@ def CalculateMakefilePath(build_file, base_name):
         makefile_path = os.path.join(
             options.toplevel_dir, options.generator_output, makefile_name
         )
-        srcdir = gyp.common.RelativePath(srcdir, options.generator_output)
+        srcdir = replace_sep(gyp.common.RelativePath(srcdir, options.generator_output))
         srcdir_prefix = "$(srcdir)/"
 
     flock_command = "flock"
     copy_archive_arguments = "-af"
     makedep_arguments = "-MMD"
+
+    # wasm-ld doesn't support --start-group/--end-group
+    link_commands = LINK_COMMANDS_LINUX
+    if flavor in ["wasi", "wasm"]:
+        link_commands = link_commands.replace(' -Wl,--start-group', '').replace(
+            ' -Wl,--end-group', ''
+        )
+
+    CC_target = replace_sep(GetEnvironFallback(("CC_target", "CC"), "$(CC)"))
+    AR_target = replace_sep(GetEnvironFallback(("AR_target", "AR"), "$(AR)"))
+    CXX_target = replace_sep(GetEnvironFallback(("CXX_target", "CXX"), "$(CXX)"))
+    LINK_target = replace_sep(GetEnvironFallback(("LINK_target", "LINK"), "$(LINK)"))
+    PLI_target = replace_sep(GetEnvironFallback(("PLI_target", "PLI"), "pli"))
+    CC_host = replace_sep(GetEnvironFallback(("CC_host", "CC"), "gcc"))
+    AR_host = replace_sep(GetEnvironFallback(("AR_host", "AR"), "ar"))
+    CXX_host = replace_sep(GetEnvironFallback(("CXX_host", "CXX"), "g++"))
+    LINK_host = replace_sep(GetEnvironFallback(("LINK_host", "LINK"), "$(CXX.host)"))
+    PLI_host = replace_sep(GetEnvironFallback(("PLI_host", "PLI"), "pli"))
+
     header_params = {
         "default_target": default_target,
         "builddir": builddir_name,
         "default_configuration": default_configuration,
         "flock": flock_command,
         "flock_index": 1,
-        "link_commands": LINK_COMMANDS_LINUX,
+        "link_commands": link_commands,
         "extra_commands": "",
         "srcdir": srcdir,
         "copy_archive_args": copy_archive_arguments,
         "makedep_args": makedep_arguments,
-        "CC.target": GetEnvironFallback(("CC_target", "CC"), "$(CC)"),
-        "AR.target": GetEnvironFallback(("AR_target", "AR"), "$(AR)"),
-        "CXX.target": GetEnvironFallback(("CXX_target", "CXX"), "$(CXX)"),
-        "LINK.target": GetEnvironFallback(("LINK_target", "LINK"), "$(LINK)"),
-        "PLI.target": GetEnvironFallback(("PLI_target", "PLI"), "pli"),
-        "CC.host": GetEnvironFallback(("CC_host", "CC"), "gcc"),
-        "AR.host": GetEnvironFallback(("AR_host", "AR"), "ar"),
-        "CXX.host": GetEnvironFallback(("CXX_host", "CXX"), "g++"),
-        "LINK.host": GetEnvironFallback(("LINK_host", "LINK"), "$(CXX.host)"),
-        "PLI.host": GetEnvironFallback(("PLI_host", "PLI"), "pli"),
+        "CC.target": CC_target,
+        "AR.target": AR_target,
+        "CXX.target": CXX_target,
+        "LINK.target": LINK_target,
+        "PLI.target": PLI_target,
+        "CC.host": CC_host,
+        "AR.host": AR_host,
+        "CXX.host": CXX_host,
+        "LINK.host": LINK_host,
+        "PLI.host": PLI_host,
     }
     if flavor == "mac":
         flock_command = "./gyp-mac-tool flock"