Skip to content

Commit

Permalink
feat: port "add support for MSVC cross-compilation" from node
Browse files Browse the repository at this point in the history 
Original commit message:

    tools,gyp: add support for MSVC cross-compilation

    This change means that GYP can now generate two sets of projects: one
    exclusively for a host x64 machine and one containing a mix of x64 and
    Arm targets. The names of host targets are fixed up to end with
    _host.exe, and any actions involving them are fixed up. This allows
    compilation of Node on an x64 server for a Windows on Arm target.

Refs: nodejs/node#32867
Closes: #40
  • Loading branch information
@targos @ryzokuken
targos authored and ryzokuken committed May 29, 2020
1 parent 44e3f3e commit 72ea11b
Showing 1 changed file with 68 additions and 30 deletions.
98 changes: 68 additions & 30 deletions pylib/gyp/generator/msvs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
# letters.
VALID_MSVS_GUID_CHARS = re.compile(r"^[A-F0-9\-]+$")

generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested()

generator_default_variables = {
"DRIVER_PREFIX": "",
Expand All @@ -50,7 +51,7 @@
"STATIC_LIB_SUFFIX": ".lib",
"SHARED_LIB_SUFFIX": ".dll",
"INTERMEDIATE_DIR": "$(IntDir)",
"SHARED_INTERMEDIATE_DIR": "$(OutDir)obj/global_intermediate",
"SHARED_INTERMEDIATE_DIR": "$(OutDir)/obj/global_intermediate",
"OS": "win",
"PRODUCT_DIR": "$(OutDir)",
"LIB_DIR": "$(OutDir)lib",
Expand Down Expand Up @@ -1005,7 +1006,7 @@ def _GetMsbuildToolsetOfProject(proj_path, spec, version):
return toolset


def _GenerateProject(project, options, version, generator_flags):
def _GenerateProject(project, options, version, generator_flags, spec):
"""Generates a vcproj file.
Arguments:
Expand All @@ -1023,7 +1024,7 @@ def _GenerateProject(project, options, version, generator_flags):
return []

if version.UsesVcxproj():
return _GenerateMSBuildProject(project, options, version, generator_flags)
return _GenerateMSBuildProject(project, options, version, generator_flags, spec)
else:
return _GenerateMSVSProject(project, options, version, generator_flags)

Expand Down Expand Up @@ -1903,6 +1904,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat):
# Convert into a tree of dicts on path.
for p in sln_projects:
gyp_file, target = gyp.common.ParseQualifiedTarget(p)[0:2]
if p.endswith("#host"):
target += "_host"
gyp_dir = os.path.dirname(gyp_file)
path_dict = _GetPathDict(root, gyp_dir)
path_dict[target + ".vcproj"] = project_objects[p]
Expand All @@ -1921,9 +1924,10 @@ def _GetPathOfProject(qualified_target, spec, options, msvs_version):
default_config = _GetDefaultConfiguration(spec)
proj_filename = default_config.get("msvs_existing_vcproj")
if not proj_filename:
proj_filename = (
spec["target_name"] + options.suffix + msvs_version.ProjectExtension()
)
proj_filename = spec["target_name"]
if spec["toolset"] == "host":
proj_filename += "_host"
proj_filename = proj_filename + options.suffix + msvs_version.ProjectExtension()

build_file = gyp.common.BuildFile(qualified_target)
proj_path = os.path.join(os.path.dirname(build_file), proj_filename)
Expand All @@ -1948,6 +1952,8 @@ def _GetPlatformOverridesOfProject(spec):
_ConfigBaseName(config_name, _ConfigPlatform(c)),
platform,
)
if spec["toolset"] == "host" and generator_supports_multiple_toolsets:
fixed_config_fullname = "%s|x64" % (config_name,)
config_platform_overrides[config_fullname] = fixed_config_fullname
return config_platform_overrides

Expand All @@ -1968,21 +1974,19 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version):
projects = {}
for qualified_target in target_list:
spec = target_dicts[qualified_target]
if spec["toolset"] != "target":
raise GypError(
"Multiple toolsets not supported in msvs build (target %s)"
% qualified_target
)
proj_path, fixpath_prefix = _GetPathOfProject(
qualified_target, spec, options, msvs_version
)
guid = _GetGuidOfProject(proj_path, spec)
overrides = _GetPlatformOverridesOfProject(spec)
build_file = gyp.common.BuildFile(qualified_target)
# Create object for this project.
target_name = spec["target_name"]
if spec["toolset"] == "host":
target_name += "_host"
obj = MSVSNew.MSVSProject(
proj_path,
name=spec["target_name"],
name=target_name,
guid=guid,
spec=spec,
build_file=build_file,
Expand Down Expand Up @@ -2161,7 +2165,10 @@ def GenerateOutput(target_list, target_dicts, data, params):
for qualified_target in target_list:
spec = target_dicts[qualified_target]
for config_name, config in spec["configurations"].items():
configs.add(_ConfigFullName(config_name, config))
config_name = _ConfigFullName(config_name, config)
configs.add(config_name)
if config_name == "Release|arm64":
configs.add("Release|x64")
configs = list(configs)

# Figure out all the projects that will be generated and their guids
Expand All @@ -2174,12 +2181,15 @@ def GenerateOutput(target_list, target_dicts, data, params):
for project in project_objects.values():
fixpath_prefix = project.fixpath_prefix
missing_sources.extend(
_GenerateProject(project, options, msvs_version, generator_flags)
_GenerateProject(project, options, msvs_version, generator_flags, spec)
)
fixpath_prefix = None

for build_file in data:
# Validate build_file extension
target_only_configs = configs
if generator_supports_multiple_toolsets:
target_only_configs = [i for i in configs if i.endswith("arm64")]
if not build_file.endswith(".gyp"):
continue
sln_path = os.path.splitext(build_file)[0] + options.suffix + ".sln"
Expand All @@ -2196,7 +2206,7 @@ def GenerateOutput(target_list, target_dicts, data, params):
sln = MSVSNew.MSVSSolution(
sln_path,
entries=root_entries,
variants=configs,
variants=target_only_configs,
websiteProperties=False,
version=msvs_version,
)
Expand Down Expand Up @@ -2930,22 +2940,24 @@ def _GenerateMSBuildRuleXmlFile(xml_path, msbuild_rules):
easy_xml.WriteXmlIfChanged(content, xml_path, pretty=True, win32=True)


def _GetConfigurationAndPlatform(name, settings):
def _GetConfigurationAndPlatform(name, settings, spec):
configuration = name.rsplit("_", 1)[0]
platform = settings.get("msvs_configuration_platform", "Win32")
if spec["toolset"] == "host" and platform == "arm64":

This comment has been minimized.

Sign in to view

Copy link
@niyas-sait

niyas-sait Mar 17, 2022

Is there any reason why arm64 host cannot be used? I've been trying to build node.js on windows on arm64 machine (without cross-compilation) and this condition forces some of the modules to build for x64 rather than arm64. Removing this condition seems to have helped.

This comment has been minimized.

Sign in to view

Copy link
@targos

targos Mar 17, 2022

Author Member

I don't know. /cc @richard-townsend-arm
platform = "x64" # Host-only tools are always built for x64
return (configuration, platform)


def _GetConfigurationCondition(name, settings):
def _GetConfigurationCondition(name, settings, spec):
return r"'$(Configuration)|$(Platform)'=='%s|%s'" % _GetConfigurationAndPlatform(
name, settings
name, settings, spec
)


def _GetMSBuildProjectConfigurations(configurations):
def _GetMSBuildProjectConfigurations(configurations, spec):
group = ["ItemGroup", {"Label": "ProjectConfigurations"}]
for (name, settings) in sorted(configurations.items()):
configuration, platform = _GetConfigurationAndPlatform(name, settings)
configuration, platform = _GetConfigurationAndPlatform(name, settings, spec)
designation = "%s|%s" % (configuration, platform)
group.append(
[
Expand Down Expand Up @@ -3033,7 +3045,7 @@ def _GetMSBuildConfigurationDetails(spec, build_file):
properties = {}
for name, settings in spec["configurations"].items():
msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file)
condition = _GetConfigurationCondition(name, settings)
condition = _GetConfigurationCondition(name, settings, spec)
character_set = msbuild_attributes.get("CharacterSet")
config_type = msbuild_attributes.get("ConfigurationType")
_AddConditionalProperty(properties, condition, "ConfigurationType", config_type)
Expand Down Expand Up @@ -3064,12 +3076,12 @@ def _GetMSBuildLocalProperties(msbuild_toolset):
return properties


def _GetMSBuildPropertySheets(configurations):
def _GetMSBuildPropertySheets(configurations, spec):
user_props = r"$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props"
additional_props = {}
props_specified = False
for name, settings in sorted(configurations.items()):
configuration = _GetConfigurationCondition(name, settings)
configuration = _GetConfigurationCondition(name, settings, spec)
if "msbuild_props" in settings:
additional_props[configuration] = _FixPaths(settings["msbuild_props"])
props_specified = True
Expand Down Expand Up @@ -3222,7 +3234,7 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file):

properties = {}
for (name, configuration) in sorted(configurations.items()):
condition = _GetConfigurationCondition(name, configuration)
condition = _GetConfigurationCondition(name, configuration, spec)
attributes = _GetMSBuildAttributes(spec, configuration, build_file)
msbuild_settings = configuration["finalized_msbuild_settings"]
_AddConditionalProperty(
Expand Down Expand Up @@ -3345,7 +3357,7 @@ def _GetMSBuildToolSettingsSections(spec, configurations):
msbuild_settings = configuration["finalized_msbuild_settings"]
group = [
"ItemDefinitionGroup",
{"Condition": _GetConfigurationCondition(name, configuration)},
{"Condition": _GetConfigurationCondition(name, configuration, spec)},
]
for tool_name, tool_settings in sorted(msbuild_settings.items()):
# Skip the tool named '' which is a holder of global settings handled
Expand Down Expand Up @@ -3625,7 +3637,7 @@ def _AddSources2(

if precompiled_source == source:
condition = _GetConfigurationCondition(
config_name, configuration
config_name, configuration, spec
)
detail.append(
["PrecompiledHeader", {"Condition": condition}, "Create"]
Expand All @@ -3652,7 +3664,21 @@ def _GetMSBuildProjectReferences(project):
references = []
if project.dependencies:
group = ["ItemGroup"]
added_dependency_set = set()
for dependency in project.dependencies:
dependency_spec = dependency.spec
should_skip_dep = False
if project.spec["toolset"] == "target":
if dependency_spec["toolset"] == "host":
if dependency_spec["type"] == "static_library":
should_skip_dep = True
if dependency.name.startswith("run_"):
should_skip_dep = False
if should_skip_dep:
continue

canonical_name = dependency.name.replace("_host", "")
added_dependency_set.add(canonical_name)
guid = dependency.guid
project_dir = os.path.split(project.path)[0]
relative_path = gyp.common.RelativePath(dependency.path, project_dir)
Expand All @@ -3675,7 +3701,7 @@ def _GetMSBuildProjectReferences(project):
return references


def _GenerateMSBuildProject(project, options, version, generator_flags):
def _GenerateMSBuildProject(project, options, version, generator_flags, spec):
spec = project.spec
configurations = spec["configurations"]
project_dir, project_file_name = os.path.split(project.path)
Expand Down Expand Up @@ -3774,7 +3800,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
},
]

content += _GetMSBuildProjectConfigurations(configurations)
content += _GetMSBuildProjectConfigurations(configurations, spec)
content += _GetMSBuildGlobalProperties(
spec, version, project.guid, project_file_name
)
Expand All @@ -3786,10 +3812,10 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
content += _GetMSBuildLocalProperties(project.msbuild_toolset)
content += import_cpp_props_section
content += import_masm_props_section
if spec.get("msvs_enable_marmasm"):
if spec.get("msvs_enable_marmasm") or True:
content += import_marmasm_props_section
content += _GetMSBuildExtensions(props_files_of_rules)
content += _GetMSBuildPropertySheets(configurations)
content += _GetMSBuildPropertySheets(configurations, spec)
content += macro_section
content += _GetMSBuildConfigurationGlobalProperties(
spec, configurations, project.build_file
Expand Down Expand Up @@ -3893,15 +3919,27 @@ def _GenerateActionsForMSBuild(spec, actions_to_add):
sources_handled_by_action = OrderedSet()
actions_spec = []
for primary_input, actions in actions_to_add.items():
if generator_supports_multiple_toolsets:
primary_input = primary_input.replace(".exe", "_host.exe")
inputs = OrderedSet()
outputs = OrderedSet()
descriptions = []
commands = []
for action in actions:

def fixup_host_exe(i):
if "$(OutDir)" in i:
i = i.replace(".exe", "_host.exe")
return i

if generator_supports_multiple_toolsets:
action["inputs"] = [fixup_host_exe(i) for i in action["inputs"]]
inputs.update(OrderedSet(action["inputs"]))
outputs.update(OrderedSet(action["outputs"]))
descriptions.append(action["description"])
cmd = action["command"]
if generator_supports_multiple_toolsets:
cmd = cmd.replace(".exe", "_host.exe")
# For most actions, add 'call' so that actions that invoke batch files
# return and continue executing. msbuild_use_call provides a way to
# disable this but I have not seen any adverse effect from doing that
Expand Down

0 comments on commit 72ea11b

Please sign in to comment.