From 0983920b96e5ac317d8e13367666686b543ef986 Mon Sep 17 00:00:00 2001 From: Richard Townsend Date: Tue, 24 Mar 2020 19:00:57 +0000 Subject: [PATCH 1/3] tools,gyp: add support for MSVC cross-compilation This change means that GYP can now generate two sets of projects: one exclusively for a host x64 machine and one containing a mix of x64 and Arm targets. The names of host targets are fixed up to end with _host.exe, and any actions involving them are fixed up. This allows compilation of Node on an x64 server for a Windows on Arm target. Change-Id: I5d9ded8a43b7ec5d5c7f501134080ac94dceacf5 --- tools/gyp/pylib/gyp/generator/msvs.py | 119 ++++++++++++++++++-------- 1 file changed, 81 insertions(+), 38 deletions(-) diff --git a/tools/gyp/pylib/gyp/generator/msvs.py b/tools/gyp/pylib/gyp/generator/msvs.py index 933042c7113c59..09abadb1bcb658 100644 --- a/tools/gyp/pylib/gyp/generator/msvs.py +++ b/tools/gyp/pylib/gyp/generator/msvs.py @@ -40,6 +40,7 @@ # letters. VALID_MSVS_GUID_CHARS = re.compile(r'^[A-F0-9\-]+$') +generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested() generator_default_variables = { 'DRIVER_PREFIX': '', @@ -51,7 +52,7 @@ 'STATIC_LIB_SUFFIX': '.lib', 'SHARED_LIB_SUFFIX': '.dll', 'INTERMEDIATE_DIR': '$(IntDir)', - 'SHARED_INTERMEDIATE_DIR': '$(OutDir)obj/global_intermediate', + 'SHARED_INTERMEDIATE_DIR': '$(OutDir)/obj/global_intermediate', 'OS': 'win', 'PRODUCT_DIR': '$(OutDir)', 'LIB_DIR': '$(OutDir)lib', @@ -286,7 +287,7 @@ def _ConfigTargetVersion(config_data): return config_data.get('msvs_target_version', 'Windows7') -def _ConfigPlatform(config_data): +def _ConfigPlatform(config_data, spec): return config_data.get('msvs_configuration_platform', 'Win32') @@ -297,8 +298,8 @@ def _ConfigBaseName(config_name, platform_name): return config_name -def _ConfigFullName(config_name, config_data): - platform_name = _ConfigPlatform(config_data) +def _ConfigFullName(config_name, config_data, spec): + platform_name = _ConfigPlatform(config_data, spec) return '%s|%s' % (_ConfigBaseName(config_name, platform_name), platform_name) @@ -951,7 +952,7 @@ def _GetMsbuildToolsetOfProject(proj_path, spec, version): return toolset -def _GenerateProject(project, options, version, generator_flags): +def _GenerateProject(project, options, version, generator_flags, spec): """Generates a vcproj file. Arguments: @@ -969,7 +970,7 @@ def _GenerateProject(project, options, version, generator_flags): return [] if version.UsesVcxproj(): - return _GenerateMSBuildProject(project, options, version, generator_flags) + return _GenerateMSBuildProject(project, options, version, generator_flags, spec) else: return _GenerateMSVSProject(project, options, version, generator_flags) @@ -1091,7 +1092,7 @@ def _GetUniquePlatforms(spec): # Gather list of unique platforms. platforms = OrderedSet() for configuration in spec['configurations']: - platforms.add(_ConfigPlatform(spec['configurations'][configuration])) + platforms.add(_ConfigPlatform(spec['configurations'][configuration], spec)) platforms = list(platforms) return platforms @@ -1801,6 +1802,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat): # Convert into a tree of dicts on path. for p in sln_projects: gyp_file, target = gyp.common.ParseQualifiedTarget(p)[0:2] + if p.endswith("#host"): + target += "_host" gyp_dir = os.path.dirname(gyp_file) path_dict = _GetPathDict(root, gyp_dir) path_dict[target + '.vcproj'] = project_objects[p] @@ -1819,7 +1822,10 @@ def _GetPathOfProject(qualified_target, spec, options, msvs_version): default_config = _GetDefaultConfiguration(spec) proj_filename = default_config.get('msvs_existing_vcproj') if not proj_filename: - proj_filename = (spec['target_name'] + options.suffix + + proj_filename = spec['target_name'] + if spec['toolset'] == 'host': + proj_filename += "_host" + proj_filename = (proj_filename + options.suffix + msvs_version.ProjectExtension()) build_file = gyp.common.BuildFile(qualified_target) @@ -1838,10 +1844,12 @@ def _GetPlatformOverridesOfProject(spec): # solution configurations for this target. config_platform_overrides = {} for config_name, c in spec['configurations'].items(): - config_fullname = _ConfigFullName(config_name, c) - platform = c.get('msvs_target_platform', _ConfigPlatform(c)) + config_fullname = _ConfigFullName(config_name, c, spec) + platform = c.get('msvs_target_platform', _ConfigPlatform(c, spec)) fixed_config_fullname = '%s|%s' % ( - _ConfigBaseName(config_name, _ConfigPlatform(c)), platform) + _ConfigBaseName(config_name, _ConfigPlatform(c, spec)), platform) + if spec['toolset'] == 'host' and generator_supports_multiple_toolsets: + fixed_config_fullname = '%s|x64' % (config_name,) config_platform_overrides[config_fullname] = fixed_config_fullname return config_platform_overrides @@ -1862,19 +1870,18 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version): projects = {} for qualified_target in target_list: spec = target_dicts[qualified_target] - if spec['toolset'] != 'target': - raise GypError( - 'Multiple toolsets not supported in msvs build (target %s)' % - qualified_target) proj_path, fixpath_prefix = _GetPathOfProject(qualified_target, spec, options, msvs_version) guid = _GetGuidOfProject(proj_path, spec) overrides = _GetPlatformOverridesOfProject(spec) build_file = gyp.common.BuildFile(qualified_target) # Create object for this project. + target_name = spec['target_name'] + if spec['toolset'] == 'host': + target_name += '_host' obj = MSVSNew.MSVSProject( proj_path, - name=spec['target_name'], + name=target_name, guid=guid, spec=spec, build_file=build_file, @@ -2041,7 +2048,10 @@ def GenerateOutput(target_list, target_dicts, data, params): for qualified_target in target_list: spec = target_dicts[qualified_target] for config_name, config in spec['configurations'].items(): - configs.add(_ConfigFullName(config_name, config)) + config_name = _ConfigFullName(config_name, config, spec) + configs.add(config_name) + if config_name == 'Release|arm64': + configs.add("Release|x64") configs = list(configs) # Figure out all the projects that will be generated and their guids @@ -2053,11 +2063,14 @@ def GenerateOutput(target_list, target_dicts, data, params): for project in project_objects.values(): fixpath_prefix = project.fixpath_prefix missing_sources.extend(_GenerateProject(project, options, msvs_version, - generator_flags)) + generator_flags, spec)) fixpath_prefix = None for build_file in data: # Validate build_file extension + target_only_configs = configs + if generator_supports_multiple_toolsets: + target_only_configs = [i for i in configs if i.endswith('arm64')] if not build_file.endswith('.gyp'): continue sln_path = os.path.splitext(build_file)[0] + options.suffix + '.sln' @@ -2072,7 +2085,7 @@ def GenerateOutput(target_list, target_dicts, data, params): # Create solution. sln = MSVSNew.MSVSSolution(sln_path, entries=root_entries, - variants=configs, + variants=target_only_configs, websiteProperties=False, version=msvs_version) sln.Write() @@ -2674,21 +2687,23 @@ def _GenerateMSBuildRuleXmlFile(xml_path, msbuild_rules): easy_xml.WriteXmlIfChanged(content, xml_path, pretty=True, win32=True) -def _GetConfigurationAndPlatform(name, settings): +def _GetConfigurationAndPlatform(name, settings, spec): configuration = name.rsplit('_', 1)[0] platform = settings.get('msvs_configuration_platform', 'Win32') + if spec['toolset'] == 'host' and platform == 'arm64': + platform = 'x64' # Host-only tools are always built for x64 return (configuration, platform) -def _GetConfigurationCondition(name, settings): +def _GetConfigurationCondition(name, settings, spec): return (r"'$(Configuration)|$(Platform)'=='%s|%s'" % - _GetConfigurationAndPlatform(name, settings)) + _GetConfigurationAndPlatform(name, settings, spec)) -def _GetMSBuildProjectConfigurations(configurations): +def _GetMSBuildProjectConfigurations(configurations, spec): group = ['ItemGroup', {'Label': 'ProjectConfigurations'}] for (name, settings) in sorted(configurations.items()): - configuration, platform = _GetConfigurationAndPlatform(name, settings) + configuration, platform = _GetConfigurationAndPlatform(name, settings, spec) designation = '%s|%s' % (configuration, platform) group.append( ['ProjectConfiguration', {'Include': designation}, @@ -2740,7 +2755,7 @@ def _GetMSBuildGlobalProperties(spec, version, guid, gyp_file_name): platform_name = None msvs_windows_sdk_version = None for configuration in spec['configurations'].values(): - platform_name = platform_name or _ConfigPlatform(configuration) + platform_name = platform_name or _ConfigPlatform(configuration, spec) msvs_windows_sdk_version = (msvs_windows_sdk_version or _ConfigWindowsTargetPlatformVersion(configuration, version)) if platform_name and msvs_windows_sdk_version: @@ -2762,7 +2777,7 @@ def _GetMSBuildConfigurationDetails(spec, build_file): properties = {} for name, settings in spec['configurations'].items(): msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file) - condition = _GetConfigurationCondition(name, settings) + condition = _GetConfigurationCondition(name, settings, spec) character_set = msbuild_attributes.get('CharacterSet') config_type = msbuild_attributes.get('ConfigurationType') _AddConditionalProperty(properties, condition, 'ConfigurationType', @@ -2790,12 +2805,12 @@ def _GetMSBuildLocalProperties(msbuild_toolset): return properties -def _GetMSBuildPropertySheets(configurations): +def _GetMSBuildPropertySheets(configurations, spec): user_props = r'$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props' additional_props = {} props_specified = False for name, settings in sorted(configurations.items()): - configuration = _GetConfigurationCondition(name, settings) + configuration = _GetConfigurationCondition(name, settings, spec) if 'msbuild_props' in settings: additional_props[configuration] = _FixPaths(settings['msbuild_props']) props_specified = True @@ -2946,7 +2961,7 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file): properties = {} for (name, configuration) in sorted(configurations.items()): - condition = _GetConfigurationCondition(name, configuration) + condition = _GetConfigurationCondition(name, configuration, spec) attributes = _GetMSBuildAttributes(spec, configuration, build_file) msbuild_settings = configuration['finalized_msbuild_settings'] _AddConditionalProperty(properties, condition, 'IntDir', @@ -3055,7 +3070,9 @@ def _GetMSBuildToolSettingsSections(spec, configurations): for (name, configuration) in sorted(configurations.items()): msbuild_settings = configuration['finalized_msbuild_settings'] group = ['ItemDefinitionGroup', - {'Condition': _GetConfigurationCondition(name, configuration)} + {'Condition': + _GetConfigurationCondition(name, configuration, spec) + } ] for tool_name, tool_settings in sorted(msbuild_settings.items()): # Skip the tool named '' which is a holder of global settings handled @@ -3277,7 +3294,9 @@ def _AddSources2(spec, sources, exclusions, grouped_sources, extensions_excluded_from_precompile = ['.c'] if precompiled_source == source: - condition = _GetConfigurationCondition(config_name, configuration) + condition = _GetConfigurationCondition( + config_name, configuration, spec + ) detail.append(['PrecompiledHeader', {'Condition': condition}, 'Create' @@ -3296,12 +3315,26 @@ def _AddSources2(spec, sources, exclusions, grouped_sources, _GetUniquePlatforms(spec)) grouped_sources[group].append([element, {'Include': source}] + detail) - -def _GetMSBuildProjectReferences(project): +def _GetMSBuildProjectReferences(project, spec): + current_configuration = spec['default_configuration'] references = [] if project.dependencies: group = ['ItemGroup'] + added_dependency_set = set() for dependency in project.dependencies: + dependency_spec = dependency.spec + should_skip_dep = False + if project.spec["toolset"] == 'target': + if dependency_spec['toolset'] == 'host': + if dependency_spec['type'] == 'static_library': + should_skip_dep = True + if dependency.name.startswith('run_'): + should_skip_dep = False + if should_skip_dep: + continue + + canonical_name = dependency.name.replace('_host', '') + added_dependency_set.add(canonical_name) guid = dependency.guid project_dir = os.path.split(project.path)[0] relative_path = gyp.common.RelativePath(dependency.path, project_dir) @@ -3323,7 +3356,7 @@ def _GetMSBuildProjectReferences(project): return references -def _GenerateMSBuildProject(project, options, version, generator_flags): +def _GenerateMSBuildProject(project, options, version, generator_flags, spec): spec = project.spec configurations = spec['configurations'] project_dir, project_file_name = os.path.split(project.path) @@ -3411,7 +3444,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): 'DefaultTargets': 'Build' }] - content += _GetMSBuildProjectConfigurations(configurations) + content += _GetMSBuildProjectConfigurations(configurations, spec) content += _GetMSBuildGlobalProperties(spec, version, project.guid, project_file_name) content += import_default_section @@ -3422,10 +3455,10 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): content += _GetMSBuildLocalProperties(project.msbuild_toolset) content += import_cpp_props_section content += import_masm_props_section - if spec.get('msvs_enable_marmasm'): + if spec.get('msvs_enable_marmasm') or True: content += import_marmasm_props_section content += _GetMSBuildExtensions(props_files_of_rules) - content += _GetMSBuildPropertySheets(configurations) + content += _GetMSBuildPropertySheets(configurations, spec) content += macro_section content += _GetMSBuildConfigurationGlobalProperties(spec, configurations, project.build_file) @@ -3433,7 +3466,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): content += _GetMSBuildSources( spec, sources, exclusions, rule_dependencies, extension_to_rule_name, actions_spec, sources_handled_by_action, list_excluded) - content += _GetMSBuildProjectReferences(project) + content += _GetMSBuildProjectReferences(project, spec) content += import_cpp_targets_section content += import_masm_targets_section if spec.get('msvs_enable_marmasm'): @@ -3516,15 +3549,25 @@ def _GenerateActionsForMSBuild(spec, actions_to_add): sources_handled_by_action = OrderedSet() actions_spec = [] for primary_input, actions in actions_to_add.items(): + if generator_supports_multiple_toolsets: + primary_input = primary_input.replace(".exe", "_host.exe") inputs = OrderedSet() outputs = OrderedSet() descriptions = [] commands = [] for action in actions: + def fixup_host_exe(i): + if "$(OutDir)" in i: + i = i.replace('.exe', '_host.exe') + return i + if generator_supports_multiple_toolsets: + action['inputs'] = [fixup_host_exe(i) for i in action['inputs']] inputs.update(OrderedSet(action['inputs'])) outputs.update(OrderedSet(action['outputs'])) descriptions.append(action['description']) cmd = action['command'] + if generator_supports_multiple_toolsets: + cmd = cmd.replace('.exe', "_host.exe") # For most actions, add 'call' so that actions that invoke batch files # return and continue executing. msbuild_use_call provides a way to # disable this but I have not seen any adverse effect from doing that From 3f582562df0c2fc51abee486b2cdef39d00f3230 Mon Sep 17 00:00:00 2001 From: Richard Townsend Date: Tue, 24 Mar 2020 18:57:08 +0000 Subject: [PATCH 2/3] build,win: add support for MSVC cross-compilation * Fixes cases in icutools where commands were issued without .exe * Changes to build scripts * Add /fp:strict flag so that MSVC's floating point behaves correctly * Enables marmasm Change-Id: Ife1fd90de271edd9dcefdef5fe439d7a9a05e8dc --- common.gypi | 5 +++++ configure.py | 4 +++- tools/icu/icu-generic.gyp | 14 +++++++------- tools/v8_gypfiles/v8.gyp | 13 +++++++++++++ vcbuild.bat | 2 +- 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/common.gypi b/common.gypi index 9fc57ba6fa189b..95443ba55df738 100644 --- a/common.gypi +++ b/common.gypi @@ -193,6 +193,11 @@ ], 'msvs_settings': { 'VCCLCompilerTool': { + 'conditions': [ + ['target_arch=="arm64"', { + 'FloatingPointModel': 1 # /fp:strict + }] + ], 'EnableFunctionLevelLinking': 'true', 'EnableIntrinsicFunctions': 'true', 'FavorSizeOrSpeed': 1, # /Ot, favor speed over size diff --git a/configure.py b/configure.py index ac26f62916cd06..d7c39275c76ea4 100755 --- a/configure.py +++ b/configure.py @@ -1040,7 +1040,9 @@ def configure_node(o): cross_compiling = (options.cross_compiling if options.cross_compiling is not None - else target_arch != host_arch) + else host_arch not in (target_arch, "x64", "ia32")) + if cross_compiling: + os.environ['GYP_CROSSCOMPILE'] = "1" if options.unused_without_snapshot: warn('building --without-snapshot is no longer possible') diff --git a/tools/icu/icu-generic.gyp b/tools/icu/icu-generic.gyp index d2d0e5a3180213..680f8528df1384 100644 --- a/tools/icu/icu-generic.gyp +++ b/tools/icu/icu-generic.gyp @@ -197,7 +197,7 @@ }], ['_toolset=="host"', { 'type': 'none', - 'dependencies': [ 'icutools' ], + 'dependencies': [ 'icutools#host' ], 'export_dependent_settings': [ 'icutools' ], }], ], @@ -221,7 +221,7 @@ 'inputs': [ '<(icu_data_in)' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], # on Windows, we can go directly to .obj file (-o) option. - 'action': [ '<(PRODUCT_DIR)/genccode', + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', # -o '-d', '<(SHARED_INTERMEDIATE_DIR)', '-n', 'icudata', @@ -258,7 +258,7 @@ 'msvs_quote_cmd': 0, 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icudt<(icu_ver_major)<(icu_endianness).dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness)_dat.<(icu_asm_ext)' ], - 'action': [ '<(PRODUCT_DIR)/genccode', + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', # -o '-d', '<(SHARED_INTERMEDIATE_DIR)/', '-n', 'icudata', @@ -284,7 +284,7 @@ 'action_name': 'icupkg', 'inputs': [ '<(icu_data_in)' ], 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)<(icu_endianness).dat' ], - 'action': [ '<(PRODUCT_DIR)/icupkg', + 'action': [ '<(PRODUCT_DIR)/icupkg<(EXECUTABLE_SUFFIX)', '-t<(icu_endianness)', '<@(_inputs)', '<@(_outputs)', @@ -305,7 +305,7 @@ 'action_name': 'icudata', 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major).dat' ], 'outputs':[ '<(SHARED_INTERMEDIATE_DIR)/icudt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - 'action': [ '<(PRODUCT_DIR)/genccode', + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '-e', 'icudt<(icu_ver_major)', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(icu_asm_opts)', @@ -350,7 +350,7 @@ 'action_name': 'genccode', 'inputs': [ '<(SHARED_INTERMEDIATE_DIR)/icutmp/icusmdt<(icu_ver_major).dat' ], 'outputs': [ '<(SHARED_INTERMEDIATE_DIR)/icusmdt<(icu_ver_major)_dat.<(icu_asm_ext)' ], - 'action': [ '<(PRODUCT_DIR)/genccode', + 'action': [ '<(PRODUCT_DIR)/genccode<(EXECUTABLE_SUFFIX)', '<@(icu_asm_opts)', '-d', '<(SHARED_INTERMEDIATE_DIR)', '<@(_inputs)' ], @@ -388,7 +388,7 @@ 'toolsets': [ 'target', 'host' ], 'conditions' : [ ['_toolset=="host"', { - 'dependencies': [ 'icutools' ], + 'dependencies': [ 'icutools#host' ], 'export_dependent_settings': [ 'icutools' ], }], ['_toolset=="target"', { diff --git a/tools/v8_gypfiles/v8.gyp b/tools/v8_gypfiles/v8.gyp index c36920efa7e958..af2ff9c602e432 100644 --- a/tools/v8_gypfiles/v8.gyp +++ b/tools/v8_gypfiles/v8.gyp @@ -470,6 +470,11 @@ 'toolsets': ['target'], 'conditions': [ ['want_separate_host_toolset', { + 'conditions': [ + ['v8_target_arch=="arm64"', { + 'msvs_enable_marmasm': 1, + }] + ], 'dependencies': [ 'generate_bytecode_builtins_list', 'run_torque', @@ -785,6 +790,14 @@ 'sources': [ ### gcmole(arch:arm64) ### ' Date: Thu, 14 May 2020 15:10:20 +0100 Subject: [PATCH 3/3] fixup! build,win: add support for MSVC cross-compilation --- configure.py | 2 +- vcbuild.bat | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/configure.py b/configure.py index d7c39275c76ea4..a8cdce4d225427 100755 --- a/configure.py +++ b/configure.py @@ -1040,7 +1040,7 @@ def configure_node(o): cross_compiling = (options.cross_compiling if options.cross_compiling is not None - else host_arch not in (target_arch, "x64", "ia32")) + else target_arch != host_arch) if cross_compiling: os.environ['GYP_CROSSCOMPILE'] = "1" if options.unused_without_snapshot: diff --git a/vcbuild.bat b/vcbuild.bat index 9cdc02e6c37f41..f1e83ead5d372c 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -196,7 +196,8 @@ if defined config_flags set configure_flags=%configure_flags% %config_flags% if defined target_arch set configure_flags=%configure_flags% --dest-cpu=%target_arch% if defined openssl_no_asm set configure_flags=%configure_flags% --openssl-no-asm if defined DEBUG_HELPER set configure_flags=%configure_flags% --verbose -if "%target_arch%" == "arm64" set configure_flags=%configure_flags% --cross-compiling +if "%target_arch%"=="x86" if "%PROCESSOR_ARCHITECTURE%"=="AMD64" set configure_flags=%configure_flags% --no-cross-compiling +if "%target_arch%"=="arm64" set configure_flags=%configure_flags% --cross-compiling if not exist "%~dp0deps\icu" goto no-depsicu if "%target%"=="Clean" echo deleting %~dp0deps\icu