From 32b8873ea89942ba0b8a35fd25d60b315390d1f4 Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Thu, 13 Oct 2022 15:06:58 -0400 Subject: [PATCH 01/10] Add source_output_file_map_aspect and DataStore symlink logic --- .../source_output_file_map_aspect.bzl | 78 +++++++++++++++++++ BazelExtensions/xchammerconfig.bzl | 5 ++ BazelExtensions/xcodeproject.bzl | 34 +++++++- Sources/XCHammer/Generator.swift | 8 +- 4 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 BazelExtensions/source_output_file_map_aspect.bzl diff --git a/BazelExtensions/source_output_file_map_aspect.bzl b/BazelExtensions/source_output_file_map_aspect.bzl new file mode 100644 index 00000000..36fbbed3 --- /dev/null +++ b/BazelExtensions/source_output_file_map_aspect.bzl @@ -0,0 +1,78 @@ +""" +Aspect to map source to output files (i.e. .o files generated during compilation). + +The mappings are generated as JSON files per target. +""" + +SourceOutputFileMapInfo = provider( + doc = "...", + fields = { + "mapping": "Dictionary where keys are source file paths and values are at the respective .o file under bazel-out", + }, +) + +def _source_output_file_map_aspect_impl(target, ctx): + source_output_file_map = ctx.actions.declare_file("{}_source_output_file_map.json".format(target.label.name)) + + mapping = {} + objc_srcs = [] + objc_objects = [] + + # List of source files to be mapped to output files + if hasattr(ctx.rule.attr, "srcs"): + objc_srcs = [ + f + for source_file in ctx.rule.attr.srcs + for f in source_file.files.to_list() + # Handling objc only for now + if f.path.endswith(".m") + # TODO: handle swift + ] + + # Get compilation outputs if present + if OutputGroupInfo in target: + if hasattr(target[OutputGroupInfo], "compilation_outputs"): + objc_objects.extend(target[OutputGroupInfo].compilation_outputs.to_list()) + + # Map source to output file + if len(objc_srcs): + if len(objc_srcs) != len(objc_objects): + fail("[ERROR] Unexpected number of object files") + for src in objc_srcs: + basename_without_ext = src.basename.replace(".%s" % src.extension, "") + obj = [o for o in objc_objects if "%s.o" % basename_without_ext == o.basename] + if len(obj) != 1: + fail("Failed to find single object file for source %s. Found: %s" % (src, obj)) + + obj = obj[0] + mapping["/{}".format(src.path)] = obj.path + + # Collect info from deps + deps = getattr(ctx.rule.attr, "deps", []) + transitive_jsons = [] + for dep in deps: + # Collect mappings from deps + if SourceOutputFileMapInfo in dep: + for k, v in dep[SourceOutputFileMapInfo].mapping.items(): + mapping[k] = v + # Collect generated JSON files from deps + if OutputGroupInfo in dep: + if hasattr(dep[OutputGroupInfo], "source_output_file_map"): + transitive_jsons.extend(dep[OutputGroupInfo].source_output_file_map.to_list()) + + # Writes JSON + ctx.actions.write(source_output_file_map, json.encode(mapping)) + + return [ + OutputGroupInfo( + source_output_file_map = depset([source_output_file_map] + transitive_jsons), + ), + SourceOutputFileMapInfo( + mapping = mapping + ), + ] + +source_output_file_map_aspect = aspect( + implementation = _source_output_file_map_aspect_impl, + attr_aspects = ["deps"], +) diff --git a/BazelExtensions/xchammerconfig.bzl b/BazelExtensions/xchammerconfig.bzl index cee81f1c..17165eea 100644 --- a/BazelExtensions/xchammerconfig.bzl +++ b/BazelExtensions/xchammerconfig.bzl @@ -66,3 +66,8 @@ def xchammer_config( projects, # [String: XCHammerProjectConfig] target_config = None): # [String: XCHammerTargetConfig]? return struct(targets = targets, targetConfig = target_config, projects = projects) + +def build_service_config( + enable_indexing = False, # Bool + relative_index_store_path = None): # String? + return struct(enableIndexing = enable_indexing, relativeIndexStorePath = relative_index_store_path) diff --git a/BazelExtensions/xcodeproject.bzl b/BazelExtensions/xcodeproject.bzl index 821d469d..da94888c 100644 --- a/BazelExtensions/xcodeproject.bzl +++ b/BazelExtensions/xcodeproject.bzl @@ -9,7 +9,8 @@ load( "@xchammer//:BazelExtensions/xchammerconfig.bzl", "xchammer_config", "gen_xchammer_config", - "project_config" + "project_config", + "build_service_config", ) load( @@ -146,7 +147,6 @@ def _xcode_project_impl(ctx): execution_requirements = { "local": "1" } ) - _xcode_project = rule( implementation=_xcode_project_impl, attrs={ @@ -169,6 +169,16 @@ get_srcroot = "\"$(cat ../../DO_NOT_BUILD_HERE)/\"" def _install_xcode_project_impl(ctx): xcodeproj = ctx.attr.xcodeproj.files.to_list()[0] output_proj = "$SRCROOT/" + xcodeproj.basename + + build_service_config = ( + ctx.attr.build_service_config + if ctx.attr.build_service_config + else build_service_config().to_json() + ) + build_service_config = json.decode(build_service_config) + enable_indexing = build_service_config.get("enableIndexing", False) + relative_index_store_path = build_service_config.get("relativeIndexStorePath", None) + command = [ "SRCROOT=" + get_srcroot, "ditto " + xcodeproj.path + " " + output_proj, @@ -185,6 +195,14 @@ def _install_xcode_project_impl(ctx): "(rm -f $SRCROOT/external && ln -sf $PWD/../../external $SRCROOT/external)", 'echo "' + output_proj + '" > ' + ctx.outputs.out.path, ] + + if enable_indexing and relative_index_store_path: + command.extend([ + "DD=$(xcodebuild -project " + output_proj + " -showBuildSettings 2>&1 | grep -e \"^\\s*BUILT_PRODUCTS_DIR = \" | cut -d'=' -f2 | xargs | sed -e 's/\\/Build.*//g')", + 'mv $DD/Index/DataStore $DD/Index/DataStore.default || true', + "mkdir -p '$SRCROOT{}'".format(relative_index_store_path), + "ln -s $(echo $SRCROOT){} $DD/Index/DataStore".format(relative_index_store_path), + ]) ctx.actions.run_shell( inputs=ctx.attr.xcodeproj.files, command=";".join(command), @@ -196,7 +214,10 @@ def _install_xcode_project_impl(ctx): _install_xcode_project = rule( implementation=_install_xcode_project_impl, - attrs={"xcodeproj": attr.label(mandatory=True)}, + attrs={ + "xcodeproj": attr.label(mandatory=True), + "build_service_config": attr.string(), + }, outputs={"out": "%{name}.dummy"}, ) @@ -236,7 +257,11 @@ def xcode_project(**kwargs): proj_args["target_config"] = "{}" proj_args["name"] = rule_name + "_impl" - proj_args["project_config"] = proj_args["project_config"].to_json() if "project_config" in proj_args else None + proj_args["project_config"] = proj_args["project_config"].to_json() if "project_config" in proj_args else None + + # For now pop to use in '_install_xcode_project', we might end up using this in the + # _xcode_project impl later on + build_service_config = proj_args.pop("build_service_config").to_json() if "build_service_config" in proj_args else None _xcode_project(**proj_args) @@ -246,4 +271,5 @@ def xcode_project(**kwargs): name=rule_name, xcodeproj=kwargs["name"], testonly=proj_args.get("testonly", False), + build_service_config = build_service_config, ) diff --git a/Sources/XCHammer/Generator.swift b/Sources/XCHammer/Generator.swift index 87e0e76e..572bac38 100644 --- a/Sources/XCHammer/Generator.swift +++ b/Sources/XCHammer/Generator.swift @@ -88,7 +88,9 @@ enum Generator { let overrides = getRepositoryOverrides(genOptions: genOptions) let bazelArgs: [String] = [ "--aspects @xchammer//:BazelExtensions/xcode_configuration_provider.bzl%pure_xcode_build_sources_aspect", - "--output_groups=xcode_project_deps" + "--output_groups=xcode_project_deps", + "--aspects @xchammer//:BazelExtensions/source_output_file_map_aspect.bzl%source_output_file_map_aspect", + "--output_groups=+source_output_file_map", ] + overrides + labels.map { $0.value } // We retry.sh the bazel command so if Xcode updates, the build still works @@ -430,7 +432,9 @@ enum Generator { ] + overrides + [ // Build xcode_project_deps for targets in question. "--aspects @xchammer//:BazelExtensions/xcode_configuration_provider.bzl%xcode_build_sources_aspect", - "--output_groups=+xcode_project_deps" + "--output_groups=+xcode_project_deps", + "--aspects @xchammer//:BazelExtensions/source_output_file_map_aspect.bzl%source_output_file_map_aspect", + "--output_groups=+source_output_file_map", ] let buildOptions = (targetConfig?.buildBazelOptions ?? "") + " " + From ef7f94ad46af97a4e22ca0a8c9de064ce2e4aab8 Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Thu, 27 Oct 2022 12:28:53 -0400 Subject: [PATCH 02/10] Configure xcbuildkit via xcconfigs --- .../source_output_file_map_aspect.bzl | 78 ---------------- BazelExtensions/xchammerconfig.bzl | 14 +-- .../xcode_configuration_provider.bzl | 73 ++++++++++++++- BazelExtensions/xcodeproject.bzl | 31 +------ Sources/XCHammer/Generator.swift | 8 +- Sources/XCHammer/XCBuildSettings.swift | 25 +++++- Sources/XCHammer/XCHammerConfig.swift | 13 +++ Sources/XCHammer/XcodeTarget.swift | 14 ++- XCHammerAssets/bazel_build_service_setup.sh | 88 +++++++++++++++++++ third_party/repositories.bzl | 2 +- 10 files changed, 225 insertions(+), 121 deletions(-) delete mode 100644 BazelExtensions/source_output_file_map_aspect.bzl create mode 100755 XCHammerAssets/bazel_build_service_setup.sh diff --git a/BazelExtensions/source_output_file_map_aspect.bzl b/BazelExtensions/source_output_file_map_aspect.bzl deleted file mode 100644 index 36fbbed3..00000000 --- a/BazelExtensions/source_output_file_map_aspect.bzl +++ /dev/null @@ -1,78 +0,0 @@ -""" -Aspect to map source to output files (i.e. .o files generated during compilation). - -The mappings are generated as JSON files per target. -""" - -SourceOutputFileMapInfo = provider( - doc = "...", - fields = { - "mapping": "Dictionary where keys are source file paths and values are at the respective .o file under bazel-out", - }, -) - -def _source_output_file_map_aspect_impl(target, ctx): - source_output_file_map = ctx.actions.declare_file("{}_source_output_file_map.json".format(target.label.name)) - - mapping = {} - objc_srcs = [] - objc_objects = [] - - # List of source files to be mapped to output files - if hasattr(ctx.rule.attr, "srcs"): - objc_srcs = [ - f - for source_file in ctx.rule.attr.srcs - for f in source_file.files.to_list() - # Handling objc only for now - if f.path.endswith(".m") - # TODO: handle swift - ] - - # Get compilation outputs if present - if OutputGroupInfo in target: - if hasattr(target[OutputGroupInfo], "compilation_outputs"): - objc_objects.extend(target[OutputGroupInfo].compilation_outputs.to_list()) - - # Map source to output file - if len(objc_srcs): - if len(objc_srcs) != len(objc_objects): - fail("[ERROR] Unexpected number of object files") - for src in objc_srcs: - basename_without_ext = src.basename.replace(".%s" % src.extension, "") - obj = [o for o in objc_objects if "%s.o" % basename_without_ext == o.basename] - if len(obj) != 1: - fail("Failed to find single object file for source %s. Found: %s" % (src, obj)) - - obj = obj[0] - mapping["/{}".format(src.path)] = obj.path - - # Collect info from deps - deps = getattr(ctx.rule.attr, "deps", []) - transitive_jsons = [] - for dep in deps: - # Collect mappings from deps - if SourceOutputFileMapInfo in dep: - for k, v in dep[SourceOutputFileMapInfo].mapping.items(): - mapping[k] = v - # Collect generated JSON files from deps - if OutputGroupInfo in dep: - if hasattr(dep[OutputGroupInfo], "source_output_file_map"): - transitive_jsons.extend(dep[OutputGroupInfo].source_output_file_map.to_list()) - - # Writes JSON - ctx.actions.write(source_output_file_map, json.encode(mapping)) - - return [ - OutputGroupInfo( - source_output_file_map = depset([source_output_file_map] + transitive_jsons), - ), - SourceOutputFileMapInfo( - mapping = mapping - ), - ] - -source_output_file_map_aspect = aspect( - implementation = _source_output_file_map_aspect_impl, - attr_aspects = ["deps"], -) diff --git a/BazelExtensions/xchammerconfig.bzl b/BazelExtensions/xchammerconfig.bzl index 17165eea..837669b4 100644 --- a/BazelExtensions/xchammerconfig.bzl +++ b/BazelExtensions/xchammerconfig.bzl @@ -56,10 +56,11 @@ def target_config( def project_config( paths, # [String]? build_bazel_platform_options = None, # [String: [String]]? + bazel_build_service_config = None, # [String: XCHammerTargetConfig]? generate_transitive_xcode_targets = None, # Bool generate_xcode_schemes = None, # Bool xcconfig_overrides = None): # : [String: String]? - return struct(paths = paths, buildBazelPlatformOptions = build_bazel_platform_options, generateTransitiveXcodeTargets = generate_transitive_xcode_targets, generateXcodeSchemes = generate_xcode_schemes, xcconfigOverrides = xcconfig_overrides) + return struct(paths = paths, buildBazelPlatformOptions = build_bazel_platform_options, bazelBuildServiceConfig = bazel_build_service_config, generateTransitiveXcodeTargets = generate_transitive_xcode_targets, generateXcodeSchemes = generate_xcode_schemes, xcconfigOverrides = xcconfig_overrides) def xchammer_config( targets, # [String] @@ -67,7 +68,10 @@ def xchammer_config( target_config = None): # [String: XCHammerTargetConfig]? return struct(targets = targets, targetConfig = target_config, projects = projects) -def build_service_config( - enable_indexing = False, # Bool - relative_index_store_path = None): # String? - return struct(enableIndexing = enable_indexing, relativeIndexStorePath = relative_index_store_path) +def bazel_build_service_config( + bep_path = None, # String? + indexing_enabled = False, # Bool + index_store_path = None, # String? + indexing_data_dir = None, # String? + progress_bar_enabled = False): # Bool + return struct(bepPath = bep_path, indexingEnabled = indexing_enabled, indexStorePath = index_store_path, indexingDataDir = indexing_data_dir, progressBarEnabled = progress_bar_enabled) diff --git a/BazelExtensions/xcode_configuration_provider.bzl b/BazelExtensions/xcode_configuration_provider.bzl index 38d13934..f2c5eba7 100644 --- a/BazelExtensions/xcode_configuration_provider.bzl +++ b/BazelExtensions/xcode_configuration_provider.bzl @@ -154,8 +154,79 @@ def _install_action(ctx, infos, itarget): ) return [output] +SourceOutputFileMapInfo = provider( + doc = "...", + fields = { + "mapping": "Dictionary where keys are source file paths and values are at the respective .o file under bazel-out", + }, +) + +def _source_output_file_map(target, ctx): + """ + Maps source code files to respective `.o` object file under bazel-out. Output group is used for indexing in xcbuildkit. + """ + source_output_file_map = ctx.actions.declare_file("{}_source_output_file_map.json".format(target.label.name)) + + mapping = {} + objc_srcs = [] + objc_objects = [] + + # List of source files to be mapped to output files + if hasattr(ctx.rule.attr, "srcs"): + objc_srcs = [ + f + for source_file in ctx.rule.attr.srcs + for f in source_file.files.to_list() + # Handling objc only for now + if f.path.endswith(".m") + # TODO: handle swift + ] + + # Get compilation outputs if present + if OutputGroupInfo in target: + if hasattr(target[OutputGroupInfo], "compilation_outputs"): + objc_objects.extend(target[OutputGroupInfo].compilation_outputs.to_list()) + + # Map source to output file + if len(objc_srcs): + if len(objc_srcs) != len(objc_objects): + fail("[ERROR] Unexpected number of object files") + for src in objc_srcs: + basename_without_ext = src.basename.replace(".%s" % src.extension, "") + obj = [o for o in objc_objects if "%s.o" % basename_without_ext == o.basename] + if len(obj) != 1: + fail("Failed to find single object file for source %s. Found: %s" % (src, obj)) + + obj = obj[0] + mapping["/{}".format(src.path)] = obj.path + + # Collect info from deps + deps = getattr(ctx.rule.attr, "deps", []) + transitive_jsons = [] + for dep in deps: + # Collect mappings from deps + if SourceOutputFileMapInfo in dep: + for k, v in dep[SourceOutputFileMapInfo].mapping.items(): + mapping[k] = v + # Collect generated JSON files from deps + if OutputGroupInfo in dep: + if hasattr(dep[OutputGroupInfo], "source_output_file_map"): + transitive_jsons.extend(dep[OutputGroupInfo].source_output_file_map.to_list()) + + # Writes JSON + ctx.actions.write(source_output_file_map, json.encode(mapping)) + + return [ + OutputGroupInfo( + source_output_file_map = depset([source_output_file_map] + transitive_jsons), + ), + SourceOutputFileMapInfo( + mapping = mapping + ), + ] def _xcode_build_sources_aspect_impl(itarget, ctx): """ Install Xcode project dependencies into the source root. + This is required as by default, Bazel only installs genfiles for those genfiles which are passed to the Bazel command line. """ @@ -192,7 +263,7 @@ def _xcode_build_sources_aspect_impl(itarget, ctx): ), ), XcodeBuildSourceInfo(values = depset(infos), transitive=[depset(compacted_transitive_files)]), - ] + ] + _source_output_file_map(itarget, ctx) # Note, that for "pure" Xcode builds we build swiftmodules with Xcode, so we diff --git a/BazelExtensions/xcodeproject.bzl b/BazelExtensions/xcodeproject.bzl index da94888c..3d441135 100644 --- a/BazelExtensions/xcodeproject.bzl +++ b/BazelExtensions/xcodeproject.bzl @@ -10,7 +10,6 @@ load( "xchammer_config", "gen_xchammer_config", "project_config", - "build_service_config", ) load( @@ -18,8 +17,8 @@ load( "XcodeProjectTargetInfo", "XcodeConfigurationAspectInfo", "target_config_aspect", - "xcode_build_sources_aspect", "XcodeBuildSourceInfo", + "SourceOutputFileMapInfo", ) non_hermetic_execution_requirements = { "no-cache": "1", "no-remote": "1", "local": "1", "no-sandbox": "1" } @@ -42,6 +41,7 @@ def _xcode_project_impl(ctx): # Collect Target configuration JSON from deps # Then, merge them to a list aggregate_target_config = {} + for dep in ctx.attr.targets: if XcodeConfigurationAspectInfo in dep: for info in dep[XcodeConfigurationAspectInfo].values: @@ -151,7 +151,7 @@ _xcode_project = rule( implementation=_xcode_project_impl, attrs={ "targets": attr.label_list( - aspects=[tulsi_sources_aspect, target_config_aspect] + aspects=[tulsi_sources_aspect, target_config_aspect], ), "project_name": attr.string(), "bazel": attr.string(default="bazel"), @@ -170,20 +170,12 @@ def _install_xcode_project_impl(ctx): xcodeproj = ctx.attr.xcodeproj.files.to_list()[0] output_proj = "$SRCROOT/" + xcodeproj.basename - build_service_config = ( - ctx.attr.build_service_config - if ctx.attr.build_service_config - else build_service_config().to_json() - ) - build_service_config = json.decode(build_service_config) - enable_indexing = build_service_config.get("enableIndexing", False) - relative_index_store_path = build_service_config.get("relativeIndexStorePath", None) - command = [ "SRCROOT=" + get_srcroot, "ditto " + xcodeproj.path + " " + output_proj, "sed -i '' \"s,__BAZEL_EXEC_ROOT__,$PWD,g\" " + output_proj + "/XCHammerAssets/bazel_build_settings.py", "sed -i '' \"s,__BAZEL_OUTPUT_BASE__,$(dirname $(dirname $PWD)),g\" " + output_proj + "/XCHammerAssets/bazel_build_settings.py", + "sed -i '' \"s,__BAZEL_EXEC_ROOT__,$PWD,g\" " + output_proj + "/XCHammerAssets/bazel_build_service_setup.sh", # This is kind of a hack for reference bazel relative to the source # directory, as bazel_build_settings.py doesn't sub Xcode build # settings. @@ -196,13 +188,6 @@ def _install_xcode_project_impl(ctx): 'echo "' + output_proj + '" > ' + ctx.outputs.out.path, ] - if enable_indexing and relative_index_store_path: - command.extend([ - "DD=$(xcodebuild -project " + output_proj + " -showBuildSettings 2>&1 | grep -e \"^\\s*BUILT_PRODUCTS_DIR = \" | cut -d'=' -f2 | xargs | sed -e 's/\\/Build.*//g')", - 'mv $DD/Index/DataStore $DD/Index/DataStore.default || true', - "mkdir -p '$SRCROOT{}'".format(relative_index_store_path), - "ln -s $(echo $SRCROOT){} $DD/Index/DataStore".format(relative_index_store_path), - ]) ctx.actions.run_shell( inputs=ctx.attr.xcodeproj.files, command=";".join(command), @@ -216,12 +201,10 @@ _install_xcode_project = rule( implementation=_install_xcode_project_impl, attrs={ "xcodeproj": attr.label(mandatory=True), - "build_service_config": attr.string(), }, outputs={"out": "%{name}.dummy"}, ) - def xcode_project(**kwargs): """ Generate an Xcode project @@ -246,7 +229,6 @@ def xcode_project(**kwargs): proj_args["project_name"] = kwargs["name"] # Build an XCHammer config Based on inputs - targets_json = [str(t) for t in kwargs.get("targets")] if "target_config" in proj_args: str_dict = {} for k in proj_args["target_config"]: @@ -259,10 +241,6 @@ def xcode_project(**kwargs): proj_args["name"] = rule_name + "_impl" proj_args["project_config"] = proj_args["project_config"].to_json() if "project_config" in proj_args else None - # For now pop to use in '_install_xcode_project', we might end up using this in the - # _xcode_project impl later on - build_service_config = proj_args.pop("build_service_config").to_json() if "build_service_config" in proj_args else None - _xcode_project(**proj_args) # Note: _xcode_project does the hermetic, reproducible bits @@ -271,5 +249,4 @@ def xcode_project(**kwargs): name=rule_name, xcodeproj=kwargs["name"], testonly=proj_args.get("testonly", False), - build_service_config = build_service_config, ) diff --git a/Sources/XCHammer/Generator.swift b/Sources/XCHammer/Generator.swift index 572bac38..cc4c8ea1 100644 --- a/Sources/XCHammer/Generator.swift +++ b/Sources/XCHammer/Generator.swift @@ -88,9 +88,7 @@ enum Generator { let overrides = getRepositoryOverrides(genOptions: genOptions) let bazelArgs: [String] = [ "--aspects @xchammer//:BazelExtensions/xcode_configuration_provider.bzl%pure_xcode_build_sources_aspect", - "--output_groups=xcode_project_deps", - "--aspects @xchammer//:BazelExtensions/source_output_file_map_aspect.bzl%source_output_file_map_aspect", - "--output_groups=+source_output_file_map", + "--output_groups=xcode_project_deps,+source_output_file_map", ] + overrides + labels.map { $0.value } // We retry.sh the bazel command so if Xcode updates, the build still works @@ -432,9 +430,7 @@ enum Generator { ] + overrides + [ // Build xcode_project_deps for targets in question. "--aspects @xchammer//:BazelExtensions/xcode_configuration_provider.bzl%xcode_build_sources_aspect", - "--output_groups=+xcode_project_deps", - "--aspects @xchammer//:BazelExtensions/source_output_file_map_aspect.bzl%source_output_file_map_aspect", - "--output_groups=+source_output_file_map", + "--output_groups=+xcode_project_deps,+source_output_file_map", ] let buildOptions = (targetConfig?.buildBazelOptions ?? "") + " " + diff --git a/Sources/XCHammer/XCBuildSettings.swift b/Sources/XCHammer/XCBuildSettings.swift index f6db842f..30d46e93 100644 --- a/Sources/XCHammer/XCBuildSettings.swift +++ b/Sources/XCHammer/XCBuildSettings.swift @@ -185,6 +185,12 @@ enum XCSettingCodingKey: String, CodingKey { case sdkRoot = "SDKROOT" case targetedDeviceFamily = "TARGETED_DEVICE_FAMILY" + // Build Service configs (xcbuildkit) + case buildServiceBEPPath = "BUILD_SERVICE_BEP_PATH" + case buildServiceIndexingEnabled = "BUILD_SERVICE_INDEXING_ENABLED" + case buildServiceIndexStorePath = "BUILD_SERVICE_INDEX_STORE_PATH" + case buildServiceIndexingDataDir = "BUILD_SERVICE_INDEXING_DATA_DIR" + case buildServiceProgressBarEnabled = "BUILD_SERVICE_PROGRESS_BAR_ENABLED" } @@ -237,6 +243,11 @@ struct XCBuildSettings: Encodable { var targetedDeviceFamily: OrderedArray = OrderedArray.empty var isBazel: First = First("NO") var diagnosticFlags: [String] = [] + var buildServiceBEPPath: First? + var buildServiceIndexingEnabled: First? + var buildServiceIndexStorePath: First? + var buildServiceIndexingDataDir: First? + var buildServiceProgressBarEnabled: First? func encode(to encoder: Encoder) throws { @@ -309,6 +320,13 @@ struct XCBuildSettings: Encodable { // XCHammer only supports Xcode projects at the root directory try container.encode("$SOURCE_ROOT", forKey: .tulsiWR) try container.encode(diagnosticFlags.joined(separator: " "), forKey: .diagnosticFlags) + + // Build Service (xcbuildkit) + try buildServiceBEPPath.map { try container.encode($0.v, forKey: .buildServiceBEPPath) } + try buildServiceIndexingEnabled.map { try container.encode($0.v, forKey: .buildServiceIndexingEnabled) } + try buildServiceIndexStorePath.map { try container.encode($0.v, forKey: .buildServiceIndexStorePath) } + try buildServiceIndexingDataDir.map { try container.encode($0.v, forKey: .buildServiceIndexingDataDir) } + try buildServiceProgressBarEnabled.map { try container.encode($0.v, forKey: .buildServiceProgressBarEnabled) } } } @@ -366,7 +384,12 @@ extension XCBuildSettings: Monoid { targetedDeviceFamily: lhs.targetedDeviceFamily <> rhs.targetedDeviceFamily, isBazel: lhs.isBazel <> rhs.isBazel, - diagnosticFlags: lhs.diagnosticFlags <> rhs.diagnosticFlags + diagnosticFlags: lhs.diagnosticFlags <> rhs.diagnosticFlags, + buildServiceBEPPath: lhs.buildServiceBEPPath <> rhs.buildServiceBEPPath, + buildServiceIndexingEnabled: lhs.buildServiceIndexingEnabled <> rhs.buildServiceIndexingEnabled, + buildServiceIndexStorePath: lhs.buildServiceIndexStorePath <> rhs.buildServiceIndexStorePath, + buildServiceIndexingDataDir: lhs.buildServiceIndexingDataDir <> rhs.buildServiceIndexingDataDir, + buildServiceProgressBarEnabled: lhs.buildServiceProgressBarEnabled <> rhs.buildServiceProgressBarEnabled ) } diff --git a/Sources/XCHammer/XCHammerConfig.swift b/Sources/XCHammer/XCHammerConfig.swift index 8daf1fd9..77135e51 100644 --- a/Sources/XCHammer/XCHammerConfig.swift +++ b/Sources/XCHammer/XCHammerConfig.swift @@ -137,6 +137,9 @@ public struct XCHammerProjectConfig: Codable { /// serialized. Spaces and escapes matter. public let buildBazelPlatformOptions: [String: [String]]? + /// Allows one to configure and pass information to the Build Service + public let bazelBuildServiceConfig: BazelBuildServiceConfig? + /// Enable generation of transitive Xcode targets. /// Defaults to `true` /// @note this is _generally_ required for Xcode projects to build with @@ -193,9 +196,19 @@ public struct XCHammerProjectConfig: Codable { xcconfigOverrides = (try container.decodeIfPresent( [String: String].self, forKey: .xcconfigOverrides)) ?? nil + bazelBuildServiceConfig = try container.decodeIfPresent( + BazelBuildServiceConfig.self, forKey: .bazelBuildServiceConfig) } } +public struct BazelBuildServiceConfig: Codable { + public let bepPath: String? + public let indexStorePath: String? + public let indexingEnabled: Bool + public let indexingDataDir: String? + public let progressBarEnabled: Bool +} + public struct XCHammerConfig: Codable { /// Labels for all targets. /// Transitve dependencies are converted into targets unless excluded by diff --git a/Sources/XCHammer/XcodeTarget.swift b/Sources/XCHammer/XcodeTarget.swift index 11eebae0..e3892f7d 100644 --- a/Sources/XCHammer/XcodeTarget.swift +++ b/Sources/XCHammer/XcodeTarget.swift @@ -1499,9 +1499,18 @@ public class XcodeTarget: Hashable, Equatable { settings.pythonPath = First("${PYTHONPATH}:$(PROJECT_FILE_PATH)/XCHammerAssets") settings <>= getDeploymentTargetSettings() + // Build Service configs (xcbuildkit) + if let bazelBuildServiceConfig = genOptions.config.projects[genOptions.projectName]?.bazelBuildServiceConfig { + settings.buildServiceBEPPath = First(bazelBuildServiceConfig.bepPath ?? "") + settings.buildServiceIndexingEnabled = First(bazelBuildServiceConfig.indexingEnabled ? "YES" : "NO") + settings.buildServiceIndexStorePath = First(bazelBuildServiceConfig.indexStorePath ?? "") + settings.buildServiceIndexingDataDir = First(bazelBuildServiceConfig.indexingDataDir ?? "") + settings.buildServiceProgressBarEnabled = First(bazelBuildServiceConfig.progressBarEnabled ? "YES" : "NO") + } + + let bazelScript = ProjectSpec.BuildScript(path: nil, script: getScriptContent(), name: "Bazel build") + let buildServiceSetupScript = ProjectSpec.BuildScript(path: nil, script: "$PROJECT_FILE_PATH/XCHammerAssets/bazel_build_service_setup.sh", name: "Build Service Setup") - let bazelScript = ProjectSpec.BuildScript(path: nil, script: getScriptContent(), - name: "Bazel build") return ProjectSpec.Target( name: xcTargetName, type: PBXProductType(rawValue: productType.rawValue)!, @@ -1510,6 +1519,7 @@ public class XcodeTarget: Hashable, Equatable { configFiles: getXCConfigFiles(for: self), sources: sources, dependencies: [], + preBuildScripts: [buildServiceSetupScript], postBuildScripts: [bazelScript] ) } diff --git a/XCHammerAssets/bazel_build_service_setup.sh b/XCHammerAssets/bazel_build_service_setup.sh new file mode 100755 index 00000000..f094dcf0 --- /dev/null +++ b/XCHammerAssets/bazel_build_service_setup.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# Values set at project generation time, for reference check `BazelExtensions/xcodeproject.bzl` => `_install_xcode_project` rule +BUILD_SERVICE_BAZEL_EXEC_ROOT=__BAZEL_EXEC_ROOT__ + +# Check `BazelExtensions/source_output_file_map_aspect.bzl`, xcbuildkit needs to know what pattern to look for to pre-load indexing information +BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX=source_output_file_map.json + +# Load DataStore related paths, used below to setup the DataStore or undo the setup if the build service is not installed +WORKSPACE_ROOT=$(echo "$BUILD_DIR" | sed -e 's/\/Build.*//g') + +if [[ ! -d $WORKSPACE_ROOT ]]; then + echo "[ERROR] Failed to setup build service. Workspace not found at $WORKSPACE_ROOT" + exit 1 +fi + +INDEX_DATA_STORE=$WORKSPACE_ROOT/Index/DataStore +INDEX_DATA_STORE_BACKUP=$WORKSPACE_ROOT/Index/DataStore.default + +undoDataStoreSetupIfNecessary () { + # Remove symlink + if [[ -L $INDEX_DATA_STORE ]]; then + rm $INDEX_DATA_STORE + + # If there's a backup put it back + if [[ -d $INDEX_DATA_STORE_BACKUP ]]; then + mv $INDEX_DATA_STORE_BACKUP $INDEX_DATA_STORE + fi + fi +} + +# Build service binary checks +XCODE_CONTENTS=$(dirname $DEVELOPER_DIR) +BUILD_SERVICE=$XCODE_CONTENTS/SharedFrameworks/XCBuild.framework/PlugIns/XCBBuildService.bundle/Contents/MacOS/XCBBuildService + +# if the build service does not exist at this location we messed something up +if [[ ! -f $BUILD_SERVICE ]]; then + echo "Could not find build service at $BUILD_SERVICE. Check your Xcode installation." + undoDataStoreSetupIfNecessary + exit 1 +fi + +# If the build service is not a symlink, xcbuildkit is not installed so there's nothing to do +if [[ ! -L $BUILD_SERVICE ]]; then + echo "Build service not installed. Nothing to do." + undoDataStoreSetupIfNecessary + exit 0 +fi + +# Ensure this folder exists, used by xcbuildkit to hold cached indexing data +mkdir -p $BUILD_SERVICE_INDEXING_DATA_DIR + +# xcbuildkit expects a config file called `xcbuildkit.config` under path/to/foo.xcodeproj +BUILD_SERVICE_CONFIG_PATH=$PROJECT_FILE_PATH/xcbuildkit.config + +cat >$BUILD_SERVICE_CONFIG_PATH < Date: Tue, 1 Nov 2022 18:23:37 -0400 Subject: [PATCH 03/10] Bump xcbuildkit --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 72e0a894..29f4cd68 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -67,7 +67,7 @@ xchammer_dependencies() # https://github.com/bazelbuild/bazel/issues/1550 git_repository( name = "xcbuildkit", - commit = "b2f0e4dd5a572b7029db3cf791d0897977f96a80", + commit = "5685d7d8bef0061bfad54f21680b5982f692440b", remote = "https://github.com/jerrymarino/xcbuildkit.git", ) From ae62d1e8a928c49b1a088042125598c31ff9f46e Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Wed, 2 Nov 2022 09:13:18 -0400 Subject: [PATCH 04/10] Minor fix to handle existing DataStore backup --- XCHammerAssets/bazel_build_service_setup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/XCHammerAssets/bazel_build_service_setup.sh b/XCHammerAssets/bazel_build_service_setup.sh index f094dcf0..f9d50692 100755 --- a/XCHammerAssets/bazel_build_service_setup.sh +++ b/XCHammerAssets/bazel_build_service_setup.sh @@ -80,6 +80,9 @@ fi # If there's an existing DataStore create a backup before symlink-ing if [[ -d $INDEX_DATA_STORE ]]; then + if [[ -d $INDEX_DATA_STORE_BACKUP ]]; then + rm -fr $INDEX_DATA_STORE_BACKUP + fi mv $INDEX_DATA_STORE $INDEX_DATA_STORE_BACKUP fi From f324e6a39f3ff688def26abd911c83ed3f020065 Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Wed, 2 Nov 2022 10:47:53 -0400 Subject: [PATCH 05/10] Bump xcbuildkit again --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 29f4cd68..1d3ba986 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -67,7 +67,7 @@ xchammer_dependencies() # https://github.com/bazelbuild/bazel/issues/1550 git_repository( name = "xcbuildkit", - commit = "5685d7d8bef0061bfad54f21680b5982f692440b", + commit = "a461724a1cf6955c343c660fdd0bf84de191c043", remote = "https://github.com/jerrymarino/xcbuildkit.git", ) From 43ce6ea91be03b8a1b2572db7fb4beb8a522ed8f Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Wed, 2 Nov 2022 12:27:13 -0400 Subject: [PATCH 06/10] Bump xcbuildkit again --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 1d3ba986..6ddac8b6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -67,7 +67,7 @@ xchammer_dependencies() # https://github.com/bazelbuild/bazel/issues/1550 git_repository( name = "xcbuildkit", - commit = "a461724a1cf6955c343c660fdd0bf84de191c043", + commit = "a03d9c7c5efb961b317f5efb95cbfcf7471b683e", remote = "https://github.com/jerrymarino/xcbuildkit.git", ) From 01efb200049f3933e4d51f9a354ecbd0ab78848a Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Fri, 4 Nov 2022 13:18:56 -0400 Subject: [PATCH 07/10] Bump xcbuildkit, handle DataStore setup there instead --- WORKSPACE | 2 +- XCHammerAssets/bazel_build_service_setup.sh | 52 +-------------------- 2 files changed, 2 insertions(+), 52 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 6ddac8b6..64ea3132 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -67,7 +67,7 @@ xchammer_dependencies() # https://github.com/bazelbuild/bazel/issues/1550 git_repository( name = "xcbuildkit", - commit = "a03d9c7c5efb961b317f5efb95cbfcf7471b683e", + commit = "1eb4ed9f2026b35cece333308ba9623acede90f3", remote = "https://github.com/jerrymarino/xcbuildkit.git", ) diff --git a/XCHammerAssets/bazel_build_service_setup.sh b/XCHammerAssets/bazel_build_service_setup.sh index f9d50692..185cb752 100755 --- a/XCHammerAssets/bazel_build_service_setup.sh +++ b/XCHammerAssets/bazel_build_service_setup.sh @@ -6,29 +6,6 @@ BUILD_SERVICE_BAZEL_EXEC_ROOT=__BAZEL_EXEC_ROOT__ # Check `BazelExtensions/source_output_file_map_aspect.bzl`, xcbuildkit needs to know what pattern to look for to pre-load indexing information BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX=source_output_file_map.json -# Load DataStore related paths, used below to setup the DataStore or undo the setup if the build service is not installed -WORKSPACE_ROOT=$(echo "$BUILD_DIR" | sed -e 's/\/Build.*//g') - -if [[ ! -d $WORKSPACE_ROOT ]]; then - echo "[ERROR] Failed to setup build service. Workspace not found at $WORKSPACE_ROOT" - exit 1 -fi - -INDEX_DATA_STORE=$WORKSPACE_ROOT/Index/DataStore -INDEX_DATA_STORE_BACKUP=$WORKSPACE_ROOT/Index/DataStore.default - -undoDataStoreSetupIfNecessary () { - # Remove symlink - if [[ -L $INDEX_DATA_STORE ]]; then - rm $INDEX_DATA_STORE - - # If there's a backup put it back - if [[ -d $INDEX_DATA_STORE_BACKUP ]]; then - mv $INDEX_DATA_STORE_BACKUP $INDEX_DATA_STORE - fi - fi -} - # Build service binary checks XCODE_CONTENTS=$(dirname $DEVELOPER_DIR) BUILD_SERVICE=$XCODE_CONTENTS/SharedFrameworks/XCBuild.framework/PlugIns/XCBBuildService.bundle/Contents/MacOS/XCBBuildService @@ -36,14 +13,12 @@ BUILD_SERVICE=$XCODE_CONTENTS/SharedFrameworks/XCBuild.framework/PlugIns/XCBBuil # if the build service does not exist at this location we messed something up if [[ ! -f $BUILD_SERVICE ]]; then echo "Could not find build service at $BUILD_SERVICE. Check your Xcode installation." - undoDataStoreSetupIfNecessary exit 1 fi # If the build service is not a symlink, xcbuildkit is not installed so there's nothing to do if [[ ! -L $BUILD_SERVICE ]]; then echo "Build service not installed. Nothing to do." - undoDataStoreSetupIfNecessary exit 0 fi @@ -63,29 +38,4 @@ BUILD_SERVICE_BAZEL_EXEC_ROOT=$BUILD_SERVICE_BAZEL_EXEC_ROOT BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX=$BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX EOL -echo "[INFO] Wrote xcbuildkit config file at $BUILD_SERVICE_CONFIG_PATH" - -# Setup DataStore - -# If for some reason there's an existing symlink and it does not match 'BUILD_SERVICE_INDEX_STORE_PATH' remove it and reconfigure below, -# otherwise exits since there's nothing to do and build service is fully configured -if [[ -L $INDEX_DATA_STORE ]]; then - if [[ "$(readlink $INDEX_DATA_STORE)" = "$BUILD_SERVICE_INDEX_STORE_PATH" ]]; then - echo "[INFO] Build service already configured. DataStore symlinked from $INDEX_DATA_STORE to $BUILD_SERVICE_INDEX_STORE_PATH." - exit 0 - else - rm $INDEX_DATA_STORE - fi -fi - -# If there's an existing DataStore create a backup before symlink-ing -if [[ -d $INDEX_DATA_STORE ]]; then - if [[ -d $INDEX_DATA_STORE_BACKUP ]]; then - rm -fr $INDEX_DATA_STORE_BACKUP - fi - mv $INDEX_DATA_STORE $INDEX_DATA_STORE_BACKUP -fi - -ln -s $BUILD_SERVICE_INDEX_STORE_PATH $INDEX_DATA_STORE - -echo "[INFO] Build service setup complete. DataStore symlinked from $INDEX_DATA_STORE to $BUILD_SERVICE_INDEX_STORE_PATH." \ No newline at end of file +echo "[INFO] Wrote xcbuildkit config file at $BUILD_SERVICE_CONFIG_PATH" \ No newline at end of file From 48a02b3790669b5801971e9268ca3249003f4f2e Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Fri, 4 Nov 2022 14:48:37 -0400 Subject: [PATCH 08/10] Support more file extensions --- BazelExtensions/xcode_configuration_provider.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BazelExtensions/xcode_configuration_provider.bzl b/BazelExtensions/xcode_configuration_provider.bzl index f2c5eba7..56b0337c 100644 --- a/BazelExtensions/xcode_configuration_provider.bzl +++ b/BazelExtensions/xcode_configuration_provider.bzl @@ -178,7 +178,7 @@ def _source_output_file_map(target, ctx): for source_file in ctx.rule.attr.srcs for f in source_file.files.to_list() # Handling objc only for now - if f.path.endswith(".m") + if f.path.endswith((".m", ".mm", ".c", ".cc", ".cpp")) # TODO: handle swift ] From 95eea020f7d0bd6ffb3155f0cbb6df4c0209aaec Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Fri, 4 Nov 2022 15:03:28 -0400 Subject: [PATCH 09/10] Better transitive deps --- BazelExtensions/xcode_configuration_provider.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BazelExtensions/xcode_configuration_provider.bzl b/BazelExtensions/xcode_configuration_provider.bzl index 56b0337c..53fba3f0 100644 --- a/BazelExtensions/xcode_configuration_provider.bzl +++ b/BazelExtensions/xcode_configuration_provider.bzl @@ -211,14 +211,14 @@ def _source_output_file_map(target, ctx): # Collect generated JSON files from deps if OutputGroupInfo in dep: if hasattr(dep[OutputGroupInfo], "source_output_file_map"): - transitive_jsons.extend(dep[OutputGroupInfo].source_output_file_map.to_list()) + transitive_jsons.append(dep[OutputGroupInfo].source_output_file_map) # Writes JSON ctx.actions.write(source_output_file_map, json.encode(mapping)) return [ OutputGroupInfo( - source_output_file_map = depset([source_output_file_map] + transitive_jsons), + source_output_file_map = depset([source_output_file_map], transitive = transitive_jsons), ), SourceOutputFileMapInfo( mapping = mapping From 3e869fc59507414fbcb84e4c6f610e8436b28cb6 Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Fri, 4 Nov 2022 16:48:57 -0400 Subject: [PATCH 10/10] Bump xcbuildkit to main branch --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 64ea3132..53e21f61 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -67,7 +67,7 @@ xchammer_dependencies() # https://github.com/bazelbuild/bazel/issues/1550 git_repository( name = "xcbuildkit", - commit = "1eb4ed9f2026b35cece333308ba9623acede90f3", + commit = "4c366afb48cb78caed268d483e3cdb308dfc1794", remote = "https://github.com/jerrymarino/xcbuildkit.git", )