From c3c83d57732d52a6778c69f4595969026a524d3e Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Fri, 7 Oct 2022 15:06:58 -0400 Subject: [PATCH 1/6] Parse source to output file map from aspect outputgroup JSON files --- Examples/BazelBuildService/main.swift | 123 ++++++++++++++++++++++++-- Sources/BKBuildService/Service.swift | 11 +-- Sources/XCBProtocol/Coder.swift | 2 +- 3 files changed, 118 insertions(+), 18 deletions(-) diff --git a/Examples/BazelBuildService/main.swift b/Examples/BazelBuildService/main.swift index 758d470..5217f28 100644 --- a/Examples/BazelBuildService/main.swift +++ b/Examples/BazelBuildService/main.swift @@ -14,7 +14,7 @@ var gStream: BEPStream? // // In `BazelBuildService` keep as `false` by default until this is ready to be enabled in all scenarios mostly to try to keep // this backwards compatible with others installing this build service to get the progress bar. -private let indexingEnabled: Bool = false +private let indexingEnabled: Bool = true // TODO: Make this part of an API to be consumed from callers // @@ -28,11 +28,13 @@ private let indexingEnabled: Bool = false // "/tests/ios/app/App/main.m": "bazel-out/ios-x86_64-min10.0-applebin_ios-ios_x86_64-dbg-ST-0f1b0425081f/bin/tests/ios/app/_objs/App_objc/arc/main.o", // "/tests/ios/app/App/Foo.m": "bazel-out/ios-x86_64-min10.0-applebin_ios-ios_x86_64-dbg-ST-0f1b0425081f/bin/tests/ios/app/_objs/App_objc/arc/Foo.o", // ], -private let outputFileForSource: [String: [String: String]] = [ +private var outputFileForSource: [String: [String: [String: String]]] = [ // Vanilla Xcode mapping for debug/testing purposes "iOSApp-frhmkkebaragakhdzyysbrsvbgtc": [ - "/CLI/main.m": "/tmp/xcbuild-out/CLI/main.o", - "/iOSApp/main.m": "/tmp/xcbuild-out/iOSApp/main.o", + "foo_source_output_file_map.json": [ + "/CLI/main.m": "/tmp/xcbuild-out/CLI/main.o", + "/iOSApp/main.m": "/tmp/xcbuild-out/iOSApp/main.o", + ] ], ] @@ -44,6 +46,15 @@ private var gXcode = "" private var workspaceHash = "" // TODO: parsed in `CreateSessionRequest`, consider a more stable approach instead of parsing `xcbuildDataPath` path there private var workspaceName = "" +// Key to identify a workspace and find its mapping of source to object files in `outputFileForSource` +private var workspaceKey: String? { + guard workspaceName.count > 0 && workspaceHash.count > 0 else { + return nil + } + return "\(workspaceName)-\(workspaceHash)" +} +// Bazel external working directory, base path used in unit files during indexing +private var bazelWorkingDir: String? // TODO: parsed in `IndexingInfoRequested`, there's probably a less hacky way to get this. // Effectively `$PWD/iOSApp` private var workingDir = "" @@ -77,6 +88,61 @@ enum BasicMessageHandler { var progressView: ProgressView? try stream.read { event in + + // XCHammer generates JSON files containing source => output file mappings. + // + // This loop looks for JSON files with a known name pattern '_source_output_file_map.json' and extracts the mapping + // information from it decoding the JSON and storing in-memory. We might want to find a way to pass this in instead. + // + // Read about the 'namedSetOfFiles' key here: https://bazel.build/remote/bep-examples#consuming-namedsetoffiles + if let json = try? JSONSerialization.jsonObject(with: event.jsonUTF8Data(), options: []) as? [String: Any] { + if let namedSetOfFiles = json["namedSetOfFiles"] as? [String: Any] { + if namedSetOfFiles.count > 0 { + if let allPairs = namedSetOfFiles["files"] as? [[String: Any]] { + for pair in allPairs { + guard let theName = pair["name"] as? String else { + continue + } + guard var jsonURI = pair["uri"] as? String else { + continue + } + guard jsonURI.hasSuffix(".json") else { + continue + } + + jsonURI = jsonURI.replacingOccurrences(of: "file://", with: "") + + // The Bazel working directory is necessary for indexing, first time we see it in the BEP + // storing in 'bazelWorkingDir' + if bazelWorkingDir == nil { + bazelWorkingDir = jsonURI.components(separatedBy: "/bazel-out").first + } + + guard let jsonData = try? Data(contentsOf: URL(fileURLWithPath:jsonURI)) else { + continue + } + guard jsonData.count > 0 else { + continue + } + guard let jsonDecoded = try? JSONSerialization.jsonObject(with: jsonData, options: [.allowFragments]) as? [String: String] else { + continue + } + + if let workspaceKey = workspaceKey, theName.contains("_source_output_file_map.json") { + if outputFileForSource[workspaceKey] == nil { + outputFileForSource[workspaceKey] = [:] + } + if outputFileForSource[workspaceKey]?[theName] == nil { + outputFileForSource[workspaceKey]?[theName] = [:] + } + outputFileForSource[workspaceKey]?[theName] = jsonDecoded + } + } + } + } + } + } + if let updatedView = ProgressView(event: event, last: progressView) { let encoder = XCBEncoder(input: startBuildInput) let response = BuildProgressUpdatedResponse(progress: @@ -115,6 +181,47 @@ enum BasicMessageHandler { return bplistData } + static func canHandleIndexing(msg: XCBProtocolMessage) -> Bool { + guard msg is IndexingInfoRequested else { + return false + } + guard let workspaceKey = workspaceKey else { + return false + } + guard indexingEnabled else { + return false + } + guard bazelWorkingDir != nil else { + return false + } + + if outputFileForSource[workspaceKey] == nil { + outputFileForSource[workspaceKey] = [:] + } + + guard (outputFileForSource[workspaceKey]?.count ?? 0) > 0 else { + return false + } + + return true + } + + static func findOutputFileForSource(filePath: String, workingDir: String) -> String? { + let sourceKey = filePath.replacingOccurrences(of: workingDir, with: "").replacingOccurrences(of: (bazelWorkingDir ?? ""), with: "") + guard let workspaceKey = workspaceKey else { + return nil + } + guard let workspaceMappings = outputFileForSource[workspaceKey] else { + return nil + } + for (_, json) in workspaceMappings { + if let objFilePath = json[sourceKey] { + return objFilePath + } + } + return nil + } + /// Proxying response handler /// Every message is written to the XCBBuildService /// This simply injects Progress messages from the BEP @@ -142,16 +249,14 @@ enum BasicMessageHandler { if let responseData = try? message.encode(encoder) { bkservice.write(responseData) } - } else if indexingEnabled && msg is IndexingInfoRequested { + } else if canHandleIndexing(msg: msg) { // Example of a custom indexing service let reqMsg = msg as! IndexingInfoRequested - workingDir = reqMsg.workingDir + workingDir = bazelWorkingDir ?? reqMsg.workingDir platform = reqMsg.platform sdk = reqMsg.sdk - let workspaceKey = "\(workspaceName)-\(workspaceHash)" - let sourceKey = reqMsg.filePath.replacingOccurrences(of: workingDir, with: "") - guard let outputFilePath = outputFileForSource[workspaceKey]?[sourceKey] else { + guard let outputFilePath = findOutputFileForSource(filePath: reqMsg.filePath, workingDir: reqMsg.workingDir) else { fatalError("Failed to find output file for source: \(reqMsg.filePath)") return } diff --git a/Sources/BKBuildService/Service.swift b/Sources/BKBuildService/Service.swift index 9769174..d41e5f9 100644 --- a/Sources/BKBuildService/Service.swift +++ b/Sources/BKBuildService/Service.swift @@ -101,14 +101,9 @@ public class BKBuildService { // // Important: Note that `ogData` still needs to be passed below so the original build service can parse `CREATE_SESSION` and // write the correct response to stdout for us for now - var inputResult = [MessagePackValue]() - var inputData = ogData - if msg is CreateSessionRequest { - inputResult = result - inputData = self.buffer - } - - messageHandler(XCBInputStream(result: inputResult, data: inputData), ogData, context) + let inputData = msg is CreateSessionRequest ? self.buffer : ogData + let ogResult = Unpacker.unpackAll(ogData) + messageHandler(XCBInputStream(result: ogResult, data: inputData), ogData, context) } // Reset all the things diff --git a/Sources/XCBProtocol/Coder.swift b/Sources/XCBProtocol/Coder.swift index 085fec7..81a0079 100644 --- a/Sources/XCBProtocol/Coder.swift +++ b/Sources/XCBProtocol/Coder.swift @@ -51,7 +51,7 @@ extension XCBDecoder { public func decodeMessage() -> XCBProtocolMessage? { do { let msg = try decodeMessageImpl() - log("decoded" + String(describing: msg)) + log("decoded: " + String(describing: msg)) return msg } catch { log("decoding failed \(error)") From e95dd8db46b458a97638582ed8ef7027139132a4 Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Thu, 27 Oct 2022 19:03:43 -0400 Subject: [PATCH 2/6] Many improvements to load and use mapping and config files --- Examples/BazelBuildService/main.swift | 303 ++++++++++++++--------- Examples/XCBBuildServiceProxy/main.swift | 2 +- Sources/BKBuildService/Service.swift | 8 +- Sources/XCBProtocol/Protocol.swift | 9 + 4 files changed, 194 insertions(+), 128 deletions(-) diff --git a/Examples/BazelBuildService/main.swift b/Examples/BazelBuildService/main.swift index 5217f28..573b158 100644 --- a/Examples/BazelBuildService/main.swift +++ b/Examples/BazelBuildService/main.swift @@ -1,6 +1,7 @@ import BKBuildService import Foundation import XCBProtocol +import BEP struct BasicMessageContext { let xcbbuildService: XCBBuildServiceProcess @@ -9,35 +10,15 @@ struct BasicMessageContext { /// FIXME: support multiple workspaces var gStream: BEPStream? -// Experimental, enables indexing buffering logic -// Make sure indexing is enabled first, i.e., run `make enable_indexing` -// -// In `BazelBuildService` keep as `false` by default until this is ready to be enabled in all scenarios mostly to try to keep -// this backwards compatible with others installing this build service to get the progress bar. -private let indexingEnabled: Bool = true - -// TODO: Make this part of an API to be consumed from callers -// -// "source file" => "output file" map, hardcoded for now, will be part of the API in the future -// Should match your local path and the values set in `Makefile > generate_custom_index_store` -// -// TODO: Should come from an aspect in Bazel -// Example of what source => object file under bazel-out mapping would look like: +// Example of what source => object file under bazel-out mapping should look like: // // "Test-XCBuildKit-cdwbwzghpxmnfadvmmhsjcdnjygy": [ -// "/tests/ios/app/App/main.m": "bazel-out/ios-x86_64-min10.0-applebin_ios-ios_x86_64-dbg-ST-0f1b0425081f/bin/tests/ios/app/_objs/App_objc/arc/main.o", -// "/tests/ios/app/App/Foo.m": "bazel-out/ios-x86_64-min10.0-applebin_ios-ios_x86_64-dbg-ST-0f1b0425081f/bin/tests/ios/app/_objs/App_objc/arc/Foo.o", +// "App_source_output_file_map.json": [ +// "/tests/ios/app/App/main.m": "bazel-out/ios-x86_64-min10.0-applebin_ios-ios_x86_64-dbg-ST-0f1b0425081f/bin/tests/ios/app/_objs/App_objc/arc/main.o", +// "/tests/ios/app/App/Foo.m": "bazel-out/ios-x86_64-min10.0-applebin_ios-ios_x86_64-dbg-ST-0f1b0425081f/bin/tests/ios/app/_objs/App_objc/arc/Foo.o", +// ], // ], -private var outputFileForSource: [String: [String: [String: String]]] = [ - // Vanilla Xcode mapping for debug/testing purposes - "iOSApp-frhmkkebaragakhdzyysbrsvbgtc": [ - "foo_source_output_file_map.json": [ - "/CLI/main.m": "/tmp/xcbuild-out/CLI/main.o", - "/iOSApp/main.m": "/tmp/xcbuild-out/iOSApp/main.o", - ] - ], -] - +private var outputFileForSource: [String: [String: [String: String]]] = [:] // Used when debugging msgs are enabled, see `XCBBuildServiceProcess.MessageDebuggingEnabled()` private var gChunkNumber = 0 // FIXME: get this from the other paths @@ -53,8 +34,6 @@ private var workspaceKey: String? { } return "\(workspaceName)-\(workspaceHash)" } -// Bazel external working directory, base path used in unit files during indexing -private var bazelWorkingDir: String? // TODO: parsed in `IndexingInfoRequested`, there's probably a less hacky way to get this. // Effectively `$PWD/iOSApp` private var workingDir = "" @@ -76,86 +55,79 @@ var sdkPath: String { return "\(gXcode)/Contents/Developer/Platforms/\(platform).platform/Developer/SDKs/\(sdk).sdk" } +// Path to .xcodeproj, used to load xcbuildkit config file from path/to/foo.xcodeproj/xcbuildkit.config +private var xcodeprojPath: String = "" +// Load configs from path/to/foo.xcodeproj/xcbuildkit.config +private var configValues: [String: Any]? { + guard let data = try? String(contentsOfFile: xcbuildkitConfigPath, encoding: .utf8) else { return nil } + + let lines = data.components(separatedBy: .newlines) + var dict: [String: Any] = [:] + for line in lines { + let split = line.components(separatedBy: "=") + guard split.count == 2 else { continue } + dict[split[0]] = split[1] + } + return dict +} +// Directory containing data used to fast load information when initializing BazelBuildService, e.g., +// .json files containing source => output file mappings generated during Xcode project generation +private var xcbuildkitDataDir: String { + return "\(xcodeprojPath)/xcbuildkit.data" +} +// File containing config values that a consumer can set, see accepted keys below. +// Format is KEY=VALUE and one config per line +// TODO: Probably better to make this a separate struct with proper validation but will do that +// once the list of accepted keys is stable +private var xcbuildkitConfigPath: String { + return "\(xcodeprojPath)/xcbuildkit.config" +} +private var sourceOutputFileMapSuffix: String? { + return configValues?["BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX"] as? String +} +private var bazelWorkingDir: String? { + return configValues?["BUILD_SERVICE_BAZEL_EXEC_ROOT"] as? String +} +private var indexingEnabled: Bool { + return (configValues?["BUILD_SERVICE_INDEXING_ENABLED"] as? String ?? "") == "YES" +} +private var progressBarEnabled: Bool { + return (configValues?["BUILD_SERVICE_PROGRESS_BAR_ENABLED"] as? String ?? "") == "YES" +} +private var configBEPPath: String? { + return configValues?["BUILD_SERVICE_BEP_PATH"] as? String +} /// This example listens to a BEP stream to display some output. /// /// All operations are delegated to XCBBuildService and we inject /// progress from BEP. enum BasicMessageHandler { + // Read info from BEP and optionally handle events static func startStream(bepPath: String, startBuildInput: XCBInputStream, bkservice: BKBuildService) throws { log("startStream " + String(describing: startBuildInput)) let stream = try BEPStream(path: bepPath) var progressView: ProgressView? try stream.read { event in - - // XCHammer generates JSON files containing source => output file mappings. - // - // This loop looks for JSON files with a known name pattern '_source_output_file_map.json' and extracts the mapping - // information from it decoding the JSON and storing in-memory. We might want to find a way to pass this in instead. - // - // Read about the 'namedSetOfFiles' key here: https://bazel.build/remote/bep-examples#consuming-namedsetoffiles - if let json = try? JSONSerialization.jsonObject(with: event.jsonUTF8Data(), options: []) as? [String: Any] { - if let namedSetOfFiles = json["namedSetOfFiles"] as? [String: Any] { - if namedSetOfFiles.count > 0 { - if let allPairs = namedSetOfFiles["files"] as? [[String: Any]] { - for pair in allPairs { - guard let theName = pair["name"] as? String else { - continue - } - guard var jsonURI = pair["uri"] as? String else { - continue - } - guard jsonURI.hasSuffix(".json") else { - continue - } - - jsonURI = jsonURI.replacingOccurrences(of: "file://", with: "") - - // The Bazel working directory is necessary for indexing, first time we see it in the BEP - // storing in 'bazelWorkingDir' - if bazelWorkingDir == nil { - bazelWorkingDir = jsonURI.components(separatedBy: "/bazel-out").first - } - - guard let jsonData = try? Data(contentsOf: URL(fileURLWithPath:jsonURI)) else { - continue - } - guard jsonData.count > 0 else { - continue - } - guard let jsonDecoded = try? JSONSerialization.jsonObject(with: jsonData, options: [.allowFragments]) as? [String: String] else { - continue - } - - if let workspaceKey = workspaceKey, theName.contains("_source_output_file_map.json") { - if outputFileForSource[workspaceKey] == nil { - outputFileForSource[workspaceKey] = [:] - } - if outputFileForSource[workspaceKey]?[theName] == nil { - outputFileForSource[workspaceKey]?[theName] = [:] - } - outputFileForSource[workspaceKey]?[theName] = jsonDecoded - } - } - } - } - } + if indexingEnabled { + parseSourceOutputFileMappingsFromBEP(event: event) } - if let updatedView = ProgressView(event: event, last: progressView) { - let encoder = XCBEncoder(input: startBuildInput) - let response = BuildProgressUpdatedResponse(progress: - updatedView.progressPercent, message: updatedView.message) - if let responseData = try? response.encode(encoder) { - bkservice.write(responseData) + if progressBarEnabled { + if let updatedView = ProgressView(event: event, last: progressView) { + let encoder = XCBEncoder(input: startBuildInput) + let response = BuildProgressUpdatedResponse(progress: + updatedView.progressPercent, message: updatedView.message) + if let responseData = try? response.encode(encoder) { + bkservice.write(responseData) + } + progressView = updatedView } - progressView = updatedView } } gStream = stream } - // Required if `outputPathOnly` is `true` in the indexing request static func outputPathOnlyData(outputFilePath: String, sourceFilePath: String) -> Data { let xml = """ @@ -180,40 +152,124 @@ enum BasicMessageHandler { return bplistData } - - static func canHandleIndexing(msg: XCBProtocolMessage) -> Bool { - guard msg is IndexingInfoRequested else { - return false + // Check many conditions that need to be met in order to handle indexing and find the respect output file, + // the call site should abort and proxy the indexing request if this returns `nil` + static func outputFileForIndexingRequest(msg: XCBProtocolMessage) -> String? { + // Nothing to do for non-indexing request types + guard let reqMsg = msg as? IndexingInfoRequested else { + return nil } - guard let workspaceKey = workspaceKey else { - return false + // TODO: handle Swift + guard reqMsg.filePath.count > 0 && reqMsg.filePath != "" && !reqMsg.filePath.hasSuffix(".swift") else { + return nil } + // Indexing needs to be enabled via config file guard indexingEnabled else { - return false + return nil } + // In `BazelBuildService` the path to the working directory (i.e. execution_root) should always + // exists guard bazelWorkingDir != nil else { - return false + fatalError("[ERROR] Path to Bazel working directory not provided. Check `BUILD_SERVICE_BAZEL_EXEC_ROOT` in your config file.") } - if outputFileForSource[workspaceKey] == nil { - outputFileForSource[workspaceKey] = [:] + guard let outputFilePath = findOutputFileForSource(filePath: reqMsg.filePath, workingDir: reqMsg.workingDir) else { + log("[WARNING] Failed to find output file for source: \(reqMsg.filePath). Indexing requests will be proxied to default build service.") + return nil } - guard (outputFileForSource[workspaceKey]?.count ?? 0) > 0 else { - return false + return outputFilePath + } + // Initialize in memory mappings from xcbuildkitDataDir if .json mappings files exist + static func initializeOutputFileMappingFromCache() { + let fm = FileManager.default + do { + let jsons = try fm.contentsOfDirectory(atPath: xcbuildkitDataDir) + + for jsonFilename in jsons { + let jsonData = try Data(contentsOf: URL(fileURLWithPath: "\(xcbuildkitDataDir)/\(jsonFilename)")) + loadSourceOutputFileMappingInfo(jsonFilename: jsonFilename, jsonData: jsonData) + } + } catch { + log("[ERROR] Failed to initialize from cache under \(xcbuildkitDataDir) with err: \(error.localizedDescription)") } + } + // Loads information into memory and optionally update the cache under xcbuildkitDataDir + static func loadSourceOutputFileMappingInfo(jsonFilename: String, jsonData: Data, updateCache: Bool = false) { + // Ensure workspace info is ready and .json can be decoded + guard let workspaceKey = workspaceKey else { return } + guard let jsonValues = try? JSONSerialization.jsonObject(with: jsonData, options: [.allowFragments]) as? [String: String] else { return } - return true + // Load .json contents into memory + initializeOutputFileForSourceIfNecessary(jsonFilename: jsonFilename) + outputFileForSource[workspaceKey]?[jsonFilename] = jsonValues + log("[INFO] Loaded mapping information into memory from: \(jsonFilename)") + + // Update .json files cached under xcbuildkitDataDir for + // fast load next time we launch Xcode + do { + guard let jsonBasename = jsonFilename.components(separatedBy: "/").last else { return } + let jsonFilePath = "\(xcbuildkitDataDir)/\(jsonBasename)" + let json = URL(fileURLWithPath: jsonFilePath) + let fm = FileManager.default + if fm.fileExists(atPath: jsonFilePath) { + try fm.removeItem(atPath: jsonFilePath) + } + try jsonData.write(to: json) + log("[INFO] Updated cache in: \(jsonFilePath)") + } catch { + log("[ERROR] Failed to update cache under \(xcbuildkitDataDir) for file \(jsonFilename) with err: \(error.localizedDescription)") + } } + // This loop looks for JSON files with a known suffix `BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX` and loads the mapping + // information from it decoding the JSON and storing in-memory. + // + // Read about the 'namedSetOfFiles' key here: https://bazel.build/remote/bep-examples#consuming-namedsetoffiles + static func parseSourceOutputFileMappingsFromBEP(event: BuildEventStream_BuildEvent) { + // Do work only if `namedSetOfFiles` is present and contain `files` + guard let json = try? JSONSerialization.jsonObject(with: event.jsonUTF8Data(), options: []) as? [String: Any] else { return } + guard let namedSetOfFiles = json["namedSetOfFiles"] as? [String: Any] else { return } + guard namedSetOfFiles.count > 0 else { return } + guard let allPairs = namedSetOfFiles["files"] as? [[String: Any]] else { return } - static func findOutputFileForSource(filePath: String, workingDir: String) -> String? { - let sourceKey = filePath.replacingOccurrences(of: workingDir, with: "").replacingOccurrences(of: (bazelWorkingDir ?? ""), with: "") - guard let workspaceKey = workspaceKey else { - return nil + for pair in allPairs { + // Only proceed if top level keys exist and a .json to be decoded is found + guard let jsonFilename = pair["name"] as? String else { continue } + guard var jsonURI = pair["uri"] as? String else { continue } + guard jsonURI.hasSuffix(".json") else { continue } + + jsonURI = jsonURI.replacingOccurrences(of: "file://", with: "") + + // Only proceed for keys holding .json files with known pattern (i.e. `BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX`) in the name + guard let jsonData = try? Data(contentsOf: URL(fileURLWithPath:jsonURI)) else { continue } + guard jsonData.count > 0 else { continue } + guard let jsonDecoded = try? JSONSerialization.jsonObject(with: jsonData, options: [.allowFragments]) as? [String: String] else { continue } + guard let sourceOutputFileMapSuffix = sourceOutputFileMapSuffix else { continue } + guard let workspaceKey = workspaceKey, jsonFilename.hasSuffix(sourceOutputFileMapSuffix) else { continue } + + // Load .json contents into memory + loadSourceOutputFileMappingInfo(jsonFilename: jsonFilename, jsonData: jsonData, updateCache: true) } - guard let workspaceMappings = outputFileForSource[workspaceKey] else { - return nil + } + // Helper to initialize in-memory mapping for workspace and give .json mappings file key + static func initializeOutputFileForSourceIfNecessary(jsonFilename: String) { + guard let workspaceKey = workspaceKey else { return } + + if outputFileForSource[workspaceKey] == nil { + outputFileForSource[workspaceKey] = [:] } + if outputFileForSource[workspaceKey]?[jsonFilename] == nil { + outputFileForSource[workspaceKey]?[jsonFilename] = [:] + } + } + // Finds output file (i.e. path to `.o` under `bazel-out`) in in-memory mapping + static func findOutputFileForSource(filePath: String, workingDir: String) -> String? { + // Create key + let sourceKey = filePath.replacingOccurrences(of: workingDir, with: "").replacingOccurrences(of: (bazelWorkingDir ?? ""), with: "") + // Ensure workspace info is loaded and mapping exists + guard let workspaceKey = workspaceKey else { return nil } + guard let workspaceMappings = outputFileForSource[workspaceKey] else { return nil } + // Loops until found for (_, json) in workspaceMappings { if let objFilePath = json[sourceKey] { return objFilePath @@ -221,7 +277,6 @@ enum BasicMessageHandler { } return nil } - /// Proxying response handler /// Every message is written to the XCBBuildService /// This simply injects Progress messages from the BEP @@ -231,37 +286,42 @@ enum BasicMessageHandler { let bkservice = basicCtx.bkservice let decoder = XCBDecoder(input: input) let encoder = XCBEncoder(input: input) + if let msg = decoder.decodeMessage() { if let createSessionRequest = msg as? CreateSessionRequest { + // Load information from `CreateSessionRequest` gXcode = createSessionRequest.xcode workspaceHash = createSessionRequest.workspaceHash workspaceName = createSessionRequest.workspaceName + xcodeprojPath = createSessionRequest.xcodeprojPath + + // Initialize build service xcbbuildService.startIfNecessary(xcode: gXcode) - } else if msg is BuildStartRequest { + + // Start reading from BEP as early as possible do { - let bepPath = "/tmp/bep.bep" + let bepPath = configBEPPath ?? "/tmp/bep.bep" try startStream(bepPath: bepPath, startBuildInput: input, bkservice: bkservice) } catch { fatalError("Failed to init stream" + error.localizedDescription) } + // Load output file mapping information from cache if it exists + initializeOutputFileMappingFromCache() + } else if msg is BuildStartRequest { let message = BuildProgressUpdatedResponse() if let responseData = try? message.encode(encoder) { bkservice.write(responseData) } - } else if canHandleIndexing(msg: msg) { - // Example of a custom indexing service + } else if let outputFilePath = outputFileForIndexingRequest(msg: msg) { + // Settings values needed to compose the payload below let reqMsg = msg as! IndexingInfoRequested workingDir = bazelWorkingDir ?? reqMsg.workingDir platform = reqMsg.platform sdk = reqMsg.sdk + log("[INFO] Handling indexing request for source \(reqMsg.filePath) and output file \(outputFilePath)") - guard let outputFilePath = findOutputFileForSource(filePath: reqMsg.filePath, workingDir: reqMsg.workingDir) else { - fatalError("Failed to find output file for source: \(reqMsg.filePath)") - return - } - log("Found output file \(outputFilePath) for source \(reqMsg.filePath)") - + // Compose the indexing response payload and emit the response message let clangXMLData = BazelBuildServiceStub.getASTArgs( targetID: reqMsg.targetID, sourceFilePath: reqMsg.filePath, @@ -280,17 +340,18 @@ enum BasicMessageHandler { clangXMLData: reqMsg.outputPathOnly ? nil : clangXMLData) if let encoded: XCBResponse = try? message.encode(encoder) { bkservice.write(encoded, msgId:message.responseChannel) + log("[INFO] Indexing response sent") return } } - } + log("[INFO] Proxying request") xcbbuildService.write(data) } } let xcbbuildService = XCBBuildServiceProcess() -let bkservice = BKBuildService(indexingEnabled: indexingEnabled) +let bkservice = BKBuildService() let context = BasicMessageContext( xcbbuildService: xcbbuildService, diff --git a/Examples/XCBBuildServiceProxy/main.swift b/Examples/XCBBuildServiceProxy/main.swift index e36766c..ae0bf9d 100644 --- a/Examples/XCBBuildServiceProxy/main.swift +++ b/Examples/XCBBuildServiceProxy/main.swift @@ -185,7 +185,7 @@ enum BasicMessageHandler { } let xcbbuildService = XCBBuildServiceProcess() -let bkservice = BKBuildService(indexingEnabled: indexingEnabled) +let bkservice = BKBuildService() let context = BasicMessageContext( xcbbuildService: xcbbuildService, diff --git a/Sources/BKBuildService/Service.swift b/Sources/BKBuildService/Service.swift index d41e5f9..ce41957 100644 --- a/Sources/BKBuildService/Service.swift +++ b/Sources/BKBuildService/Service.swift @@ -59,14 +59,10 @@ public class BKBuildService { private var readLen: Int32 = 0 private var msgId: UInt64 = 0 - // This is highly experimental - private var indexingEnabled: Bool = false - // TODO: Move record mode out private var chunkId = 0 - public init(indexingEnabled: Bool=false) { - self.indexingEnabled = indexingEnabled + public init() { self.shouldDump = CommandLine.arguments.contains("--dump") self.shouldDumpHumanReadable = CommandLine.arguments.contains("--dump_h") } @@ -87,7 +83,7 @@ public class BKBuildService { ogData.append(self.bufferContentSize) ogData.append(self.buffer) - if msg is IndexingInfoRequested && self.indexingEnabled { + if msg is IndexingInfoRequested { // Indexing msgs require a PING on the msgId before passing the payload // doing this here so proxy writers don't have to worry about this impl detail write([ diff --git a/Sources/XCBProtocol/Protocol.swift b/Sources/XCBProtocol/Protocol.swift index e2f272e..b626033 100644 --- a/Sources/XCBProtocol/Protocol.swift +++ b/Sources/XCBProtocol/Protocol.swift @@ -66,6 +66,7 @@ public struct CreateSessionRequest: XCBProtocolMessage { public let workspaceName: String public let workspaceHash: String public let xcode: String + public let xcodeprojPath: String public let xcbuildDataPath: String init(input: XCBInputStream) throws { @@ -114,9 +115,13 @@ public struct CreateSessionRequest: XCBProtocolMessage { self.workspaceName = "" } + // Parse path to .xcodeproj used to load xcbuildkit config as early as possible + self.xcodeprojPath = self.workspace.components(separatedBy:"path:\'").last?.components(separatedBy:"/project.xcworkspace").first ?? "" + log("Found XCBuildData path: \(self.xcbuildDataPath)") log("Parsed workspaceHash: \(self.workspaceHash)") log("Parsed workspaceName: \(self.workspaceName)") + log("Parsed xcodeprojPath: \(self.xcodeprojPath)") } } @@ -139,6 +144,8 @@ public struct SetSessionUserInfoRequest: XCBProtocolMessage { public struct CreateBuildRequest: XCBProtocolMessage { public let configuredTargets: [String] + public let containerPath: String + public init(input: XCBInputStream) throws { var minput = input guard let next = minput.next(), @@ -149,6 +156,8 @@ public struct CreateBuildRequest: XCBProtocolMessage { throw XCBProtocolError.unexpectedInput(for: input) } let requestJSON = json["request"] as? [String: Any] ?? [:] + self.containerPath = requestJSON["containerPath"] as? String ?? "" + log("info: got containerPath \(self.containerPath)") if let ct = requestJSON["configuredTargets"] as? [[String: Any]] { self.configuredTargets = ct.compactMap { ctInfo in return ctInfo["guid"] as? String From 5685d7d8bef0061bfad54f21680b5982f692440b Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Tue, 1 Nov 2022 16:50:51 -0400 Subject: [PATCH 3/6] Bug fixes and more logs --- Examples/BazelBuildService/main.swift | 78 +++++++++++++++------------ Sources/XCBProtocol/Packer.swift | 13 +++++ 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/Examples/BazelBuildService/main.swift b/Examples/BazelBuildService/main.swift index 573b158..d6a5594 100644 --- a/Examples/BazelBuildService/main.swift +++ b/Examples/BazelBuildService/main.swift @@ -70,11 +70,6 @@ private var configValues: [String: Any]? { } return dict } -// Directory containing data used to fast load information when initializing BazelBuildService, e.g., -// .json files containing source => output file mappings generated during Xcode project generation -private var xcbuildkitDataDir: String { - return "\(xcodeprojPath)/xcbuildkit.data" -} // File containing config values that a consumer can set, see accepted keys below. // Format is KEY=VALUE and one config per line // TODO: Probably better to make this a separate struct with proper validation but will do that @@ -82,21 +77,26 @@ private var xcbuildkitDataDir: String { private var xcbuildkitConfigPath: String { return "\(xcodeprojPath)/xcbuildkit.config" } -private var sourceOutputFileMapSuffix: String? { - return configValues?["BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX"] as? String -} -private var bazelWorkingDir: String? { - return configValues?["BUILD_SERVICE_BAZEL_EXEC_ROOT"] as? String -} private var indexingEnabled: Bool { return (configValues?["BUILD_SERVICE_INDEXING_ENABLED"] as? String ?? "") == "YES" } +// Directory containing data used to fast load information when initializing BazelBuildService, e.g., +// .json files containing source => output file mappings generated during Xcode project generation +private var xcbuildkitDataDir: String? { + return configValues?["BUILD_SERVICE_INDEXING_DATA_DIR"] as? String +} private var progressBarEnabled: Bool { return (configValues?["BUILD_SERVICE_PROGRESS_BAR_ENABLED"] as? String ?? "") == "YES" } private var configBEPPath: String? { return configValues?["BUILD_SERVICE_BEP_PATH"] as? String } +private var sourceOutputFileMapSuffix: String? { + return configValues?["BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX"] as? String +} +private var bazelWorkingDir: String? { + return configValues?["BUILD_SERVICE_BAZEL_EXEC_ROOT"] as? String +} /// This example listens to a BEP stream to display some output. /// @@ -152,25 +152,27 @@ enum BasicMessageHandler { return bplistData } - // Check many conditions that need to be met in order to handle indexing and find the respect output file, + // Check many conditions that need to be met in order to handle indexing and returns the respect output file, // the call site should abort and proxy the indexing request if this returns `nil` - static func outputFileForIndexingRequest(msg: XCBProtocolMessage) -> String? { + static func canHandleIndexingWithOutfile(msg: XCBProtocolMessage) -> String? { // Nothing to do for non-indexing request types guard let reqMsg = msg as? IndexingInfoRequested else { return nil } - // TODO: handle Swift - guard reqMsg.filePath.count > 0 && reqMsg.filePath != "" && !reqMsg.filePath.hasSuffix(".swift") else { + // Nothing to do if indexing is disabled + guard indexingEnabled else { return nil } - // Indexing needs to be enabled via config file - guard indexingEnabled else { + // TODO: handle Swift + guard reqMsg.filePath.count > 0 && reqMsg.filePath != "" && !reqMsg.filePath.hasSuffix(".swift") else { + log("[WARNING] Unsupported filePath for indexing: \(reqMsg.filePath)") return nil } // In `BazelBuildService` the path to the working directory (i.e. execution_root) should always // exists guard bazelWorkingDir != nil else { - fatalError("[ERROR] Path to Bazel working directory not provided. Check `BUILD_SERVICE_BAZEL_EXEC_ROOT` in your config file.") + log("[WARNING] Could not find bazel working directory. Make sure `BUILD_SERVICE_BAZEL_EXEC_ROOT` is set in the config file.") + return nil } guard let outputFilePath = findOutputFileForSource(filePath: reqMsg.filePath, workingDir: reqMsg.workingDir) else { @@ -182,6 +184,7 @@ enum BasicMessageHandler { } // Initialize in memory mappings from xcbuildkitDataDir if .json mappings files exist static func initializeOutputFileMappingFromCache() { + guard let xcbuildkitDataDir = xcbuildkitDataDir else { return } let fm = FileManager.default do { let jsons = try fm.contentsOfDirectory(atPath: xcbuildkitDataDir) @@ -198,27 +201,30 @@ enum BasicMessageHandler { static func loadSourceOutputFileMappingInfo(jsonFilename: String, jsonData: Data, updateCache: Bool = false) { // Ensure workspace info is ready and .json can be decoded guard let workspaceKey = workspaceKey else { return } + guard let xcbuildkitDataDir = xcbuildkitDataDir else { return } guard let jsonValues = try? JSONSerialization.jsonObject(with: jsonData, options: [.allowFragments]) as? [String: String] else { return } // Load .json contents into memory initializeOutputFileForSourceIfNecessary(jsonFilename: jsonFilename) outputFileForSource[workspaceKey]?[jsonFilename] = jsonValues - log("[INFO] Loaded mapping information into memory from: \(jsonFilename)") + log("[INFO] Loaded mapping information into memory for file: \(jsonFilename)") // Update .json files cached under xcbuildkitDataDir for // fast load next time we launch Xcode - do { - guard let jsonBasename = jsonFilename.components(separatedBy: "/").last else { return } - let jsonFilePath = "\(xcbuildkitDataDir)/\(jsonBasename)" - let json = URL(fileURLWithPath: jsonFilePath) - let fm = FileManager.default - if fm.fileExists(atPath: jsonFilePath) { - try fm.removeItem(atPath: jsonFilePath) + if updateCache { + do { + guard let jsonBasename = jsonFilename.components(separatedBy: "/").last else { return } + let jsonFilePath = "\(xcbuildkitDataDir)/\(jsonBasename)" + let json = URL(fileURLWithPath: jsonFilePath) + let fm = FileManager.default + if fm.fileExists(atPath: jsonFilePath) { + try fm.removeItem(atPath: jsonFilePath) + } + try jsonData.write(to: json) + log("[INFO] Updated cache for file \(jsonFilePath)") + } catch { + log("[ERROR] Failed to update cache under \(xcbuildkitDataDir) for file \(jsonFilename) with err: \(error.localizedDescription)") } - try jsonData.write(to: json) - log("[INFO] Updated cache in: \(jsonFilePath)") - } catch { - log("[ERROR] Failed to update cache under \(xcbuildkitDataDir) for file \(jsonFilename) with err: \(error.localizedDescription)") } } // This loop looks for JSON files with a known suffix `BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX` and loads the mapping @@ -248,6 +254,7 @@ enum BasicMessageHandler { guard let workspaceKey = workspaceKey, jsonFilename.hasSuffix(sourceOutputFileMapSuffix) else { continue } // Load .json contents into memory + log("[INFO] Parsed \(jsonFilename) from BEP.") loadSourceOutputFileMappingInfo(jsonFilename: jsonFilename, jsonData: jsonData, updateCache: true) } } @@ -286,6 +293,7 @@ enum BasicMessageHandler { let bkservice = basicCtx.bkservice let decoder = XCBDecoder(input: input) let encoder = XCBEncoder(input: input) + let identifier = input.identifier ?? "" if let msg = decoder.decodeMessage() { if let createSessionRequest = msg as? CreateSessionRequest { @@ -313,13 +321,12 @@ enum BasicMessageHandler { if let responseData = try? message.encode(encoder) { bkservice.write(responseData) } - } else if let outputFilePath = outputFileForIndexingRequest(msg: msg) { + } else if let outputFilePath = canHandleIndexingWithOutfile(msg: msg) { // Settings values needed to compose the payload below let reqMsg = msg as! IndexingInfoRequested workingDir = bazelWorkingDir ?? reqMsg.workingDir platform = reqMsg.platform sdk = reqMsg.sdk - log("[INFO] Handling indexing request for source \(reqMsg.filePath) and output file \(outputFilePath)") // Compose the indexing response payload and emit the response message let clangXMLData = BazelBuildServiceStub.getASTArgs( @@ -340,12 +347,15 @@ enum BasicMessageHandler { clangXMLData: reqMsg.outputPathOnly ? nil : clangXMLData) if let encoded: XCBResponse = try? message.encode(encoder) { bkservice.write(encoded, msgId:message.responseChannel) - log("[INFO] Indexing response sent") + log("[INFO] Handling \(identifier) for source \(reqMsg.filePath) and output file \(outputFilePath)") return } } } - log("[INFO] Proxying request") + log("[INFO] Proxying request with type: \(identifier)") + if indexingEnabled && identifier == "INDEXING_INFO_REQUESTED" { + log("[WARNING] BazelBuildService failed to handle indexing request, message will be proxied instead.") + } xcbbuildService.write(data) } } diff --git a/Sources/XCBProtocol/Packer.swift b/Sources/XCBProtocol/Packer.swift index 0e14fba..ba8c332 100644 --- a/Sources/XCBProtocol/Packer.swift +++ b/Sources/XCBProtocol/Packer.swift @@ -117,6 +117,19 @@ public struct XCBInputStream { public let data: Data public var stream: IndexingIterator<[MessagePackValue]> public let first: MessagePackValue + // Used for debugging only, assumes the first .string found is the identifier for this msg + public var identifier: String? { + var mutableSelf = self + while let value = mutableSelf.next() { + switch value { + case let XCBRawValue.string(str): + return str + default: + continue + } + } + return nil + } // This thing reads in a result - maybe it will not do this. public init (result: [MessagePackValue], data: Data) { From a461724a1cf6955c343c660fdd0bf84de191c043 Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Wed, 2 Nov 2022 10:46:38 -0400 Subject: [PATCH 4/6] Handle edge case when loading mapping into memory --- Examples/BazelBuildService/main.swift | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Examples/BazelBuildService/main.swift b/Examples/BazelBuildService/main.swift index d6a5594..f2213f7 100644 --- a/Examples/BazelBuildService/main.swift +++ b/Examples/BazelBuildService/main.swift @@ -317,6 +317,15 @@ enum BasicMessageHandler { // Load output file mapping information from cache if it exists initializeOutputFileMappingFromCache() } else if msg is BuildStartRequest { + // Attempt to initialize in-memory mapping if empty + // It's possible that indexing data is not ready yet in `CreateSessionRequest` above + // so retry to load info into memory at `BuildStartRequest` time + if let workspaceKey = workspaceKey { + if (outputFileForSource[workspaceKey]?.count ?? 0) == 0 { + initializeOutputFileMappingFromCache() + } + } + let message = BuildProgressUpdatedResponse() if let responseData = try? message.encode(encoder) { bkservice.write(responseData) @@ -355,6 +364,8 @@ enum BasicMessageHandler { log("[INFO] Proxying request with type: \(identifier)") if indexingEnabled && identifier == "INDEXING_INFO_REQUESTED" { log("[WARNING] BazelBuildService failed to handle indexing request, message will be proxied instead.") + // If we hit this means that something went wrong with indexing, logging this in-memory mapping is useful for troubleshooting + log("[INFO] outputFileForSource: \(outputFileForSource)") } xcbbuildService.write(data) } From a03d9c7c5efb961b317f5efb95cbfcf7471b683e Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Wed, 2 Nov 2022 12:18:11 -0400 Subject: [PATCH 5/6] Cleanup and move BEP stream initialization back to build_start --- Examples/BazelBuildService/BEPStream.swift | 10 +++++++-- Examples/BazelBuildService/main.swift | 24 ++++++++++++---------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Examples/BazelBuildService/BEPStream.swift b/Examples/BazelBuildService/BEPStream.swift index 6094fbe..5dfa098 100644 --- a/Examples/BazelBuildService/BEPStream.swift +++ b/Examples/BazelBuildService/BEPStream.swift @@ -18,9 +18,10 @@ public class BEPStream { deinit { // According to close should automatically happen, but it does't under a // few cases: - // https://developer.apple.com/documentation/foundation/filehandle/1413393-closefile - fileHandle?.closeFile() + // https://developer.apple.com/documentation/foundation/filehandle/3172525-close + try? fileHandle?.close() fileHandle?.readabilityHandler = nil + log("BEPStream: dealloc") } /// Reads data from a BEP stream /// @param eventAvailableHandler - this is called with _every_ BEP event @@ -55,6 +56,7 @@ public class BEPStream { log("BEPStream: failed to allocate \(path)") return } + self.fileHandle = fileHandle fileHandle.readabilityHandler = { handle in @@ -83,5 +85,9 @@ public class BEPStream { } } } + + // The file handle works without this in debug mode (i.e. manually launching xcbuildkit), + // but when installed on a machine as a pkg this call is necessary to get events in the `readabilityHandler` block above. + fileHandle.waitForDataInBackgroundAndNotify() } } diff --git a/Examples/BazelBuildService/main.swift b/Examples/BazelBuildService/main.swift index f2213f7..88a4adc 100644 --- a/Examples/BazelBuildService/main.swift +++ b/Examples/BazelBuildService/main.swift @@ -105,7 +105,7 @@ private var bazelWorkingDir: String? { enum BasicMessageHandler { // Read info from BEP and optionally handle events static func startStream(bepPath: String, startBuildInput: XCBInputStream, bkservice: BKBuildService) throws { - log("startStream " + String(describing: startBuildInput)) + log("[INFO] Will start BEP stream at path \(bepPath) with input" + String(describing: startBuildInput)) let stream = try BEPStream(path: bepPath) var progressView: ProgressView? try stream.read { @@ -249,9 +249,8 @@ enum BasicMessageHandler { // Only proceed for keys holding .json files with known pattern (i.e. `BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX`) in the name guard let jsonData = try? Data(contentsOf: URL(fileURLWithPath:jsonURI)) else { continue } guard jsonData.count > 0 else { continue } - guard let jsonDecoded = try? JSONSerialization.jsonObject(with: jsonData, options: [.allowFragments]) as? [String: String] else { continue } guard let sourceOutputFileMapSuffix = sourceOutputFileMapSuffix else { continue } - guard let workspaceKey = workspaceKey, jsonFilename.hasSuffix(sourceOutputFileMapSuffix) else { continue } + guard jsonFilename.hasSuffix(sourceOutputFileMapSuffix) else { continue } // Load .json contents into memory log("[INFO] Parsed \(jsonFilename) from BEP.") @@ -306,17 +305,20 @@ enum BasicMessageHandler { // Initialize build service xcbbuildService.startIfNecessary(xcode: gXcode) - // Start reading from BEP as early as possible - do { - let bepPath = configBEPPath ?? "/tmp/bep.bep" - try startStream(bepPath: bepPath, startBuildInput: input, bkservice: bkservice) - } catch { - fatalError("Failed to init stream" + error.localizedDescription) - } - // Load output file mapping information from cache if it exists initializeOutputFileMappingFromCache() } else if msg is BuildStartRequest { + // Start reading from BEP as early as possible + if let bepPath = configBEPPath { + do { + try startStream(bepPath: bepPath, startBuildInput: input, bkservice: bkservice) + } catch { + // fatalError("[ERROR] Failed to start BEP stream with error: " + error.localizedDescription) + log("[ERROR] Failed to start BEP stream with error: " + error.localizedDescription) + } + } else { + log("[WARNING] BEP string config key 'BUILD_SERVICE_BEP_PATH' empty. Bazel information won't be available during a build.") + } // Attempt to initialize in-memory mapping if empty // It's possible that indexing data is not ready yet in `CreateSessionRequest` above // so retry to load info into memory at `BuildStartRequest` time From 1eb4ed9f2026b35cece333308ba9623acede90f3 Mon Sep 17 00:00:00 2001 From: Thiago Cruz Date: Thu, 3 Nov 2022 16:18:22 -0400 Subject: [PATCH 6/6] Support multiple workspaces and add DataStore setup logic --- Examples/BazelBuildService/BEPService.swift | 70 ++++ .../BazelBuildServiceConfig.swift | 55 +++ .../BazelBuildServiceStub.swift | 26 ++ .../BazelBuildService/IndexingService.swift | 211 ++++++++++ Examples/BazelBuildService/Messages.swift | 1 - .../BazelBuildService/WorkspaceInfo.swift | 60 +++ .../WorkspaceInfoKeyable.swift | 15 + Examples/BazelBuildService/main.swift | 380 +++--------------- Examples/XCBBuildServiceProxy/main.swift | 10 +- Sources/XCBProtocol/Protocol.swift | 127 +++--- 10 files changed, 586 insertions(+), 369 deletions(-) create mode 100644 Examples/BazelBuildService/BEPService.swift create mode 100644 Examples/BazelBuildService/BazelBuildServiceConfig.swift create mode 100644 Examples/BazelBuildService/IndexingService.swift delete mode 100644 Examples/BazelBuildService/Messages.swift create mode 100644 Examples/BazelBuildService/WorkspaceInfo.swift create mode 100644 Examples/BazelBuildService/WorkspaceInfoKeyable.swift diff --git a/Examples/BazelBuildService/BEPService.swift b/Examples/BazelBuildService/BEPService.swift new file mode 100644 index 0000000..e9b78bb --- /dev/null +++ b/Examples/BazelBuildService/BEPService.swift @@ -0,0 +1,70 @@ +import BEP +import BKBuildService +import Foundation +import XCBProtocol + +// Handles BEP related tasks for all incoming message types (progress bar, indexing, etc.). +// In the future we might want to delegate the message specific work to a diff service, +// keep all logic here now for simplicity. +class BEPService { + // Read info from BEP and optionally handle events + func startStream(msg: WorkspaceInfoKeyable, bepPath: String, startBuildInput: XCBInputStream, ctx: BasicMessageContext) throws { + guard let info = ctx.indexingService.infos[msg.workspaceKey] else { return } + + log("[INFO] Will start BEP stream at path \(bepPath) with input" + String(describing: startBuildInput)) + let bkservice = ctx.bkservice + let stream = try BEPStream(path: bepPath) + var progressView: ProgressView? + try stream.read { + event in + if info.config.indexingEnabled { + self.parseSourceOutputFileMappingsFromBEP(msg: msg, event: event, ctx: ctx) + } + + if info.config.progressBarEnabled { + if let updatedView = ProgressView(event: event, last: progressView) { + let encoder = XCBEncoder(input: startBuildInput) + let response = BuildProgressUpdatedResponse(progress: + updatedView.progressPercent, message: updatedView.message) + if let responseData = try? response.encode(encoder) { + bkservice.write(responseData) + } + progressView = updatedView + } + } + } + info.bepStream = stream + } + + // This loop looks for JSON files with a known suffix `BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX` and loads the mapping + // information from it decoding the JSON and storing in-memory. + // + // Read about the 'namedSetOfFiles' key here: https://bazel.build/remote/bep-examples#consuming-namedsetoffiles + func parseSourceOutputFileMappingsFromBEP(msg: WorkspaceInfoKeyable, event: BuildEventStream_BuildEvent, ctx: BasicMessageContext) { + guard let info = ctx.indexingService.infos[msg.workspaceKey] else { return } + // Do work only if `namedSetOfFiles` is present and contain `files` + guard let json = try? JSONSerialization.jsonObject(with: event.jsonUTF8Data(), options: []) as? [String: Any] else { return } + guard let namedSetOfFiles = json["namedSetOfFiles"] as? [String: Any] else { return } + guard namedSetOfFiles.count > 0 else { return } + guard let allPairs = namedSetOfFiles["files"] as? [[String: Any]] else { return } + + for pair in allPairs { + // Only proceed if top level keys exist and a .json to be decoded is found + guard let jsonFilename = pair["name"] as? String else { continue } + guard var jsonURI = pair["uri"] as? String else { continue } + guard jsonURI.hasSuffix(".json") else { continue } + + jsonURI = jsonURI.replacingOccurrences(of: "file://", with: "") + + // Only proceed for keys holding .json files with known pattern (i.e. `BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX`) in the name + guard let jsonData = try? Data(contentsOf: URL(fileURLWithPath:jsonURI)) else { continue } + guard jsonData.count > 0 else { continue } + guard let sourceOutputFileMapSuffix = info.config.sourceOutputFileMapSuffix else { continue } + guard jsonFilename.hasSuffix(sourceOutputFileMapSuffix) else { continue } + + // Load .json contents into memory + log("[INFO] Parsed \(jsonFilename) from BEP.") + ctx.indexingService.loadSourceOutputFileMappingInfo(msg: msg, jsonFilename: jsonFilename, jsonData: jsonData, updateCache: true) + } + } +} \ No newline at end of file diff --git a/Examples/BazelBuildService/BazelBuildServiceConfig.swift b/Examples/BazelBuildService/BazelBuildServiceConfig.swift new file mode 100644 index 0000000..931a836 --- /dev/null +++ b/Examples/BazelBuildService/BazelBuildServiceConfig.swift @@ -0,0 +1,55 @@ +// Encapsulates the config file used to control xcbuildkit's behaviour. +// +// Format is: +// +// KEY1=VALUE1 +// KEY2=VALUE2 +// +// so one config per line. This is intentionally simple at the moment but we can add more validation or a more reliable file format later. +// +// TODO: Codable? +// TODO: Not load from disk all the time but still detect changes in the file and refresh in-memory values? +struct BazelBuildServiceConfig { + private enum ConfigKeys: String { + case indexingEnabled = "BUILD_SERVICE_INDEXING_ENABLED" + case indexStorePath = "BUILD_SERVICE_INDEX_STORE_PATH" + case xcbuildkitDataDir = "BUILD_SERVICE_INDEXING_DATA_DIR" + case progressBarEnabled = "BUILD_SERVICE_PROGRESS_BAR_ENABLED" + case configBEPPath = "BUILD_SERVICE_BEP_PATH" + case sourceOutputFileMapSuffix = "BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX" + case bazelWorkingDir = "BUILD_SERVICE_BAZEL_EXEC_ROOT" + } + + var indexingEnabled: Bool { return self.value(for: .indexingEnabled) == "YES" } + var indexStorePath: String? { return self.value(for: .indexStorePath) } + var xcbuildkitDataDir: String? { return self.value(for: .xcbuildkitDataDir) } + var progressBarEnabled: Bool { return self.value(for: .progressBarEnabled) == "YES" } + var configBEPPath: String? { return self.value(for: .configBEPPath) ?? "/tmp/bep.bep" } + var sourceOutputFileMapSuffix: String? { return self.value(for: .sourceOutputFileMapSuffix) } + var bazelWorkingDir: String? { return self.value(for: .bazelWorkingDir) } + + let configPath: String + + init(configPath: String) { + self.configPath = configPath + } + + private var loadConfigFile: [String: Any]? { + guard let data = try? String(contentsOfFile: self.configPath, encoding: .utf8) else { + return nil + } + + let lines = data.components(separatedBy: .newlines) + var dict: [String: Any] = [:] + for line in lines { + let split = line.components(separatedBy: "=") + guard split.count == 2 else { continue } + dict[split[0]] = split[1] + } + return dict + } + + private func value(for config: ConfigKeys) -> String? { + return loadConfigFile?[config.rawValue] as? String + } +} \ No newline at end of file diff --git a/Examples/BazelBuildService/BazelBuildServiceStub.swift b/Examples/BazelBuildService/BazelBuildServiceStub.swift index 07d9ef4..1098a1d 100644 --- a/Examples/BazelBuildService/BazelBuildServiceStub.swift +++ b/Examples/BazelBuildService/BazelBuildServiceStub.swift @@ -176,5 +176,31 @@ public enum BazelBuildServiceStub { return BPlistConverter(xml: clangXML)?.convertToBinary() ?? Data() } + + // Required if `outputPathOnly` is `true` in the indexing request + public static func outputPathOnlyData(outputFilePath: String, + sourceFilePath: String) -> Data { + let xml = """ + + + + + outputFilePath + \(outputFilePath) + sourceFilePath + \(sourceFilePath) + + + + """ + guard let converter = BPlistConverter(xml: xml) else { + fatalError("Failed to allocate converter") + } + guard let bplistData = converter.convertToBinary() else { + fatalError("Failed to convert XML to binary plist data") + } + + return bplistData + } } diff --git a/Examples/BazelBuildService/IndexingService.swift b/Examples/BazelBuildService/IndexingService.swift new file mode 100644 index 0000000..5b82f22 --- /dev/null +++ b/Examples/BazelBuildService/IndexingService.swift @@ -0,0 +1,211 @@ +import Foundation +import XCBProtocol + +class IndexingService { + // Holds list of `WorkspaceInfo` for each opened workspace + var infos: [String: WorkspaceInfo] = [:] + + // Finds output file (i.e. path to `.o` under `bazel-out`) in in-memory mapping + func findOutputFileForSource(msg: WorkspaceInfoKeyable, filePath: String, workingDir: String) -> String? { + guard let info = self.infos[msg.workspaceKey] else { return nil } + // Create key + let sourceKey = filePath.replacingOccurrences(of: workingDir, with: "").replacingOccurrences(of: (info.config.bazelWorkingDir ?? ""), with: "") + // Loops until found + for (_, json) in info.outputFileForSource { + if let objFilePath = json[sourceKey] { + return objFilePath + } + } + return nil + } + + // Initialize in memory mappings from xcbuildkitDataDir if .json mappings files exist + func initializeOutputFileMappingFromCache(msg: WorkspaceInfoKeyable) { + guard let info = self.infos[msg.workspaceKey] else { return } + guard let xcbuildkitDataDir = info.config.xcbuildkitDataDir else { return } + let fm = FileManager.default + do { + let jsons = try fm.contentsOfDirectory(atPath: xcbuildkitDataDir) + + for jsonFilename in jsons { + let jsonData = try Data(contentsOf: URL(fileURLWithPath: "\(xcbuildkitDataDir)/\(jsonFilename)")) + self.loadSourceOutputFileMappingInfo(msg: msg, jsonFilename: jsonFilename, jsonData: jsonData) + } + } catch { + log("[ERROR] Failed to initialize from cache under \(xcbuildkitDataDir) with err: \(error.localizedDescription)") + } + } + + // Check many conditions that need to be met in order to handle indexing and returns the respect output file, + // the call site should abort and proxy the indexing request if this returns `nil` + func indexingOutputFilePath(msg: IndexingInfoRequested) -> String? { + // Load workspace info + guard let info = self.infos[msg.workspaceKey] else { + log("[WARNING] Workspace info not found for key \(msg.workspaceKey).") + return nil + } + // Nothing to do if indexing is disabled + guard info.config.indexingEnabled else { + return nil + } + // TODO: handle Swift + guard msg.filePath.count > 0 && msg.filePath != "" && !msg.filePath.hasSuffix(".swift") else { + log("[WARNING] Unsupported filePath for indexing: \(msg.filePath)") + return nil + } + // In `BazelBuildService` the path to the working directory (i.e. execution_root) should always + // exists + guard info.config.bazelWorkingDir != nil else { + log("[WARNING] Could not find bazel working directory. Make sure `BUILD_SERVICE_BAZEL_EXEC_ROOT` is set in the config file.") + return nil + } + // Find .o file under `bazel-out` for source `msg.filePath` + guard let outputFilePath = self.findOutputFileForSource(msg: msg, filePath: msg.filePath, workingDir: info.workingDir) else { + log("[WARNING] Failed to find output file for source: \(msg.filePath). Indexing requests will be proxied to default build service.") + return nil + } + + return outputFilePath + } + + // Loads information into memory and optionally update the cache under xcbuildkitDataDir + func loadSourceOutputFileMappingInfo(msg: WorkspaceInfoKeyable, jsonFilename: String, jsonData: Data, updateCache: Bool = false) { + guard let info = self.infos[msg.workspaceKey] else { return } + // Ensure workspace info is ready and .json can be decoded + guard let xcbuildkitDataDir = info.config.xcbuildkitDataDir else { return } + guard let jsonValues = try? JSONSerialization.jsonObject(with: jsonData, options: [.allowFragments]) as? [String: String] else { return } + + // Load .json contents into memory + if info.outputFileForSource[jsonFilename] == nil { + info.outputFileForSource[jsonFilename] = [:] + } + info.outputFileForSource[jsonFilename] = jsonValues + log("[INFO] Loaded mapping information into memory for file: \(jsonFilename)") + + // Update .json files cached under xcbuildkitDataDir for + // fast load next time we launch Xcode + if updateCache { + do { + guard let jsonBasename = jsonFilename.components(separatedBy: "/").last else { return } + let jsonFilePath = "\(xcbuildkitDataDir)/\(jsonBasename)" + let json = URL(fileURLWithPath: jsonFilePath) + let fm = FileManager.default + if fm.fileExists(atPath: jsonFilePath) { + try fm.removeItem(atPath: jsonFilePath) + } + try jsonData.write(to: json) + log("[INFO] Updated cache for file \(jsonFilePath)") + } catch { + log("[ERROR] Failed to update cache under \(xcbuildkitDataDir) for file \(jsonFilename) with err: \(error.localizedDescription)") + } + } + } + + // Helper method to compose sdk path for given sdk and platform + // This is not an instance variable because if might change for different targets under the same workspace + func sdkPath(msg: IndexingInfoRequested) -> String { + guard let info = self.infos[msg.workspaceKey] else { + fatalError("[ERROR] Workspace info not found") + } + guard msg.sdk.count > 0 else { + fatalError("[ERROR] Failed to build SDK path, sdk name is empty.") + } + guard msg.platform.count > 0 else { + fatalError("[ERROR] Failed to build SDK path, platform is empty.") + } + + return "\(info.xcode)/Contents/Developer/Platforms/\(msg.platform).platform/Developer/SDKs/\(msg.sdk).sdk" + } + + // Xcode will try to find the data store under DerivedData so we need to symlink it to `BazelBuildServiceConfig.indexStorePath` + // if that value was set in the config file. + // + // This function manages creating a symlink if it doesn't exist and it creates a backup with the `.default` suffix before doing that. Additionally, + // it restores the backup if it's present and indexing is disabled. + // + // p.s.: Maybe there's a way to trick Xcode to try to find the DataStore under a an arbitrary path but it's not clear to me how + func setupDataStore(msg: CreateBuildRequest) { + guard let info = self.infos[msg.workspaceKey] else { + log("[ERROR] Failed to setup indexing data store. Workspace information not found.") + return + } + + let fm = FileManager.default + let ogDataStorePath = info.indexDataStoreFolderPath + let dataStoreBackupPath = info.indexDataStoreFolderPath + ".default" + // Used to check if certain directories exist + var isDirectory = ObjCBool(true) + + // Only proceed if indexing is enabled and a Bazel index store path was specified in the config file + // Otherwise try to restore the DataStore backup if it exists + guard info.config.indexingEnabled, let indexStorePath = info.config.indexStorePath else { + log("[INFO] Indexing disabled. Skipping DataStore setup.") + + // DataStore restore backup code + // If a symlink exists, remove it and restore the backup + if let existingSymlink = try? FileManager.default.destinationOfSymbolicLink(atPath: ogDataStorePath) { + do { + try fm.removeItem(atPath: ogDataStorePath) + } catch { + log("[ERROR] Failed to remove existing DataStore symlink with error: \(error.localizedDescription)") + } + if fm.fileExists(atPath: dataStoreBackupPath, isDirectory: &isDirectory) { + do { + try fm.moveItem(atPath: dataStoreBackupPath, toPath: ogDataStorePath) + } catch { + log("[ERROR] Failed to restore DataStore backup with error: \(error.localizedDescription)") + } + } + } + return + } + + // Prep work before creating the symlink. Handles two scenarios: + // + // (1) A symlink already exists (in this case it has to match what is in the config file, it will be removed otherwise) + // (2) A symlink does not exist, in this case a backup will be created + if let existingSymlink = try? FileManager.default.destinationOfSymbolicLink(atPath: ogDataStorePath) { + if existingSymlink == indexStorePath { + // Nothing to do, a symlink already exists and points to the correct path + log("[INFO] DataStore symlink already setup. Nothing to do.") + return + } else { + do { + try fm.removeItem(atPath: ogDataStorePath) + } catch { + log("[ERROR] Failed to remove existing DataStore symlink with error: \(error.localizedDescription)") + return + } + } + } else { + if fm.fileExists(atPath: ogDataStorePath, isDirectory: &isDirectory) { + // Remove existing backup if it exists + if fm.fileExists(atPath: dataStoreBackupPath, isDirectory: &isDirectory) { + do { + try fm.removeItem(atPath: dataStoreBackupPath) + } catch { + log("[ERROR] Failed to remove existing DataStore backup with error: \(error.localizedDescription)") + return + } + } + // Backup DataStore + do { + try fm.moveItem(atPath: ogDataStorePath, toPath: dataStoreBackupPath) + } catch { + log("[ERROR] Failed to backup DataStore with error: \(error.localizedDescription)") + return + } + } + } + + // If all the above went fine, create a symlink using the value from the config file + do { + try fm.createSymbolicLink(atPath: ogDataStorePath, withDestinationPath: indexStorePath) + } catch { + log("[ERROR] Failed to symlink DataStore with error: \(error.localizedDescription)") + return + } + + log("[INFO] DataStore symlink setup complete.") + } +} \ No newline at end of file diff --git a/Examples/BazelBuildService/Messages.swift b/Examples/BazelBuildService/Messages.swift deleted file mode 100644 index 8b13789..0000000 --- a/Examples/BazelBuildService/Messages.swift +++ /dev/null @@ -1 +0,0 @@ - diff --git a/Examples/BazelBuildService/WorkspaceInfo.swift b/Examples/BazelBuildService/WorkspaceInfo.swift new file mode 100644 index 0000000..59f0ee5 --- /dev/null +++ b/Examples/BazelBuildService/WorkspaceInfo.swift @@ -0,0 +1,60 @@ +import XCBProtocol +import BEP + +// Holds information about a workspace, encapsulates many aspects relevant for +// build service implementation. +// +// Exposes a method to define a key for this workspace +class WorkspaceInfo { + // Values expected to be set during initialization (e.g. on `CreateSessionRequest`) + let xcode: String + let workspaceHash: String + let workspaceName: String + let xcodeprojPath: String + let config: BazelBuildServiceConfig + + // Values expected to be update later (e.g. on `CreateBuildRequest`, `IndexingInfoRequested`, etc.) + var workingDir: String = "" + var derivedDataPath: String = "" + var indexDataStoreFolderPath: String = "" + var bepStream: BEPStream? + + // Dictionary that holds mapping from source file to respective `.o` file under `bazel-out`. Used to respond to indexing requests. + // + // Example: + // + // "Test-XCBuildKit-cdwbwzghpxmnfadvmmhsjcdnjygy": [ + // "App_source_output_file_map.json": [ + // "/tests/ios/app/App/main.m": "bazel-out/ios-x86_64-min10.0-applebin_ios-ios_x86_64-dbg-ST-0f1b0425081f/bin/tests/ios/app/_objs/App_objc/arc/main.o", + // "/tests/ios/app/App/Foo.m": "bazel-out/ios-x86_64-min10.0-applebin_ios-ios_x86_64-dbg-ST-0f1b0425081f/bin/tests/ios/app/_objs/App_objc/arc/Foo.o", + // ], + // ], + var outputFileForSource: [String: [String: String]] = [:] + + + init(xcode: String, workspaceName: String, workspaceHash: String, xcodeprojPath: String) { + // Fail early if key values are not present + guard xcode.count > 0 else { fatalError("[ERROR] Xcode path should not be empty.") } + self.xcode = xcode + + guard xcodeprojPath.count > 0 else { fatalError("[ERROR] Xcode project path should not be empty.") } + self.xcodeprojPath = xcodeprojPath + + guard workspaceName.count > 0 && workspaceHash.count > 0 else { + fatalError("[ERROR] Workspace info not found. Both workspace name and hash should not be empty: workspaceName=\(workspaceName), workspaceHash=\(workspaceHash)") + } + self.workspaceName = workspaceName + self.workspaceHash = workspaceHash + + // Hard coded value for now for the expected config path + // TODO: Find a way to pass this from Xcode + self.config = BazelBuildServiceConfig(configPath: "\(self.xcodeprojPath)/xcbuildkit.config") + } + + // Key to uniquely identify a workspace + // The reason this is a static method is for other components to be able to call it + // without duplicating key generation code + static func workspaceKey(workspaceName: String, workspaceHash: String) -> String { + return "\(workspaceName)-\(workspaceHash)" + } +} \ No newline at end of file diff --git a/Examples/BazelBuildService/WorkspaceInfoKeyable.swift b/Examples/BazelBuildService/WorkspaceInfoKeyable.swift new file mode 100644 index 0000000..1a1d17f --- /dev/null +++ b/Examples/BazelBuildService/WorkspaceInfoKeyable.swift @@ -0,0 +1,15 @@ +import XCBProtocol + +protocol WorkspaceInfoKeyable { + var workspaceName: String { get } + var workspaceHash: String { get } + var workspaceKey: String { get } +} + +extension WorkspaceInfoKeyable { + var workspaceKey: String { WorkspaceInfo.workspaceKey(workspaceName: self.workspaceName, workspaceHash: self.workspaceHash) } +} + +extension CreateSessionRequest: WorkspaceInfoKeyable {} +extension CreateBuildRequest: WorkspaceInfoKeyable {} +extension IndexingInfoRequested: WorkspaceInfoKeyable {} \ No newline at end of file diff --git a/Examples/BazelBuildService/main.swift b/Examples/BazelBuildService/main.swift index 88a4adc..ce0b3a0 100644 --- a/Examples/BazelBuildService/main.swift +++ b/Examples/BazelBuildService/main.swift @@ -6,96 +6,8 @@ import BEP struct BasicMessageContext { let xcbbuildService: XCBBuildServiceProcess let bkservice: BKBuildService -} - -/// FIXME: support multiple workspaces -var gStream: BEPStream? -// Example of what source => object file under bazel-out mapping should look like: -// -// "Test-XCBuildKit-cdwbwzghpxmnfadvmmhsjcdnjygy": [ -// "App_source_output_file_map.json": [ -// "/tests/ios/app/App/main.m": "bazel-out/ios-x86_64-min10.0-applebin_ios-ios_x86_64-dbg-ST-0f1b0425081f/bin/tests/ios/app/_objs/App_objc/arc/main.o", -// "/tests/ios/app/App/Foo.m": "bazel-out/ios-x86_64-min10.0-applebin_ios-ios_x86_64-dbg-ST-0f1b0425081f/bin/tests/ios/app/_objs/App_objc/arc/Foo.o", -// ], -// ], -private var outputFileForSource: [String: [String: [String: String]]] = [:] -// Used when debugging msgs are enabled, see `XCBBuildServiceProcess.MessageDebuggingEnabled()` -private var gChunkNumber = 0 -// FIXME: get this from the other paths -private var gXcode = "" -// TODO: parsed in `CreateSessionRequest`, consider a more stable approach instead of parsing `xcbuildDataPath` path there -private var workspaceHash = "" -// TODO: parsed in `CreateSessionRequest`, consider a more stable approach instead of parsing `xcbuildDataPath` path there -private var workspaceName = "" -// Key to identify a workspace and find its mapping of source to object files in `outputFileForSource` -private var workspaceKey: String? { - guard workspaceName.count > 0 && workspaceHash.count > 0 else { - return nil - } - return "\(workspaceName)-\(workspaceHash)" -} -// TODO: parsed in `IndexingInfoRequested`, there's probably a less hacky way to get this. -// Effectively `$PWD/iOSApp` -private var workingDir = "" -// TODO: parsed in `IndexingInfoRequested` and it's lowercased there, might not be stable in different OSes -private var sdk = "" -// TODO: parsed in `IndexingInfoRequested` and it's lowercased there, might not be stable in different OSes -private var platform = "" -// TODO: parse the relative path to the SDK from somewhere -var sdkPath: String { - guard gXcode.count > 0 else { - fatalError("Failed to build SDK path, Xcode path is empty.") - } - guard sdk.count > 0 else { - fatalError("Failed to build SDK path, sdk name is empty.") - } - guard platform.count > 0 else { - fatalError("Failed to build SDK path, platform is empty.") - } - - return "\(gXcode)/Contents/Developer/Platforms/\(platform).platform/Developer/SDKs/\(sdk).sdk" -} -// Path to .xcodeproj, used to load xcbuildkit config file from path/to/foo.xcodeproj/xcbuildkit.config -private var xcodeprojPath: String = "" -// Load configs from path/to/foo.xcodeproj/xcbuildkit.config -private var configValues: [String: Any]? { - guard let data = try? String(contentsOfFile: xcbuildkitConfigPath, encoding: .utf8) else { return nil } - - let lines = data.components(separatedBy: .newlines) - var dict: [String: Any] = [:] - for line in lines { - let split = line.components(separatedBy: "=") - guard split.count == 2 else { continue } - dict[split[0]] = split[1] - } - return dict -} -// File containing config values that a consumer can set, see accepted keys below. -// Format is KEY=VALUE and one config per line -// TODO: Probably better to make this a separate struct with proper validation but will do that -// once the list of accepted keys is stable -private var xcbuildkitConfigPath: String { - return "\(xcodeprojPath)/xcbuildkit.config" -} -private var indexingEnabled: Bool { - return (configValues?["BUILD_SERVICE_INDEXING_ENABLED"] as? String ?? "") == "YES" -} -// Directory containing data used to fast load information when initializing BazelBuildService, e.g., -// .json files containing source => output file mappings generated during Xcode project generation -private var xcbuildkitDataDir: String? { - return configValues?["BUILD_SERVICE_INDEXING_DATA_DIR"] as? String -} -private var progressBarEnabled: Bool { - return (configValues?["BUILD_SERVICE_PROGRESS_BAR_ENABLED"] as? String ?? "") == "YES" -} -private var configBEPPath: String? { - return configValues?["BUILD_SERVICE_BEP_PATH"] as? String -} -private var sourceOutputFileMapSuffix: String? { - return configValues?["BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX"] as? String -} -private var bazelWorkingDir: String? { - return configValues?["BUILD_SERVICE_BAZEL_EXEC_ROOT"] as? String + let indexingService: IndexingService + let bepService: BEPService } /// This example listens to a BEP stream to display some output. @@ -103,282 +15,110 @@ private var bazelWorkingDir: String? { /// All operations are delegated to XCBBuildService and we inject /// progress from BEP. enum BasicMessageHandler { - // Read info from BEP and optionally handle events - static func startStream(bepPath: String, startBuildInput: XCBInputStream, bkservice: BKBuildService) throws { - log("[INFO] Will start BEP stream at path \(bepPath) with input" + String(describing: startBuildInput)) - let stream = try BEPStream(path: bepPath) - var progressView: ProgressView? - try stream.read { - event in - if indexingEnabled { - parseSourceOutputFileMappingsFromBEP(event: event) - } - - if progressBarEnabled { - if let updatedView = ProgressView(event: event, last: progressView) { - let encoder = XCBEncoder(input: startBuildInput) - let response = BuildProgressUpdatedResponse(progress: - updatedView.progressPercent, message: updatedView.message) - if let responseData = try? response.encode(encoder) { - bkservice.write(responseData) - } - progressView = updatedView - } - } - } - gStream = stream - } - // Required if `outputPathOnly` is `true` in the indexing request - static func outputPathOnlyData(outputFilePath: String, sourceFilePath: String) -> Data { - let xml = """ - - - - - outputFilePath - \(outputFilePath) - sourceFilePath - \(sourceFilePath) - - - - """ - guard let converter = BPlistConverter(xml: xml) else { - fatalError("Failed to allocate converter") - } - guard let bplistData = converter.convertToBinary() else { - fatalError("Failed to convert XML to binary plist data") - } - - return bplistData - } - // Check many conditions that need to be met in order to handle indexing and returns the respect output file, - // the call site should abort and proxy the indexing request if this returns `nil` - static func canHandleIndexingWithOutfile(msg: XCBProtocolMessage) -> String? { - // Nothing to do for non-indexing request types - guard let reqMsg = msg as? IndexingInfoRequested else { - return nil - } - // Nothing to do if indexing is disabled - guard indexingEnabled else { - return nil - } - // TODO: handle Swift - guard reqMsg.filePath.count > 0 && reqMsg.filePath != "" && !reqMsg.filePath.hasSuffix(".swift") else { - log("[WARNING] Unsupported filePath for indexing: \(reqMsg.filePath)") - return nil - } - // In `BazelBuildService` the path to the working directory (i.e. execution_root) should always - // exists - guard bazelWorkingDir != nil else { - log("[WARNING] Could not find bazel working directory. Make sure `BUILD_SERVICE_BAZEL_EXEC_ROOT` is set in the config file.") - return nil - } - - guard let outputFilePath = findOutputFileForSource(filePath: reqMsg.filePath, workingDir: reqMsg.workingDir) else { - log("[WARNING] Failed to find output file for source: \(reqMsg.filePath). Indexing requests will be proxied to default build service.") - return nil - } - - return outputFilePath - } - // Initialize in memory mappings from xcbuildkitDataDir if .json mappings files exist - static func initializeOutputFileMappingFromCache() { - guard let xcbuildkitDataDir = xcbuildkitDataDir else { return } - let fm = FileManager.default - do { - let jsons = try fm.contentsOfDirectory(atPath: xcbuildkitDataDir) - - for jsonFilename in jsons { - let jsonData = try Data(contentsOf: URL(fileURLWithPath: "\(xcbuildkitDataDir)/\(jsonFilename)")) - loadSourceOutputFileMappingInfo(jsonFilename: jsonFilename, jsonData: jsonData) - } - } catch { - log("[ERROR] Failed to initialize from cache under \(xcbuildkitDataDir) with err: \(error.localizedDescription)") - } - } - // Loads information into memory and optionally update the cache under xcbuildkitDataDir - static func loadSourceOutputFileMappingInfo(jsonFilename: String, jsonData: Data, updateCache: Bool = false) { - // Ensure workspace info is ready and .json can be decoded - guard let workspaceKey = workspaceKey else { return } - guard let xcbuildkitDataDir = xcbuildkitDataDir else { return } - guard let jsonValues = try? JSONSerialization.jsonObject(with: jsonData, options: [.allowFragments]) as? [String: String] else { return } - - // Load .json contents into memory - initializeOutputFileForSourceIfNecessary(jsonFilename: jsonFilename) - outputFileForSource[workspaceKey]?[jsonFilename] = jsonValues - log("[INFO] Loaded mapping information into memory for file: \(jsonFilename)") - - // Update .json files cached under xcbuildkitDataDir for - // fast load next time we launch Xcode - if updateCache { - do { - guard let jsonBasename = jsonFilename.components(separatedBy: "/").last else { return } - let jsonFilePath = "\(xcbuildkitDataDir)/\(jsonBasename)" - let json = URL(fileURLWithPath: jsonFilePath) - let fm = FileManager.default - if fm.fileExists(atPath: jsonFilePath) { - try fm.removeItem(atPath: jsonFilePath) - } - try jsonData.write(to: json) - log("[INFO] Updated cache for file \(jsonFilePath)") - } catch { - log("[ERROR] Failed to update cache under \(xcbuildkitDataDir) for file \(jsonFilename) with err: \(error.localizedDescription)") - } - } - } - // This loop looks for JSON files with a known suffix `BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX` and loads the mapping - // information from it decoding the JSON and storing in-memory. - // - // Read about the 'namedSetOfFiles' key here: https://bazel.build/remote/bep-examples#consuming-namedsetoffiles - static func parseSourceOutputFileMappingsFromBEP(event: BuildEventStream_BuildEvent) { - // Do work only if `namedSetOfFiles` is present and contain `files` - guard let json = try? JSONSerialization.jsonObject(with: event.jsonUTF8Data(), options: []) as? [String: Any] else { return } - guard let namedSetOfFiles = json["namedSetOfFiles"] as? [String: Any] else { return } - guard namedSetOfFiles.count > 0 else { return } - guard let allPairs = namedSetOfFiles["files"] as? [[String: Any]] else { return } - - for pair in allPairs { - // Only proceed if top level keys exist and a .json to be decoded is found - guard let jsonFilename = pair["name"] as? String else { continue } - guard var jsonURI = pair["uri"] as? String else { continue } - guard jsonURI.hasSuffix(".json") else { continue } - - jsonURI = jsonURI.replacingOccurrences(of: "file://", with: "") - - // Only proceed for keys holding .json files with known pattern (i.e. `BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX`) in the name - guard let jsonData = try? Data(contentsOf: URL(fileURLWithPath:jsonURI)) else { continue } - guard jsonData.count > 0 else { continue } - guard let sourceOutputFileMapSuffix = sourceOutputFileMapSuffix else { continue } - guard jsonFilename.hasSuffix(sourceOutputFileMapSuffix) else { continue } - - // Load .json contents into memory - log("[INFO] Parsed \(jsonFilename) from BEP.") - loadSourceOutputFileMappingInfo(jsonFilename: jsonFilename, jsonData: jsonData, updateCache: true) - } - } - // Helper to initialize in-memory mapping for workspace and give .json mappings file key - static func initializeOutputFileForSourceIfNecessary(jsonFilename: String) { - guard let workspaceKey = workspaceKey else { return } - - if outputFileForSource[workspaceKey] == nil { - outputFileForSource[workspaceKey] = [:] - } - if outputFileForSource[workspaceKey]?[jsonFilename] == nil { - outputFileForSource[workspaceKey]?[jsonFilename] = [:] - } - } - // Finds output file (i.e. path to `.o` under `bazel-out`) in in-memory mapping - static func findOutputFileForSource(filePath: String, workingDir: String) -> String? { - // Create key - let sourceKey = filePath.replacingOccurrences(of: workingDir, with: "").replacingOccurrences(of: (bazelWorkingDir ?? ""), with: "") - // Ensure workspace info is loaded and mapping exists - guard let workspaceKey = workspaceKey else { return nil } - guard let workspaceMappings = outputFileForSource[workspaceKey] else { return nil } - // Loops until found - for (_, json) in workspaceMappings { - if let objFilePath = json[sourceKey] { - return objFilePath - } - } - return nil - } /// Proxying response handler /// Every message is written to the XCBBuildService /// This simply injects Progress messages from the BEP static func respond(input: XCBInputStream, data: Data, context: Any?) { - let basicCtx = context as! BasicMessageContext - let xcbbuildService = basicCtx.xcbbuildService - let bkservice = basicCtx.bkservice + let ctx = context as! BasicMessageContext + let xcbbuildService = ctx.xcbbuildService + let bkservice = ctx.bkservice + let indexingService = ctx.indexingService + let bepService = ctx.bepService let decoder = XCBDecoder(input: input) let encoder = XCBEncoder(input: input) let identifier = input.identifier ?? "" if let msg = decoder.decodeMessage() { if let createSessionRequest = msg as? CreateSessionRequest { - // Load information from `CreateSessionRequest` - gXcode = createSessionRequest.xcode - workspaceHash = createSessionRequest.workspaceHash - workspaceName = createSessionRequest.workspaceName - xcodeprojPath = createSessionRequest.xcodeprojPath + // Initialize workspace info for the current workspace + let workspaceInfo = WorkspaceInfo( + xcode: createSessionRequest.xcode, + workspaceName: createSessionRequest.workspaceName, + workspaceHash: createSessionRequest.workspaceHash, + xcodeprojPath: createSessionRequest.xcodeprojPath + ) + indexingService.infos[createSessionRequest.workspaceKey] = workspaceInfo // Initialize build service - xcbbuildService.startIfNecessary(xcode: gXcode) + xcbbuildService.startIfNecessary(xcode: workspaceInfo.xcode) - // Load output file mapping information from cache if it exists - initializeOutputFileMappingFromCache() + // Initialize indexing information + indexingService.initializeOutputFileMappingFromCache(msg: createSessionRequest) } else if msg is BuildStartRequest { - // Start reading from BEP as early as possible - if let bepPath = configBEPPath { + let message = BuildProgressUpdatedResponse() + if let responseData = try? message.encode(encoder) { + bkservice.write(responseData) + } + } else if let createBuildRequest = msg as? CreateBuildRequest, let workspaceInfo = indexingService.infos[createBuildRequest.workspaceKey] { + // Start streaming from the BEP + if let bepPath = workspaceInfo.config.configBEPPath { do { - try startStream(bepPath: bepPath, startBuildInput: input, bkservice: bkservice) + try bepService.startStream(msg: createBuildRequest, bepPath: bepPath, startBuildInput: input, ctx: ctx) } catch { - // fatalError("[ERROR] Failed to start BEP stream with error: " + error.localizedDescription) log("[ERROR] Failed to start BEP stream with error: " + error.localizedDescription) } } else { log("[WARNING] BEP string config key 'BUILD_SERVICE_BEP_PATH' empty. Bazel information won't be available during a build.") } + + // This information was not explicitly available in `CreateSessionRequest`, parse from `CreateBuildRequest` instead + // Necessary for indexing and potentially for other things in the future. This is effectively $SRCROOT. + workspaceInfo.workingDir = createBuildRequest.workingDir + workspaceInfo.derivedDataPath = createBuildRequest.derivedDataPath + workspaceInfo.indexDataStoreFolderPath = createBuildRequest.indexDataStoreFolderPath + // Attempt to initialize in-memory mapping if empty // It's possible that indexing data is not ready yet in `CreateSessionRequest` above - // so retry to load info into memory at `BuildStartRequest` time - if let workspaceKey = workspaceKey { - if (outputFileForSource[workspaceKey]?.count ?? 0) == 0 { - initializeOutputFileMappingFromCache() - } + if workspaceInfo.outputFileForSource.count == 0 { + indexingService.initializeOutputFileMappingFromCache(msg: createBuildRequest) } - let message = BuildProgressUpdatedResponse() - if let responseData = try? message.encode(encoder) { - bkservice.write(responseData) - } - } else if let outputFilePath = canHandleIndexingWithOutfile(msg: msg) { - // Settings values needed to compose the payload below - let reqMsg = msg as! IndexingInfoRequested - workingDir = bazelWorkingDir ?? reqMsg.workingDir - platform = reqMsg.platform - sdk = reqMsg.sdk - + // Setup DataStore for indexing with Bazel + indexingService.setupDataStore(msg: createBuildRequest) + } else if let indexingInfoRequest = msg as? IndexingInfoRequested, + let outputFilePath = indexingService.indexingOutputFilePath(msg: indexingInfoRequest), + let workspaceInfo = indexingService.infos[indexingInfoRequest.workspaceKey] { // Compose the indexing response payload and emit the response message + // Note that information is combined from different places (workspace info, incoming indexing request, indexing service helper methods) let clangXMLData = BazelBuildServiceStub.getASTArgs( - targetID: reqMsg.targetID, - sourceFilePath: reqMsg.filePath, + targetID: indexingInfoRequest.targetID, + sourceFilePath: indexingInfoRequest.filePath, outputFilePath: outputFilePath, - derivedDataPath: reqMsg.derivedDataPath, - workspaceHash: workspaceHash, - workspaceName: workspaceName, - sdkPath: sdkPath, - sdkName: sdk, - workingDir: workingDir) + derivedDataPath: workspaceInfo.derivedDataPath, + workspaceHash: workspaceInfo.workspaceHash, + workspaceName: workspaceInfo.workspaceName, + sdkPath: ctx.indexingService.sdkPath(msg: indexingInfoRequest), + sdkName: indexingInfoRequest.sdk, + workingDir: workspaceInfo.config.bazelWorkingDir ?? workspaceInfo.workingDir) let message = IndexingInfoReceivedResponse( - targetID: reqMsg.targetID, - data: reqMsg.outputPathOnly ? outputPathOnlyData(outputFilePath: outputFilePath, sourceFilePath: reqMsg.filePath) : nil, - responseChannel: UInt64(reqMsg.responseChannel), - clangXMLData: reqMsg.outputPathOnly ? nil : clangXMLData) + targetID: indexingInfoRequest.targetID, + data: indexingInfoRequest.outputPathOnly ? BazelBuildServiceStub.outputPathOnlyData(outputFilePath: outputFilePath, sourceFilePath: indexingInfoRequest.filePath) : nil, + responseChannel: UInt64(indexingInfoRequest.responseChannel), + clangXMLData: indexingInfoRequest.outputPathOnly ? nil : clangXMLData) + if let encoded: XCBResponse = try? message.encode(encoder) { bkservice.write(encoded, msgId:message.responseChannel) - log("[INFO] Handling \(identifier) for source \(reqMsg.filePath) and output file \(outputFilePath)") + log("[INFO] Handling \(identifier) for source \(indexingInfoRequest.filePath) and output file \(outputFilePath)") return } } } log("[INFO] Proxying request with type: \(identifier)") - if indexingEnabled && identifier == "INDEXING_INFO_REQUESTED" { - log("[WARNING] BazelBuildService failed to handle indexing request, message will be proxied instead.") - // If we hit this means that something went wrong with indexing, logging this in-memory mapping is useful for troubleshooting - log("[INFO] outputFileForSource: \(outputFileForSource)") - } xcbbuildService.write(data) } } let xcbbuildService = XCBBuildServiceProcess() let bkservice = BKBuildService() +let indexingService = IndexingService() +let bepService = BEPService() let context = BasicMessageContext( xcbbuildService: xcbbuildService, - bkservice: bkservice + bkservice: bkservice, + indexingService: indexingService, + bepService: bepService ) bkservice.start(messageHandler: BasicMessageHandler.respond, context: context) diff --git a/Examples/XCBBuildServiceProxy/main.swift b/Examples/XCBBuildServiceProxy/main.swift index ae0bf9d..f6cd68d 100644 --- a/Examples/XCBBuildServiceProxy/main.swift +++ b/Examples/XCBBuildServiceProxy/main.swift @@ -68,6 +68,8 @@ private var workspaceName = "" // TODO: parsed in `IndexingInfoRequested`, there's probably a less hacky way to get this. // Effectively `$PWD/iOSApp` private var workingDir = "" +// Path to derived data for the current workspace +private var derivedDataPath = "" // TODO: parsed in `IndexingInfoRequested` and it's lowercased there, might not be stable in different OSes private var sdk = "" // TODO: parsed in `IndexingInfoRequested` and it's lowercased there, might not be stable in different OSes @@ -132,10 +134,14 @@ enum BasicMessageHandler { workspaceHash = createSessionRequest.workspaceHash workspaceName = createSessionRequest.workspaceName xcbbuildService.startIfNecessary(xcode: gXcode) + } else if let createBuildRequest = msg as? CreateBuildRequest { + // This information was not explicitly available in `CreateSessionRequest`, parse from `CreateBuildRequest` instead + // Necessary for indexing and potentially for other things in the future. This is effectively $SRCROOT. + workingDir = createBuildRequest.workingDir + derivedDataPath = createBuildRequest.derivedDataPath } else if !XCBBuildServiceProcess.MessageDebuggingEnabled() && indexingEnabled && msg is IndexingInfoRequested { // Example of a custom indexing service let reqMsg = msg as! IndexingInfoRequested - workingDir = reqMsg.workingDir platform = reqMsg.platform sdk = reqMsg.sdk @@ -151,7 +157,7 @@ enum BasicMessageHandler { targetID: reqMsg.targetID, sourceFilePath: reqMsg.filePath, outputFilePath: outputFilePath, - derivedDataPath: reqMsg.derivedDataPath, + derivedDataPath: derivedDataPath, workspaceHash: workspaceHash, workspaceName: workspaceName, sdkPath: sdkPath, diff --git a/Sources/XCBProtocol/Protocol.swift b/Sources/XCBProtocol/Protocol.swift index b626033..3744b9a 100644 --- a/Sources/XCBProtocol/Protocol.swift +++ b/Sources/XCBProtocol/Protocol.swift @@ -60,6 +60,28 @@ private extension XCBEncoder { return id - offset } } +// `path` should look something like this (note that `/path/to/DerivedData` can also be a custom path): +// +// path/to/DerivedData/App-/any/other/path/here +// +private func workspaceInfoFromPath(path: String) -> (workspaceName: String, workspaceHash: String) { + var name: String = "" + var hash: String = "" + + let componentsByDash = path.components(separatedBy: "-") + let parsedWorkspaceHash = componentsByDash.last!.components(separatedBy: "/").first ?? "" + hash = parsedWorkspaceHash + + // Workspace names can contain `-` characters too + let componentsByForwardSlash = path.components(separatedBy: "/") + if let workspaceNameComponent = componentsByForwardSlash.filter({ $0.contains(parsedWorkspaceHash) }).first { + var workspaceNameComponentsByDash = workspaceNameComponent.components(separatedBy: "-") + workspaceNameComponentsByDash.removeLast() + name = String(workspaceNameComponentsByDash.joined(separator: "-")) + } + + return (name, hash) +} public struct CreateSessionRequest: XCBProtocolMessage { public let workspace: String @@ -97,23 +119,8 @@ public struct CreateSessionRequest: XCBProtocolMessage { self.xcbuildDataPath = "" } - // `self.xcbuildDataPath` looks something like this (note that `/path/to/DerivedData` can also be a custom path): - // - // /path/to/DerivedData/iOSApp-frhmkkebaragakhdzyysbrsvbgtc/Build/Intermediates.noindex/XCBuildData - // - let componentsByDash = self.xcbuildDataPath.components(separatedBy: "-") - let parsedWorkspaceHash = componentsByDash.last!.components(separatedBy: "/").first ?? "" - self.workspaceHash = parsedWorkspaceHash - - // Workspace names can contain `-` characters too - let componentsByForwardSlash = self.xcbuildDataPath.components(separatedBy: "/") - if let workspaceNameComponent = componentsByForwardSlash.filter { $0.contains(parsedWorkspaceHash) }.first as? String { - var workspaceNameComponentsByDash = workspaceNameComponent.components(separatedBy: "-") - workspaceNameComponentsByDash.removeLast() - self.workspaceName = String(workspaceNameComponentsByDash.joined(separator: "-")) - } else { - self.workspaceName = "" - } + // Use helper method to extract this from `/path/to/DerivedData/App-/Build/Intermediates.noindex/XCBuildData` + (self.workspaceName, self.workspaceHash) = workspaceInfoFromPath(path: self.xcbuildDataPath) // Parse path to .xcodeproj used to load xcbuildkit config as early as possible self.xcodeprojPath = self.workspace.components(separatedBy:"path:\'").last?.components(separatedBy:"/project.xcworkspace").first ?? "" @@ -145,6 +152,11 @@ public struct SetSessionUserInfoRequest: XCBProtocolMessage { public struct CreateBuildRequest: XCBProtocolMessage { public let configuredTargets: [String] public let containerPath: String + public let workspaceName: String + public let workspaceHash: String + public let workingDir: String + public let derivedDataPath: String + public let indexDataStoreFolderPath: String public init(input: XCBInputStream) throws { var minput = input @@ -152,21 +164,41 @@ public struct CreateBuildRequest: XCBProtocolMessage { case let .binary(jsonData) = next else { throw XCBProtocolError.unexpectedInput(for: input) } - guard let json = try JSONSerialization.jsonObject(with: jsonData, options: []) as? [String: Any] else { + guard let json = try JSONSerialization.jsonObject(with: jsonData, options: []) as? [String: Any] else { throw XCBProtocolError.unexpectedInput(for: input) - } - let requestJSON = json["request"] as? [String: Any] ?? [:] - self.containerPath = requestJSON["containerPath"] as? String ?? "" - log("info: got containerPath \(self.containerPath)") - if let ct = requestJSON["configuredTargets"] as? [[String: Any]] { - self.configuredTargets = ct.compactMap { ctInfo in - return ctInfo["guid"] as? String - } - log("info: got configured targets \(self.configuredTargets)") - } else { - log("warning: malformd configured targets\(json["configuredTargets"])") - self.configuredTargets = [] - } + } + let requestJSON = json["request"] as? [String: Any] ?? [:] + // Remove last word of `$PWD/iOSApp/iOSApp.xcodeproj` to get `workingDir` + self.containerPath = requestJSON["containerPath"] as? String ?? "" + self.workingDir = Array(containerPath.components(separatedBy: "/").dropLast()).joined(separator: "/") + log("info: got workingDir \(self.workingDir)") + + let parameters = requestJSON["parameters"] as? [String: Any] ?? [:] + let arenaInfo = parameters["arenaInfo"] as? [String: Any] ?? [:] + self.derivedDataPath = arenaInfo["derivedDataPath"] as? String ?? "" + self.indexDataStoreFolderPath = arenaInfo["indexDataStoreFolderPath"] as? String ?? "" + log("info: got derivedDataPath \(self.derivedDataPath)") + log("info: got indexDataStoreFolderPath \(self.indexDataStoreFolderPath)") + + if let indexDataStoreFolderPath = arenaInfo["indexDataStoreFolderPath"] as? String { + (self.workspaceName, self.workspaceHash) = workspaceInfoFromPath(path: indexDataStoreFolderPath) + } else { + self.workspaceName = "" + self.workspaceHash = "" + } + log("info: got workspaceName \(self.workspaceName)") + log("info: got workspaceHash \(self.workspaceHash)") + + log("info: got containerPath \(self.containerPath)") + if let ct = requestJSON["configuredTargets"] as? [[String: Any]] { + self.configuredTargets = ct.compactMap { ctInfo in + return ctInfo["guid"] as? String + } + log("info: got configured targets \(self.configuredTargets)") + } else { + log("warning: malformd configured targets\(json["configuredTargets"] ?? [])") + self.configuredTargets = [] + } } } @@ -208,10 +240,10 @@ public struct IndexingInfoRequested: XCBProtocolMessage { public let targetID: String public let outputPathOnly: Bool public let filePath: String - public let derivedDataPath: String - public let workingDir: String public let sdk: String public let platform: String + public let workspaceName: String + public let workspaceHash: String public init(input: XCBInputStream) throws { var minput = input @@ -222,10 +254,10 @@ public struct IndexingInfoRequested: XCBProtocolMessage { self.filePath = "_internal_stub_" self.outputPathOnly = false self.responseChannel = -1 - self.derivedDataPath = "" - self.workingDir = "" self.sdk = "" self.platform = "" + self.workspaceName = "" + self.workspaceHash = "" return } @@ -242,18 +274,14 @@ public struct IndexingInfoRequested: XCBProtocolMessage { self.outputPathOnly = json["outputPathOnly"] as? Bool ?? false let requestJSON = json["request"] as? [String: Any] ?? [:] - - // Remove last word of `$PWD/iOSApp/iOSApp.xcodeproj` to get `workingDir` - let containerPath = requestJSON["containerPath"] as? String ?? "" - self.workingDir = Array(containerPath.components(separatedBy: "/").dropLast()).joined(separator: "/") - let jsonRep64Str = requestJSON["jsonRepresentation"] as? String ?? "" let jsonRepData = Data.fromBase64(jsonRep64Str) ?? Data() guard let jsonJSON = try JSONSerialization.jsonObject(with: jsonRepData, options: []) as? [String: Any] else { log("warning: missing rep str") - self.derivedDataPath = "" self.sdk = "" self.platform = "" + self.workspaceName = "" + self.workspaceHash = "" log("RequestReceived \(self)") return } @@ -261,16 +289,23 @@ public struct IndexingInfoRequested: XCBProtocolMessage { let parameters = jsonJSON["parameters"] as? [String: Any] ?? [:] let arenaInfo = parameters["arenaInfo"] as? [String: Any] ?? [:] - self.derivedDataPath = arenaInfo["derivedDataPath"] as? String ?? "" - let activeRunDestination = parameters["activeRunDestination"] as? [String: Any] ?? [:] self.sdk = activeRunDestination["sdk"] as? String ?? "" self.platform = activeRunDestination["platform"] as? String ?? "" + // Use helper method to extract this from `/path/to/DerivedData/App-/Build/Intermediates.noindex/Index/DataStore` + if let indexDataStoreFolderPath = arenaInfo["indexDataStoreFolderPath"] as? String { + (self.workspaceName, self.workspaceHash) = workspaceInfoFromPath(path: indexDataStoreFolderPath) + } else { + self.workspaceName = "" + self.workspaceHash = "" + } + log("RequestReceived \(self)") - log("Parsed derivedDataPath \(self.derivedDataPath)") - log("Parsed sdk \(self.sdk)") - log("Parsed platform \(self.platform)") + log("Parsed(indexing) sdk \(self.sdk)") + log("Parsed(indexing) platform \(self.platform)") + log("Parsed(indexing) workspaceName \(self.workspaceName)") + log("Parsed(indexing) workspaceHash \(self.workspaceHash)") } }