Parse source to output file map from aspect outputgroup JSON files

Examples/BazelBuildService/BEPService.swift

@@ -0,0 +1,70 @@
+import BEP
+import BKBuildService
+import Foundation
+import XCBProtocol
+
+// Handles BEP related tasks for all incoming message types (progress bar, indexing, etc.).
+// In the future we might want to delegate the message specific work to a diff service,
+// keep all logic here now for simplicity.
+class BEPService {
+  // Read info from BEP and optionally handle events
+  func startStream(msg: WorkspaceInfoKeyable, bepPath: String, startBuildInput: XCBInputStream, ctx: BasicMessageContext) throws {
+        guard let info = ctx.indexingService.infos[msg.workspaceKey] else { return }
+
+        log("[INFO] Will start BEP stream at path \(bepPath) with input" + String(describing: startBuildInput))
+        let bkservice = ctx.bkservice
+        let stream = try BEPStream(path: bepPath)
+        var progressView: ProgressView?
+        try stream.read {
+            event in
+            if info.config.indexingEnabled {
+                self.parseSourceOutputFileMappingsFromBEP(msg: msg, event: event, ctx: ctx)
+            }
+
+            if info.config.progressBarEnabled {
+                if let updatedView = ProgressView(event: event, last: progressView) {
+                    let encoder = XCBEncoder(input: startBuildInput)
+                    let response = BuildProgressUpdatedResponse(progress:
+                        updatedView.progressPercent, message: updatedView.message)
+                    if let responseData = try? response.encode(encoder) {
+                        bkservice.write(responseData)
+                    }
+                    progressView = updatedView
+                }
+            }
+        }
+        info.bepStream = stream
+    }
+
+  // This loop looks for JSON files with a known suffix `BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX` and loads the mapping
+  // information from it decoding the JSON and storing in-memory.
+  //
+  // Read about the 'namedSetOfFiles' key here: https://bazel.build/remote/bep-examples#consuming-namedsetoffiles
+  func parseSourceOutputFileMappingsFromBEP(msg: WorkspaceInfoKeyable, event: BuildEventStream_BuildEvent, ctx: BasicMessageContext) {
+      guard let info = ctx.indexingService.infos[msg.workspaceKey] else { return }
+      // Do work only if `namedSetOfFiles` is present and contain `files`
+      guard let json = try? JSONSerialization.jsonObject(with: event.jsonUTF8Data(), options: []) as? [String: Any] else { return }
+      guard let namedSetOfFiles = json["namedSetOfFiles"] as? [String: Any] else { return }
+      guard namedSetOfFiles.count > 0 else { return }
+      guard let allPairs = namedSetOfFiles["files"] as? [[String: Any]] else { return }
+
+      for pair in allPairs {
+          // Only proceed if top level keys exist and a .json to be decoded is found
+          guard let jsonFilename = pair["name"] as? String else { continue }
+          guard var jsonURI = pair["uri"] as? String else { continue }
+          guard jsonURI.hasSuffix(".json") else { continue }
+
+          jsonURI = jsonURI.replacingOccurrences(of: "file://", with: "")
+
+          // Only proceed for keys holding .json files with known pattern (i.e. `BUILD_SERVICE_SOURE_OUTPUT_FILE_MAP_SUFFIX`) in the name
+          guard let jsonData = try? Data(contentsOf: URL(fileURLWithPath:jsonURI)) else { continue }
+          guard jsonData.count > 0 else { continue }
+          guard let sourceOutputFileMapSuffix = info.config.sourceOutputFileMapSuffix else { continue }
+          guard jsonFilename.hasSuffix(sourceOutputFileMapSuffix) else { continue }
+
+          // Load .json contents into memory
+          log("[INFO] Parsed \(jsonFilename) from BEP.")
+          ctx.indexingService.loadSourceOutputFileMappingInfo(msg: msg, jsonFilename: jsonFilename, jsonData: jsonData, updateCache: true)
+      }
+  }
+}

Examples/BazelBuildService/BEPStream.swift

@@ -18,9 +18,10 @@ public class BEPStream {
     deinit {
         // According to close should automatically happen, but it does't under a
         // few cases:
-        // https://developer.apple.com/documentation/foundation/filehandle/1413393-closefile
-        fileHandle?.closeFile()
+        // https://developer.apple.com/documentation/foundation/filehandle/3172525-close
+        try? fileHandle?.close()
         fileHandle?.readabilityHandler = nil
+        log("BEPStream: dealloc")
     }
     /// Reads data from a BEP stream
     /// @param eventAvailableHandler - this is called with _every_ BEP event
@@ -55,6 +56,7 @@ public class BEPStream {
             log("BEPStream: failed to allocate \(path)")
             return
         }
+
         self.fileHandle = fileHandle
         fileHandle.readabilityHandler = {
             handle in
@@ -83,5 +85,9 @@ public class BEPStream {
                 }
             }
         }
+
+        // The file handle works without this in debug mode (i.e. manually launching xcbuildkit),
+        // but when installed on a machine as a pkg this call is necessary to get events in the `readabilityHandler` block above.
+        fileHandle.waitForDataInBackgroundAndNotify()
     }
 }

Examples/BazelBuildService/BazelBuildServiceConfig.swift

@@ -0,0 +1,55 @@
+// Encapsulates the config file used to control xcbuildkit's behaviour.
+//
+// Format is:
+//
+// KEY1=VALUE1
+// KEY2=VALUE2
+//
+// so one config per line. This is intentionally simple at the moment but we can add more validation or a more reliable file format later.
+//
+// TODO: Codable?
+// TODO: Not load from disk all the time but still detect changes in the file and refresh in-memory values?
+struct BazelBuildServiceConfig {
+  private enum ConfigKeys: String {
+    case indexingEnabled = "BUILD_SERVICE_INDEXING_ENABLED"
+    case indexStorePath = "BUILD_SERVICE_INDEX_STORE_PATH"
+    case xcbuildkitDataDir = "BUILD_SERVICE_INDEXING_DATA_DIR"
+    case progressBarEnabled = "BUILD_SERVICE_PROGRESS_BAR_ENABLED"
+    case configBEPPath = "BUILD_SERVICE_BEP_PATH"
+    case sourceOutputFileMapSuffix = "BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX"
+    case bazelWorkingDir = "BUILD_SERVICE_BAZEL_EXEC_ROOT"
+  }
+
+  var indexingEnabled: Bool {  return self.value(for: .indexingEnabled) == "YES" }
+  var indexStorePath: String? {  return self.value(for: .indexStorePath) }
+  var xcbuildkitDataDir: String? {  return self.value(for: .xcbuildkitDataDir) }
+  var progressBarEnabled: Bool {  return self.value(for: .progressBarEnabled) == "YES" }
+  var configBEPPath: String? {  return self.value(for: .configBEPPath) ?? "/tmp/bep.bep" }
+  var sourceOutputFileMapSuffix: String? {  return self.value(for: .sourceOutputFileMapSuffix) }
+  var bazelWorkingDir: String? {  return self.value(for: .bazelWorkingDir) }
+
+  let configPath: String
+
+  init(configPath: String) {
+    self.configPath = configPath
+  }
+
+  private var loadConfigFile: [String: Any]? {
+      guard let data = try? String(contentsOfFile: self.configPath, encoding: .utf8) else {
+        return nil
+      }
+
+      let lines = data.components(separatedBy: .newlines)
+      var dict: [String: Any] = [:]
+      for line in lines {
+          let split = line.components(separatedBy: "=")
+          guard split.count == 2 else { continue }
+          dict[split[0]] = split[1]
+      }
+      return dict
+  }
+
+  private func value(for config: ConfigKeys) -> String? {
+    return loadConfigFile?[config.rawValue] as? String
+  }
+}

Examples/BazelBuildService/BazelBuildServiceStub.swift

@@ -176,5 +176,31 @@ public enum BazelBuildServiceStub {
 
                 return BPlistConverter(xml: clangXML)?.convertToBinary() ?? Data()
         }
+
+        // Required if `outputPathOnly` is `true` in the indexing request
+        public static func outputPathOnlyData(outputFilePath: String,
+                                              sourceFilePath: String) -> Data {
+                let xml = """
+                <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+                <plist version="1.0">
+                <array>
+                        <dict>
+                        <key>outputFilePath</key>
+                        <string>\(outputFilePath)</string>
+                        <key>sourceFilePath</key>
+                        <string>\(sourceFilePath)</string>
+                        </dict>
+                </array>
+                </plist>
+                """
+                guard let converter = BPlistConverter(xml: xml) else {
+                fatalError("Failed to allocate converter")
+                }
+                guard let bplistData = converter.convertToBinary() else {
+                fatalError("Failed to convert XML to binary plist data")
+                }
+
+                return bplistData
+        }
 }