Add a test discovery tool that uses SymbolKit to detect XCTestCase …

…subclasses and their methods from symbol graphs and generate the necessary runner.

This is similar to the logic used by Swift Package Manager to do test discovery, except that SPM uses the index store. We use symbol graphs since JSON is much easier to process than index store records and they can be easily extracted from modules after they've been built.

Note to open-source rules maintainers: You will need to provide BUILD definitions for swift-argument-parser and swift-docc-symbolkit in order to populate these dependencies.

PiperOrigin-RevId: 430518889

tools/test_discoverer/BUILD

@@ -0,0 +1,13 @@
+load("//swift:swift.bzl", "swift_binary")
+
+swift_binary(
+    name = "test_discoverer",
+    srcs = [
+        "DiscoveredTests.swift",
+        "SymbolCollector.swift",
+        "SymbolKitExtensions.swift",
+        "TestDiscoverer.swift",
+        "TestPrinter.swift",
+    ],
+    visibility = ["//visibility:public"],
+)

tools/test_discoverer/DiscoveredTests.swift

@@ -0,0 +1,55 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Structured information about test classes and methods discovered by scanning symbol graphs.
+struct DiscoveredTests {
+  /// The modules containing test classes/methods that were discovered in the symbol graph, keyed by
+  /// the module name.
+  var modules: [String: Module] = [:]
+}
+
+extension DiscoveredTests {
+  /// Information about a module discovered in the symbol graphs that contains tests.
+  struct Module {
+    /// The name of the module.
+    var name: String
+
+    /// The `XCTestCase`-inheriting classes (or extensions to `XCTestCase`-inheriting classes) in
+    /// the module, keyed by the class name.
+    var classes: [String: Class] = [:]
+  }
+}
+
+extension DiscoveredTests {
+  /// Information about a class or class extension discovered in the symbol graphs that inherits
+  /// (directly or indirectly) from `XCTestCase`.
+  struct Class {
+    /// The name of the `XCTestCase`-inheriting class.
+    var name: String
+
+    /// The methods that were discovered in the class to represent tests.
+    var methods: [Method] = []
+  }
+}
+
+extension DiscoveredTests {
+  /// Information about a discovered test method in an `XCTestCase` subclass.
+  struct Method {
+    /// The name of the discovered test method.
+    var name: String
+
+    /// Indicates whether the test method was declared `async` or not.
+    var isAsync: Bool
+  }
+}

tools/test_discoverer/SymbolCollector.swift

@@ -0,0 +1,186 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import SymbolKit
+
+/// The precise identifier of the `XCTest.XCTestCase` class.
+///
+/// This is the mangled name of `XCTest.XCTestCase` with the leading "$s" replaced by "s:" (a common
+/// notation used in Clang/Swift UIDs used for indexing).
+private let xcTestCasePreciseIdentifier = "s:6XCTest0A4CaseC"
+
+/// Collects information from one or more symbol graphs in order to determine which classes and
+/// methods correspond to `XCTest`-style test cases.
+final class SymbolCollector {
+  /// `inheritsFrom` relationships collected from symbol graphs, keyed by their source identifier
+  /// (i.e., the subclass in the relationship).
+  private var inheritanceRelationships: [String: SymbolGraph.Relationship] = [:]
+
+  /// `memberOf` relationships collected from symbol graphs, keyed by their source identifier (i.e.,
+  /// the nested or contained declaration).
+  private var memberRelationships: [String: SymbolGraph.Relationship] = [:]
+
+  /// A mapping from class identifiers to Boolean values indicating whether or not the class is
+  /// known to be or not be a test class (that is, inherit directly or indirectly from
+  /// `XCTestCase`).
+  ///
+  /// If the value for a class identifier is true, the class is known to inherit from `XCTestCase`.
+  /// If the value is false, the class is known to not inherit from `XCTestCase`. If the class
+  /// identifier is not present in the map, then its state is not yet known.
+  private var testCaseClassCache: [String: Bool] = [:]
+
+  /// A mapping from discovered class identifiers to their symbol graph data.
+  private var possibleTestClasses: [String: SymbolGraph.Symbol] = [:]
+
+  /// A collection of methods that match heuristics to be considered as test methods -- their names
+  /// begin with "test", they take no arguments, and they return `Void`.
+  private var possibleTestMethods: [SymbolGraph.Symbol] = []
+
+  /// A mapping from class (or class extension) identifiers to the module name where they were
+  /// declared.
+  private var modulesForClassIdentifiers: [String: String] = [:]
+
+  /// Collects information from the given symbol graph that is needed to discover test classes and
+  /// test methods in the module.
+  func consume(_ symbolGraph: SymbolGraph) {
+    // First, collect all the inheritance and member relationships from the graph. We cannot filter
+    // them at this time, since they only contain the identifiers and might reference symbols in
+    // modules whose graphs haven't been processed yet.
+    for relationship in symbolGraph.relationships {
+      switch relationship.kind {
+      case .inheritsFrom:
+        inheritanceRelationships[relationship.source] = relationship
+      case .memberOf:
+        memberRelationships[relationship.source] = relationship
+      default:
+        break
+      }
+    }
+
+    // Next, collect classes and methods that might be tests. We can do limited filtering here, as
+    // described below.
+    symbolLoop: for (preciseIdentifier, symbol) in symbolGraph.symbols {
+      switch symbol.kind.identifier {
+      case "swift.class":
+        // Keep track of all classes for now; their inheritance relationships will be resolved
+        // on-demand once we have all the symbol graphs loaded.
+        possibleTestClasses[preciseIdentifier] = symbol
+        modulesForClassIdentifiers[preciseIdentifier] = symbolGraph.module.name
+
+      case "swift.method":
+        // Swift Package Manager uses the index store to discover tests; index-while-building writes
+        // a unit-test property for any method that satisfies this method:
+        // https://github.com/apple/swift/blob/da3856c45b7149730d6e5fdf528ac82b43daccac/lib/Index/IndexSymbol.cpp#L40-L82
+        // We duplicate that logic here.
+
+        guard symbol.swiftGenerics == nil else {
+          // Generic methods cannot be tests.
+          continue symbolLoop
+        }
+
+        guard symbol.functionSignature?.isTestLike == true else {
+          // Functions with parameters or which return something other than `Void` cannot be tests.
+          continue symbolLoop
+        }
+
+        let lastComponent = symbol.pathComponents.last!
+        guard lastComponent.hasPrefix("test") else {
+          // Test methods must be named `test*`.
+          continue symbolLoop
+        }
+
+        // If we got this far, record the symbol as a possible test method. We still need to make
+        // sure later that it is a member of a class that inherits from `XCTestCase`.
+        possibleTestMethods.append(symbol)
+
+      default:
+        break
+      }
+    }
+  }
+
+  /// Returns a `DiscoveredTests` value containing structured information about the tests discovered
+  /// in the symbol graph.
+  func discoveredTests() -> DiscoveredTests {
+    var discoveredTests = DiscoveredTests()
+
+    for method in possibleTestMethods {
+      if let classSymbol = testClassSymbol(for: method),
+        let moduleName = modulesForClassIdentifiers[classSymbol.identifier.precise]
+      {
+        let className = classSymbol.pathComponents.last!
+
+        let lastMethodComponent = method.pathComponents.last!
+        let methodName =
+          lastMethodComponent.hasSuffix("()")
+          ? String(lastMethodComponent.dropLast(2))
+          : lastMethodComponent
+
+        discoveredTests.modules[moduleName, default: DiscoveredTests.Module(name: moduleName)]
+          .classes[className, default: DiscoveredTests.Class(name: className)]
+          .methods.append(
+            DiscoveredTests.Method(name: methodName, isAsync: method.isAsyncDeclaration))
+      }
+    }
+
+    return discoveredTests
+  }
+}
+
+extension SymbolCollector {
+  /// Returns the symbol graph symbol information for the class (or class extension) that contains
+  /// the given method if and only if the class or class extension is a test class.
+  ///
+  /// If the containing class is unknown or it is not a test class, this method returns nil.
+  private func testClassSymbol(for method: SymbolGraph.Symbol) -> SymbolGraph.Symbol? {
+    guard let memberRelationship = memberRelationships[method.identifier.precise] else {
+      return nil
+    }
+
+    let classIdentifier = memberRelationship.target
+    guard isTestClass(classIdentifier) else {
+      return nil
+    }
+
+    return possibleTestClasses[classIdentifier]
+  }
+
+  /// Returns a value indicating whether or not the class with the given identifier extends
+  /// `XCTestCase` (or if the identifier is a class extension, whether it extends a subclass of
+  /// `XCTestCase`).
+  private func isTestClass(_ preciseIdentifier: String) -> Bool {
+    if let known = testCaseClassCache[preciseIdentifier] {
+      return known
+    }
+
+    guard let inheritanceRelationship = inheritanceRelationships[preciseIdentifier] else {
+      // If there are no inheritance relationships with the identifier as the source, then the class
+      // is either a root class or we didn't process the symbol graph for the module that declares
+      // it. In either case, we can't go any further so we mark the class as not-a-test.
+      testCaseClassCache[preciseIdentifier] = false
+      return false
+    }
+
+    if inheritanceRelationship.target == xcTestCasePreciseIdentifier {
+      // If the inheritance relationship has the precise identifier for `XCTest.XCTestCase` as its
+      // target, then we know definitively that the class is a direct subclass of `XCTestCase`.
+      testCaseClassCache[preciseIdentifier] = true
+      return true
+    }
+
+    // If the inheritance relationship had some other class as its target (the superclass), then
+    // (inductively) the source (subclass) is a test class if the superclass is.
+    return isTestClass(inheritanceRelationship.target)
+  }
+}

tools/test_discoverer/SymbolKitExtensions.swift

@@ -0,0 +1,54 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import SymbolKit
+
+extension SymbolGraph.Symbol {
+  /// Returns true if the given symbol represents an `async` declaration, or false otherwise.
+  var isAsyncDeclaration: Bool {
+    guard let mixin = declarationFragments else { return false }
+
+    return mixin.declarationFragments.contains { fragment in
+      fragment.kind == .keyword && fragment.spelling == "async"
+    }
+  }
+
+  /// Returns the symbol's `DeclarationFragments` mixin, or `nil` if it does not exist.
+  var declarationFragments: DeclarationFragments? {
+    mixins[DeclarationFragments.mixinKey] as? DeclarationFragments
+  }
+
+  /// Returns the symbol's `FunctionSignature` mixin, or `nil` if it does not exist.
+  var functionSignature: FunctionSignature? {
+    mixins[FunctionSignature.mixinKey] as? FunctionSignature
+  }
+
+  /// Returns the symbol's `Swift.Generics` mixin, or `nil` if it does not exist.
+  var swiftGenerics: Swift.Generics? {
+    mixins[Swift.Generics.mixinKey] as? Swift.Generics
+  }
+}
+
+extension SymbolGraph.Symbol.FunctionSignature {
+  /// Returns true if the given function signature satisfies the requirements to be a test function;
+  /// that is, it has no parameters and returns `Void`.
+  var isTestLike: Bool {
+    // TODO(b/220940013): Do we need to support the `Void` spelling here too, if someone writes
+    // `Void` specifically instead of omitting the return type?
+    parameters.isEmpty
+      && returns.count == 1
+      && returns[0].kind == .text
+      && returns[0].spelling == "()"
+  }
+}