Support resolving code file references in ConvertService (swiftlang#570)

* Support resolving code file references in ConvertService

rdar://107965493

* Support highlighting lines in resolved file assets

Sources/SwiftDocC/Model/Rendering/References/FileReference.swift

@@ -47,12 +47,21 @@ public struct FileReference: RenderReference, Equatable {
     ///   - fileType: The type of file, typically represented by its file extension.
     ///   - syntax: The syntax of the file's content.
     ///   - content: The line-by-line contents of the file.
-    public init(identifier: RenderReferenceIdentifier, fileName: String, fileType: String, syntax: String, content: [String]) {
+    ///   - highlights: The line highlights for this file.
+    public init(
+        identifier: RenderReferenceIdentifier,
+        fileName: String,
+        fileType: String,
+        syntax: String,
+        content: [String],
+        highlights: [LineHighlighter.Highlight] = []
+    ) {
         self.identifier = identifier
         self.fileName = fileName
         self.fileType = fileType
         self.syntax = syntax
         self.content = content
+        self.highlights = highlights
     }
 
     public init(from decoder: Decoder) throws {

Sources/SwiftDocC/Model/Rendering/RenderNodeTranslator.swift

@@ -55,11 +55,13 @@ public struct RenderNodeTranslator: SemanticVisitor {
 
     public mutating func visitCode(_ code: Code) -> RenderTree? {
         let fileType = NSString(string: code.fileName).pathExtension
-        let fileReference = code.fileReference
+        guard let fileIdentifier = context.identifier(forAssetName: code.fileReference.path, in: identifier) else {
+            return nil
+        }
 
-        guard let fileData = try? context.resource(with: code.fileReference),
-            let fileContents = String(data: fileData, encoding: .utf8) else {
-            return RenderReferenceIdentifier("")
+        let fileReference = ResourceReference(bundleIdentifier: code.fileReference.bundleIdentifier, path: fileIdentifier)
+        guard let fileContents = fileContents(with: fileReference) else {
+            return nil
         }
 
         let assetReference = RenderReferenceIdentifier(fileReference.path)
@@ -74,6 +76,21 @@ public struct RenderNodeTranslator: SemanticVisitor {
         return assetReference
     }
 
+    private func fileContents(with fileReference: ResourceReference) -> String? {
+        // Check if the file is a local asset that can be read directly from the context
+        if let fileData = try? context.resource(with: fileReference) {
+            return String(data: fileData, encoding: .utf8)
+        }
+        // Check if the file needs to be resolved to read its content
+        else if let asset = context.resolveAsset(named: fileReference.path, in: identifier) {
+            return try? String(contentsOf: asset.data(bestMatching: DataTraitCollection()).url, encoding: .utf8)
+        }
+        // Couldn't find the file reference's content
+        else {
+            return nil
+        }
+    }
+
     public mutating func visitSteps(_ steps: Steps) -> RenderTree? {
         let stepsContent = steps.content.flatMap { child -> [RenderBlockContent] in
             return visit(child) as! [RenderBlockContent]
@@ -107,7 +124,7 @@ public struct RenderNodeTranslator: SemanticVisitor {
             stepsContent = []
         }
 
-        let highlightsPerFile = LineHighlighter(context: context, tutorialSection: tutorialSection).highlights
+        let highlightsPerFile = LineHighlighter(context: context, tutorialSection: tutorialSection, tutorialReference: identifier).highlights
 
         // Add the highlights to the file references.
         for result in highlightsPerFile {

Sources/SwiftDocC/Model/Rendering/Tutorial/LineHighlighter.swift

@@ -90,23 +90,38 @@ public struct LineHighlighter {
     /// The ``TutorialSection`` whose ``Steps`` will be analyzed for their code highlights.
     private let tutorialSection: TutorialSection
 
-    init(context: DocumentationContext, tutorialSection: TutorialSection) {
+    /// The topic reference of the tutorial whose section will be analyzed for their code highlights.
+    private let tutorialReference: ResolvedTopicReference
+
+    init(context: DocumentationContext, tutorialSection: TutorialSection, tutorialReference: ResolvedTopicReference) {
         self.context = context
         self.tutorialSection = tutorialSection
+        self.tutorialReference = tutorialReference
     }
 
     /// The lines in the `resource` file.
-    private func lines(of resource: ResourceReference) throws -> [String] {
-        let data = try context.resource(with: ResourceReference(bundleIdentifier: resource.bundleIdentifier, path: resource.path))
-        return String(data: data, encoding: .utf8)?.splitByNewlines ?? []
+    private func lines(of resource: ResourceReference) -> [String]? {
+        let fileContent: String?
+        // Check if the file is a local asset that can be read directly from the context
+        if let fileData = try? context.resource(with: resource) {
+            fileContent = String(data: fileData, encoding: .utf8)
+        }
+        // Check if the file needs to be resolved to read its content
+        else if let asset = context.resolveAsset(named: resource.path, in: tutorialReference) {
+            fileContent = try? String(contentsOf: asset.data(bestMatching: DataTraitCollection()).url, encoding: .utf8)
+        }
+        // Couldn't find the file reference's content
+        else {
+            fileContent = nil
+        }
+        return fileContent?.splitByNewlines
     }
 
     /// Returns the line highlights between two files.
     private func lineHighlights(old: ResourceReference, new: ResourceReference) -> Result {
         // Retrieve the contents of the current file and the file we're comparing against.
-        guard let oldLines = try? lines(of: old),
-            let newLines = try? lines(of: new) else {
-                return Result(file: new, highlights: [])
+        guard let oldLines = lines(of: old), let newLines = lines(of: new) else {
+            return Result(file: new, highlights: [])
         }
 
         let diff = newLines.difference(from: oldLines)

Tests/SwiftDocCTests/DocumentationService/ConvertService/ConvertServiceTests.swift

@@ -12,6 +12,7 @@ import XCTest
 import Foundation
 @testable import SwiftDocC
 import SymbolKit
+import SwiftDocCTestUtilities
 
 class ConvertServiceTests: XCTestCase {
     private let testBundleInfo = DocumentationBundle.Info(
@@ -835,6 +836,177 @@ class ConvertServiceTests: XCTestCase {
         }
     }
 
+    func testConvertTutorialWithCode() throws {
+        let tutorialContent = """
+        @Tutorial(time: 99) {
+            @Intro(title: "Tutorial Title") {
+                Tutorial intro.
+            }
+            @Section(title: "Section title") {
+                This section has one step with a code file reference.
+                
+                @Steps {
+                    @Step {
+                        Start with this
+                        
+                        @Code(name: "Something.swift", file: before.swift)
+                    }
+        
+                    @Step {
+                        Add this
+                        
+                        @Code(name: "Something.swift", file: after.swift)
+                    }
+                }
+            }
+        }
+        """
+
+        let tempURL = try createTempFolder(content: [
+            Folder(name: "TutorialWithCodeTest.docc", content: [
+                TextFile(name: "Something.tutorial", utf8Content: tutorialContent),
+
+                TextFile(name: "before.swift", utf8Content: """
+                    // This is an example swift file
+                    """),
+                TextFile(name: "after.swift", utf8Content: """
+                    // This is an example swift file
+                    let something = 0
+                    """),
+            ])
+        ])
+        let catalog = tempURL.appendingPathComponent("TutorialWithCodeTest.docc")
+
+        let request = ConvertRequest(
+            bundleInfo: testBundleInfo,
+            externalIDsToConvert: nil,
+            documentPathsToConvert: nil,
+            bundleLocation: nil,
+            symbolGraphs: [],
+            knownDisambiguatedSymbolPathComponents: nil,
+            markupFiles: [],
+            tutorialFiles: [tutorialContent.data(using: .utf8)!],
+            miscResourceURLs: []
+        )
+
+        let server = DocumentationServer()
+
+        let mockLinkResolvingService = LinkResolvingService { message in
+            XCTAssertEqual(message.type, "resolve-reference")
+            XCTAssert(message.identifier.hasPrefix("SwiftDocC"))
+            do {
+                let payload = try XCTUnwrap(message.payload)
+                let request = try JSONDecoder()
+                    .decode(
+                        ConvertRequestContextWrapper<OutOfProcessReferenceResolver.Request>.self,
+                        from: payload
+                    )
+
+                XCTAssertEqual(request.convertRequestIdentifier, "test-identifier")
+
+                switch request.payload {
+                case .topic(let url):
+                    XCTFail("Unexpected topic request: \(url.absoluteString.singleQuoted)")
+                    // Fail to resolve every topic
+                    return DocumentationServer.Message(
+                        type: "resolve-reference-response",
+                        payload: try JSONEncoder().encode(
+                            OutOfProcessReferenceResolver.Response.errorMessage("Unexpected topic request")
+                        )
+                    )
+
+                case .symbol(let preciseIdentifier):
+                    XCTFail("Unexpected symbol request: \(preciseIdentifier)")
+                    // Fail to resolve every symbol
+                    return DocumentationServer.Message(
+                        type: "resolve-reference-response",
+                        payload: try JSONEncoder().encode(
+                            OutOfProcessReferenceResolver.Response.errorMessage("Unexpected symbol request")
+                        )
+                    )
+
+                case .asset(let assetReference):
+                    print(assetReference)
+                    switch (assetReference.assetName, assetReference.bundleIdentifier) {
+                    case (let assetName, "identifier") where ["before.swift", "after.swift"].contains(assetName):
+                        var asset = DataAsset()
+                        asset.register(
+                            catalog.appendingPathComponent(assetName),
+                            with: DataTraitCollection()
+                        )
+
+                        return DocumentationServer.Message(
+                            type: "resolve-reference-response",
+                            payload: try JSONEncoder().encode(
+                                OutOfProcessReferenceResolver.Response
+                                    .asset(asset)
+                            )
+                        )
+
+                    default:
+                        XCTFail("Unexpected asset request: \(assetReference.assetName)")
+                        // Fail to resolve all other assets
+                        return DocumentationServer.Message(
+                            type: "resolve-reference-response",
+                            payload: try JSONEncoder().encode(
+                                OutOfProcessReferenceResolver.Response.errorMessage("Unexpected topic request")
+                            )
+                        )
+                    }
+                }
+            } catch {
+                XCTFail(error.localizedDescription)
+                return nil
+            }
+        }
+
+        server.register(service: mockLinkResolvingService)
+
+        try processAndAssert(request: request, linkResolvingServer: server) { message in
+            XCTAssertEqual(message.type, "convert-response")
+            XCTAssertEqual(message.identifier, "test-identifier-response")
+
+            let response = try JSONDecoder().decode(
+                ConvertResponse.self, from: XCTUnwrap(message.payload)
+            )
+
+            XCTAssertEqual(response.renderNodes.count, 1)
+            let data = try XCTUnwrap(response.renderNodes.first)
+            let renderNode = try JSONDecoder().decode(RenderNode.self, from: data)
+
+            let beforeIdentifier = RenderReferenceIdentifier("before.swift")
+            let afterIdentifier = RenderReferenceIdentifier("after.swift")
+
+            XCTAssertEqual(
+                renderNode.references["before.swift"] as? FileReference,
+                FileReference(identifier: beforeIdentifier, fileName: "Something.swift", fileType: "swift", syntax: "swift", content: [
+                    "// This is an example swift file",
+                ], highlights: [])
+            )
+            XCTAssertEqual(
+                renderNode.references["after.swift"] as? FileReference,
+                FileReference(identifier: afterIdentifier, fileName: "Something.swift", fileType: "swift", syntax: "swift", content: [
+                    "// This is an example swift file",
+                    "let something = 0",
+                ], highlights: [.init(line: 2)])
+            )
+
+            let stepsSection = try XCTUnwrap(renderNode.sections.compactMap { $0 as? TutorialSectionsRenderSection }.first?.tasks.first?.stepsSection)
+            XCTAssertEqual(stepsSection.count, 2)
+            if case .step(let step) = stepsSection.first {
+                XCTAssertEqual(step.code, beforeIdentifier)
+            } else {
+                XCTFail("Unexpected kind of step")
+            }
+
+            if case .step(let step) = stepsSection.last {
+                XCTAssertEqual(step.code, afterIdentifier)
+            } else {
+                XCTFail("Unexpected kind of step")
+            }
+        }
+    }
+
     func testConvertArticleWithImageReferencesAndDetailedGridLinks() throws {
         let articleData = try XCTUnwrap("""
             # First article

Tests/SwiftDocCTests/Model/LineHighlighterTests.swift

@@ -68,7 +68,7 @@ class LineHighlighterTests: XCTestCase {
         let tutorialReference = ResolvedTopicReference(bundleIdentifier: bundle.identifier, path: "/tutorials/Line-Highlighter-Tests/Tutorial", fragment: nil, sourceLanguage: .swift)
         let tutorial = try context.entity(with: tutorialReference).semantic as! Tutorial
         let section = tutorial.sections.first!
-        return LineHighlighter(context: context, tutorialSection: section).highlights
+        return LineHighlighter(context: context, tutorialSection: section, tutorialReference: tutorialReference).highlights
     }
 
     func testNoSteps() throws {