More Cargo improvements

plugins/package-managers/cargo/src/main/kotlin/Cargo.kt

@@ -45,14 +45,13 @@ import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
 import org.ossreviewtoolkit.model.config.RepositoryConfiguration
 import org.ossreviewtoolkit.model.orEmpty
 import org.ossreviewtoolkit.utils.common.CommandLineTool
-import org.ossreviewtoolkit.utils.common.splitOnWhitespace
 import org.ossreviewtoolkit.utils.common.unquote
 import org.ossreviewtoolkit.utils.common.withoutPrefix
 import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor
-import org.ossreviewtoolkit.utils.ort.ProcessedDeclaredLicense
 import org.ossreviewtoolkit.utils.spdx.SpdxConstants
 import org.ossreviewtoolkit.utils.spdx.SpdxOperator
 
+private const val DEFAULT_KIND_NAME = "normal"
 private const val DEV_KIND_NAME = "dev"
 private const val BUILD_KIND_NAME = "build"
 
@@ -102,57 +101,30 @@ class Cargo(
         }
 
         val contents = lockfile.reader().use { toml.decodeFromNativeReader<CargoLockfile>(it) }
+
+        if (contents.version == null) {
+            val checksumMetadata = contents.metadata.mapNotNull { (k, v) ->
+                // Lockfile version 1 uses strings like:
+                // "checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)"
+                k.unquote().withoutPrefix("checksum ")?.let { it to v }
+            }.toMap()
+
+            if (checksumMetadata.isNotEmpty()) return checksumMetadata
+        }
+
         return when (contents.version) {
-            3 -> {
+            null, 2, 3 -> {
                 contents.packages.mapNotNull { pkg ->
                     pkg.checksum?.let { checksum ->
+                        // Use the same key format as for version 1, see above.
                         val key = "${pkg.name} ${pkg.version} (${pkg.source})"
                         key to checksum
                     }
-                }
-            }
-
-            else -> {
-                contents.metadata.mapNotNull { (k, v) ->
-                    k.unquote().withoutPrefix("checksum ")?.let { it to v }
-                }
+                }.toMap()
             }
-        }.toMap()
-    }
 
-    /**
-     * Check if a package is a project. All path dependencies inside the analyzer root are treated as project
-     * dependencies.
-     */
-    private fun isProjectDependency(id: String) =
-        PATH_DEPENDENCY_REGEX.matchEntire(id)?.groups?.get(1)?.let { match ->
-            val packageDir = File(match.value)
-            packageDir.startsWith(analysisRoot)
-        } == true
-
-    private fun buildDependencyTree(
-        name: String,
-        version: String,
-        packages: Map<String, Package>,
-        metadata: CargoMetadata
-    ): PackageReference {
-        val node = metadata.packages.single { it.name == name && it.version == version }
-
-        val dependencies = node.dependencies.filter {
-            // Filter dev and build dependencies, because they are not transitive.
-            it.kind != DEV_KIND_NAME && it.kind != BUILD_KIND_NAME
-        }.mapNotNullTo(mutableSetOf()) {
-            // TODO: Handle renamed dependencies here, see:
-            //       https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#renaming-dependencies-in-cargotoml
-            getResolvedVersion(name, version, it.name, metadata)?.let { dependencyVersion ->
-                buildDependencyTree(it.name, dependencyVersion, packages, metadata)
-            }
+            else -> throw IllegalArgumentException("Unsupported lockfile version ${contents.version}.")
         }
-
-        val pkg = packages.getValue(node.id)
-        val linkage = if (isProjectDependency(node.id)) PackageLinkage.PROJECT_STATIC else PackageLinkage.STATIC
-
-        return pkg.toReference(linkage, dependencies)
     }
 
     override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> {
@@ -164,146 +136,133 @@ class Cargo(
             "Virtual workspaces are not supported."
         }
 
-        val projectNode = metadata.packages.single { it.id == projectId }
-        val groupedDependencies = projectNode.dependencies.groupBy { it.kind.orEmpty() }
+        val projectNode = metadata.resolve.nodes.single { it.id == projectId }
+        val depNodesByKind = mutableMapOf<String, MutableList<CargoMetadata.Node>>()
+        projectNode.deps.forEach { dep ->
+            val depNode = metadata.resolve.nodes.single { it.id == dep.pkg }
 
-        val hashes = readHashes(resolveLockfile(metadata))
-        val packages = metadata.packages.associateBy(
-            { it.id },
-            { parsePackage(it, hashes) }
-        )
-
-        fun getTransitiveDependencies(directDependencies: List<CargoMetadata.Dependency>?, scope: String): Scope? {
-            if (directDependencies == null) return null
+            dep.depKinds.forEach { depKind ->
+                depNodesByKind.getOrPut(depKind.kind ?: DEFAULT_KIND_NAME) { mutableListOf() } += depNode
+            }
+        }
 
-            val transitiveDependencies = directDependencies
-                .mapNotNull { dependency ->
-                    val version = getResolvedVersion(projectNode.name, projectNode.version, dependency.name, metadata)
-                    version?.let { Pair(dependency.name, it) }
-                }
-                .mapTo(mutableSetOf()) {
-                    buildDependencyTree(name = it.first, version = it.second, packages = packages, metadata = metadata)
+        val packageById = metadata.packages.associateBy { it.id }
+
+        fun Collection<CargoMetadata.Node>.toPackageReferences(): Set<PackageReference> =
+            mapNotNullTo(mutableSetOf()) { node ->
+                // TODO: Handle renamed dependencies here, see:
+                //       https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#renaming-dependencies-in-cargotoml
+                val dependencyNodes = node.deps.filter { dep ->
+                    // Only normal dependencies are transitive.
+                    dep.depKinds.any { it.kind == null }
+                }.map { dep ->
+                    metadata.resolve.nodes.single { it.id == dep.pkg }
                 }
 
-            return Scope(scope, transitiveDependencies)
-        }
+                val pkg = packageById.getValue(node.id)
+                PackageReference(
+                    id = Identifier("Crate", "", pkg.name, pkg.version),
+                    linkage = if (pkg.isProject()) PackageLinkage.PROJECT_STATIC else PackageLinkage.STATIC,
+                    dependencies = dependencyNodes.toPackageReferences()
+                )
+            }
 
         val scopes = setOfNotNull(
-            getTransitiveDependencies(groupedDependencies[""], "dependencies"),
-            getTransitiveDependencies(groupedDependencies[DEV_KIND_NAME], "dev-dependencies"),
-            getTransitiveDependencies(groupedDependencies[BUILD_KIND_NAME], "build-dependencies")
+            depNodesByKind[DEFAULT_KIND_NAME]?.let { Scope("dependencies", it.toPackageReferences()) },
+            depNodesByKind[DEV_KIND_NAME]?.let { Scope("dev-dependencies", it.toPackageReferences()) },
+            depNodesByKind[BUILD_KIND_NAME]?.let { Scope("build-dependencies", it.toPackageReferences()) }
         )
 
-        val projectPkg = packages.getValue(projectId).let { it.copy(id = it.id.copy(type = managerName)) }
+        val hashes = readHashes(resolveLockfile(metadata))
+        val projectPkg = packageById.getValue(projectId).let { cargoPkg ->
+            cargoPkg.toPackage(hashes).let { it.copy(id = it.id.copy(type = managerName)) }
+        }
 
         val project = Project(
             id = projectPkg.id,
             definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path,
             authors = projectPkg.authors,
             declaredLicenses = projectPkg.declaredLicenses,
-            declaredLicensesProcessed = processDeclaredLicenses(projectPkg.declaredLicenses),
+            declaredLicensesProcessed = projectPkg.declaredLicensesProcessed,
             vcs = projectPkg.vcs,
             vcsProcessed = processProjectVcs(workingDir, projectPkg.vcs, projectPkg.homepageUrl),
             homepageUrl = projectPkg.homepageUrl,
             scopeDependencies = scopes
         )
 
-        val nonProjectPackages = packages
-            .filterNot { isProjectDependency(it.key) }
-            .mapTo(mutableSetOf()) { it.value }
+        val nonProjectPackages = packageById.values.mapNotNullTo(mutableSetOf()) { cargoPkg ->
+            cargoPkg.takeUnless { it.isProject() }?.toPackage(hashes)
+        }
 
         return listOf(ProjectAnalyzerResult(project, nonProjectPackages))
     }
 }
 
-private val PATH_DEPENDENCY_REGEX = Regex("""^.*\(path\+file://(.*)\)$""")
+private fun CargoMetadata.Package.isProject() = source == null
 
-private fun parseDeclaredLicenses(pkg: CargoMetadata.Package): Set<String> {
-    val declaredLicenses = pkg.license.orEmpty().split('/')
-        .map { it.trim() }
-        .filterTo(mutableSetOf()) { it.isNotEmpty() }
+private fun CargoMetadata.Package.toPackage(hashes: Map<String, String>): Package {
+    val declaredLicenses = parseDeclaredLicenses()
 
-    // Cargo allows declaring non-SPDX licenses only by referencing a license file. If a license file is specified, add
-    // an unknown declared license to indicate that there is a declared license, but we cannot know which it is at this
-    // point.
-    // See: https://doc.rust-lang.org/cargo/reference/manifest.html#the-license-and-license-file-fields
-    if (pkg.licenseFile.orEmpty().isNotBlank()) {
-        declaredLicenses += SpdxConstants.NOASSERTION
-    }
-
-    return declaredLicenses
-}
-
-private fun processDeclaredLicenses(licenses: Set<String>): ProcessedDeclaredLicense =
     // While the previously used "/" was not explicit about the intended license operator, the community consensus
     // seems to be that an existing "/" should be interpreted as "OR", see e.g. the discussions at
     // https://github.com/rust-lang/cargo/issues/2039
     // https://github.com/rust-lang/cargo/pull/4920
-    DeclaredLicenseProcessor.process(licenses, operator = SpdxOperator.OR)
-
-private fun parsePackage(pkg: CargoMetadata.Package, hashes: Map<String, String>): Package {
-    val declaredLicenses = parseDeclaredLicenses(pkg)
-    val declaredLicensesProcessed = processDeclaredLicenses(declaredLicenses)
+    val declaredLicensesProcessed = DeclaredLicenseProcessor.process(declaredLicenses, operator = SpdxOperator.OR)
 
     return Package(
         id = Identifier(
             type = "Crate",
             // Note that Rust / Cargo do not support package namespaces, see:
             // https://samsieber.tech/posts/2020/09/registry-structure-influence/
             namespace = "",
-            name = pkg.name,
-            version = pkg.version
+            name = name,
+            version = version
         ),
-        authors = pkg.authors.mapNotNullTo(mutableSetOf()) { parseAuthorString(it) },
+        authors = authors.mapNotNullTo(mutableSetOf()) { parseAuthorString(it) },
         declaredLicenses = declaredLicenses,
         declaredLicensesProcessed = declaredLicensesProcessed,
-        description = pkg.description.orEmpty(),
+        description = description.orEmpty(),
         binaryArtifact = RemoteArtifact.EMPTY,
-        sourceArtifact = parseSourceArtifact(pkg, hashes).orEmpty(),
-        homepageUrl = pkg.homepage.orEmpty(),
-        vcs = VcsHost.parseUrl(pkg.repository.orEmpty())
+        sourceArtifact = parseSourceArtifact(hashes).orEmpty(),
+        homepageUrl = homepage.orEmpty(),
+        vcs = VcsHost.parseUrl(repository.orEmpty())
     )
 }
 
+private fun CargoMetadata.Package.parseDeclaredLicenses(): Set<String> {
+    val declaredLicenses = license.orEmpty().split('/')
+        .map { it.trim() }
+        .filterTo(mutableSetOf()) { it.isNotEmpty() }
+
+    // Cargo allows declaring non-SPDX licenses only by referencing a license file. If a license file is specified, add
+    // an unknown declared license to indicate that there is a declared license, but we cannot know which it is at this
+    // point.
+    // See: https://doc.rust-lang.org/cargo/reference/manifest.html#the-license-and-license-file-fields
+    if (licenseFile.orEmpty().isNotBlank()) {
+        declaredLicenses += SpdxConstants.NOASSERTION
+    }
+
+    return declaredLicenses
+}
+
 // Match source dependencies that directly reference git repositories. The specified tag or branch
 // name is ignored (i.e. not captured) in favor of the actual commit hash that they currently refer
 // to.
 // See https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#specifying-dependencies-from-git-repositories
 // for the specification for this kind of dependency.
 private val GIT_DEPENDENCY_REGEX = Regex("git\\+(https://.*)\\?(?:rev|tag|branch)=.+#([0-9a-zA-Z]+)")
 
-private fun parseSourceArtifact(pkg: CargoMetadata.Package, hashes: Map<String, String>): RemoteArtifact? {
-    val source = pkg.source ?: return null
+private fun CargoMetadata.Package.parseSourceArtifact(hashes: Map<String, String>): RemoteArtifact? {
+    val source = source ?: return null
 
     if (source == "registry+https://github.com/rust-lang/crates.io-index") {
-        val url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"
-        val hash = Hash.create(hashes[pkg.id].orEmpty())
+        val url = "https://crates.io/api/v1/crates/$name/$version/download"
+        val key = "$name $version ($source)"
+        val hash = Hash.create(hashes[key].orEmpty())
         return RemoteArtifact(url, hash)
     }
 
     val match = GIT_DEPENDENCY_REGEX.matchEntire(source) ?: return null
     val (url, hash) = match.destructured
     return RemoteArtifact(url, Hash.create(hash))
 }
-
-private fun getResolvedVersion(
-    parentName: String,
-    parentVersion: String,
-    dependencyName: String,
-    metadata: CargoMetadata
-): String? {
-    val node = metadata.resolve.nodes.single { it.id.startsWith("$parentName $parentVersion") }
-
-    // This is empty if the dependency is optional and the feature was not enabled. In that case the version was not
-    // resolved and the dependency should not appear in the dependency tree. An example for a dependency string is
-    // "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", for more details see
-    // https://doc.rust-lang.org/cargo/commands/cargo-metadata.html.
-    node.dependencies.forEach {
-        val substrings = it.splitOnWhitespace()
-        require(substrings.size > 1) { "Unexpected format while parsing dependency '$it'." }
-
-        if (substrings[0] == dependencyName) return substrings[1]
-    }
-
-    return null
-}

plugins/package-managers/cargo/src/main/kotlin/CargoLockfile.kt

@@ -31,7 +31,7 @@ internal val toml = Toml { ignoreUnknownKeys = true }
  */
 @Serializable
 internal data class CargoLockfile(
-    val version: Int = 1,
+    val version: Int? = null,
 
     @SerialName("package")
     val packages: List<Package>,

plugins/package-managers/cargo/src/main/kotlin/CargoMetadata.kt

@@ -93,6 +93,30 @@ internal data class CargoMetadata(
         /** The Package ID of this node. */
         val id: String,
         /** The dependencies of this package, an array of Package IDs. */
-        val dependencies: List<String>
+        val dependencies: List<String>,
+
+        /**
+         * The dependencies of this package. This is an alternative to "dependencies" which contains additional
+         * information. In particular, this handles renamed dependencies.
+         */
+        val deps: List<Dep>
+    )
+
+    @Serializable
+    data class Dep(
+        /** The name of the dependency's library target. If this is a renamed dependency, this is the new name. */
+        val name: String,
+        /** The Package ID of the dependency. */
+        val pkg: String,
+        /** Array of dependency kinds. Added in Cargo 1.40. */
+        val depKinds: List<DepKind>
+    )
+
+    @Serializable
+    data class DepKind(
+        /** The dependency kind. "dev", "build", or null for a normal dependency. */
+        val kind: String? = null,
+        /** The target platform for the dependency. null if not a target dependency. */
+        val target: String? = null
     )
 }