From 4ba16b693cb89f62deef79791ff4f111e529f7a6 Mon Sep 17 00:00:00 2001 From: Onur Demirci Date: Thu, 7 Jul 2022 18:44:49 +0200 Subject: [PATCH] ExperimentalScanner: Catch `DownloadException` when creating archives The `ExperimentalScanner` downloads missing archives for packages and, whenever the `Downloader` fails, throws a `DownloadException` which stops the scanner execution. This commit moves the downloader call in a runCatching block and creates an `OrtIssue` on failure. Signed-off-by: Onur Demirci --- .../experimental/ExperimentalScanner.kt | 30 +++++++++++++------ .../kotlin/experimental/ScanController.kt | 21 ++++++++++--- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/scanner/src/main/kotlin/experimental/ExperimentalScanner.kt b/scanner/src/main/kotlin/experimental/ExperimentalScanner.kt index 4193c49baf145..c6a7f528fe26d 100644 --- a/scanner/src/main/kotlin/experimental/ExperimentalScanner.kt +++ b/scanner/src/main/kotlin/experimental/ExperimentalScanner.kt @@ -153,7 +153,7 @@ class ExperimentalScanner( runProvenanceScanners(controller, context) runPathScanners(controller, context) - createMissingArchives(controller.getNestedProvenancesByPackage()) + createMissingArchives(controller) val results = controller.getNestedScanResultsByPackage().entries.associateTo(sortedMapOf()) { it.key.id to it.value.merge() @@ -548,7 +548,7 @@ class ExperimentalScanner( } } - private fun createMissingArchives(nestedProvenances: Map) { + private fun createMissingArchives(controller: ScanController) { // TODO: The archives are currently created in a way compatible with the existing implementation in the // PathScanner. This allows to keep using existing file archives without changing the logic used to // access those archives in the reporter. To achieve this nested provenances are downloaded recursively, @@ -561,17 +561,29 @@ class ExperimentalScanner( return } - val provenancesWithMissingArchives = nestedProvenances.filterNot { (_, nestedProvenance) -> - archiver.hasArchive(nestedProvenance.root) - } + val provenancesWithMissingArchives = controller.getNestedProvenancesByPackage() + .filterNot { (_, nestedProvenance) -> archiver.hasArchive(nestedProvenance.root) } logger.info { "Creating file archives for ${provenancesWithMissingArchives.size} package(s)." } val duration = measureTime { - provenancesWithMissingArchives.forEach { (_, nestedProvenance) -> - val dir = downloadRecursively(nestedProvenance) - archiver.archive(dir, nestedProvenance.root) - dir.safeDeleteRecursively(force = true) + provenancesWithMissingArchives.forEach { (pkg, nestedProvenance) -> + runCatching { + downloadRecursively(nestedProvenance) + }.onSuccess { dir -> + archiver.archive(dir, nestedProvenance.root) + dir.safeDeleteRecursively(force = true) + }.onFailure { + controller.addIssue( + pkg.id, + OrtIssue( + source = "Downloader", + message = "Could not create file archive for " + + "'${pkg.id.toCoordinates()}': ${it.collectMessages()}", + severity = Severity.ERROR + ) + ) + } } } diff --git a/scanner/src/main/kotlin/experimental/ScanController.kt b/scanner/src/main/kotlin/experimental/ScanController.kt index 3d11e80685e0d..487ba841c58d3 100644 --- a/scanner/src/main/kotlin/experimental/ScanController.kt +++ b/scanner/src/main/kotlin/experimental/ScanController.kt @@ -58,6 +58,11 @@ class ScanController( */ private val provenanceResolutionIssues = mutableMapOf>() + /** + * TODO: Docs + */ + private val issues = mutableMapOf>() + /** * A map of [KnownProvenance]s to their resolved [NestedProvenance]s. */ @@ -84,6 +89,10 @@ class ScanController( provenanceResolutionIssues.getOrPut(id) { mutableListOf() } += issue } + fun addIssue(id: Identifier, issue: OrtIssue) { + issues.getOrPut(id) { mutableListOf() } += issue + } + /** * Add an entry to [packageProvenances] and [packageProvenancesWithoutVcsPath], overwriting any existing values. */ @@ -158,7 +167,7 @@ class ScanController( * Get the [NestedProvenanceScanResult] for the provided [id]. */ fun getNestedScanResult(id: Identifier): NestedProvenanceScanResult = - buildNestedProvenanceScanResult(packageProvenancesWithoutVcsPath.getValue(id)) + buildNestedProvenanceScanResult(packageProvenancesWithoutVcsPath.getValue(id), emptyList()) /** * Get the [NestedProvenanceScanResult] for each [Package], filtered by the VCS path for each package and the @@ -167,7 +176,8 @@ class ScanController( fun getNestedScanResultsByPackage(): Map = // TODO: Return map containing all packages with issues for packages that could not be completely scanned. packageProvenancesWithoutVcsPath.entries.associate { (id, provenance) -> - packages.first { it.id == id } to buildNestedProvenanceScanResult(provenance) + val issues = issues[id].orEmpty() + packages.first { it.id == id } to buildNestedProvenanceScanResult(provenance, issues) }.filterByVcsPath().filterByIgnorePatterns() /** @@ -276,10 +286,13 @@ class ScanController( fun hasScanResult(scanner: ScannerWrapper, provenance: Provenance) = scanResults[scanner]?.get(provenance)?.isNotEmpty() == true - private fun buildNestedProvenanceScanResult(root: KnownProvenance): NestedProvenanceScanResult { + private fun buildNestedProvenanceScanResult( + root: KnownProvenance, + issues: List + ): NestedProvenanceScanResult { val nestedProvenance = nestedProvenances.getValue(root) val scanResults = nestedProvenance.getProvenances().associateWith { provenance -> - getScanResults(provenance) + getScanResults(provenance).map { it.copy(summary = it.summary.copy(issues = it.summary.issues + issues)) } } return NestedProvenanceScanResult(nestedProvenance, scanResults)