ExperimentalScanner: Catch DownloadException when creating archives

The `ExperimentalScanner` downloads missing archives for packages and, whenever
the `Downloader` fails, throws a `DownloadException` which stops the scanner
execution. This commit moves the downloader call in a runCatching block
and creates an `OrtIssue` on failure.

Signed-off-by: Onur Demirci <onur.demirci@bosch.io>

scanner/src/main/kotlin/experimental/ExperimentalScanner.kt

@@ -153,7 +153,7 @@ class ExperimentalScanner(
         runProvenanceScanners(controller, context)
         runPathScanners(controller, context)
 
-        createMissingArchives(controller.getNestedProvenancesByPackage())
+        createMissingArchives(controller)
 
         val results = controller.getNestedScanResultsByPackage().entries.associateTo(sortedMapOf()) {
             it.key.id to it.value.merge()
@@ -548,7 +548,7 @@ class ExperimentalScanner(
         }
     }
 
-    private fun createMissingArchives(nestedProvenances: Map<Package, NestedProvenance>) {
+    private fun createMissingArchives(controller: ScanController) {
         // TODO: The archives are currently created in a way compatible with the existing implementation in the
         //       PathScanner. This allows to keep using existing file archives without changing the logic used to
         //       access those archives in the reporter. To achieve this nested provenances are downloaded recursively,
@@ -561,17 +561,29 @@ class ExperimentalScanner(
             return
         }
 
-        val provenancesWithMissingArchives = nestedProvenances.filterNot { (_, nestedProvenance) ->
-            archiver.hasArchive(nestedProvenance.root)
-        }
+        val provenancesWithMissingArchives = controller.getNestedProvenancesByPackage()
+            .filterNot { (_, nestedProvenance) -> archiver.hasArchive(nestedProvenance.root) }
 
         logger.info { "Creating file archives for ${provenancesWithMissingArchives.size} package(s)." }
 
         val duration = measureTime {
-            provenancesWithMissingArchives.forEach { (_, nestedProvenance) ->
-                val dir = downloadRecursively(nestedProvenance)
-                archiver.archive(dir, nestedProvenance.root)
-                dir.safeDeleteRecursively(force = true)
+            provenancesWithMissingArchives.forEach { (pkg, nestedProvenance) ->
+                runCatching {
+                    downloadRecursively(nestedProvenance)
+                }.onSuccess { dir ->
+                    archiver.archive(dir, nestedProvenance.root)
+                    dir.safeDeleteRecursively(force = true)
+                }.onFailure {
+                    controller.addIssue(
+                        pkg.id,
+                        OrtIssue(
+                            source = "Downloader",
+                            message = "Could not create file archive for " +
+                                    "'${pkg.id.toCoordinates()}': ${it.collectMessages()}",
+                            severity = Severity.ERROR
+                        )
+                    )
+                }
             }
         }
 

scanner/src/main/kotlin/experimental/ScanController.kt

@@ -58,6 +58,11 @@ class ScanController(
      */
     private val provenanceResolutionIssues = mutableMapOf<Identifier, MutableList<OrtIssue>>()
 
+    /**
+     * TODO: Docs
+     */
+    private val issues = mutableMapOf<Identifier, MutableList<OrtIssue>>()
+
     /**
      * A map of [KnownProvenance]s to their resolved [NestedProvenance]s.
      */
@@ -84,6 +89,10 @@ class ScanController(
         provenanceResolutionIssues.getOrPut(id) { mutableListOf() } += issue
     }
 
+    fun addIssue(id: Identifier, issue: OrtIssue) {
+        issues.getOrPut(id) { mutableListOf() } += issue
+    }
+
     /**
      * Add an entry to [packageProvenances] and [packageProvenancesWithoutVcsPath], overwriting any existing values.
      */
@@ -158,7 +167,7 @@ class ScanController(
      * Get the [NestedProvenanceScanResult] for the provided [id].
      */
     fun getNestedScanResult(id: Identifier): NestedProvenanceScanResult =
-        buildNestedProvenanceScanResult(packageProvenancesWithoutVcsPath.getValue(id))
+        buildNestedProvenanceScanResult(packageProvenancesWithoutVcsPath.getValue(id), emptyList())
 
     /**
      * Get the [NestedProvenanceScanResult] for each [Package], filtered by the VCS path for each package and the
@@ -167,7 +176,8 @@ class ScanController(
     fun getNestedScanResultsByPackage(): Map<Package, NestedProvenanceScanResult> =
         // TODO: Return map containing all packages with issues for packages that could not be completely scanned.
         packageProvenancesWithoutVcsPath.entries.associate { (id, provenance) ->
-            packages.first { it.id == id } to buildNestedProvenanceScanResult(provenance)
+            val issues = issues[id].orEmpty()
+            packages.first { it.id == id } to buildNestedProvenanceScanResult(provenance, issues)
         }.filterByVcsPath().filterByIgnorePatterns()
 
     /**
@@ -276,10 +286,13 @@ class ScanController(
     fun hasScanResult(scanner: ScannerWrapper, provenance: Provenance) =
         scanResults[scanner]?.get(provenance)?.isNotEmpty() == true
 
-    private fun buildNestedProvenanceScanResult(root: KnownProvenance): NestedProvenanceScanResult {
+    private fun buildNestedProvenanceScanResult(
+        root: KnownProvenance,
+        issues: List<OrtIssue>
+    ): NestedProvenanceScanResult {
         val nestedProvenance = nestedProvenances.getValue(root)
         val scanResults = nestedProvenance.getProvenances().associateWith { provenance ->
-            getScanResults(provenance)
+            getScanResults(provenance).map { it.copy(summary = it.summary.copy(issues = it.summary.issues + issues)) }
         }
 
         return NestedProvenanceScanResult(nestedProvenance, scanResults)