From 711179d7c928ae9e6b73b0b5f31202c09f2e5eec Mon Sep 17 00:00:00 2001 From: Marcel Bochtler Date: Mon, 1 Mar 2021 11:56:39 +0100 Subject: [PATCH 1/8] RepositoryConfiguration: Add data model for license choice This specifies the structure for license choices by packages in the repository configuration. Signed-off-by: Marcel Bochtler --- .../src/main/kotlin/config/LicenseChoices.kt | 46 +++++++++++++++ .../kotlin/config/RepositoryConfiguration.kt | 14 ++++- .../config/RepositoryConfigurationTest.kt | 24 ++++++++ .../src/main/kotlin/model/LicenseChoice.kt | 56 +++++++++++++++++++ 4 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 model/src/main/kotlin/config/LicenseChoices.kt create mode 100644 spdx-utils/src/main/kotlin/model/LicenseChoice.kt diff --git a/model/src/main/kotlin/config/LicenseChoices.kt b/model/src/main/kotlin/config/LicenseChoices.kt new file mode 100644 index 0000000000000..421d3d077ae04 --- /dev/null +++ b/model/src/main/kotlin/config/LicenseChoices.kt @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2021 Bosch.IO GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model.config + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonInclude + +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.spdx.model.LicenseChoice + +/** + * The license choices configured for a repository. + */ +data class LicenseChoices( + @JsonInclude(JsonInclude.Include.NON_EMPTY) + val packageLicenseChoices: List = emptyList() +) { + @JsonIgnore + fun isEmpty() = packageLicenseChoices.isEmpty() +} + +/** + * [LicenseChoice]s defined for an artifact. + */ +data class PackageLicenseChoice( + val packageId: Identifier, + @JsonInclude(JsonInclude.Include.NON_EMPTY) + val licenseChoices: List = emptyList() +) diff --git a/model/src/main/kotlin/config/RepositoryConfiguration.kt b/model/src/main/kotlin/config/RepositoryConfiguration.kt index 8841a217c6f26..dab63f580c89b 100644 --- a/model/src/main/kotlin/config/RepositoryConfiguration.kt +++ b/model/src/main/kotlin/config/RepositoryConfiguration.kt @@ -45,7 +45,13 @@ data class RepositoryConfiguration( * Defines curations for artifacts contained in this repository. */ @JsonInclude(value = JsonInclude.Include.CUSTOM, valueFilter = CurationsFilter::class) - val curations: Curations = Curations() + val curations: Curations = Curations(), + + /** + * Defines license choices within this repository. + */ + @JsonInclude(value = JsonInclude.Include.CUSTOM, valueFilter = LicenseChoiceFilter::class) + val licenseChoices: LicenseChoices = LicenseChoices() ) @Suppress("EqualsOrHashCode", "EqualsWithHashCodeExist") // The class is not supposed to be used with hashing. @@ -68,3 +74,9 @@ private class CurationsFilter { override fun equals(other: Any?): Boolean = if (other is Curations) other.licenseFindings.isEmpty() else false } + +@Suppress("EqualsOrHashCode", "EqualsWithHashCodeExist") // The class is not supposed to be used with hashing. +private class LicenseChoiceFilter { + override fun equals(other: Any?): Boolean = + other is LicenseChoices && other.isEmpty() +} diff --git a/model/src/test/kotlin/config/RepositoryConfigurationTest.kt b/model/src/test/kotlin/config/RepositoryConfigurationTest.kt index c48f2791c2a0e..4ae08b7542547 100644 --- a/model/src/test/kotlin/config/RepositoryConfigurationTest.kt +++ b/model/src/test/kotlin/config/RepositoryConfigurationTest.kt @@ -26,7 +26,9 @@ import io.kotest.matchers.collections.haveSize import io.kotest.matchers.should import io.kotest.matchers.shouldBe +import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.yamlMapper +import org.ossreviewtoolkit.spdx.toSpdx class RepositoryConfigurationTest : WordSpec({ "RepositoryConfiguration" should { @@ -67,6 +69,13 @@ class RepositoryConfigurationTest : WordSpec({ - id: "vulnerability id" reason: "INEFFECTIVE_VULNERABILITY" comment: "vulnerability comment" + license_choices: + package_license_choices: + - package_id: "Maven:com.example:lib:0.0.1" + license_choices: + - given: MPL-2.0 or EPL-1.0 + choice: MPL-2.0 + - choice: MPL-2.0 AND MIT """.trimIndent() val repositoryConfiguration = yamlMapper.readValue(configuration) @@ -110,6 +119,21 @@ class RepositoryConfigurationTest : WordSpec({ reason shouldBe VulnerabilityResolutionReason.INEFFECTIVE_VULNERABILITY comment shouldBe "vulnerability comment" } + + val packageLicenseChoices = repositoryConfiguration.licenseChoices.packageLicenseChoices + packageLicenseChoices should haveSize(1) + with(packageLicenseChoices.first()) { + packageId shouldBe Identifier("Maven:com.example:lib:0.0.1") + with(licenseChoices.first()) { + given shouldBe "MPL-2.0 or EPL-1.0".toSpdx() + choice shouldBe "MPL-2.0".toSpdx() + } + + with(licenseChoices[1]) { + given shouldBe null + choice shouldBe "MPL-2.0 AND MIT".toSpdx() + } + } } } }) diff --git a/spdx-utils/src/main/kotlin/model/LicenseChoice.kt b/spdx-utils/src/main/kotlin/model/LicenseChoice.kt new file mode 100644 index 0000000000000..ff9969c17a197 --- /dev/null +++ b/spdx-utils/src/main/kotlin/model/LicenseChoice.kt @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2021 Bosch.IO GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.spdx.model + +import org.ossreviewtoolkit.spdx.SpdxExpression + +/** + * An individual license choice. + * + * [given] is the complete license expression, or a sub-expression of the license, where [choice] is going to be applied + * on. If no [given] is supplied, the [choice] will be applied to the complete expression of the package. + * + * e.g.: with [given] as complete expression + * ``` + * -> Complete license expression: (A OR B) AND C + * given: (A OR B) AND C + * choice: A AND C + * -> result: A AND C + * ``` + * + * e.g.: with [given] as sub-expression + * ``` + * -> Complete license expression: (A OR B) AND C + * given: (A OR B) + * choice: A + * -> result: A AND C + * ``` + * + * e.g.: without [given] + * ``` + * -> Complete license expression: (A OR B) AND (C OR D) + * choice: A AND C + * -> result: A AND C + * ``` + */ +data class LicenseChoice( + val given: SpdxExpression?, + val choice: SpdxExpression, +) From cdcec0d590a2d58283a2e98e2da8dee5406374d1 Mon Sep 17 00:00:00 2001 From: Marcel Bochtler Date: Wed, 10 Mar 2021 14:34:50 +0100 Subject: [PATCH 2/8] SpdxExpression: Add functionality to apply multiple license choices Signed-off-by: Marcel Bochtler --- spdx-utils/src/main/kotlin/SpdxExpression.kt | 31 ++++++++- .../src/test/kotlin/SpdxExpressionTest.kt | 65 +++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/spdx-utils/src/main/kotlin/SpdxExpression.kt b/spdx-utils/src/main/kotlin/SpdxExpression.kt index 3400186e81972..932767b7d99cf 100644 --- a/spdx-utils/src/main/kotlin/SpdxExpression.kt +++ b/spdx-utils/src/main/kotlin/SpdxExpression.kt @@ -29,6 +29,7 @@ import org.antlr.v4.runtime.CommonTokenStream import org.ossreviewtoolkit.spdx.SpdxConstants.DOCUMENT_REF_PREFIX import org.ossreviewtoolkit.spdx.SpdxConstants.LICENSE_REF_PREFIX +import org.ossreviewtoolkit.spdx.model.LicenseChoice /** * An SPDX expression as defined by version 2.1 of the [SPDX specification, appendix IV][1]. @@ -155,6 +156,11 @@ sealed class SpdxExpression { */ fun isValidChoice(choice: SpdxExpression): Boolean = !choice.offersChoice() && choice in validChoices() + /** + * Return true if [subExpression] is a valid sub-expression of [this][SpdxExpression]. + */ + open fun isSubExpression(subExpression: SpdxExpression?): Boolean = false + /** * Return true if this expression offers a license choice. This can only be true if this expression contains the * [OR operator][SpdxOperator.OR]. @@ -177,6 +183,25 @@ sealed class SpdxExpression { return this } + /** + * Apply [licenseChoices] in the given order to [this][SpdxExpression]. + */ + fun applyChoices(licenseChoices: List): SpdxExpression { + if (validChoices().size == 1) return this + + var currentExpression = this + + licenseChoices.forEach { + if (it.given == null && currentExpression.isValidChoice(it.choice)) { + currentExpression = currentExpression.applyChoice(it.choice) + } else if (currentExpression.isSubExpression(it.given)) { + currentExpression = currentExpression.applyChoice(it.choice, it.given!!) + } + } + + return currentExpression + } + /** * Concatenate [this][SpdxExpression] and [other] using [SpdxOperator.AND]. */ @@ -267,7 +292,7 @@ class SpdxCompoundExpression( ) } - if (!isValidSubExpression(subExpression)) { + if (!isSubExpression(subExpression)) { throw InvalidSubExpressionException("$subExpression is not not a valid subExpression of $this") } @@ -293,7 +318,9 @@ class SpdxCompoundExpression( } } - private fun isValidSubExpression(subExpression: SpdxExpression): Boolean { + override fun isSubExpression(subExpression: SpdxExpression?): Boolean { + if (subExpression == null) return false + val expressionString = toString() val subExpressionString = subExpression.toString() diff --git a/spdx-utils/src/test/kotlin/SpdxExpressionTest.kt b/spdx-utils/src/test/kotlin/SpdxExpressionTest.kt index 4f2ef5482a45e..cb2d779cdced5 100644 --- a/spdx-utils/src/test/kotlin/SpdxExpressionTest.kt +++ b/spdx-utils/src/test/kotlin/SpdxExpressionTest.kt @@ -34,6 +34,7 @@ import io.kotest.matchers.shouldNotBe import org.ossreviewtoolkit.spdx.SpdxExpression.Strictness import org.ossreviewtoolkit.spdx.SpdxLicense.* import org.ossreviewtoolkit.spdx.SpdxLicenseException.* +import org.ossreviewtoolkit.spdx.model.LicenseChoice class SpdxExpressionTest : WordSpec() { private val yamlMapper = YAMLMapper() @@ -493,6 +494,70 @@ class SpdxExpressionTest : WordSpec() { } } + "applyChoices()" should { + "return the correct result if a single choice is applied" { + val expression = "a OR b OR c OR d".toSpdx() + + val choices = listOf(LicenseChoice(expression, "a".toSpdx())) + + val result = expression.applyChoices(choices) + + result shouldBe "a".toSpdx() + } + + "return the correct result if multiple simple choices are applied" { + val expression = "a OR b AND c OR d".toSpdx() + + val choices = listOf( + LicenseChoice("a OR b".toSpdx(), "a".toSpdx()), + LicenseChoice("c OR d".toSpdx(), "c".toSpdx()) + ) + + val result = expression.applyChoices(choices) + + result shouldBe "a AND c".toSpdx() + } + + "ignore invalid sub-expressions and return the correct result for valid choices" { + val expression = "a OR b OR c OR d".toSpdx() + + val choices = listOf( + LicenseChoice("a OR b".toSpdx(), "b".toSpdx()), // b OR c OR d + LicenseChoice("a OR c".toSpdx(), "a".toSpdx()) // not applied + ) + + val result = expression.applyChoices(choices) + + result shouldBe "b OR c OR d".toSpdx() + } + + "apply the second choice to the effective license after the first choice" { + val expression = "a OR b OR c OR d".toSpdx() + + val choices = listOf( + LicenseChoice("a OR b".toSpdx(), "b".toSpdx()), // b OR c OR d + LicenseChoice("b OR c".toSpdx(), "b".toSpdx()) // b OR d + ) + + val result = expression.applyChoices(choices) + + result shouldBe "b OR d".toSpdx() + } + + "apply a single choice to multiple expressions" { + val expression = "(a OR b) AND (c OR d) AND (a OR e)".toSpdx() + + val choices = listOf( + LicenseChoice("a OR b".toSpdx(), "a".toSpdx()), + LicenseChoice("a OR e".toSpdx(), "a".toSpdx()) + ) + + val result = expression.applyChoices(choices) + + result shouldBe "a AND (c OR d) AND a".toSpdx() + } + } + "equals()" should { "return true for semantically equal expressions" { "a".toSpdx() shouldBe "a".toSpdx() From e0044601e2f9bc0aee6c99d759cdbebeb0ad238b Mon Sep 17 00:00:00 2001 From: Stephanie Neubauer Date: Thu, 11 Mar 2021 12:36:06 +0100 Subject: [PATCH 3/8] ResolvedLicenseInfo: Add effective license function The effective license filters all licenses from all sources by the LicenseView with the LicenseChoices applied. Signed-off-by: Stephanie Neubauer --- model/build.gradle.kts | 2 + .../kotlin/licenses/ResolvedLicenseInfo.kt | 18 ++ .../licenses/ResolvedLicenseInfoTest.kt | 158 ++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt diff --git a/model/build.gradle.kts b/model/build.gradle.kts index 755e19b45e0e7..ced25943aeeea 100644 --- a/model/build.gradle.kts +++ b/model/build.gradle.kts @@ -24,6 +24,7 @@ val hopliteVersion: String by project val jacksonVersion: String by project val postgresEmbeddedVersion: String by project val postgresVersion: String by project +val mockkVersion: String by project val semverVersion: String by project plugins { @@ -54,4 +55,5 @@ dependencies { implementation("org.postgresql:postgresql:$postgresVersion") testImplementation("com.opentable.components:otj-pg-embedded:$postgresEmbeddedVersion") + testImplementation("io.mockk:mockk:$mockkVersion") } diff --git a/model/src/main/kotlin/licenses/ResolvedLicenseInfo.kt b/model/src/main/kotlin/licenses/ResolvedLicenseInfo.kt index 5be17985827a4..5a3f46faa5f3d 100644 --- a/model/src/main/kotlin/licenses/ResolvedLicenseInfo.kt +++ b/model/src/main/kotlin/licenses/ResolvedLicenseInfo.kt @@ -29,6 +29,7 @@ import org.ossreviewtoolkit.model.config.LicenseFindingCuration import org.ossreviewtoolkit.model.config.PathExclude import org.ossreviewtoolkit.spdx.SpdxExpression import org.ossreviewtoolkit.spdx.SpdxSingleLicenseExpression +import org.ossreviewtoolkit.spdx.model.LicenseChoice import org.ossreviewtoolkit.utils.CopyrightStatementsProcessor import org.ossreviewtoolkit.utils.DeclaredLicenseProcessor @@ -65,6 +66,23 @@ data class ResolvedLicenseInfo( ) : Iterable by licenses { operator fun get(license: SpdxSingleLicenseExpression): ResolvedLicense? = find { it.license == license } + /** + * Return the effective [SpdxExpression] of this [ResolvedLicenseInfo] based on their [licenses] filtered by the + * [licenseView] and the applied [licenseChoices]. Effective, in this context, refers to an [SpdxExpression] that + * can be used as a final license of this [ResolvedLicenseInfo]. + */ + fun effectiveLicense(licenseView: LicenseView, licenseChoices: List = emptyList()): SpdxExpression? { + val resolvedLicenseInfo = filter(licenseView, filterSources = true) + + return resolvedLicenseInfo.licenses.flatMap { it.originalExpressions.values } + .flatten() + .toSet() + .reduceOrNull(SpdxExpression::and) + ?.applyChoices(licenseChoices) + ?.validChoices() + ?.reduceOrNull(SpdxExpression::or) + } + /** * Return all copyright statements associated to this license info. Copyright findings that are excluded by * [PathExclude]s are [omitted][omitExcluded] by default. The copyrights are [processed][process] by default diff --git a/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt b/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt new file mode 100644 index 0000000000000..04998f9733221 --- /dev/null +++ b/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2021 Bosch.IO GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model.licenses + +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.shouldBe + +import io.mockk.mockk + +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.LicenseSource +import org.ossreviewtoolkit.spdx.SpdxSingleLicenseExpression +import org.ossreviewtoolkit.spdx.model.LicenseChoice +import org.ossreviewtoolkit.spdx.toSpdx + +class ResolvedLicenseInfoTest : WordSpec() { + private val mit = "MIT" + private val apache = "Apache-2.0 WITH LLVM-exception" + private val gpl = "GPL-2.0-only" + private val bsd = "0BSD" + + init { + "effectiveLicense()" should { + "apply choices for LicenseView.ALL on all resolved licenses" { + // All: (Apache-2.0 WITH LLVM-exception OR MIT) AND (MIT OR GPL-2.0-only) AND (0BSD OR GPL-2.0-only) + val choices = listOf( + LicenseChoice("$apache OR $mit".toSpdx(), mit.toSpdx()), + LicenseChoice("$mit OR $gpl".toSpdx(), mit.toSpdx()), + LicenseChoice("$bsd OR $gpl".toSpdx(), bsd.toSpdx()) + ) + + val effectiveLicense = createResolvedLicenseInfo().effectiveLicense(LicenseView.ALL, choices) + + effectiveLicense shouldBe "$mit AND $bsd".toSpdx() + } + + "apply a choice for a sub-expression only" { + // Declared: Apache-2.0 WITH LLVM-exception OR MIT OR GPL-2.0-only + val resolvedLicenseInfo = ResolvedLicenseInfo( + id = Identifier.EMPTY, + licenseInfo = mockk(), + licenses = listOf( + ResolvedLicense( + license = apache.toSpdx() as SpdxSingleLicenseExpression, + originalDeclaredLicenses = setOf("$apache OR $mit"), + originalExpressions = mapOf( + LicenseSource.DECLARED to setOf("$apache OR $mit OR $gpl".toSpdx()) + ), + locations = emptySet() + ) + ), + copyrightGarbage = emptyMap(), + unmatchedCopyrights = emptyMap() + ) + + val choices = listOf( + LicenseChoice("$apache OR $mit".toSpdx(), mit.toSpdx()) + ) + + val effectiveLicense = resolvedLicenseInfo.effectiveLicense(LicenseView.ONLY_DECLARED, choices) + + effectiveLicense shouldBe "$mit OR $gpl".toSpdx() + } + + "apply choices for LicenseView.CONCLUDED_OR_DECLARED_AND_DETECTED" { + // Concluded: 0BSD OR GPL-2.0-only + val choices = listOf( + LicenseChoice("$bsd OR $gpl".toSpdx(), bsd.toSpdx()) + ) + + val effectiveLicense = createResolvedLicenseInfo().effectiveLicense( + LicenseView.CONCLUDED_OR_DECLARED_AND_DETECTED, + choices + ) + + effectiveLicense shouldBe bsd.toSpdx() + } + + "apply choices for LicenseView.ONLY_DECLARED" { + // Declared: Apache-2.0 WITH LLVM-exception OR MIT + val choices = listOf( + LicenseChoice("$apache OR $mit".toSpdx(), mit.toSpdx()) + ) + + val effectiveLicense = createResolvedLicenseInfo().effectiveLicense( + LicenseView.ONLY_DECLARED, + choices + ) + + effectiveLicense shouldBe mit.toSpdx() + } + } + } + + private fun createResolvedLicenseInfo(): ResolvedLicenseInfo { + val resolvedLicenses = listOf( + ResolvedLicense( + license = apache.toSpdx() as SpdxSingleLicenseExpression, + originalDeclaredLicenses = setOf("$apache OR $mit"), + originalExpressions = mapOf( + LicenseSource.DECLARED to setOf("$apache OR $mit".toSpdx()) + ), + locations = emptySet() + ), + ResolvedLicense( + license = mit.toSpdx() as SpdxSingleLicenseExpression, + originalDeclaredLicenses = setOf("$apache OR $mit"), + originalExpressions = mapOf( + LicenseSource.DECLARED to setOf("$apache OR $mit".toSpdx()), + LicenseSource.DETECTED to setOf("$mit OR $gpl".toSpdx()) + ), + locations = emptySet() + ), + ResolvedLicense( + license = gpl.toSpdx() as SpdxSingleLicenseExpression, + originalDeclaredLicenses = emptySet(), + originalExpressions = mapOf( + LicenseSource.DETECTED to setOf("$mit OR $gpl".toSpdx()), + LicenseSource.CONCLUDED to setOf("$bsd OR $gpl".toSpdx()) + ), + locations = emptySet() + ), + ResolvedLicense( + license = bsd.toSpdx() as SpdxSingleLicenseExpression, + originalDeclaredLicenses = emptySet(), + originalExpressions = mapOf( + LicenseSource.CONCLUDED to setOf("$bsd OR $gpl".toSpdx()) + ), + locations = emptySet() + ) + ) + + return ResolvedLicenseInfo( + id = Identifier.EMPTY, + licenseInfo = mockk(), + licenses = resolvedLicenses, + copyrightGarbage = emptyMap(), + unmatchedCopyrights = emptyMap() + ) + } +} From 0f5de06ea4afe845c0767c6fa40b22458adb17c0 Mon Sep 17 00:00:00 2001 From: Stephanie Neubauer Date: Thu, 11 Mar 2021 15:02:10 +0100 Subject: [PATCH 4/8] ResolvedLicenseInfo: add applyChoices function for license choices For later use. Signed-off-by: Stephanie Neubauer --- .../kotlin/licenses/ResolvedLicenseInfo.kt | 9 ++++ .../licenses/LicenseInfoResolverTest.kt | 14 +----- .../licenses/ResolvedLicenseInfoTest.kt | 18 ++++++++ model/src/test/kotlin/licenses/TestUtils.kt | 43 +++++++++++++++++++ 4 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 model/src/test/kotlin/licenses/TestUtils.kt diff --git a/model/src/main/kotlin/licenses/ResolvedLicenseInfo.kt b/model/src/main/kotlin/licenses/ResolvedLicenseInfo.kt index 5a3f46faa5f3d..8ad87cafc0af8 100644 --- a/model/src/main/kotlin/licenses/ResolvedLicenseInfo.kt +++ b/model/src/main/kotlin/licenses/ResolvedLicenseInfo.kt @@ -114,6 +114,15 @@ data class ResolvedLicenseInfo( } } + /** + * Apply [licenseChoices] on the effective license of [LicenseView.ALL]. + */ + fun applyChoices(licenseChoices: List): ResolvedLicenseInfo { + val licenses = effectiveLicense(LicenseView.ALL, licenseChoices)?.decompose().orEmpty() + + return this.copy(licenses = this.licenses.filter { it.license in licenses }) + } + /** * Filter all excluded licenses and copyrights. Licenses are removed if they are only * [detected][LicenseSource.DETECTED] and all [locations][ResolvedLicense.locations] have diff --git a/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt b/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt index fa71f10be8319..8023a16bdb992 100644 --- a/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt +++ b/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt @@ -51,6 +51,7 @@ import org.ossreviewtoolkit.model.config.LicenseFindingCuration import org.ossreviewtoolkit.model.config.LicenseFindingCurationReason import org.ossreviewtoolkit.model.config.PathExclude import org.ossreviewtoolkit.model.config.PathExcludeReason +import org.ossreviewtoolkit.model.licenses.TestUtils.containLicensesExactly import org.ossreviewtoolkit.model.utils.FileArchiver import org.ossreviewtoolkit.spdx.SpdxExpression import org.ossreviewtoolkit.spdx.SpdxSingleLicenseExpression @@ -763,19 +764,6 @@ fun containLicenseExpressionsExactlyBySource( ) } -fun containLicensesExactly(vararg licenses: String): Matcher?> = - neverNullMatcher { value -> - val expected = licenses.map { SpdxExpression.parse(it) as SpdxSingleLicenseExpression }.toSet() - val actual = value.map { it.license }.toSet() - - MatcherResult( - expected == actual, - "ResolvedLicenseInfo should contain exactly licenses ${expected.show().value}, but has " + - actual.show().value, - "ResolvedLicenseInfo should not contain exactly ${expected.show().value}" - ) - } - fun containNumberOfLocationsForLicense(license: String, count: Int): Matcher = neverNullMatcher { value -> val actualCount = value[SpdxSingleLicenseExpression.parse(license)]?.locations?.size ?: 0 diff --git a/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt b/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt index 04998f9733221..92d2cfce9d1e4 100644 --- a/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt +++ b/model/src/test/kotlin/licenses/ResolvedLicenseInfoTest.kt @@ -20,12 +20,14 @@ package org.ossreviewtoolkit.model.licenses import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.should import io.kotest.matchers.shouldBe import io.mockk.mockk import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.LicenseSource +import org.ossreviewtoolkit.model.licenses.TestUtils.containLicensesExactly import org.ossreviewtoolkit.spdx.SpdxSingleLicenseExpression import org.ossreviewtoolkit.spdx.model.LicenseChoice import org.ossreviewtoolkit.spdx.toSpdx @@ -107,6 +109,22 @@ class ResolvedLicenseInfoTest : WordSpec() { effectiveLicense shouldBe mit.toSpdx() } } + + "applyChoices(licenseChoices)" should { + "apply license choices on all licenses" { + val resolvedLicenseInfo = createResolvedLicenseInfo() + + val choices = listOf( + LicenseChoice("$apache OR $mit".toSpdx(), mit.toSpdx()), + LicenseChoice("$mit OR $gpl".toSpdx(), mit.toSpdx()), + LicenseChoice("$bsd OR $gpl".toSpdx(), bsd.toSpdx()) + ) + + val filteredResolvedLicenseInfo = resolvedLicenseInfo.applyChoices(choices) + + filteredResolvedLicenseInfo.licenses should containLicensesExactly(mit, bsd) + } + } } private fun createResolvedLicenseInfo(): ResolvedLicenseInfo { diff --git a/model/src/test/kotlin/licenses/TestUtils.kt b/model/src/test/kotlin/licenses/TestUtils.kt new file mode 100644 index 0000000000000..a8b659cf4dd5e --- /dev/null +++ b/model/src/test/kotlin/licenses/TestUtils.kt @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2021 Bosch.IO GmbH. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model.licenses + +import io.kotest.assertions.show.show +import io.kotest.matchers.Matcher +import io.kotest.matchers.MatcherResult +import io.kotest.matchers.neverNullMatcher + +import org.ossreviewtoolkit.spdx.SpdxExpression +import org.ossreviewtoolkit.spdx.SpdxSingleLicenseExpression + +object TestUtils { + fun containLicensesExactly(vararg licenses: String): Matcher?> = + neverNullMatcher { value -> + val expected = licenses.map { SpdxExpression.parse(it) as SpdxSingleLicenseExpression }.toSet() + val actual = value.map { it.license }.toSet() + + MatcherResult( + expected == actual, + "ResolvedLicenseInfo should contain exactly licenses ${expected.show().value}, but has " + + actual.show().value, + "ResolvedLicenseInfo should not contain exactly ${expected.show().value}" + ) + } +} From fbd6ba75f087559fce3b979129d19a1d8e5ddc79 Mon Sep 17 00:00:00 2001 From: Stephanie Neubauer Date: Thu, 11 Mar 2021 15:06:46 +0100 Subject: [PATCH 5/8] PackageRule: Add filtering by licenseChoices to licenseRule The new filter applies the license choices from the ORT result to all licenses in the ResolvedLicenseInfo when using a licenseRule. Signed-off-by: Stephanie Neubauer --- evaluator/src/main/kotlin/PackageRule.kt | 3 ++- evaluator/src/test/kotlin/RuleSetTest.kt | 21 +++++++++++++++++++++ evaluator/src/test/kotlin/TestData.kt | 15 ++++++++++++++- model/src/main/kotlin/OrtResult.kt | 7 +++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/evaluator/src/main/kotlin/PackageRule.kt b/evaluator/src/main/kotlin/PackageRule.kt index c34b106eaf7ef..30ab16ec3cf86 100644 --- a/evaluator/src/main/kotlin/PackageRule.kt +++ b/evaluator/src/main/kotlin/PackageRule.kt @@ -135,7 +135,8 @@ open class PackageRule( * A DSL function to configure a [LicenseRule] and add it to this rule. */ fun licenseRule(name: String, licenseView: LicenseView, block: LicenseRule.() -> Unit) { - resolvedLicenseInfo.filter(licenseView, filterSources = true).forEach { resolvedLicense -> + resolvedLicenseInfo.filter(licenseView, filterSources = true) + .applyChoices(ruleSet.ortResult.getLicenseChoices(pkg.id)).forEach { resolvedLicense -> resolvedLicense.sources.forEach { licenseSource -> licenseRules += LicenseRule(name, resolvedLicense, licenseSource).apply(block) } diff --git a/evaluator/src/test/kotlin/RuleSetTest.kt b/evaluator/src/test/kotlin/RuleSetTest.kt index ea74577da6ce9..146f1fad244b4 100644 --- a/evaluator/src/test/kotlin/RuleSetTest.kt +++ b/evaluator/src/test/kotlin/RuleSetTest.kt @@ -24,6 +24,7 @@ import io.kotest.matchers.collections.haveSize import io.kotest.matchers.should import org.ossreviewtoolkit.model.licenses.LicenseView +import org.ossreviewtoolkit.spdx.toSpdx class RuleSetTest : WordSpec() { private val errorMessage = "error message" @@ -120,6 +121,26 @@ class RuleSetTest : WordSpec() { ruleSet.violations should haveSize(4) } + + "add no license errors if license is removed by license choice" { + val ruleSet = ruleSet(ortResult) { + dependencyRule("test") { + licenseRule("test", LicenseView.CONCLUDED_OR_DECLARED_AND_DETECTED) { + require { + object : RuleMatcher { + override val description = "containsLicense(license)" + + override fun matches() = license == "LicenseRef-b".toSpdx() + } + } + + error(errorMessage, howToFix) + } + } + } + + ruleSet.violations should haveSize(5) + } } } } diff --git a/evaluator/src/test/kotlin/TestData.kt b/evaluator/src/test/kotlin/TestData.kt index 93850a8f64dcb..62880f5ccc17a 100644 --- a/evaluator/src/test/kotlin/TestData.kt +++ b/evaluator/src/test/kotlin/TestData.kt @@ -43,15 +43,18 @@ import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.config.AnalyzerConfiguration import org.ossreviewtoolkit.model.config.Excludes +import org.ossreviewtoolkit.model.config.LicenseChoices +import org.ossreviewtoolkit.model.config.PackageLicenseChoice import org.ossreviewtoolkit.model.config.PathExclude import org.ossreviewtoolkit.model.config.PathExcludeReason import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.model.config.ScannerConfiguration +import org.ossreviewtoolkit.spdx.model.LicenseChoice import org.ossreviewtoolkit.spdx.toSpdx import org.ossreviewtoolkit.utils.DeclaredLicenseProcessor import org.ossreviewtoolkit.utils.Environment -val concludedLicense = "LicenseRef-a AND LicenseRef-b".toSpdx() +val concludedLicense = "LicenseRef-a OR LicenseRef-b".toSpdx() val declaredLicenses = sortedSetOf("Apache-2.0", "MIT") val declaredLicensesProcessed = DeclaredLicenseProcessor.process(declaredLicenses) @@ -162,6 +165,16 @@ val ortResult = OrtResult( comment = "excluded" ) ) + ), + licenseChoices = LicenseChoices( + packageLicenseChoices = listOf( + PackageLicenseChoice( + packageId = Identifier("Maven:org.ossreviewtoolkit:package-with-only-concluded-license:1.0"), + licenseChoices = listOf( + LicenseChoice("LicenseRef-a OR LicenseRef-b".toSpdx(), "LicenseRef-a".toSpdx()) + ) + ) + ) ) ) ), diff --git a/model/src/main/kotlin/OrtResult.kt b/model/src/main/kotlin/OrtResult.kt index bd946bb157b3b..9ef08f245f417 100644 --- a/model/src/main/kotlin/OrtResult.kt +++ b/model/src/main/kotlin/OrtResult.kt @@ -32,6 +32,7 @@ import org.ossreviewtoolkit.model.config.LicenseFindingCuration import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.model.config.Resolutions import org.ossreviewtoolkit.model.config.orEmpty +import org.ossreviewtoolkit.spdx.model.LicenseChoice import org.ossreviewtoolkit.utils.log import org.ossreviewtoolkit.utils.perf import org.ossreviewtoolkit.utils.zipWithDefault @@ -385,6 +386,12 @@ data class OrtResult( !omitExcluded || !isExcluded(id) } + /** + * Return all [LicenseChoice]s for the [Package] with [id]. + */ + fun getLicenseChoices(id: Identifier): List = + repository.config.licenseChoices.packageLicenseChoices.find { it.packageId == id }?.licenseChoices.orEmpty() + /** * Return the list of [AdvisorResult]s for the given [id]. */ From ffad00d8236abefe9de3fbbd9855256907504153 Mon Sep 17 00:00:00 2001 From: Marcel Bochtler Date: Thu, 11 Mar 2021 15:01:32 +0100 Subject: [PATCH 6/8] StaticHtmlReporter: Add effective license Signed-off-by: Marcel Bochtler --- ...d-model-reporter-test-expected-output.json | 174 ++++++++++++++---- ...ed-model-reporter-test-expected-output.yml | 161 ++++++++++++---- ...ic-html-reporter-test-expected-output.html | 69 ++++++- .../static-html-reporter-test-input.yml | 66 +++++++ .../kotlin/reporters/StaticHtmlReporter.kt | 5 + .../src/main/kotlin/utils/ReportTableModel.kt | 6 + .../kotlin/utils/ReportTableModelMapper.kt | 5 + 7 files changed, 407 insertions(+), 79 deletions(-) diff --git a/reporter/src/funTest/assets/evaluated-model-reporter-test-expected-output.json b/reporter/src/funTest/assets/evaluated-model-reporter-test-expected-output.json index 75c6434507b21..e7ffc2ff66735 100644 --- a/reporter/src/funTest/assets/evaluated-model-reporter-test-expected-output.json +++ b/reporter/src/funTest/assets/evaluated-model-reporter-test-expected-output.json @@ -55,9 +55,15 @@ "id" : "EPL-1.0" }, { "_id" : 11, - "id" : "Apache License, Version 2.0" + "id" : "MPL 2.0 or EPL 1.0" }, { "_id" : 12, + "id" : "MPL-2.0" + }, { + "_id" : 13, + "id" : "Apache License, Version 2.0" + }, { + "_id" : 14, "id" : "New BSD License" } ], "scopes" : [ { @@ -146,6 +152,27 @@ "package_verification_code" : "0000000000000000000000000000000000000000" }, { "_id" : 3, + "provenance" : { + "download_time" : "1970-01-01T00:00:00Z", + "source_artifact" : { + "url" : "https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200-sources.jar", + "hash" : { + "value" : "3b5883b7a5a05b932c699760f0854ca565785a84", + "algorithm" : "SHA-1" + } + } + }, + "scanner" : { + "name" : "FileCounter", + "version" : "1.0", + "configuration" : "" + }, + "start_time" : "1970-01-01T00:00:00Z", + "end_time" : "1970-01-01T00:00:00Z", + "file_count" : 42, + "package_verification_code" : "0000000000000000000000000000000000000000" + }, { + "_id" : 4, "provenance" : { "download_time" : "1970-01-01T00:00:00Z", "source_artifact" : { @@ -166,7 +193,7 @@ "file_count" : 168, "package_verification_code" : "0000000000000000000000000000000000000000" }, { - "_id" : 4, + "_id" : 5, "provenance" : { "download_time" : "1970-01-01T00:00:00Z", "source_artifact" : { @@ -187,7 +214,7 @@ "file_count" : 80, "package_verification_code" : "0000000000000000000000000000000000000000" }, { - "_id" : 5, + "_id" : 6, "provenance" : { "download_time" : "1970-01-01T00:00:00Z", "source_artifact" : { @@ -440,11 +467,64 @@ "scope_excludes" : [ 0 ] }, { "_id" : 3, + "id" : "Maven:com.h2database:h2:1.4.200", + "is_project" : false, + "definition_file_path" : "", + "purl" : "pkg:maven/com.h2database/h2@1.4.200", + "declared_licenses" : [ 11 ], + "declared_licenses_processed" : { + "spdx_expression" : "MPL-2.0 OR EPL-1.0", + "mapped_licenses" : [ 12, 10 ] + }, + "concluded_license" : "MPL-2.0 OR EPL-1.0", + "description" : "H2 Database Engine", + "homepage_url" : "https://h2database.com", + "binary_artifact" : { + "url" : "https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200.jar", + "hash" : { + "value" : "f7533fe7cb8e99c87a43d325a77b4b678ad9031a", + "algorithm" : "SHA-1" + } + }, + "source_artifact" : { + "url" : "https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200-sources.jar", + "hash" : { + "value" : "3b5883b7a5a05b932c699760f0854ca565785a84", + "algorithm" : "SHA-1" + } + }, + "vcs" : { + "type" : "Git", + "url" : "https://github.com/h2database/h2database", + "revision" : "", + "path" : "" + }, + "vcs_processed" : { + "type" : "Git", + "url" : "https://github.com/h2database/h2database.git", + "revision" : "", + "path" : "" + }, + "curations" : [ { + "base" : { }, + "curation" : { + "concluded_license" : "MPL-2.0 OR EPL-1.0", + "comment" : "H2 database offers a license choice" + } + } ], + "paths" : [ 1 ], + "levels" : [ 1 ], + "scopes" : [ 1 ], + "scan_results" : [ 3 ], + "is_excluded" : true, + "scope_excludes" : [ 0 ] + }, { + "_id" : 4, "id" : "Maven:org.apache.commons:commons-lang3:3.5", "is_project" : false, "definition_file_path" : "", "purl" : "pkg:maven/org.apache.commons/commons-lang3@3.5", - "declared_licenses" : [ 11 ], + "declared_licenses" : [ 13 ], "declared_licenses_processed" : { "spdx_expression" : "Apache-2.0", "mapped_licenses" : [ 8 ] @@ -477,18 +557,18 @@ "revision" : "", "path" : "" }, - "paths" : [ 1, 2 ], + "paths" : [ 2, 3 ], "levels" : [ 1 ], "scopes" : [ 0, 1 ], - "scan_results" : [ 3 ], + "scan_results" : [ 4 ], "is_excluded" : false }, { - "_id" : 4, + "_id" : 5, "id" : "Maven:org.apache.commons:commons-text:1.1", "is_project" : false, "definition_file_path" : "", "purl" : "pkg:maven/org.apache.commons/commons-text@1.1", - "declared_licenses" : [ 11 ], + "declared_licenses" : [ 13 ], "declared_licenses_processed" : { "spdx_expression" : "Apache-2.0", "mapped_licenses" : [ 8 ] @@ -521,18 +601,18 @@ "revision" : "", "path" : "" }, - "paths" : [ 3, 4 ], + "paths" : [ 4, 5 ], "levels" : [ 0 ], "scopes" : [ 0, 1 ], - "scan_results" : [ 4 ], + "scan_results" : [ 5 ], "is_excluded" : false }, { - "_id" : 5, + "_id" : 6, "id" : "Maven:org.hamcrest:hamcrest-core:1.3", "is_project" : false, "definition_file_path" : "", "purl" : "pkg:maven/org.hamcrest/hamcrest-core@1.3", - "declared_licenses" : [ 12 ], + "declared_licenses" : [ 14 ], "declared_licenses_processed" : { "spdx_expression" : "BSD-3-Clause", "mapped_licenses" : [ 1 ] @@ -565,10 +645,10 @@ "revision" : "", "path" : "" }, - "paths" : [ 5 ], + "paths" : [ 6 ], "levels" : [ 1 ], "scopes" : [ 1 ], - "scan_results" : [ 5 ], + "scan_results" : [ 6 ], "is_excluded" : true, "scope_excludes" : [ 0 ] } ], @@ -582,31 +662,37 @@ "_id" : 1, "pkg" : 3, "project" : 0, - "scope" : 0, - "path" : [ 4 ] + "scope" : 1, + "path" : [ 2 ] }, { "_id" : 2, - "pkg" : 3, + "pkg" : 4, "project" : 0, - "scope" : 1, - "path" : [ 4 ] + "scope" : 0, + "path" : [ 5 ] }, { "_id" : 3, "pkg" : 4, "project" : 0, - "scope" : 0, - "path" : [ ] + "scope" : 1, + "path" : [ 5 ] }, { "_id" : 4, - "pkg" : 4, + "pkg" : 5, "project" : 0, - "scope" : 1, + "scope" : 0, "path" : [ ] }, { "_id" : 5, "pkg" : 5, "project" : 0, "scope" : 1, + "path" : [ ] + }, { + "_id" : 6, + "pkg" : 6, + "project" : 0, + "scope" : 1, "path" : [ 2 ] } ], "dependency_trees" : [ { @@ -618,11 +704,11 @@ "children" : [ { "key" : 2, "linkage" : "DYNAMIC", - "pkg" : 4, + "pkg" : 5, "children" : [ { "key" : 3, "linkage" : "DYNAMIC", - "pkg" : 3 + "pkg" : 4 } ] } ] }, { @@ -636,21 +722,25 @@ "children" : [ { "key" : 6, "linkage" : "DYNAMIC", - "pkg" : 5 + "pkg" : 3 + }, { + "key" : 7, + "linkage" : "DYNAMIC", + "pkg" : 6 } ] }, { - "key" : 7, + "key" : 8, "linkage" : "DYNAMIC", - "pkg" : 4, + "pkg" : 5, "children" : [ { - "key" : 8, + "key" : 9, "linkage" : "DYNAMIC", - "pkg" : 3 + "pkg" : 4 } ] } ] } ] }, { - "key" : 9, + "key" : 10, "pkg" : 1, "path_excludes" : [ 0 ] } ], @@ -672,7 +762,7 @@ }, { "_id" : 1, "rule" : "rule 2", - "pkg" : 4, + "pkg" : 5, "license" : 8, "license_source" : "DECLARED", "severity" : "HINT", @@ -682,7 +772,7 @@ }, { "_id" : 2, "rule" : "rule 3", - "pkg" : 5, + "pkg" : 6, "license" : 1, "license_source" : "CONCLUDED", "severity" : "WARNING", @@ -704,7 +794,7 @@ "included_projects" : 1, "excluded_projects" : 1, "included_packages" : 2, - "excludes_packages" : 2, + "excludes_packages" : 3, "total_tree_depth" : 2, "included_tree_depth" : 2, "included_scopes" : [ "compile" ], @@ -715,8 +805,9 @@ "Apache-2.0" : 2, "BSD-3-Clause" : 1, "CC-BY-NC-3.0" : 1, - "EPL-1.0" : 1, - "GPL-3.0-only WITH GCC-exception-3.1" : 1 + "EPL-1.0" : 2, + "GPL-3.0-only WITH GCC-exception-3.1" : 1, + "MPL-2.0" : 1 }, "detected" : { "Apache-2.0" : 1, @@ -756,10 +847,19 @@ }, "resolutions" : { "rule_violations" : [ 0 ] + }, + "license_choices" : { + "package_license_choices" : [ { + "package_id" : "Maven:com.h2database:h2:1.4.200", + "license_choices" : [ { + "given" : "MPL-2.0 OR EPL-1.0", + "choice" : "MPL-2.0" + } ] + } ] } } }, - "repository_configuration" : "---\nexcludes:\n paths:\n - pattern: \"sub/module/project/build.gradle\"\n reason: \"EXAMPLE_OF\"\n comment: \"The project is an example.\"\n - pattern: \"**.java\"\n reason: \"EXAMPLE_OF\"\n comment: \"These are example files.\"\n scopes:\n - pattern: \"testCompile\"\n reason: \"TEST_DEPENDENCY_OF\"\n comment: \"The scope only contains test dependencies.\"\nresolutions:\n rule_violations:\n - message: \"Apache-2.0 hint\"\n reason: \"CANT_FIX_EXCEPTION\"\n comment: \"Apache-2 is not an issue.\"\n", + "repository_configuration" : "---\nexcludes:\n paths:\n - pattern: \"sub/module/project/build.gradle\"\n reason: \"EXAMPLE_OF\"\n comment: \"The project is an example.\"\n - pattern: \"**.java\"\n reason: \"EXAMPLE_OF\"\n comment: \"These are example files.\"\n scopes:\n - pattern: \"testCompile\"\n reason: \"TEST_DEPENDENCY_OF\"\n comment: \"The scope only contains test dependencies.\"\nresolutions:\n rule_violations:\n - message: \"Apache-2.0 hint\"\n reason: \"CANT_FIX_EXCEPTION\"\n comment: \"Apache-2 is not an issue.\"\nlicense_choices:\n package_license_choices:\n - package_id: \"Maven:com.h2database:h2:1.4.200\"\n license_choices:\n - given: \"MPL-2.0 OR EPL-1.0\"\n choice: \"MPL-2.0\"\n", "labels" : { "job_parameters.JOB_PARAM_1" : "label job param 1", "job_parameters.JOB_PARAM_2" : "label job param 2", diff --git a/reporter/src/funTest/assets/evaluated-model-reporter-test-expected-output.yml b/reporter/src/funTest/assets/evaluated-model-reporter-test-expected-output.yml index 45902b78fa066..827306843ff9e 100644 --- a/reporter/src/funTest/assets/evaluated-model-reporter-test-expected-output.yml +++ b/reporter/src/funTest/assets/evaluated-model-reporter-test-expected-output.yml @@ -40,8 +40,12 @@ licenses: - _id: 10 id: "EPL-1.0" - _id: 11 - id: "Apache License, Version 2.0" + id: "MPL 2.0 or EPL 1.0" - _id: 12 + id: "MPL-2.0" +- _id: 13 + id: "Apache License, Version 2.0" +- _id: 14 id: "New BSD License" scopes: - _id: 0 @@ -117,6 +121,22 @@ scan_results: file_count: 234 package_verification_code: "0000000000000000000000000000000000000000" - _id: 3 + provenance: + download_time: "1970-01-01T00:00:00Z" + source_artifact: + url: "https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200-sources.jar" + hash: + value: "3b5883b7a5a05b932c699760f0854ca565785a84" + algorithm: "SHA-1" + scanner: + name: "FileCounter" + version: "1.0" + configuration: "" + start_time: "1970-01-01T00:00:00Z" + end_time: "1970-01-01T00:00:00Z" + file_count: 42 + package_verification_code: "0000000000000000000000000000000000000000" +- _id: 4 provenance: download_time: "1970-01-01T00:00:00Z" source_artifact: @@ -132,7 +152,7 @@ scan_results: end_time: "1970-01-01T00:00:00Z" file_count: 168 package_verification_code: "0000000000000000000000000000000000000000" -- _id: 4 +- _id: 5 provenance: download_time: "1970-01-01T00:00:00Z" source_artifact: @@ -148,7 +168,7 @@ scan_results: end_time: "1970-01-01T00:00:00Z" file_count: 80 package_verification_code: "0000000000000000000000000000000000000000" -- _id: 5 +- _id: 6 provenance: download_time: "1970-01-01T00:00:00Z" source_artifact: @@ -388,12 +408,63 @@ packages: scope_excludes: - 0 - _id: 3 + id: "Maven:com.h2database:h2:1.4.200" + is_project: false + definition_file_path: "" + purl: "pkg:maven/com.h2database/h2@1.4.200" + declared_licenses: + - 11 + declared_licenses_processed: + spdx_expression: "MPL-2.0 OR EPL-1.0" + mapped_licenses: + - 12 + - 10 + concluded_license: "MPL-2.0 OR EPL-1.0" + description: "H2 Database Engine" + homepage_url: "https://h2database.com" + binary_artifact: + url: "https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200.jar" + hash: + value: "f7533fe7cb8e99c87a43d325a77b4b678ad9031a" + algorithm: "SHA-1" + source_artifact: + url: "https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200-sources.jar" + hash: + value: "3b5883b7a5a05b932c699760f0854ca565785a84" + algorithm: "SHA-1" + vcs: + type: "Git" + url: "https://github.com/h2database/h2database" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/h2database/h2database.git" + revision: "" + path: "" + curations: + - base: {} + curation: + concluded_license: "MPL-2.0 OR EPL-1.0" + comment: "H2 database offers a license choice" + paths: + - 1 + levels: + - 1 + scopes: + - 1 + scan_results: + - 3 + is_excluded: true + scope_excludes: + - 0 +- _id: 4 id: "Maven:org.apache.commons:commons-lang3:3.5" is_project: false definition_file_path: "" purl: "pkg:maven/org.apache.commons/commons-lang3@3.5" declared_licenses: - - 11 + - 13 declared_licenses_processed: spdx_expression: "Apache-2.0" mapped_licenses: @@ -423,23 +494,23 @@ packages: revision: "" path: "" paths: - - 1 - 2 + - 3 levels: - 1 scopes: - 0 - 1 scan_results: - - 3 + - 4 is_excluded: false -- _id: 4 +- _id: 5 id: "Maven:org.apache.commons:commons-text:1.1" is_project: false definition_file_path: "" purl: "pkg:maven/org.apache.commons/commons-text@1.1" declared_licenses: - - 11 + - 13 declared_licenses_processed: spdx_expression: "Apache-2.0" mapped_licenses: @@ -468,23 +539,23 @@ packages: revision: "" path: "" paths: - - 3 - 4 + - 5 levels: - 0 scopes: - 0 - 1 scan_results: - - 4 + - 5 is_excluded: false -- _id: 5 +- _id: 6 id: "Maven:org.hamcrest:hamcrest-core:1.3" is_project: false definition_file_path: "" purl: "pkg:maven/org.hamcrest/hamcrest-core@1.3" declared_licenses: - - 12 + - 14 declared_licenses_processed: spdx_expression: "BSD-3-Clause" mapped_licenses: @@ -514,13 +585,13 @@ packages: revision: "" path: "" paths: - - 5 + - 6 levels: - 1 scopes: - 1 scan_results: - - 5 + - 6 is_excluded: true scope_excludes: - 0 @@ -533,29 +604,35 @@ paths: - _id: 1 pkg: 3 project: 0 - scope: 0 + scope: 1 path: - - 4 + - 2 - _id: 2 - pkg: 3 + pkg: 4 project: 0 - scope: 1 + scope: 0 path: - - 4 + - 5 - _id: 3 pkg: 4 project: 0 - scope: 0 - path: [] + scope: 1 + path: + - 5 - _id: 4 - pkg: 4 + pkg: 5 project: 0 - scope: 1 + scope: 0 path: [] - _id: 5 pkg: 5 project: 0 scope: 1 + path: [] +- _id: 6 + pkg: 6 + project: 0 + scope: 1 path: - 2 dependency_trees: @@ -567,11 +644,11 @@ dependency_trees: children: - key: 2 linkage: "DYNAMIC" - pkg: 4 + pkg: 5 children: - key: 3 linkage: "DYNAMIC" - pkg: 3 + pkg: 4 - key: 4 scope: 1 scope_excludes: @@ -583,15 +660,18 @@ dependency_trees: children: - key: 6 linkage: "DYNAMIC" - pkg: 5 - - key: 7 + pkg: 3 + - key: 7 + linkage: "DYNAMIC" + pkg: 6 + - key: 8 linkage: "DYNAMIC" - pkg: 4 + pkg: 5 children: - - key: 8 + - key: 9 linkage: "DYNAMIC" - pkg: 3 -- key: 9 + pkg: 4 +- key: 10 pkg: 1 path_excludes: - 0 @@ -612,7 +692,7 @@ rule_violations: \ that overflow:scroll is working as expected.\n```" - _id: 1 rule: "rule 2" - pkg: 4 + pkg: 5 license: 8 license_source: "DECLARED" severity: "HINT" @@ -623,7 +703,7 @@ rule_violations: - 0 - _id: 2 rule: "rule 3" - pkg: 5 + pkg: 6 license: 1 license_source: "CONCLUDED" severity: "WARNING" @@ -643,7 +723,7 @@ statistics: included_projects: 1 excluded_projects: 1 included_packages: 2 - excludes_packages: 2 + excludes_packages: 3 total_tree_depth: 2 included_tree_depth: 2 included_scopes: @@ -655,8 +735,9 @@ statistics: Apache-2.0: 2 BSD-3-Clause: 1 CC-BY-NC-3.0: 1 - EPL-1.0: 1 + EPL-1.0: 2 GPL-3.0-only WITH GCC-exception-3.1: 1 + MPL-2.0: 1 detected: Apache-2.0: 1 BSD-3-Clause: 1 @@ -691,13 +772,21 @@ repository: resolutions: rule_violations: - 0 + license_choices: + package_license_choices: + - package_id: "Maven:com.h2database:h2:1.4.200" + license_choices: + - given: "MPL-2.0 OR EPL-1.0" + choice: "MPL-2.0" repository_configuration: "---\nexcludes:\n paths:\n - pattern: \"sub/module/project/build.gradle\"\ \n reason: \"EXAMPLE_OF\"\n comment: \"The project is an example.\"\n - pattern:\ \ \"**.java\"\n reason: \"EXAMPLE_OF\"\n comment: \"These are example files.\"\ \n scopes:\n - pattern: \"testCompile\"\n reason: \"TEST_DEPENDENCY_OF\"\n\ \ comment: \"The scope only contains test dependencies.\"\nresolutions:\n rule_violations:\n\ \ - message: \"Apache-2.0 hint\"\n reason: \"CANT_FIX_EXCEPTION\"\n comment:\ - \ \"Apache-2 is not an issue.\"\n" + \ \"Apache-2 is not an issue.\"\nlicense_choices:\n package_license_choices:\n\ + \ - package_id: \"Maven:com.h2database:h2:1.4.200\"\n license_choices:\n \ + \ - given: \"MPL-2.0 OR EPL-1.0\"\n choice: \"MPL-2.0\"\n" labels: job_parameters.JOB_PARAM_1: "label job param 1" job_parameters.JOB_PARAM_2: "label job param 2" diff --git a/reporter/src/funTest/assets/static-html-reporter-test-expected-output.html b/reporter/src/funTest/assets/static-html-reporter-test-expected-output.html index b1e37957a3989..4dabec88bf178 100644 --- a/reporter/src/funTest/assets/static-html-reporter-test-expected-output.html +++ b/reporter/src/funTest/assets/static-html-reporter-test-expected-output.html @@ -620,6 +620,10 @@

Packages

MIT (link to the location)
+ Effective License: +
+
GPL-2.0-only WITH Classpath-exception-2.0 AND BSD-3-Clause AND LicenseRef-test-Apache-2.0-multi-line AND LicenseRef-test-Apache-2.0-single-line OR MIT AND LicenseRef-test-Apache-2.0-multi-line AND LicenseRef-test-Apache-2.0-single-line
+
    @@ -642,14 +646,45 @@

    Packages

    EPL-1.0
    + Effective License: +
    +
    EPL-1.0
    +
        - - 3Maven:org.apache.commons:commons-lang3:3.5 + + 3Maven:com.h2database:h2:1.4.200 +
          +
        • testCompile
          Excluded: TEST_DEPENDENCY_OF - The scope only contains test dependencies.
          +
          • +
        + Concluded License: +
        +
        MPL-2.0 OR EPL-1.0
        +
        + Declared Licenses: +
        +
        +
        EPL-1.0
        +
        MPL-2.0
        +
        +
        + Effective License: +
        +
        MPL-2.0
        +
        + +
          + +
            + + + + 4Maven:org.apache.commons:commons-lang3:3.5
            • compile
            • testCompile
              Excluded: TEST_DEPENDENCY_OF - The scope only contains test dependencies.
              @@ -661,14 +696,18 @@

              Packages

              Apache-2.0
              + Effective License: +
              +
              Apache-2.0
              +
                  - - 4Maven:org.apache.commons:commons-text:1.1 + + 5Maven:org.apache.commons:commons-text:1.1
                  • compile
                  • testCompile
                    Excluded: TEST_DEPENDENCY_OF - The scope only contains test dependencies.
                    @@ -680,14 +719,18 @@

                    Packages

                    Apache-2.0
                    + Effective License: +
                    +
                    Apache-2.0
                    +
                        - - 5Maven:org.hamcrest:hamcrest-core:1.3 + + 6Maven:org.hamcrest:hamcrest-core:1.3
                        • testCompile
                          Excluded: TEST_DEPENDENCY_OF - The scope only contains test dependencies.
                          • @@ -698,6 +741,10 @@

                          Packages

                          BSD-3-Clause
                          + Effective License: +
                          +
                          BSD-3-Clause
                          +
                            @@ -752,6 +799,10 @@

                            Packages

                            MIT (Excluded: EXAMPLE_OF - These are example files.)
                            + Effective License: +
                            +
                            GPL-3.0-only WITH GCC-exception-3.1 AND CC-BY-NC-3.0 AND Apache-2.0 AND MIT
                            +
                              @@ -781,6 +832,12 @@

                              Repository Configuration

                              - message: "Apache-2.0 hint" reason: "CANT_FIX_EXCEPTION" comment: "Apache-2 is not an issue." +license_choices: + package_license_choices: + - package_id: "Maven:com.h2database:h2:1.4.200" + license_choices: + - given: "MPL-2.0 OR EPL-1.0" + choice: "MPL-2.0" diff --git a/reporter/src/funTest/assets/static-html-reporter-test-input.yml b/reporter/src/funTest/assets/static-html-reporter-test-input.yml index c12745844cd3d..2b5d02f4653f2 100644 --- a/reporter/src/funTest/assets/static-html-reporter-test-input.yml +++ b/reporter/src/funTest/assets/static-html-reporter-test-input.yml @@ -34,6 +34,12 @@ repository: - message: "Apache-2.0 hint" reason: "CANT_FIX_EXCEPTION" comment: "Apache-2 is not an issue." + license_choices: + package_license_choices: + - package_id: "Maven:com.h2database:h2:1.4.200" + license_choices: + - given: "MPL-2.0 OR EPL-1.0" + choice: "MPL-2.0" analyzer: start_time: "1970-01-01T00:00:00Z" end_time: "1970-01-01T00:00:00Z" @@ -74,6 +80,7 @@ analyzer: - id: "Ant:junit:junit:4.12" dependencies: - id: "Maven:org.hamcrest:hamcrest-core:1.3" + - id: "Maven:com.h2database:h2:1.4.200" - id: "Maven:org.apache.commons:commons-text:1.1" dependencies: - id: "Maven:org.apache.commons:commons-lang3:3.5" @@ -228,6 +235,45 @@ analyzer: revision: "" path: "" curations: [] + - package: + id: "Maven:com.h2database:h2:1.4.200" + purl: "pkg:maven/com.h2database/h2@1.4.200" + authors: + - "Thomas Mueller" + declared_licenses: + - "MPL 2.0 or EPL 1.0" + declared_licenses_processed: + spdx_expression: "MPL-2.0 OR EPL-1.0" + mapped: + MPL 2.0 or EPL 1.0: "MPL-2.0 OR EPL-1.0" + concluded_license: "MPL-2.0 OR EPL-1.0" + description: "H2 Database Engine" + homepage_url: "https://h2database.com" + binary_artifact: + url: "https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200.jar" + hash: + value: "f7533fe7cb8e99c87a43d325a77b4b678ad9031a" + algorithm: "SHA-1" + source_artifact: + url: "https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200-sources.jar" + hash: + value: "3b5883b7a5a05b932c699760f0854ca565785a84" + algorithm: "SHA-1" + vcs: + type: "Git" + url: "https://github.com/h2database/h2database" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/h2database/h2database.git" + revision: "" + path: "" + curations: + - base: { } + curation: + concluded_license: "MPL-2.0 OR EPL-1.0" + comment: "H2 database offers a license choice" has_issues: false scanner: start_time: "1970-01-01T00:00:00Z" @@ -435,6 +481,26 @@ scanner: package_verification_code: "0000000000000000000000000000000000000000" licenses: [] copyrights: [] + - id: "Maven:com.h2database:h2:1.4.200" + results: + - provenance: + download_time: "1970-01-01T00:00:00Z" + source_artifact: + url: "https://repo.maven.apache.org/maven2/com/h2database/h2/1.4.200/h2-1.4.200-sources.jar" + hash: + value: "3b5883b7a5a05b932c699760f0854ca565785a84" + algorithm: "SHA-1" + scanner: + name: "FileCounter" + version: "1.0" + configuration: "" + summary: + start_time: "1970-01-01T00:00:00Z" + end_time: "1970-01-01T00:00:00Z" + file_count: 42 + package_verification_code: "0000000000000000000000000000000000000000" + licenses: [ ] + copyrights: [ ] storage_stats: num_reads: 5 num_hits: 0 diff --git a/reporter/src/main/kotlin/reporters/StaticHtmlReporter.kt b/reporter/src/main/kotlin/reporters/StaticHtmlReporter.kt index 7891106736fa8..7441a7cebaf5f 100644 --- a/reporter/src/main/kotlin/reporters/StaticHtmlReporter.kt +++ b/reporter/src/main/kotlin/reporters/StaticHtmlReporter.kt @@ -573,6 +573,11 @@ class StaticHtmlReporter : Reporter { } } } + + if (row.effectiveLicense != null) { + em { +"Effective License:" } + dl { dd { +"${row.effectiveLicense}" } } + } } td { issueList(row.analyzerIssues) } diff --git a/reporter/src/main/kotlin/utils/ReportTableModel.kt b/reporter/src/main/kotlin/utils/ReportTableModel.kt index 1d42d61ba7d56..ad73035133e0d 100644 --- a/reporter/src/main/kotlin/utils/ReportTableModel.kt +++ b/reporter/src/main/kotlin/utils/ReportTableModel.kt @@ -134,6 +134,12 @@ data class ReportTableModel( */ val detectedLicenses: List, + /** + * The effective license of the package derived from the licenses of the license sources chosen by a + * LicenseView, with optional choices applied. + */ + val effectiveLicense: SpdxExpression?, + /** * All analyzer issues related to this package. */ diff --git a/reporter/src/main/kotlin/utils/ReportTableModelMapper.kt b/reporter/src/main/kotlin/utils/ReportTableModelMapper.kt index b96a280e292a7..da7bf9082da2b 100644 --- a/reporter/src/main/kotlin/utils/ReportTableModelMapper.kt +++ b/reporter/src/main/kotlin/utils/ReportTableModelMapper.kt @@ -30,6 +30,7 @@ import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.config.Excludes import org.ossreviewtoolkit.model.config.ScopeExclude import org.ossreviewtoolkit.model.licenses.LicenseInfoResolver +import org.ossreviewtoolkit.model.licenses.LicenseView import org.ossreviewtoolkit.model.utils.ResolutionProvider import org.ossreviewtoolkit.reporter.HowToFixTextProvider import org.ossreviewtoolkit.reporter.utils.ReportTableModel.DependencyRow @@ -159,6 +160,10 @@ class ReportTableModelMapper( concludedLicense = concludedLicense, declaredLicenses = declaredLicenses, detectedLicenses = detectedLicenses, + effectiveLicense = resolvedLicenseInfo.effectiveLicense( + LicenseView.CONCLUDED_OR_DECLARED_AND_DETECTED, + ortResult.getLicenseChoices(id) + ), analyzerIssues = analyzerIssues.map { it.toResolvableIssue() }, scanIssues = scanIssues.map { it.toResolvableIssue() } ).also { row -> From ddff3615bad5865f336128f08c4826147845ce10 Mon Sep 17 00:00:00 2001 From: Marcel Bochtler Date: Tue, 16 Mar 2021 13:35:34 +0100 Subject: [PATCH 7/8] LicenseChoice: Add init function to check valid choice Signed-off-by: Marcel Bochtler --- spdx-utils/src/main/kotlin/model/LicenseChoice.kt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/spdx-utils/src/main/kotlin/model/LicenseChoice.kt b/spdx-utils/src/main/kotlin/model/LicenseChoice.kt index ff9969c17a197..5b1b8ad872276 100644 --- a/spdx-utils/src/main/kotlin/model/LicenseChoice.kt +++ b/spdx-utils/src/main/kotlin/model/LicenseChoice.kt @@ -19,6 +19,7 @@ package org.ossreviewtoolkit.spdx.model +import org.ossreviewtoolkit.spdx.InvalidLicenseChoiceException import org.ossreviewtoolkit.spdx.SpdxExpression /** @@ -53,4 +54,12 @@ import org.ossreviewtoolkit.spdx.SpdxExpression data class LicenseChoice( val given: SpdxExpression?, val choice: SpdxExpression, -) +) { + init { + if (given?.isValidChoice(choice) == false) { + throw InvalidLicenseChoiceException( + "$choice is not a valid choice for $given. Valid choices are: ${given.validChoices()}." + ) + } + } +} From 9a0f405912b75ebeee830fdf81121dc61c0b7ed8 Mon Sep 17 00:00:00 2001 From: Stephanie Neubauer Date: Wed, 17 Mar 2021 09:59:35 +0100 Subject: [PATCH 8/8] config-file-ort-yml: Add documentation for license choices Signed-off-by: Stephanie Neubauer --- docs/config-file-ort-yml.md | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/config-file-ort-yml.md b/docs/config-file-ort-yml.md index 00b9bd7ea43dd..bcd40666cbc90 100644 --- a/docs/config-file-ort-yml.md +++ b/docs/config-file-ort-yml.md @@ -260,3 +260,57 @@ resolutions: reason: "INEFFECTIVE_VULNERABILITY" comment: "CVE-9999-9999 is a false positive" ``` + +# License Choices + +### When to Use License Choices + +For multi-licensed dependencies a specific license can be selected. +The license choice can be applied to a package. +A choice is only valid for licenses combined with the SPDX operator `OR`. +The choices are applied in the evaluator, and the reporter to the effective license of a package, which is calculated +by the chosen [LicenseView](../model/src/main/kotlin/licenses/LicenseView.kt). + +### License Choice by Package + +To select a license from a multi-licensed dependency, specified by its `packageId`, an SPDX expression for a `choice` +must be provided. +The `choice` is either applied to the whole effective SPDX expression of the package or to an optional `given` SPDX +expression that can represent only a sub-expression of the whole effective SPDX expression. + +e.g. +```yaml +license_choices: + package_license_choice: + - package_id: "Maven:com.example:first:0.0.1" + license_choices: + # The input of the calculated effective license would be: (A OR B) AND ((C OR D) AND E) + - given: A OR B + choice: A + # The result would be: A AND ((C OR D) AND E) + # The input of the current effective license would be: A AND ((C OR D) AND E) + - given: (C OR D) AND E + choice: C AND E + # The result would be: A AND C AND E + - package_id: "Maven:com.example:second:2.3.4" + license_choices: + # Without a 'given', the 'choice' is applied to the effective license expression if it is a valid choice. + # The input from the calculated effective license would be: (C OR D) AND E + - choice: C AND E + # The result would be: C AND E +``` + +--- +**NOTE** + +The choice will be applied to the WHOLE `given` license. +If the choice does not provide a valid result, an exception will be thrown upon deserialization. + +e.g. invalid configuration: +```yaml +# This is invalid, as 'E' must be in the resulting license. +- given: (C OR D) AND E + choice: C +``` + +---