diff --git a/helper-cli/build.gradle.kts b/helper-cli/build.gradle.kts index 5d0bd5d3dbc26..1730e521cb58a 100644 --- a/helper-cli/build.gradle.kts +++ b/helper-cli/build.gradle.kts @@ -43,8 +43,6 @@ dependencies { implementation(libs.clikt) implementation(libs.commonsCompress) - implementation(libs.exposed.core) - implementation(libs.hikari) implementation(libs.jackson.module.kotlin) implementation(libs.jslt) implementation(libs.log4j.api) diff --git a/helper-cli/src/main/kotlin/HelperMain.kt b/helper-cli/src/main/kotlin/HelperMain.kt index 935be5c2012cc..7ca5a4a1a5f6b 100644 --- a/helper-cli/src/main/kotlin/HelperMain.kt +++ b/helper-cli/src/main/kotlin/HelperMain.kt @@ -40,7 +40,6 @@ import org.ossreviewtoolkit.helper.commands.packageconfig.PackageConfigurationCo import org.ossreviewtoolkit.helper.commands.packagecuration.PackageCurationsCommand import org.ossreviewtoolkit.helper.commands.provenancestorage.ProvenanceStorageCommand import org.ossreviewtoolkit.helper.commands.repoconfig.RepositoryConfigurationCommand -import org.ossreviewtoolkit.helper.commands.scanstorage.ScanStorageCommand import org.ossreviewtoolkit.helper.utils.ORTH_NAME import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.ort.printStackTrace @@ -100,7 +99,6 @@ internal class HelperMain : CliktCommand( PackageCurationsCommand(), ProvenanceStorageCommand(), RepositoryConfigurationCommand(), - ScanStorageCommand(), SetDependencyRepresentationCommand(), SetLabelsCommand(), TransformResultCommand(), diff --git a/helper-cli/src/main/kotlin/commands/scanstorage/DeleteCommand.kt b/helper-cli/src/main/kotlin/commands/scanstorage/DeleteCommand.kt deleted file mode 100644 index 5239ac9f33f9a..0000000000000 --- a/helper-cli/src/main/kotlin/commands/scanstorage/DeleteCommand.kt +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (C) 2021 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.helper.commands.scanstorage - -import com.github.ajalt.clikt.core.CliktCommand -import com.github.ajalt.clikt.parameters.options.associate -import com.github.ajalt.clikt.parameters.options.convert -import com.github.ajalt.clikt.parameters.options.default -import com.github.ajalt.clikt.parameters.options.flag -import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.options.split -import com.github.ajalt.clikt.parameters.types.enum -import com.github.ajalt.clikt.parameters.types.file - -import org.apache.logging.log4j.kotlin.logger - -import org.jetbrains.exposed.sql.Database -import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList -import org.jetbrains.exposed.sql.SqlExpressionBuilder.isNotNull -import org.jetbrains.exposed.sql.SqlExpressionBuilder.regexp -import org.jetbrains.exposed.sql.compoundAnd -import org.jetbrains.exposed.sql.compoundOr -import org.jetbrains.exposed.sql.deleteWhere -import org.jetbrains.exposed.sql.selectAll - -import org.ossreviewtoolkit.helper.utils.ORTH_NAME -import org.ossreviewtoolkit.helper.utils.execAndMap -import org.ossreviewtoolkit.model.SourceCodeOrigin -import org.ossreviewtoolkit.model.config.OrtConfiguration -import org.ossreviewtoolkit.model.config.PostgresStorageConfiguration -import org.ossreviewtoolkit.model.utils.DatabaseUtils -import org.ossreviewtoolkit.model.utils.DatabaseUtils.transaction -import org.ossreviewtoolkit.model.utils.rawParam -import org.ossreviewtoolkit.scanner.storages.utils.ScanResults -import org.ossreviewtoolkit.utils.common.expandTilde -import org.ossreviewtoolkit.utils.ort.ORT_CONFIG_FILENAME -import org.ossreviewtoolkit.utils.ort.ortConfigDirectory - -internal class DeleteCommand : CliktCommand( - help = "Removes stored scan results matching the options or all results if no options are given." -) { - private val configFile by option( - "--config", - help = "The path to the ORT configuration file that configures the scan results storage." - ).convert { it.expandTilde() } - .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) - .convert { it.absoluteFile.normalize() } - .default(ortConfigDirectory.resolve(ORT_CONFIG_FILENAME)) - - private val configArguments by option( - "-P", - help = "Override a key-value pair in the configuration file. For example: " + - "-P ort.scanner.storages.postgres.connection.schema=testSchema" - ).associate() - - private val sourceCodeOrigins by option( - "--source-code-origins", - help = "The origin of the scan results that should be deleted." - ).enum().split(",").default(emptyList()) - - private val packageId by option( - "--package-id", - help = "A regular expression for matching the package id." - ) - - private val license by option( - "--license", - help = "The license expression of a detected license of a package that should be deleted." - ) - - private val dryRun by option( - "--dry-run", - help = "Perform a dry run without actually deleting anything." - ).flag() - - override fun run() { - val database = connectPostgresStorage(OrtConfiguration.load(configArguments, configFile)) - - val provenanceKeys = sourceCodeOrigins.map { origin -> - when (origin) { - SourceCodeOrigin.ARTIFACT -> "source_artifact" - SourceCodeOrigin.VCS -> "vcs_info" - } - } - - val identifierCondition = packageId?.let { ScanResults.identifier regexp it } - - val licenseCondition = license?.let { - println("Searching for stored scan results with license '$license'. This might take some time.") - val ids = database.transaction { - // language=PostgreSQL - """ - SELECT ${ScanResults.id.name} - FROM ${ScanResults.tableName} - WHERE (${ScanResults.scanResult.name} -> 'summary' -> 'licenses')::jsonb @> - '[{"license": "$license"}]'::jsonb - """.trimIndent().execAndMap { - it.getInt(ScanResults.id.name) - } - } - - ScanResults.id inList ids - } - - val provenanceConditions = provenanceKeys.map { key -> - rawParam("scan_result->'provenance'->>'$key'").isNotNull() - }.takeIf { it.isNotEmpty() }?.compoundOr() - - val conditions = listOfNotNull(identifierCondition, licenseCondition, provenanceConditions) - if (conditions.isEmpty()) { - // Default to the safe option to not delete anything if no conditions are given. - println("Not specified what entries to delete. Not deleting anything.") - - return - } - - val condition = conditions.compoundAnd() - - if (dryRun) { - val count = database.transaction { - ScanResults.selectAll().where { condition }.count() - } - - println("Would delete $count scan result(s).") - - if (logger.delegate.isDebugEnabled) { - database.transaction { - ScanResults.select(ScanResults.identifier).where { condition } - .forEach(logger::debug) - } - } - } else { - val count = database.transaction { - ScanResults.deleteWhere { condition } - } - - println("Successfully deleted $count stored scan result(s).") - } - } - - private fun connectPostgresStorage(config: OrtConfiguration): Database { - val storageConfig = config.scanner.storages?.get("postgresStorage") as? PostgresStorageConfiguration - ?: throw IllegalArgumentException("postgresStorage not configured.") - - logger.info { - "Using Postgres storage with URL ${storageConfig.connection.url} and schema " + - "'${storageConfig.connection.schema}'." - } - - val dataSource = DatabaseUtils.createHikariDataSource( - config = storageConfig.connection, - applicationNameSuffix = ORTH_NAME, - maxPoolSize = 1 - ) - - return Database.connect(dataSource.value) - } -} diff --git a/helper-cli/src/main/kotlin/commands/scanstorage/ScanStorageCommand.kt b/helper-cli/src/main/kotlin/commands/scanstorage/ScanStorageCommand.kt deleted file mode 100644 index 8bb6012ba65d0..0000000000000 --- a/helper-cli/src/main/kotlin/commands/scanstorage/ScanStorageCommand.kt +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2021 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.helper.commands.scanstorage - -import com.github.ajalt.clikt.core.NoOpCliktCommand -import com.github.ajalt.clikt.core.subcommands - -internal class ScanStorageCommand : NoOpCliktCommand( - help = "Commands for working with scan storages." -) { - init { - subcommands( - DeleteCommand() - ) - } -} diff --git a/helper-cli/src/main/kotlin/utils/Extensions.kt b/helper-cli/src/main/kotlin/utils/Extensions.kt index 1887319336f86..7c78a3a9c8dd7 100644 --- a/helper-cli/src/main/kotlin/utils/Extensions.kt +++ b/helper-cli/src/main/kotlin/utils/Extensions.kt @@ -26,12 +26,9 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator import java.io.File import java.nio.file.Paths -import java.sql.ResultSet import kotlin.io.path.createTempDirectory -import org.jetbrains.exposed.sql.transactions.TransactionManager - import org.ossreviewtoolkit.analyzer.PackageManagerFactory import org.ossreviewtoolkit.downloader.Downloader import org.ossreviewtoolkit.model.ArtifactProvenance @@ -294,20 +291,6 @@ internal fun String.wrapAt(column: Int): String = } }.trimEnd() -/** - * Execute the raw SQL statement and map it using [transform]. - */ -internal fun String.execAndMap(transform: (ResultSet) -> T): List { - val result = mutableListOf() - TransactionManager.current().exec(this) { resultSet -> - while (resultSet.next()) { - result += transform(resultSet) - } - } - - return result -} - /** * Return a copy of this [OrtResult] with the [Repository.config] with the content of the given * [repositoryConfigurationFile].