From 3f50356f8b744f756442c0c0d33892e30057f936 Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Tue, 1 Oct 2024 09:16:47 +0200 Subject: [PATCH] chore(helper-cli): Drop the obsolete command to create analyzer results The command had been introduced years ago to support migrating to a newer ScanCode version, e.g. scan already scanned packages with a newer ScanCode version to avoid performance temporary poor execution times due to no chache hits. Nowadays the scanning is provenance based, so triggering a re-scan would be better done by provenance, as opposed to by package. So, remove the command, because it is no more appropriate. Signed-off-by: Frank Viernau --- helper-cli/src/main/kotlin/HelperMain.kt | 1 - .../commands/CreateAnalyzerResultCommand.kt | 237 ------------------ 2 files changed, 238 deletions(-) delete mode 100644 helper-cli/src/main/kotlin/commands/CreateAnalyzerResultCommand.kt diff --git a/helper-cli/src/main/kotlin/HelperMain.kt b/helper-cli/src/main/kotlin/HelperMain.kt index 3d36fe46c6f2..11a44b90b514 100644 --- a/helper-cli/src/main/kotlin/HelperMain.kt +++ b/helper-cli/src/main/kotlin/HelperMain.kt @@ -76,7 +76,6 @@ internal class HelperMain : CliktCommand(ORTH_NAME) { subcommands( ConvertOrtFileCommand(), - CreateAnalyzerResultCommand(), CreateAnalyzerResultFromPackageListCommand(), DevCommand(), ExtractRepositoryConfigurationCommand(), diff --git a/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultCommand.kt b/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultCommand.kt deleted file mode 100644 index ac9c53997aa6..000000000000 --- a/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultCommand.kt +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Copyright (C) 2021 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.helper.commands - -import com.github.ajalt.clikt.parameters.options.associate -import com.github.ajalt.clikt.parameters.options.convert -import com.github.ajalt.clikt.parameters.options.default -import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.options.required -import com.github.ajalt.clikt.parameters.types.file - -import java.sql.Connection -import java.time.Instant - -import org.ossreviewtoolkit.helper.utils.ORTH_NAME -import org.ossreviewtoolkit.helper.utils.OrtHelperCommand -import org.ossreviewtoolkit.helper.utils.writeOrtResult -import org.ossreviewtoolkit.model.AnalyzerResult -import org.ossreviewtoolkit.model.AnalyzerRun -import org.ossreviewtoolkit.model.ArtifactProvenance -import org.ossreviewtoolkit.model.Identifier -import org.ossreviewtoolkit.model.OrtResult -import org.ossreviewtoolkit.model.Package -import org.ossreviewtoolkit.model.Provenance -import org.ossreviewtoolkit.model.RemoteArtifact -import org.ossreviewtoolkit.model.RepositoryProvenance -import org.ossreviewtoolkit.model.VcsInfo -import org.ossreviewtoolkit.model.config.AnalyzerConfiguration -import org.ossreviewtoolkit.model.config.OrtConfiguration -import org.ossreviewtoolkit.model.config.PostgresStorageConfiguration -import org.ossreviewtoolkit.model.config.StorageType -import org.ossreviewtoolkit.model.jsonMapper -import org.ossreviewtoolkit.model.utils.DatabaseUtils -import org.ossreviewtoolkit.scanner.provenance.PackageProvenanceResolutionResult -import org.ossreviewtoolkit.scanner.provenance.ResolvedArtifactProvenance -import org.ossreviewtoolkit.scanner.provenance.ResolvedRepositoryProvenance -import org.ossreviewtoolkit.utils.common.expandTilde -import org.ossreviewtoolkit.utils.ort.Environment -import org.ossreviewtoolkit.utils.ort.ORT_CONFIG_FILENAME -import org.ossreviewtoolkit.utils.ort.ortConfigDirectory - -internal class CreateAnalyzerResultCommand : OrtHelperCommand( - help = "Creates an analyzer result that contains packages for the given list of package ids. The result contains " + - "only packages which have a corresponding ScanCode scan result in the postgres storage." -) { - private val packageIdsFile by option( - "--package-ids-file", - help = "The list of package ids to put into the output analyzer result in plain text with one entry per line." - ).convert { it.expandTilde() } - .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) - .convert { it.absoluteFile.normalize() } - .required() - - private val ortFile by option( - "--ort-file", "-o", - help = "The ORT file to write the generated synthetic analyzer result to." - ).convert { it.expandTilde() } - .file(mustExist = false, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false) - .convert { it.absoluteFile.normalize() } - .required() - - private val configFile by option( - "--config", - help = "The path to the ORT configuration file that configures the scan results storage." - ).convert { it.expandTilde() } - .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) - .convert { it.absoluteFile.normalize() } - .default(ortConfigDirectory.resolve(ORT_CONFIG_FILENAME)) - - private val configArguments by option( - "-P", - help = "Override a key-value pair in the configuration file. For example: " + - "-P ort.scanner.storages.postgres.connection.schema=testSchema" - ).associate() - - private val scancodeVersion by option( - "--scancode-version", - help = "The ScanCode version to match for in the scan results. If blank, any ScanCode version is matched." - ) - - override fun run() { - val ids = packageIdsFile.readLines().filterNot { it.isBlank() }.map { Identifier(it.trim()) } - val packages = openDatabaseConnection().use { connection -> - getScannedPackages(connection, ids, scancodeVersion?.takeIf { it.isNotBlank() }) - }.filterMaxByRowId() - - val ortResult = createAnalyzerResult(packages) - - println("Writing analyzer result with ${packages.size} packages to '${ortFile.absolutePath}'.") - writeOrtResult(ortResult, ortFile) - } - - private fun openDatabaseConnection(): Connection { - val ortConfig = OrtConfiguration.load(configArguments, configFile) - - val storageConfig = ortConfig.scanner.storages.orEmpty().values - .filterIsInstance() - .firstOrNull { it.type == StorageType.PROVENANCE_BASED } - ?: throw IllegalArgumentException("postgresStorage not configured.") - - val dataSource = DatabaseUtils.createHikariDataSource( - config = storageConfig.connection, - applicationNameSuffix = ORTH_NAME, - maxPoolSize = 1 - ) - - return dataSource.value.connection - } -} - -private data class ScannedPackage( - val rowId: Int, - val id: Identifier, - val provenance: Provenance - -) - -private fun getScannedPackages( - connection: Connection, - ids: Collection, - scanCodeVersion: String? -): List { - val whereClause = listOfNotNull( - "p.identifier = ANY(?)", - "p.vcs_type IS NOT DISTINCT FROM s.vcs_type", - "p.vcs_url IS NOT DISTINCT FROM s.vcs_url", - "p.vcs_revision IS NOT DISTINCT FROM s.vcs_revision", - "p.artifact_url IS NOT DISTINCT FROM s.artifact_url", - "p.artifact_hash IS NOT DISTINCT FROM s.artifact_hash", - "s.scanner_name = 'ScanCode'", - scanCodeVersion?.takeUnless { it.isEmpty() }?.let { "s.scanner_version = '$it'" } - ).joinToString(" AND ") - - val query = """ - SELECT DISTINCT - p.id, - p.identifier, - p.result - FROM - package_provenances p, provenance_scan_results s - WHERE - $whereClause; - """.trimIndent() - - val resultSet = connection.prepareStatement(query).apply { - val array = connection.createArrayOf("VARCHAR", ids.distinct().map { it.toCoordinates() }.toTypedArray()) - setArray(1, array) - }.executeQuery() - - val result = mutableListOf() - - while (resultSet.next()) { - val rowId = resultSet.getInt("id") - val id = Identifier(resultSet.getString("identifier")) - - val provenanceResolutionResult = jsonMapper.readValue( - resultSet.getString("result"), - PackageProvenanceResolutionResult::class.java - ) - - val provenance = when (provenanceResolutionResult) { - is ResolvedRepositoryProvenance -> provenanceResolutionResult.provenance - is ResolvedArtifactProvenance -> provenanceResolutionResult.provenance - else -> continue - } - - result += ScannedPackage(rowId, id, provenance) - } - - return result.distinct() -} - -private fun createAnalyzerResult(packages: Collection) = - OrtResult.EMPTY.copy( - analyzer = AnalyzerRun( - startTime = Instant.now(), - endTime = Instant.now(), - environment = Environment(), - config = AnalyzerConfiguration(), - result = AnalyzerResult( - projects = emptySet(), - packages = packages.mapTo(mutableSetOf()) { it.toPackage() } - ) - ) - ) - -private fun Collection.filterMaxByRowId(): List { - fun filterMaxByRowId(predicate: (ScannedPackage) -> Boolean) = - filter(predicate).groupBy { it.id }.mapNotNull { (_, packages) -> - packages.maxByOrNull { it.rowId } - } - - return filterMaxByRowId { it.provenance is RepositoryProvenance } + - filterMaxByRowId { it.provenance is ArtifactProvenance } -} - -private fun ScannedPackage.toPackage(): Package { - val sourceArtifact = if (provenance is ArtifactProvenance) { - provenance.sourceArtifact - } else { - RemoteArtifact.EMPTY - } - - val vcs = if (provenance is RepositoryProvenance) { - provenance.vcsInfo.copy(revision = provenance.resolvedRevision) - } else { - VcsInfo.EMPTY - } - - return Package( - id = id, - declaredLicenses = emptySet(), - description = "", - homepageUrl = "", - binaryArtifact = RemoteArtifact.EMPTY, - sourceArtifact = sourceArtifact, - vcs = vcs, - vcsProcessed = vcs - ) -}