Skip to content

Commit

Permalink
fix(scanner): Filter scan results by scanner name
Browse files Browse the repository at this point in the history
When a project scanner is configured alongside a package scanner, the
scan results could be duplicated if the provenances of those packages are
overlapping (e.g. in the case of repositories with Git submodules).
This commit addresses the bug by constructing a map of packages with their
scanners and filtering the scan results using this map.

Fixes oss-review-toolkit#7231.

Signed-off-by: Nicolas Nobelis <nicolas.nobelis@bosch.io>
  • Loading branch information
nnobelis committed Jul 12, 2023
1 parent 8e3b457 commit b90e2a3
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 9 deletions.
12 changes: 11 additions & 1 deletion model/src/main/kotlin/ScannerRun.kt
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.ossreviewtoolkit.model

import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.annotation.JsonPropertyOrder
import com.fasterxml.jackson.databind.annotation.JsonSerialize

import java.io.File
Expand Down Expand Up @@ -72,6 +73,12 @@ data class ScannerRun(
@JsonSerialize(converter = ScanResultSortedSetConverter::class)
val scanResults: Set<ScanResult>,

/**
* The scanners for each scanned package.
*/
@JsonPropertyOrder(alphabetic = true)
val scanners: Map<Identifier, Set<String>>,

/**
* The list of files for each resolved provenance.
*/
Expand All @@ -90,7 +97,8 @@ data class ScannerRun(
config = ScannerConfiguration(),
provenances = emptySet(),
scanResults = emptySet(),
files = emptySet()
files = emptySet(),
scanners = emptyMap()
)
}

Expand Down Expand Up @@ -196,6 +204,8 @@ data class ScannerRun(

val scanResultsByPath = resolutionResult.getKnownProvenancesWithoutVcsPath().mapValues { (_, provenance) ->
scanResultsByProvenance[provenance].orEmpty()
}.mapValues { scanResults ->
scanResults.value.filter { it.scanner.name in scanners[id].orEmpty() }
}

val scanResults = mergeScanResultsByScanner(scanResultsByPath).map { scanResult ->
Expand Down
124 changes: 124 additions & 0 deletions scanner/src/funTest/assets/analyzer-result_for_multiple_scanners.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
---
repository:
vcs:
type: "Git"
url: "https://github.com/nnobelis/spdx-project.git"
revision: "8fae7f2b0efa8addeadfd15041e3437d92ff6b71"
path: ""
vcs_processed:
type: "Git"
url: "https://github.com/nnobelis/spdx-project.git"
revision: "8fae7f2b0efa8addeadfd15041e3437d92ff6b71"
path: ""
nested_repositories:
spdx-dependency:
type: "Git"
url: "https://github.com/nnobelis/spdx-dependency.git"
revision: "78c8a5b12ae2d0f613fff25f2d1eb746a768ece7"
path: ""
config: {}
analyzer:
start_time: "2023-07-10T11:03:42.648265Z"
end_time: "2023-07-10T11:03:43.379659Z"
environment:
ort_version: "9e03ee5"
java_version: "11.0.19"
os: "Linux"
processors: 8
max_memory: 12884901888
variables:
JAVA_HOME: "/opt/java/openjdk"
ANDROID_HOME: "/opt/android-sdk"
GOPATH: "/tmp/go"
tool_versions: {}
config:
allow_dynamic_versions: false
package_managers:
NPM:
options:
legacyPeerDeps: "true"
NuGet:
options:
directDependenciesOnly: "true"
skip_excluded: false
result:
projects:
- id: "SpdxDocumentFile::spdx-project:"
definition_file_path: "project.spdx.yml"
authors:
- "Nicolas Nobelis"
declared_licenses:
- "LicenseRef-Bosch-Proprietary"
declared_licenses_processed:
spdx_expression: "LicenseRef-Bosch-Proprietary"
vcs:
type: "Git"
url: "https://github.com/nnobelis/spdx-project.git"
revision: "8fae7f2b0efa8addeadfd15041e3437d92ff6b71"
path: ""
vcs_processed:
type: "Git"
url: "https://github.com/nnobelis/spdx-project.git"
revision: "8fae7f2b0efa8addeadfd15041e3437d92ff6b71"
path: ""
homepage_url: ""
scope_names:
- "default"
packages:
- id: "SpdxDocumentFile::spdx-dependency:"
purl: "pkg:spdxdocumentfile/spdx-dependency@"
authors:
- "Nicolas Nobelis"
declared_licenses:
- "LicenseRef-Bosch-Proprietary"
declared_licenses_processed:
spdx_expression: "LicenseRef-Bosch-Proprietary"
description: ""
homepage_url: ""
binary_artifact:
url: ""
hash:
value: ""
algorithm: ""
source_artifact:
url: ""
hash:
value: ""
algorithm: ""
vcs:
type: "Git"
url: "https://github.com/nnobelis/spdx-dependency.git"
revision: "78c8a5b12ae2d0f613fff25f2d1eb746a768ece7"
path: ""
vcs_processed:
type: "Git"
url: "https://github.com/nnobelis/spdx-dependency.git"
revision: "78c8a5b12ae2d0f613fff25f2d1eb746a768ece7"
path: ""
dependency_graphs:
SpdxDocumentFile:
packages:
- "SpdxDocumentFile::spdx-dependency:"
scopes:
:spdx-project::default:
- root: 0
nodes:
- linkage: "STATIC"
edges: []
scanner: null
advisor: null
evaluator: null
resolved_configuration:
package_curations:
- provider:
id: "RepositoryConfiguration"
curations: []
- provider:
id: "File"
curations: []
- provider:
id: "OrtConfig"
curations: []
- provider:
id: "ClearlyDefined"
curations: []
54 changes: 54 additions & 0 deletions scanner/src/funTest/kotlin/scanners/MultipleScannersTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Copyright (C) 2023 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.scanner.scanners

import io.kotest.core.spec.style.WordSpec
import io.kotest.matchers.shouldBe

import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.OrtResult
import org.ossreviewtoolkit.model.PackageType
import org.ossreviewtoolkit.model.fromYaml
import org.ossreviewtoolkit.utils.test.getAssetFile
import org.ossreviewtoolkit.utils.test.shouldNotBeNull

class MultipleScannersTest : WordSpec({
"Scanning a set of packages with overlapping provenance" should {
val analyzerResult =
getAssetFile("analyzer-result_for_multiple_scanners.yml").readText().fromYaml<OrtResult>()

val dummyScanners = DummyScanner()
val ortResult = createScanner().copy(
scannerWrappers = mapOf(
PackageType.PROJECT to listOf(dummyScanners.copy(name = "Dummy2")),
PackageType.PACKAGE to listOf(dummyScanners)
)
).scan(analyzerResult, skipExcluded = false, emptyMap())

"return the expected ORT result" {
ortResult.scanner.shouldNotBeNull {
val results = getScanResults(Identifier("SpdxDocumentFile", "", "spdx-project", ""))
results.map { it.scanner.name }.toSet() shouldBe setOf("Dummy2")
val results2 = getScanResults(Identifier("SpdxDocumentFile", "", "spdx-dependency", ""))
results2.map { it.scanner.name }.toSet() shouldBe setOf("Dummy")
}
}
}
})
12 changes: 6 additions & 6 deletions scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ class ScannerIntegrationFunTest : WordSpec({
}
})

private fun createScanner(): Scanner {
internal fun createScanner(): Scanner {
val downloaderConfiguration = DownloaderConfiguration()
val workingTreeCache = DefaultWorkingTreeCache()
val provenanceDownloader = DefaultProvenanceDownloader(downloaderConfiguration, workingTreeCache)
Expand Down Expand Up @@ -210,11 +210,11 @@ private val pkg4 = createPackage(
)
)

private class DummyScanner : PathScannerWrapper {
override val name = "Dummy"
override val version = "1.0.0"
override val configuration = ""

internal data class DummyScanner(
override val name: String = "Dummy",
override val version: String = "1.0.0",
override val configuration: String = ""
) : PathScannerWrapper {
override val criteria = ScannerCriteria.forDetails(details)

override fun scanPath(path: File, context: ScanContext): ScanSummary {
Expand Down
12 changes: 10 additions & 2 deletions scanner/src/main/kotlin/Scanner.kt
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ import org.ossreviewtoolkit.utils.spdx.toSpdx
const val TOOL_NAME = "scanner"

@Suppress("TooManyFunctions")
class Scanner(
data class Scanner(
val scannerConfig: ScannerConfiguration,
val downloaderConfig: DownloaderConfiguration,
val provenanceDownloader: ProvenanceDownloader,
Expand Down Expand Up @@ -111,12 +111,16 @@ class Scanner(

val projectScannerWrappers = scannerWrappers[PackageType.PROJECT].orEmpty()
val packageScannerWrappers = scannerWrappers[PackageType.PACKAGE].orEmpty()
val scannersByPackages = mutableMapOf<Identifier, Set<String>>()

val projectResults = if (projectScannerWrappers.isNotEmpty()) {
val packages = ortResult.getProjects(skipExcluded).mapTo(mutableSetOf()) { it.toPackage() }

logger.info { "Scanning ${packages.size} project(s) with ${projectScannerWrappers.size} scanner(s)." }

val projectScanners = projectScannerWrappers.map { it.name }.toSet()
packages.associateByTo(scannersByPackages, { it.id }) { projectScanners }

scan(
packages,
ScanContext(
Expand All @@ -137,6 +141,9 @@ class Scanner(

logger.info { "Scanning ${packages.size} package(s) with ${packageScannerWrappers.size} scanner(s)." }

val packageScanners = packageScannerWrappers.map { it.name }.toSet()
packages.associateByTo(scannersByPackages, { it.id }) { packageScanners }

scan(packages, ScanContext(ortResult.labels, PackageType.PACKAGE, ortResult.repository.config.excludes))
} else {
logger.info { "Skipping package scan as no package scanner is configured." }
Expand Down Expand Up @@ -169,7 +176,8 @@ class Scanner(
config = filteredScannerConfig,
provenances = projectResults.provenances + packageResults.provenances,
scanResults = projectResults.scanResults + packageResults.scanResults,
files = projectResults.files + packageResults.files
files = projectResults.files + packageResults.files,
scanners = scannersByPackages
)

return ortResult.copy(scanner = scannerRun)
Expand Down

0 comments on commit b90e2a3

Please sign in to comment.