Skip to content

Commit

Permalink
fix(scanner): Filter scan results by scanner name
Browse files Browse the repository at this point in the history
When a project scanner is configured alongside a package scanner, the
scan results could be duplicated if the provenances of those packages are
overlapping (e.g. in the case of repositories with Git submodules).
This commit addresses the bug by constructing a map of packages with their
scanners and filtering the scan results using this map.

Fixes oss-review-toolkit#7231.

Signed-off-by: Nicolas Nobelis <nicolas.nobelis@bosch.io>
  • Loading branch information
nnobelis committed Jul 12, 2023
1 parent 8e3b457 commit 187571e
Show file tree
Hide file tree
Showing 5 changed files with 146 additions and 10 deletions.
12 changes: 11 additions & 1 deletion model/src/main/kotlin/ScannerRun.kt
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import org.ossreviewtoolkit.model.config.ScannerConfiguration
import org.ossreviewtoolkit.model.utils.FileListSortedSetConverter
import org.ossreviewtoolkit.model.utils.ProvenanceResolutionResultSortedSetConverter
import org.ossreviewtoolkit.model.utils.ScanResultSortedSetConverter
import org.ossreviewtoolkit.model.utils.ScannersMapConverter
import org.ossreviewtoolkit.model.utils.getKnownProvenancesWithoutVcsPath
import org.ossreviewtoolkit.model.utils.mergeScanResultsByScanner
import org.ossreviewtoolkit.model.utils.prependPath
Expand Down Expand Up @@ -72,6 +73,12 @@ data class ScannerRun(
@JsonSerialize(converter = ScanResultSortedSetConverter::class)
val scanResults: Set<ScanResult>,

/**
* The scanners for each scanned package.
*/
@JsonSerialize(converter = ScannersMapConverter::class)
val scanners: Map<Identifier, Set<String>>,

/**
* The list of files for each resolved provenance.
*/
Expand All @@ -90,7 +97,8 @@ data class ScannerRun(
config = ScannerConfiguration(),
provenances = emptySet(),
scanResults = emptySet(),
files = emptySet()
files = emptySet(),
scanners = emptyMap()
)
}

Expand Down Expand Up @@ -196,6 +204,8 @@ data class ScannerRun(

val scanResultsByPath = resolutionResult.getKnownProvenancesWithoutVcsPath().mapValues { (_, provenance) ->
scanResultsByProvenance[provenance].orEmpty()
}.mapValues { scanResults ->
scanResults.value.filter { it.scanner.name in scanners[id].orEmpty() }
}

val scanResults = mergeScanResultsByScanner(scanResultsByPath).map { scanResult ->
Expand Down
6 changes: 6 additions & 0 deletions model/src/main/kotlin/utils/SortedSetConverters.kt
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import java.util.SortedSet
import org.ossreviewtoolkit.model.ArtifactProvenance
import org.ossreviewtoolkit.model.CopyrightFinding
import org.ossreviewtoolkit.model.FileList
import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.LicenseFinding
import org.ossreviewtoolkit.model.Package
import org.ossreviewtoolkit.model.PackageReference
Expand Down Expand Up @@ -74,6 +75,11 @@ class ProvenanceResolutionResultSortedSetConverter :
override fun convert(value: Set<ProvenanceResolutionResult>) = value.toSortedSet(compareBy { it.id })
}

class ScannersMapConverter : StdConverter<Map<Identifier, Set<String>>, Map<Identifier, Set<String>>>() {
override fun convert(value: Map<Identifier, Set<String>>) =
value.mapValues { it.value.toSortedSet() }.toSortedMap(compareBy { it })
}

/** Do not convert to SortedSet in order to not require a comparator consistent with equals */
class ScanResultSortedSetConverter : StdConverter<Set<ScanResult>, Set<ScanResult>>() {
override fun convert(value: Set<ScanResult>) = value.sortedBy { it.provenance.getSortKey() }.toSet()
Expand Down
114 changes: 114 additions & 0 deletions scanner/src/funTest/kotlin/scanners/MultipleScannersTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
* Copyright (C) 2023 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.scanner.scanners

import io.kotest.core.spec.style.WordSpec
import io.kotest.matchers.shouldBe

import org.ossreviewtoolkit.model.AnalyzerResult
import org.ossreviewtoolkit.model.AnalyzerRun
import org.ossreviewtoolkit.model.DependencyGraph
import org.ossreviewtoolkit.model.DependencyGraphNode
import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.OrtResult
import org.ossreviewtoolkit.model.Package
import org.ossreviewtoolkit.model.PackageLinkage
import org.ossreviewtoolkit.model.PackageType
import org.ossreviewtoolkit.model.Project
import org.ossreviewtoolkit.model.RootDependencyIndex
import org.ossreviewtoolkit.model.VcsInfo
import org.ossreviewtoolkit.model.VcsType
import org.ossreviewtoolkit.utils.test.shouldNotBeNull

private const val DEFAULT_NS = "SpdxDocumentFile"
private const val DEFAULT_VERSION = ""

class MultipleScannersTest : WordSpec({
"Scanning a project and a package with overlapping provenance and non overlapping scanners" should {
val analyzerResult = createAnalyzerResult()

val dummyScanners = DummyScanner()
val ortResult = createScanner().copy(
scannerWrappers = mapOf(
PackageType.PROJECT to listOf(dummyScanners.copy(name = "Dummy2")),
PackageType.PACKAGE to listOf(dummyScanners)
)
).scan(analyzerResult, skipExcluded = false, emptyMap())

"return scan results with non-overlapping scanners" {
ortResult.scanner.shouldNotBeNull {
val results = getScanResults(Identifier(DEFAULT_NS, "", "spdx-project", DEFAULT_VERSION))
results.map { it.scanner.name }.toSet() shouldBe setOf("Dummy2")
val results2 = getScanResults(Identifier(DEFAULT_NS, "", "spdx-dependency", DEFAULT_VERSION))
results2.map { it.scanner.name }.toSet() shouldBe setOf("Dummy")
}
}
}
})

private fun createAnalyzerResult(): OrtResult {
val vcsPackage = VcsInfo(
type = VcsType.GIT,
url = "https://github.com/nnobelis/spdx-dependency.git",
revision = "78c8a5b12ae2d0f613fff25f2d1eb746a768ece7",
path = ""
)

val pkg = Package.EMPTY.copy(
id = createId("spdx-dependency"),
vcs = vcsPackage,
vcsProcessed = vcsPackage.normalize()
)

val vcsProject = VcsInfo(
type = VcsType.GIT,
url = "https://github.com/nnobelis/spdx-project.git",
revision = "8fae7f2b0efa8addeadfd15041e3437d92ff6b71",
path = ""
)

val project = Project.EMPTY.copy(
id = createId("spdx-project"),
vcs = vcsProject,
vcsProcessed = vcsProject.normalize()
)

val scopes = mapOf(":spdx-project::default" to listOf(RootDependencyIndex(0, 0)))
val dependencyGraph = mapOf(
DEFAULT_NS to DependencyGraph(
listOf(pkg.id),
scopes = scopes,
nodes = listOf(
DependencyGraphNode(0, 0, PackageLinkage.STATIC)
)
)
)
val analyzerRun = AnalyzerRun.EMPTY.copy(
result = AnalyzerResult.EMPTY.copy(
projects = setOf(project),
packages = setOf(pkg),
dependencyGraphs = dependencyGraph
)
)

return OrtResult.EMPTY.copy(analyzer = analyzerRun)
}

private fun createId(name: String): Identifier = Identifier("$DEFAULT_NS::$name:$DEFAULT_VERSION")
12 changes: 6 additions & 6 deletions scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ class ScannerIntegrationFunTest : WordSpec({
}
})

private fun createScanner(): Scanner {
internal fun createScanner(): Scanner {
val downloaderConfiguration = DownloaderConfiguration()
val workingTreeCache = DefaultWorkingTreeCache()
val provenanceDownloader = DefaultProvenanceDownloader(downloaderConfiguration, workingTreeCache)
Expand Down Expand Up @@ -210,11 +210,11 @@ private val pkg4 = createPackage(
)
)

private class DummyScanner : PathScannerWrapper {
override val name = "Dummy"
override val version = "1.0.0"
override val configuration = ""

internal data class DummyScanner(
override val name: String = "Dummy",
override val version: String = "1.0.0",
override val configuration: String = ""
) : PathScannerWrapper {
override val criteria = ScannerCriteria.forDetails(details)

override fun scanPath(path: File, context: ScanContext): ScanSummary {
Expand Down
12 changes: 9 additions & 3 deletions scanner/src/main/kotlin/Scanner.kt
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ import org.ossreviewtoolkit.utils.spdx.toSpdx
const val TOOL_NAME = "scanner"

@Suppress("TooManyFunctions")
class Scanner(
data class Scanner(
val scannerConfig: ScannerConfiguration,
val downloaderConfig: DownloaderConfiguration,
val provenanceDownloader: ProvenanceDownloader,
Expand Down Expand Up @@ -169,7 +169,8 @@ class Scanner(
config = filteredScannerConfig,
provenances = projectResults.provenances + packageResults.provenances,
scanResults = projectResults.scanResults + packageResults.scanResults,
files = projectResults.files + packageResults.files
files = projectResults.files + packageResults.files,
scanners = projectResults.scanners + packageResults.scanners
)

return ortResult.copy(scanner = scannerRun)
Expand Down Expand Up @@ -238,11 +239,16 @@ class Scanner(
}
}

val scannersByPackages = mutableMapOf<Identifier, Set<String>>()
val scannerNames = scanners.map { it.name }.toSet()
packages.associateByTo(scannersByPackages, { it.id }) { scannerNames }

return ScannerRun.EMPTY.copy(
config = scannerConfig,
provenances = provenances,
scanResults = scanResults,
files = files
files = files,
scanners = scannersByPackages
)
}

Expand Down

0 comments on commit 187571e

Please sign in to comment.