Skip to content

Commit

Permalink
fix(scanner): Filter scan results by scanner name
Browse files Browse the repository at this point in the history
The scanner may use different set of scanners for scanning different
packages. The same holds for projects. `ScannerRun` does not contain that
information, and `ScannerRun.getScanResultsById()` just returns all scans
matching the package (nested) provenance, disregarding which scanners are
to be used for the particular package.
Extend the data model to contain that information and use it to adjust the
internals of `getScanResultsById()` to adhere to the set of scanners to be
used for the particular project or package.

Fixes oss-review-toolkit#7231.

Signed-off-by: Nicolas Nobelis <nicolas.nobelis@bosch.io>
  • Loading branch information
nnobelis committed Jul 14, 2023
1 parent 8e3b457 commit 141d5c2
Show file tree
Hide file tree
Showing 5 changed files with 129 additions and 10 deletions.
12 changes: 11 additions & 1 deletion model/src/main/kotlin/ScannerRun.kt
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import org.ossreviewtoolkit.model.config.ScannerConfiguration
import org.ossreviewtoolkit.model.utils.FileListSortedSetConverter
import org.ossreviewtoolkit.model.utils.ProvenanceResolutionResultSortedSetConverter
import org.ossreviewtoolkit.model.utils.ScanResultSortedSetConverter
import org.ossreviewtoolkit.model.utils.ScannersMapConverter
import org.ossreviewtoolkit.model.utils.getKnownProvenancesWithoutVcsPath
import org.ossreviewtoolkit.model.utils.mergeScanResultsByScanner
import org.ossreviewtoolkit.model.utils.prependPath
Expand Down Expand Up @@ -72,6 +73,12 @@ data class ScannerRun(
@JsonSerialize(converter = ScanResultSortedSetConverter::class)
val scanResults: Set<ScanResult>,

/**
* The scanners for each scanned package.
*/
@JsonSerialize(converter = ScannersMapConverter::class)
val scanners: Map<Identifier, Set<String>>,

/**
* The list of files for each resolved provenance.
*/
Expand All @@ -90,7 +97,8 @@ data class ScannerRun(
config = ScannerConfiguration(),
provenances = emptySet(),
scanResults = emptySet(),
files = emptySet()
files = emptySet(),
scanners = emptyMap()
)
}

Expand Down Expand Up @@ -196,6 +204,8 @@ data class ScannerRun(

val scanResultsByPath = resolutionResult.getKnownProvenancesWithoutVcsPath().mapValues { (_, provenance) ->
scanResultsByProvenance[provenance].orEmpty()
}.mapValues { scanResults ->
scanResults.value.filter { it.scanner.name in scanners[id].orEmpty() }
}

val scanResults = mergeScanResultsByScanner(scanResultsByPath).map { scanResult ->
Expand Down
6 changes: 6 additions & 0 deletions model/src/main/kotlin/utils/SortedSetConverters.kt
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import java.util.SortedSet
import org.ossreviewtoolkit.model.ArtifactProvenance
import org.ossreviewtoolkit.model.CopyrightFinding
import org.ossreviewtoolkit.model.FileList
import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.LicenseFinding
import org.ossreviewtoolkit.model.Package
import org.ossreviewtoolkit.model.PackageReference
Expand Down Expand Up @@ -74,6 +75,11 @@ class ProvenanceResolutionResultSortedSetConverter :
override fun convert(value: Set<ProvenanceResolutionResult>) = value.toSortedSet(compareBy { it.id })
}

class ScannersMapConverter : StdConverter<Map<Identifier, Set<String>>, Map<Identifier, Set<String>>>() {
override fun convert(value: Map<Identifier, Set<String>>) =
value.mapValues { it.value.toSortedSet() }.toSortedMap(compareBy { it })
}

/** Do not convert to SortedSet in order to not require a comparator consistent with equals */
class ScanResultSortedSetConverter : StdConverter<Set<ScanResult>, Set<ScanResult>>() {
override fun convert(value: Set<ScanResult>) = value.sortedBy { it.provenance.getSortKey() }.toSet()
Expand Down
97 changes: 97 additions & 0 deletions scanner/src/funTest/kotlin/scanners/MultipleScannersTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
* Copyright (C) 2023 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.scanner.scanners

import io.kotest.core.spec.style.WordSpec
import io.kotest.matchers.collections.containExactly
import io.kotest.matchers.should

import org.ossreviewtoolkit.model.AnalyzerResult
import org.ossreviewtoolkit.model.AnalyzerRun
import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.OrtResult
import org.ossreviewtoolkit.model.Package
import org.ossreviewtoolkit.model.PackageType
import org.ossreviewtoolkit.model.Project
import org.ossreviewtoolkit.model.VcsInfo
import org.ossreviewtoolkit.model.VcsType
import org.ossreviewtoolkit.utils.test.shouldNotBeNull

private val PROJECT_IDENTIFIER = Identifier("Dummy", "", "project", "1.0.0")
private val PACKAGE_IDENTIFIER = Identifier("Dummy", "", "pkg1", "1.0.0")

class MultipleScannersTest : WordSpec({
"Scanning a project and a package with overlapping provenance and non overlapping scanners" should {
val analyzerResult = createAnalyzerResult()

val dummyScanner = DummyScanner()
val scannerWrappers = mapOf(
PackageType.PROJECT to listOf(dummyScanner.copy(name = "Dummy2")),
PackageType.PACKAGE to listOf(dummyScanner)
)
val ortResult = createScanner(scannerWrappers).scan(analyzerResult, skipExcluded = false, emptyMap())

"return scan results with non-overlapping scanners" {
ortResult.scanner.shouldNotBeNull {
val results = getScanResults(PROJECT_IDENTIFIER)
results.map { it.scanner.name }.toSet() should containExactly("Dummy2")
val results2 = getScanResults(PACKAGE_IDENTIFIER)
results2.map { it.scanner.name }.toSet() should containExactly("Dummy")
}
}
}
})

private fun createAnalyzerResult(): OrtResult {
val vcsPackage = VcsInfo(
type = VcsType.GIT,
url = "https://github.com/oss-review-toolkit/ort-test-data-scanner.git",
revision = "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec",
path = ""
)

val pkg = Package.EMPTY.copy(
id = PACKAGE_IDENTIFIER,
vcs = vcsPackage,
vcsProcessed = vcsPackage.normalize()
)

val vcsProject = VcsInfo(
type = VcsType.GIT,
url = "https://github.com/oss-review-toolkit/ort-test-data-scanner.git",
revision = "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec",
path = ""
)

val project = Project.EMPTY.copy(
id = PROJECT_IDENTIFIER,
vcs = vcsProject,
vcsProcessed = vcsProject.normalize()
)

val analyzerRun = AnalyzerRun.EMPTY.copy(
result = AnalyzerResult.EMPTY.copy(
projects = setOf(project),
packages = setOf(pkg)
)
)

return OrtResult.EMPTY.copy(analyzer = analyzerRun)
}
15 changes: 8 additions & 7 deletions scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import org.ossreviewtoolkit.scanner.PathScannerWrapper
import org.ossreviewtoolkit.scanner.ScanContext
import org.ossreviewtoolkit.scanner.Scanner
import org.ossreviewtoolkit.scanner.ScannerCriteria
import org.ossreviewtoolkit.scanner.ScannerWrapper
import org.ossreviewtoolkit.scanner.provenance.DefaultNestedProvenanceResolver
import org.ossreviewtoolkit.scanner.provenance.DefaultPackageProvenanceResolver
import org.ossreviewtoolkit.scanner.provenance.DefaultProvenanceDownloader
Expand Down Expand Up @@ -100,7 +101,7 @@ class ScannerIntegrationFunTest : WordSpec({
}
})

private fun createScanner(): Scanner {
internal fun createScanner(scannerWrappers: Map<PackageType, List<ScannerWrapper>>? = null): Scanner {
val downloaderConfiguration = DownloaderConfiguration()
val workingTreeCache = DefaultWorkingTreeCache()
val provenanceDownloader = DefaultProvenanceDownloader(downloaderConfiguration, workingTreeCache)
Expand All @@ -118,7 +119,7 @@ private fun createScanner(): Scanner {
storageWriters = emptyList(),
packageProvenanceResolver = packageProvenanceResolver,
nestedProvenanceResolver = nestedProvenanceResolver,
scannerWrappers = mapOf(
scannerWrappers = scannerWrappers ?: mapOf(
PackageType.PROJECT to listOf(dummyScanner),
PackageType.PACKAGE to listOf(dummyScanner)
)
Expand Down Expand Up @@ -210,11 +211,11 @@ private val pkg4 = createPackage(
)
)

private class DummyScanner : PathScannerWrapper {
override val name = "Dummy"
override val version = "1.0.0"
override val configuration = ""

internal data class DummyScanner(
override val name: String = "Dummy",
override val version: String = "1.0.0",
override val configuration: String = ""
) : PathScannerWrapper {
override val criteria = ScannerCriteria.forDetails(details)

override fun scanPath(path: File, context: ScanContext): ScanSummary {
Expand Down
9 changes: 7 additions & 2 deletions scanner/src/main/kotlin/Scanner.kt
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,8 @@ class Scanner(
config = filteredScannerConfig,
provenances = projectResults.provenances + packageResults.provenances,
scanResults = projectResults.scanResults + packageResults.scanResults,
files = projectResults.files + packageResults.files
files = projectResults.files + packageResults.files,
scanners = projectResults.scanners + packageResults.scanners
)

return ortResult.copy(scanner = scannerRun)
Expand Down Expand Up @@ -238,11 +239,15 @@ class Scanner(
}
}

val scannerNames = scanners.mapTo(mutableSetOf()) { it.name }
val scannerNamesByPackageId = packages.associateBy({ it.id }) { scannerNames }

return ScannerRun.EMPTY.copy(
config = scannerConfig,
provenances = provenances,
scanResults = scanResults,
files = files
files = files,
scanners = scannerNamesByPackageId
)
}

Expand Down

0 comments on commit 141d5c2

Please sign in to comment.