From 2f4c0a4c05e71d0dde03acccb57c944075ba0693 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Tue, 3 Sep 2024 17:39:30 +0200 Subject: [PATCH 1/4] refactor(gradle): Move `OrtDependency` extension functions to the model Prepare for reuse and also perform some minor generalizations. As part of that, hard-code the "Gradle" project type in order to not be dependent on a package manager instance. Signed-off-by: Sebastian Schuberth --- .../src/main/kotlin/Extensions.kt | 38 +++++++++++++++++++ .../main/kotlin/GradleDependencyHandler.kt | 19 +--------- .../kotlin/GradleDependencyHandlerTest.kt | 24 +++--------- 3 files changed, 46 insertions(+), 35 deletions(-) create mode 100644 plugins/package-managers/gradle-model/src/main/kotlin/Extensions.kt diff --git a/plugins/package-managers/gradle-model/src/main/kotlin/Extensions.kt b/plugins/package-managers/gradle-model/src/main/kotlin/Extensions.kt new file mode 100644 index 0000000000000..f12699d0d3dca --- /dev/null +++ b/plugins/package-managers/gradle-model/src/main/kotlin/Extensions.kt @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2021 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.gradlemodel + +import OrtDependency + +/** + * Determine the type of this Gradle dependency. In case of a project, it is "Gradle". Otherwise it is "Maven" unless + * there is no POM, then it is "Unknown". + */ +fun OrtDependency.dependencyType(): String = + if (isProjectDependency()) { + "Gradle" + } else { + pomFile?.let { "Maven" } ?: "Unknown" + } + +/** + * Return true if this Gradle dependency refers to another project, or false if it refers to a package. + */ +fun OrtDependency.isProjectDependency() = localPath != null diff --git a/plugins/package-managers/gradle/src/main/kotlin/GradleDependencyHandler.kt b/plugins/package-managers/gradle/src/main/kotlin/GradleDependencyHandler.kt index 78a9ecd9806d6..8be4489e65c25 100644 --- a/plugins/package-managers/gradle/src/main/kotlin/GradleDependencyHandler.kt +++ b/plugins/package-managers/gradle/src/main/kotlin/GradleDependencyHandler.kt @@ -34,6 +34,8 @@ import org.ossreviewtoolkit.model.PackageLinkage import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.createAndLogIssue import org.ossreviewtoolkit.model.utils.DependencyHandler +import org.ossreviewtoolkit.plugins.packagemanagers.gradlemodel.dependencyType +import org.ossreviewtoolkit.plugins.packagemanagers.gradlemodel.isProjectDependency import org.ossreviewtoolkit.plugins.packagemanagers.maven.utils.MavenSupport import org.ossreviewtoolkit.plugins.packagemanagers.maven.utils.identifier import org.ossreviewtoolkit.utils.common.collectMessages @@ -117,21 +119,4 @@ internal class GradleDependencyHandler( } } } - - /** - * Determine the type of this dependency. This manager implementation uses Maven to resolve packages, so - * the type of dependencies to packages is typically _Maven_ unless no pom is available. Only for module - * dependencies, the type of this manager is used. - */ - private fun OrtDependency.dependencyType(): String = - if (isProjectDependency()) { - managerName - } else { - pomFile?.let { "Maven" } ?: "Unknown" - } } - -/** - * Return a flag whether this dependency references another project in the current build. - */ -private fun OrtDependency.isProjectDependency() = localPath != null diff --git a/plugins/package-managers/gradle/src/test/kotlin/GradleDependencyHandlerTest.kt b/plugins/package-managers/gradle/src/test/kotlin/GradleDependencyHandlerTest.kt index b12914b2c36e7..234acb00e9522 100644 --- a/plugins/package-managers/gradle/src/test/kotlin/GradleDependencyHandlerTest.kt +++ b/plugins/package-managers/gradle/src/test/kotlin/GradleDependencyHandlerTest.kt @@ -57,6 +57,7 @@ import org.ossreviewtoolkit.model.Scope import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.utils.DependencyGraphBuilder +import org.ossreviewtoolkit.plugins.packagemanagers.gradlemodel.dependencyType import org.ossreviewtoolkit.plugins.packagemanagers.maven.utils.MavenSupport /** @@ -131,7 +132,7 @@ class GradleDependencyHandlerTest : WordSpec({ val scopes = graph.createScopes() - scopeDependencies(scopes, scope).single { it.id.type == NAME } + scopeDependencies(scopes, scope).single { it.id.type == "Gradle" } } "collect information about packages" { @@ -312,13 +313,13 @@ class GradleDependencyHandlerTest : WordSpec({ val issues = mutableListOf() every { maven.parsePackage(any(), any(), useReposFromDependencies = false) } throws exception - val handler = GradleDependencyHandler(NAME, maven) + val handler = GradleDependencyHandler("Gradle", maven) handler.createPackage(dep, issues) should beNull() issues should haveSize(1) with(issues.first()) { - source shouldBe NAME + source shouldBe "Gradle" severity shouldBe Severity.ERROR message should contain("${dep.groupId}:${dep.artifactId}:${dep.version}") } @@ -326,9 +327,6 @@ class GradleDependencyHandlerTest : WordSpec({ } }) -/** The name of the package manager. */ -private const val NAME = "GradleTest" - /** Remote repositories used by the test. */ private val remoteRepositories = listOf(mockk()) @@ -361,7 +359,7 @@ private fun createDependency( * this class. */ private fun createGraphBuilder(): DependencyGraphBuilder { - val dependencyHandler = GradleDependencyHandler(NAME, createMavenSupport()) + val dependencyHandler = GradleDependencyHandler("Gradle", createMavenSupport()) dependencyHandler.repositories = remoteRepositories return DependencyGraphBuilder(dependencyHandler) } @@ -389,20 +387,10 @@ private fun createMavenSupport(): MavenSupport { return maven } -/** - * Determine the type of the [Identifier] for this dependency. - */ -private fun OrtDependency.type(): String = - if (localPath != null) { - NAME - } else { - "Maven" - } - /** * Returns an [Identifier] for this [OrtDependency]. */ -private fun OrtDependency.toId() = Identifier(type(), groupId, artifactId, version) +private fun OrtDependency.toId() = Identifier(dependencyType(), groupId, artifactId, version) /** * Return the package references from the given [scopes] associated with the scope with the given [scopeName]. From 383232f0aad4a80cd7d314739bc4ed140d4e5300 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Tue, 3 Sep 2024 17:48:56 +0200 Subject: [PATCH 2/4] refactor(gradle): Turn extension functions into properties This simplifies the code a bit. Signed-off-by: Sebastian Schuberth --- .../gradle-model/src/main/kotlin/Extensions.kt | 13 +++++++------ .../src/main/kotlin/GradleDependencyHandler.kt | 6 +++--- .../src/test/kotlin/GradleDependencyHandlerTest.kt | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/plugins/package-managers/gradle-model/src/main/kotlin/Extensions.kt b/plugins/package-managers/gradle-model/src/main/kotlin/Extensions.kt index f12699d0d3dca..82b3d91021296 100644 --- a/plugins/package-managers/gradle-model/src/main/kotlin/Extensions.kt +++ b/plugins/package-managers/gradle-model/src/main/kotlin/Extensions.kt @@ -22,17 +22,18 @@ package org.ossreviewtoolkit.plugins.packagemanagers.gradlemodel import OrtDependency /** - * Determine the type of this Gradle dependency. In case of a project, it is "Gradle". Otherwise it is "Maven" unless - * there is no POM, then it is "Unknown". + * The type of this Gradle dependency. In case of a project, it is "Gradle". Otherwise it is "Maven" unless there is no + * POM, then it is "Unknown". */ -fun OrtDependency.dependencyType(): String = - if (isProjectDependency()) { +val OrtDependency.dependencyType: String + get() = if (isProjectDependency) { "Gradle" } else { pomFile?.let { "Maven" } ?: "Unknown" } /** - * Return true if this Gradle dependency refers to another project, or false if it refers to a package. + * A flag to indicate whether this Gradle dependency refers to a project, or to a package. */ -fun OrtDependency.isProjectDependency() = localPath != null +val OrtDependency.isProjectDependency: Boolean + get() = localPath != null diff --git a/plugins/package-managers/gradle/src/main/kotlin/GradleDependencyHandler.kt b/plugins/package-managers/gradle/src/main/kotlin/GradleDependencyHandler.kt index 8be4489e65c25..3595fd7d15638 100644 --- a/plugins/package-managers/gradle/src/main/kotlin/GradleDependencyHandler.kt +++ b/plugins/package-managers/gradle/src/main/kotlin/GradleDependencyHandler.kt @@ -60,7 +60,7 @@ internal class GradleDependencyHandler( override fun identifierFor(dependency: OrtDependency): Identifier = Identifier( - type = dependency.dependencyType(), + type = dependency.dependencyType, namespace = dependency.groupId, name = dependency.artifactId, version = dependency.version @@ -88,11 +88,11 @@ internal class GradleDependencyHandler( ) override fun linkageFor(dependency: OrtDependency): PackageLinkage = - if (dependency.isProjectDependency()) PackageLinkage.PROJECT_DYNAMIC else PackageLinkage.DYNAMIC + if (dependency.isProjectDependency) PackageLinkage.PROJECT_DYNAMIC else PackageLinkage.DYNAMIC override fun createPackage(dependency: OrtDependency, issues: MutableCollection): Package? { // Only look for a package if there was no error resolving the dependency and it is no project dependency. - if (dependency.error != null || dependency.isProjectDependency()) return null + if (dependency.error != null || dependency.isProjectDependency) return null val artifact = DefaultArtifact( dependency.groupId, dependency.artifactId, dependency.classifier, diff --git a/plugins/package-managers/gradle/src/test/kotlin/GradleDependencyHandlerTest.kt b/plugins/package-managers/gradle/src/test/kotlin/GradleDependencyHandlerTest.kt index 234acb00e9522..664bc57922a0d 100644 --- a/plugins/package-managers/gradle/src/test/kotlin/GradleDependencyHandlerTest.kt +++ b/plugins/package-managers/gradle/src/test/kotlin/GradleDependencyHandlerTest.kt @@ -390,7 +390,7 @@ private fun createMavenSupport(): MavenSupport { /** * Returns an [Identifier] for this [OrtDependency]. */ -private fun OrtDependency.toId() = Identifier(dependencyType(), groupId, artifactId, version) +private fun OrtDependency.toId() = Identifier(dependencyType, groupId, artifactId, version) /** * Return the package references from the given [scopes] associated with the scope with the given [scopeName]. From 50dbf07fe04ee1ac22cb85508265ea880d5f3d8b Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Tue, 3 Sep 2024 17:51:15 +0200 Subject: [PATCH 3/4] refactor(gradle-inspector): Make use of `OrtDependency` extensions Signed-off-by: Sebastian Schuberth --- .../src/main/kotlin/GradleInspector.kt | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleInspector.kt b/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleInspector.kt index 51704f6620ce4..f40a709636007 100644 --- a/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleInspector.kt +++ b/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleInspector.kt @@ -350,14 +350,11 @@ private fun Collection.toPackageRefs( packageDependencies: MutableCollection ): Set = mapTo(mutableSetOf()) { dep -> - val (id, linkage) = if (dep.localPath != null) { - val id = Identifier("Gradle", dep.groupId, dep.artifactId, dep.version) - id to PackageLinkage.PROJECT_DYNAMIC + val id = Identifier(dep.dependencyType, dep.groupId, dep.artifactId, dep.version) + val linkage = if (dep.isProjectDependency) { + PackageLinkage.PROJECT_DYNAMIC } else { - packageDependencies += dep - - val id = Identifier("Maven", dep.groupId, dep.artifactId, dep.version) - id to PackageLinkage.DYNAMIC + PackageLinkage.DYNAMIC } PackageReference(id, linkage, dep.dependencies.toPackageRefs(packageDependencies)) From 96d39163e016ef3daa0c429bee888e5f669e81dc Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Tue, 3 Sep 2024 18:08:54 +0200 Subject: [PATCH 4/4] refactor(gradle-inspector): Migrate the code to use the dependency graph See [1] for context. [1]: https://github.com/oss-review-toolkit/ort/issues/3825 Signed-off-by: Sebastian Schuberth --- ...radle-expected-output-lib-without-repo.yml | 10 +- .../main/kotlin/GradleDependencyHandler.kt | 271 ++++++++++++++++++ .../src/main/kotlin/GradleInspector.kt | 261 ++--------------- 3 files changed, 289 insertions(+), 253 deletions(-) create mode 100644 plugins/package-managers/gradle-inspector/src/main/kotlin/GradleDependencyHandler.kt diff --git a/plugins/package-managers/gradle-inspector/src/funTest/assets/projects/synthetic/gradle-expected-output-lib-without-repo.yml b/plugins/package-managers/gradle-inspector/src/funTest/assets/projects/synthetic/gradle-expected-output-lib-without-repo.yml index 54287574ef3cd..ed98762ff10ec 100644 --- a/plugins/package-managers/gradle-inspector/src/funTest/assets/projects/synthetic/gradle-expected-output-lib-without-repo.yml +++ b/plugins/package-managers/gradle-inspector/src/funTest/assets/projects/synthetic/gradle-expected-output-lib-without-repo.yml @@ -15,15 +15,7 @@ project: revision: "" path: "plugins/package-managers/gradle/src/funTest/assets/projects/synthetic/gradle/lib-without-repo" homepage_url: "" - scopes: - - name: "compileClasspath" - dependencies: [] - - name: "runtimeClasspath" - dependencies: [] - - name: "testCompileClasspath" - dependencies: [] - - name: "testRuntimeClasspath" - dependencies: [] + scopes: [] packages: [] issues: - timestamp: "1970-01-01T00:00:00Z" diff --git a/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleDependencyHandler.kt b/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleDependencyHandler.kt new file mode 100644 index 0000000000000..ccb854f80d664 --- /dev/null +++ b/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleDependencyHandler.kt @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.gradleinspector + +import OrtDependency + +import java.lang.invoke.MethodHandles + +import org.apache.logging.log4j.kotlin.logger +import org.apache.logging.log4j.kotlin.loggerOf + +import org.ossreviewtoolkit.analyzer.PackageManager.Companion.processPackageVcs +import org.ossreviewtoolkit.downloader.VcsHost +import org.ossreviewtoolkit.model.Hash +import org.ossreviewtoolkit.model.HashAlgorithm +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.Issue +import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.PackageLinkage +import org.ossreviewtoolkit.model.RemoteArtifact +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.createAndLogIssue +import org.ossreviewtoolkit.model.orEmpty +import org.ossreviewtoolkit.model.utils.DependencyHandler +import org.ossreviewtoolkit.model.utils.parseRepoManifestPath +import org.ossreviewtoolkit.plugins.packagemanagers.gradlemodel.dependencyType +import org.ossreviewtoolkit.plugins.packagemanagers.gradlemodel.isProjectDependency +import org.ossreviewtoolkit.utils.common.splitOnWhitespace +import org.ossreviewtoolkit.utils.common.withoutPrefix +import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor +import org.ossreviewtoolkit.utils.ort.downloadText +import org.ossreviewtoolkit.utils.ort.okHttpClient +import org.ossreviewtoolkit.utils.spdx.SpdxOperator + +/** + * A specialized [DependencyHandler] implementation for Gradle's dependency model. + */ +internal class GradleDependencyHandler : DependencyHandler { + override fun identifierFor(dependency: OrtDependency): Identifier = + with(dependency) { Identifier(dependencyType, groupId, artifactId, version) } + + override fun dependenciesFor(dependency: OrtDependency): List = dependency.dependencies + + override fun linkageFor(dependency: OrtDependency): PackageLinkage = + if (dependency.isProjectDependency) PackageLinkage.PROJECT_DYNAMIC else PackageLinkage.DYNAMIC + + override fun createPackage(dependency: OrtDependency, issues: MutableCollection): Package? { + // Only look for a package if there was no error resolving the dependency and it is no project dependency. + if (dependency.error != null || dependency.isProjectDependency) return null + + val id = identifierFor(dependency) + val model = dependency.mavenModel ?: run { + issues += createAndLogIssue( + source = "Gradle", + message = "No Maven model available for '${id.toCoordinates()}'." + ) + + return null + } + + val isSpringMetadataProject = with(id) { + listOf("boot", "cloud").any { + namespace == "org.springframework.$it" + && (name.startsWith("spring-$it-starter") || name.startsWith("spring-$it-contract-spec")) + } + } + + val isMetadataOnly = dependency.extension == "pom" || isSpringMetadataProject + + val binaryArtifact = when { + isMetadataOnly -> RemoteArtifact.EMPTY + else -> with(dependency) { + createRemoteArtifact(pomFile, classifier, extension.takeUnless { it == "bundle" }) + } + } + + val sourceArtifact = when { + isMetadataOnly -> RemoteArtifact.EMPTY + else -> createRemoteArtifact(dependency.pomFile, "sources", "jar") + } + + val vcs = dependency.toVcsInfo() + val vcsFallbackUrls = listOfNotNull(model.vcs?.browsableUrl, model.homepageUrl).toTypedArray() + val vcsProcessed = processPackageVcs(vcs, *vcsFallbackUrls) + + return Package( + id = id, + authors = model.authors, + declaredLicenses = model.licenses, + declaredLicensesProcessed = DeclaredLicenseProcessor.process( + model.licenses, + // See http://maven.apache.org/ref/3.6.3/maven-model/maven.html#project saying: "If multiple + // licenses are listed, it is assumed that the user can select any of them, not that they must + // accept all." + operator = SpdxOperator.OR + ), + description = model.description.orEmpty(), + homepageUrl = model.homepageUrl.orEmpty(), + binaryArtifact = binaryArtifact, + sourceArtifact = sourceArtifact, + vcs = vcs, + vcsProcessed = vcsProcessed, + isMetadataOnly = isMetadataOnly + ) + } +} + +// See http://maven.apache.org/pom.html#SCM. +private val SCM_REGEX = Regex("scm:(?[^:@]+):(?.+)") +private val USER_HOST_REGEX = Regex("scm:(?[^:@]+)@(?[^:]+)[:/](?.+)") + +private val logger = loggerOf(MethodHandles.lookup().lookupClass()) + +private fun OrtDependency.toVcsInfo(): VcsInfo = + mavenModel?.vcs?.run { + @Suppress("UnsafeCallOnNullableType") + SCM_REGEX.matchEntire(connection)?.let { match -> + val type = match.groups["type"]!!.value + val url = match.groups["url"]!!.value + + handleValidScmInfo(type, url, tag) + } ?: handleInvalidScmInfo(connection, tag) + }.orEmpty() + +private fun OrtDependency.handleValidScmInfo(type: String, url: String, tag: String): VcsInfo = + when { + // Maven does not officially support git-repo as an SCM, see http://maven.apache.org/scm/scms-overview.html, so + // come up with the convention to use the "manifest" query parameter for the path to the manifest inside the + // repository. An earlier version of this workaround expected the query string to be only the path to the + // manifest, for backward compatibility convert such URLs to the new syntax. + type == "git-repo" -> { + val manifestPath = url.parseRepoManifestPath() + ?: url.substringAfter('?').takeIf { it.isNotBlank() && it.endsWith(".xml") } + val urlWithManifest = url.takeIf { manifestPath == null } + ?: "${url.substringBefore('?')}?manifest=$manifestPath" + + VcsInfo( + type = VcsType.GIT_REPO, + url = urlWithManifest, + revision = tag + ) + } + + type == "svn" -> { + val revision = tag.takeIf { it.isEmpty() } ?: "tags/$tag" + VcsInfo(type = VcsType.SUBVERSION, url = url, revision = revision) + } + + url.startsWith("//") -> { + // Work around the common mistake to omit the Maven SCM provider. + val fixedUrl = "$type:$url" + + // Try to detect the Maven SCM provider from the URL only, e.g. by looking at the host or special URL paths. + VcsHost.parseUrl(fixedUrl).copy(revision = tag).also { + logger.info { + "Fixed up invalid SCM connection without a provider in '$groupId:$artifactId:$version' to $it." + } + } + } + + else -> { + val trimmedUrl = if (!url.startsWith("git://")) url.removePrefix("git:") else url + + VcsHost.fromUrl(trimmedUrl)?.let { host -> + host.toVcsInfo(trimmedUrl)?.let { vcsInfo -> + // Fixup paths that are specified as part of the URL and contain the project name as a prefix. + val projectPrefix = "${host.getProject(trimmedUrl)}-" + vcsInfo.path.withoutPrefix(projectPrefix)?.let { path -> + vcsInfo.copy(path = path) + } + } + } ?: VcsInfo(type = VcsType.forName(type), url = trimmedUrl, revision = tag) + } + } + +private fun OrtDependency.handleInvalidScmInfo(connection: String, tag: String): VcsInfo = + @Suppress("UnsafeCallOnNullableType") + USER_HOST_REGEX.matchEntire(connection)?.let { match -> + // Some projects omit the provider and use the SCP-like Git URL syntax, for example + // "scm:git@github.com:facebook/facebook-android-sdk.git". + val user = match.groups["user"]!!.value + val host = match.groups["host"]!!.value + val path = match.groups["path"]!!.value + + if (user == "git" || host.startsWith("git")) { + VcsInfo(type = VcsType.GIT, url = "https://$host/$path", revision = tag) + } else { + VcsInfo.EMPTY + } + } ?: run { + val dep = "$groupId:$artifactId:$version" + + if (connection.startsWith("git://") || connection.endsWith(".git")) { + // It is a common mistake to omit the "scm:[provider]:" prefix. Add fall-backs for nevertheless clear + // cases. + logger.info { + "Maven SCM connection '$connection' in '$dep' lacks the required 'scm' prefix." + } + + VcsInfo(type = VcsType.GIT, url = connection, revision = tag) + } else { + if (connection.isNotEmpty()) { + logger.info { + "Ignoring Maven SCM connection '$connection' in '$dep' due to an unexpected format." + } + } + + VcsInfo.EMPTY + } + } + +/** + * Create a [RemoteArtifact] based on the given [pomUrl], [classifier] and [extension]. The hash value is retrieved + * remotely. + */ +private fun createRemoteArtifact( + pomUrl: String?, + classifier: String? = null, + extension: String? = null +): RemoteArtifact { + val algorithm = "sha1" + val artifactBaseUrl = pomUrl?.removeSuffix(".pom") ?: return RemoteArtifact.EMPTY + + val artifactUrl = buildString { + append(artifactBaseUrl) + if (!classifier.isNullOrEmpty()) append("-$classifier") + if (!extension.isNullOrEmpty()) append(".$extension") else append(".jar") + } + + // TODO: How to handle authentication for private repositories here, or rely on Gradle for the download? + val checksum = okHttpClient.downloadText("$artifactUrl.$algorithm") + .getOrElse { return RemoteArtifact.EMPTY } + + val hash = parseChecksum(checksum, algorithm) + + // Ignore file with zero byte size, because it cannot be a valid archive. + if (hash.value == HashAlgorithm.SHA1.emptyValue) { + logger.info { "Ignoring zero byte size artifact: $artifactUrl" } + return RemoteArtifact.EMPTY + } + + return RemoteArtifact(artifactUrl, hash) +} + +/** + * Split the provided [checksum] by whitespace and return a [Hash] for the first element that matches the provided + * algorithm. If no element matches, return [Hash.NONE]. This works around the issue that Maven checksum files sometimes + * contain arbitrary strings before or after the actual checksum. + */ +private fun parseChecksum(checksum: String, algorithm: String) = + checksum.splitOnWhitespace().firstNotNullOfOrNull { + runCatching { Hash(it, algorithm) }.getOrNull() + } ?: Hash.NONE diff --git a/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleInspector.kt b/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleInspector.kt index f40a709636007..e4c21d9843cfd 100644 --- a/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleInspector.kt +++ b/plugins/package-managers/gradle-inspector/src/main/kotlin/GradleInspector.kt @@ -19,7 +19,6 @@ package org.ossreviewtoolkit.plugins.packagemanagers.gradleinspector -import OrtDependency import OrtDependencyTreeModel import java.io.ByteArrayOutputStream @@ -36,38 +35,25 @@ import org.gradle.tooling.model.build.BuildEnvironment import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory import org.ossreviewtoolkit.analyzer.PackageManager -import org.ossreviewtoolkit.downloader.VcsHost +import org.ossreviewtoolkit.analyzer.PackageManagerResult import org.ossreviewtoolkit.downloader.VersionControlSystem -import org.ossreviewtoolkit.model.Hash -import org.ossreviewtoolkit.model.HashAlgorithm +import org.ossreviewtoolkit.model.DependencyGraph import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.Issue -import org.ossreviewtoolkit.model.Package -import org.ossreviewtoolkit.model.PackageLinkage -import org.ossreviewtoolkit.model.PackageReference import org.ossreviewtoolkit.model.Project import org.ossreviewtoolkit.model.ProjectAnalyzerResult -import org.ossreviewtoolkit.model.RemoteArtifact -import org.ossreviewtoolkit.model.Scope import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.VcsInfo -import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.AnalyzerConfiguration import org.ossreviewtoolkit.model.config.PackageManagerConfiguration import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.model.createAndLogIssue -import org.ossreviewtoolkit.model.orEmpty -import org.ossreviewtoolkit.model.utils.parseRepoManifestPath +import org.ossreviewtoolkit.model.utils.DependencyGraphBuilder import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.safeMkdirs import org.ossreviewtoolkit.utils.common.splitOnWhitespace import org.ossreviewtoolkit.utils.common.unquote -import org.ossreviewtoolkit.utils.common.withoutPrefix -import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor -import org.ossreviewtoolkit.utils.ort.downloadText -import org.ossreviewtoolkit.utils.ort.okHttpClient import org.ossreviewtoolkit.utils.ort.ortToolsDirectory -import org.ossreviewtoolkit.utils.spdx.SpdxOperator import org.semver4j.Semver @@ -117,6 +103,7 @@ class GradleInspector( ) = GradleInspector(type, analysisRoot, analyzerConfig, repoConfig) } + private val graphBuilder = DependencyGraphBuilder(GradleDependencyHandler()) private val initScriptFile by lazy { extractInitScript() } private fun extractInitScript(): File { @@ -232,12 +219,15 @@ class GradleInspector( version = dependencyTreeModel.version ) - val packageDependencies = mutableSetOf() - - val scopes = dependencyTreeModel.configurations.filterNot { + dependencyTreeModel.configurations.filterNot { excludes.isScopeExcluded(it.name) - }.mapTo(mutableSetOf()) { - Scope(name = it.name, dependencies = it.dependencies.toPackageRefs(packageDependencies)) + }.forEach { configuration -> + configuration.dependencies.forEach { dependency -> + graphBuilder.addDependency( + DependencyGraph.qualifyScope(projectId, configuration.name), + dependency + ) + } } val project = Project( @@ -248,69 +238,15 @@ class GradleInspector( vcs = VcsInfo.EMPTY, vcsProcessed = processProjectVcs(definitionFile.parentFile), homepageUrl = "", - scopeDependencies = scopes + scopeNames = graphBuilder.scopesFor(projectId) ) - val packages = packageDependencies.associateBy { - // Deduplicate OrtDependency serialization proxy objects by Identifier. - Identifier("Maven", it.groupId, it.artifactId, it.version) - }.mapNotNullTo(mutableSetOf()) { (id, dep) -> - val model = dep.mavenModel ?: run { - issues += createAndLogIssue( - source = "Gradle", - message = "No Maven model available for '${id.toCoordinates()}'." - ) - - return@mapNotNullTo Package.EMPTY.copy(id = id) - } - - val isSpringMetadataProject = with(id) { - listOf("boot", "cloud").any { - namespace == "org.springframework.$it" - && (name.startsWith("spring-$it-starter") || name.startsWith("spring-$it-contract-spec")) - } - } - - val isMetadataOnly = dep.extension == "pom" || isSpringMetadataProject - - val binaryArtifact = when { - isMetadataOnly -> RemoteArtifact.EMPTY - else -> createRemoteArtifact(dep.pomFile, dep.classifier, dep.extension.takeUnless { it == "bundle" }) - } - - val sourceArtifact = when { - isMetadataOnly -> RemoteArtifact.EMPTY - else -> createRemoteArtifact(dep.pomFile, "sources", "jar") - } - - val vcs = dep.toVcsInfo() - val vcsFallbackUrls = listOfNotNull(model.vcs?.browsableUrl, model.homepageUrl).toTypedArray() - val vcsProcessed = processPackageVcs(vcs, *vcsFallbackUrls) - - Package( - id = id, - authors = model.authors, - declaredLicenses = model.licenses, - declaredLicensesProcessed = DeclaredLicenseProcessor.process( - model.licenses, - // See http://maven.apache.org/ref/3.6.3/maven-model/maven.html#project saying: "If multiple - // licenses are listed, it is assumed that the user can select any of them, not that they must - // accept all." - operator = SpdxOperator.OR - ), - description = model.description.orEmpty(), - homepageUrl = model.homepageUrl.orEmpty(), - binaryArtifact = binaryArtifact, - sourceArtifact = sourceArtifact, - vcs = vcs, - vcsProcessed = vcsProcessed, - isMetadataOnly = isMetadataOnly - ) - } - - val result = ProjectAnalyzerResult(project, packages, issues) + val result = ProjectAnalyzerResult(project, emptySet(), issues) return listOf(result) } + + override fun createPackageManagerResult(projectResults: Map>) = + PackageManagerResult(projectResults, graphBuilder.build(), graphBuilder.packages()) } /** @@ -341,166 +277,3 @@ private fun readGradleProperties(projectDir: File): Map { return gradleProperties.toMap() } - -/** - * Recursively convert a collection of [OrtDependency] objects to a set of [PackageReference] objects for use in [Scope] - * while flattening all dependencies into the [packageDependencies] collection. - */ -private fun Collection.toPackageRefs( - packageDependencies: MutableCollection -): Set = - mapTo(mutableSetOf()) { dep -> - val id = Identifier(dep.dependencyType, dep.groupId, dep.artifactId, dep.version) - val linkage = if (dep.isProjectDependency) { - PackageLinkage.PROJECT_DYNAMIC - } else { - PackageLinkage.DYNAMIC - } - - PackageReference(id, linkage, dep.dependencies.toPackageRefs(packageDependencies)) - } - -/** - * Create a [RemoteArtifact] based on the given [pomUrl], [classifier] and [extension]. The hash value is retrieved - * remotely. - */ -private fun GradleInspector.createRemoteArtifact( - pomUrl: String?, - classifier: String? = null, - extension: String? = null -): RemoteArtifact { - val algorithm = "sha1" - val artifactBaseUrl = pomUrl?.removeSuffix(".pom") ?: return RemoteArtifact.EMPTY - - val artifactUrl = buildString { - append(artifactBaseUrl) - if (!classifier.isNullOrEmpty()) append("-$classifier") - if (!extension.isNullOrEmpty()) append(".$extension") else append(".jar") - } - - // TODO: How to handle authentication for private repositories here, or rely on Gradle for the download? - val checksum = okHttpClient.downloadText("$artifactUrl.$algorithm") - .getOrElse { return RemoteArtifact.EMPTY } - - val hash = parseChecksum(checksum, algorithm) - - // Ignore file with zero byte size, because it cannot be a valid archive. - if (hash.value == HashAlgorithm.SHA1.emptyValue) { - logger.info { "Ignoring zero byte size artifact: $artifactUrl" } - return RemoteArtifact.EMPTY - } - - return RemoteArtifact(artifactUrl, hash) -} - -/** - * Split the provided [checksum] by whitespace and return a [Hash] for the first element that matches the provided - * algorithm. If no element matches, return [Hash.NONE]. This works around the issue that Maven checksum files sometimes - * contain arbitrary strings before or after the actual checksum. - */ -private fun parseChecksum(checksum: String, algorithm: String) = - checksum.splitOnWhitespace().firstNotNullOfOrNull { - runCatching { Hash(it, algorithm) }.getOrNull() - } ?: Hash.NONE - -// See http://maven.apache.org/pom.html#SCM. -private val SCM_REGEX = Regex("scm:(?[^:@]+):(?.+)") -private val USER_HOST_REGEX = Regex("scm:(?[^:@]+)@(?[^:]+)[:/](?.+)") - -private fun OrtDependency.toVcsInfo() = - mavenModel?.vcs?.run { - @Suppress("UnsafeCallOnNullableType") - SCM_REGEX.matchEntire(connection)?.let { match -> - val type = match.groups["type"]!!.value - val url = match.groups["url"]!!.value - - handleValidScmInfo(type, url, tag) - } ?: handleInvalidScmInfo(connection, tag) - }.orEmpty() - -private fun OrtDependency.handleValidScmInfo(type: String, url: String, tag: String) = - when { - // Maven does not officially support git-repo as an SCM, see http://maven.apache.org/scm/scms-overview.html, so - // come up with the convention to use the "manifest" query parameter for the path to the manifest inside the - // repository. An earlier version of this workaround expected the query string to be only the path to the - // manifest, for backward compatibility convert such URLs to the new syntax. - type == "git-repo" -> { - val manifestPath = url.parseRepoManifestPath() - ?: url.substringAfter('?').takeIf { it.isNotBlank() && it.endsWith(".xml") } - val urlWithManifest = url.takeIf { manifestPath == null } - ?: "${url.substringBefore('?')}?manifest=$manifestPath" - - VcsInfo( - type = VcsType.GIT_REPO, - url = urlWithManifest, - revision = tag - ) - } - - type == "svn" -> { - val revision = tag.takeIf { it.isEmpty() } ?: "tags/$tag" - VcsInfo(type = VcsType.SUBVERSION, url = url, revision = revision) - } - - url.startsWith("//") -> { - // Work around the common mistake to omit the Maven SCM provider. - val fixedUrl = "$type:$url" - - // Try to detect the Maven SCM provider from the URL only, e.g. by looking at the host or special URL paths. - VcsHost.parseUrl(fixedUrl).copy(revision = tag).also { - logger.info { - "Fixed up invalid SCM connection without a provider in '$groupId:$artifactId:$version' to $it." - } - } - } - - else -> { - val trimmedUrl = if (!url.startsWith("git://")) url.removePrefix("git:") else url - - VcsHost.fromUrl(trimmedUrl)?.let { host -> - host.toVcsInfo(trimmedUrl)?.let { vcsInfo -> - // Fixup paths that are specified as part of the URL and contain the project name as a prefix. - val projectPrefix = "${host.getProject(trimmedUrl)}-" - vcsInfo.path.withoutPrefix(projectPrefix)?.let { path -> - vcsInfo.copy(path = path) - } - } - } ?: VcsInfo(type = VcsType.forName(type), url = trimmedUrl, revision = tag) - } - } - -private fun OrtDependency.handleInvalidScmInfo(connection: String, tag: String) = - @Suppress("UnsafeCallOnNullableType") - USER_HOST_REGEX.matchEntire(connection)?.let { match -> - // Some projects omit the provider and use the SCP-like Git URL syntax, for example - // "scm:git@github.com:facebook/facebook-android-sdk.git". - val user = match.groups["user"]!!.value - val host = match.groups["host"]!!.value - val path = match.groups["path"]!!.value - - if (user == "git" || host.startsWith("git")) { - VcsInfo(type = VcsType.GIT, url = "https://$host/$path", revision = tag) - } else { - VcsInfo.EMPTY - } - } ?: run { - val dep = "$groupId:$artifactId:$version" - - if (connection.startsWith("git://") || connection.endsWith(".git")) { - // It is a common mistake to omit the "scm:[provider]:" prefix. Add fall-backs for nevertheless clear - // cases. - logger.info { - "Maven SCM connection '$connection' in '$dep' lacks the required 'scm' prefix." - } - - VcsInfo(type = VcsType.GIT, url = connection, revision = tag) - } else { - if (connection.isNotEmpty()) { - logger.info { - "Ignoring Maven SCM connection '$connection' in '$dep' due to an unexpected format." - } - } - - VcsInfo.EMPTY - } - }