Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Resolve multiple Maven dependencies at once #432

Merged
merged 1 commit into from
Sep 25, 2023
Merged

Resolve multiple Maven dependencies at once #432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions gradle/libs.versions.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
jvmTarget = "1.8"

kotlin = "1.9.0"
kotlinMavenResolver = "1.9.30-dev-3705"
stableKotlin = "1.9.0"
gradleKotlin = "1.9.0"
ksp = "1.9.0-1.0.12"
Expand Down Expand Up @@ -51,8 +52,8 @@ kotlin-dev-scriptingCompilerImplUnshaded = { group = "org.jetbrains.kotlin", nam
kotlin-dev-scriptingCompilerImplEmbeddable = { group = "org.jetbrains.kotlin", name = "kotlin-scripting-compiler-impl-embeddable", version.ref = "kotlin" }
kotlin-dev-scriptingCompilerEmbeddable = { group = "org.jetbrains.kotlin", name = "kotlin-scripting-compiler-embeddable", version.ref = "kotlin" }
kotlin-dev-scriptingIdeServices = { group = "org.jetbrains.kotlin", name = "kotlin-scripting-ide-services", version.ref = "kotlin" }
kotlin-dev-scriptingDependencies = { group = "org.jetbrains.kotlin", name = "kotlin-scripting-dependencies", version.ref = "kotlin" }
kotlin-dev-scriptingDependenciesMavenAll = { group = "org.jetbrains.kotlin", name = "kotlin-scripting-dependencies-maven-all", version.ref = "kotlin" }
kotlin-dev-scriptingDependencies = { group = "org.jetbrains.kotlin", name = "kotlin-scripting-dependencies", version.ref = "kotlinMavenResolver" }
kotlin-dev-scriptingDependenciesMavenAll = { group = "org.jetbrains.kotlin", name = "kotlin-scripting-dependencies-maven-all", version.ref = "kotlinMavenResolver" }
kotlin-dev-scriptingCommon = { group = "org.jetbrains.kotlin", name = "kotlin-scripting-common", version.ref = "kotlin" }
kotlin-dev-scriptingJvm = { group = "org.jetbrains.kotlin", name = "kotlin-scripting-jvm", version.ref = "kotlin" }
kotlin-dev-scriptRuntime = { group = "org.jetbrains.kotlin", name = "kotlin-script-runtime", version.ref = "kotlin" }
Expand Down
84 changes: 52 additions & 32 deletions ...otlin/org/jetbrains/kotlinx/jupyter/dependencies/JupyterScriptDependenciesResolverImpl.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import kotlin.script.experimental.api.ScriptDiagnostic
import kotlin.script.experimental.api.asSuccess
import kotlin.script.experimental.api.makeFailureResult
import kotlin.script.experimental.api.valueOrNull
import kotlin.script.experimental.dependencies.ArtifactWithLocation
import kotlin.script.experimental.dependencies.CompoundDependenciesResolver
import kotlin.script.experimental.dependencies.ExternalDependenciesResolver
import kotlin.script.experimental.dependencies.ExternalDependenciesResolver.Options
Expand Down Expand Up @@ -58,7 +59,7 @@ open class JupyterScriptDependenciesResolverImpl(mavenRepositories: List<Reposit
init {
resolver = CompoundDependenciesResolver(
FileSystemDependenciesResolver(),
RemoteResolverWrapper(MavenDependenciesResolver()),
RemoteResolverWrapper(MavenDependenciesResolver(true)),
)
mavenRepositories.forEach { addRepository(Repo(it)) }
}
Expand Down Expand Up @@ -95,6 +96,7 @@ open class JupyterScriptDependenciesResolverImpl(mavenRepositories: List<Reposit
val scriptDiagnostics = mutableListOf<ScriptDiagnostic>()
val classpath = mutableListOf<File>()
var existingRepositories: List<Repo>? = null
val dependsOnAnnotations = mutableListOf<String>()

script.annotations.forEach { annotation ->
when (annotation) {
Expand All @@ -121,72 +123,82 @@ open class JupyterScriptDependenciesResolverImpl(mavenRepositories: List<Reposit
existingRepositories?.forEach { addRepository(it) }
}
is DependsOn -> {
log.info("Resolving ${annotation.value}")
try {
tryResolve(
annotation.value,
scriptDiagnostics,
classpath,
)
} catch (e: Exception) {
val diagnostic = ScriptDiagnostic(ScriptDiagnostic.unspecifiedError, "Unhandled exception during resolve", exception = e)
log.error(diagnostic.message, e)
scriptDiagnostics.add(diagnostic)
}
// Hack: after first resolution add "standard" Central repo to the end of the list, giving it the lowest priority
addRepository(CENTRAL_REPO)
dependsOnAnnotations.add(annotation.value)
}
else -> throw Exception("Unknown annotation ${annotation.javaClass}")
}
}

return if (scriptDiagnostics.isEmpty()) classpath.asSuccess()
else makeFailureResult(scriptDiagnostics)
try {
tryResolve(
dependsOnAnnotations,
scriptDiagnostics,
classpath,
)
} catch (e: Exception) {
val diagnostic = ScriptDiagnostic(ScriptDiagnostic.unspecifiedError, "Unhandled exception during resolve", exception = e)
log.error(diagnostic.message, e)
scriptDiagnostics.add(diagnostic)
}
// Hack: after first resolution add "standard" Central repo to the end of the list, giving it the lowest priority
addRepository(CENTRAL_REPO)

return makeResolutionResult(classpath, scriptDiagnostics)
}

private suspend fun resolveWithOptions(annotationArgs: List<String>, options: Options): ResultWithDiagnostics<List<File>> {
return resolver.resolve(annotationArgs.map { ArtifactWithLocation(it, null) }, options)
}

private fun tryResolve(
annotationArg: String,
annotationArgs: List<String>,
scriptDiagnosticsResult: MutableList<ScriptDiagnostic>,
classpathResult: MutableList<File>,
) {
if (annotationArgs.isEmpty()) return

log.info("Resolving $annotationArgs")
doResolve(
{ resolver.resolve(annotationArg, resolverOptions, null) },
{ resolveWithOptions(annotationArgs, resolverOptions) },
onResolved = { files ->
addedClasspath.addAll(files)
classpathResult.addAll(files)
},
onFailure = { result ->
val diagnostics = ScriptDiagnostic(ScriptDiagnostic.unspecifiedError, "Failed to resolve $annotationArg:\n" + result.reports.joinToString("\n") { it.message })
val diagnostics = ScriptDiagnostic(ScriptDiagnostic.unspecifiedError, "Failed to resolve $annotationArgs:\n" + result.reports.joinToString("\n") { it.message })
log.warn(diagnostics.message, diagnostics.exception)
scriptDiagnosticsResult.add(diagnostics)
},
)

if (resolveSources) {
doResolve(
{ resolver.resolve(annotationArg, sourcesResolverOptions, null) },
{ resolveWithOptions(annotationArgs, sourcesResolverOptions) },
onResolved = { files ->
addedSourcesClasspath.addAll(files)
},
onFailure = { result ->
log.warn("Failed to resolve sources for $annotationArg:\n" + result.reports.joinToString("\n") { it.message })
log.warn("Failed to resolve sources for $annotationArgs:\n" + result.reports.joinToString("\n") { it.message })
},
)
}

if (resolveMpp) {
doResolve(
{ resolver.resolve(annotationArg, mppResolverOptions, null) },
{ resolveWithOptions(annotationArgs, mppResolverOptions) },
onResolved = { files ->
val resolvedArtifacts = mutableSetOf(annotationArg)
val resolvedArtifacts = mutableSetOf<String>()
resolvedArtifacts.addAll(annotationArgs)
resolveMpp(files) { artifactCoordinates ->
if (resolvedArtifacts.add(artifactCoordinates)) {
tryResolve(
artifactCoordinates,
scriptDiagnosticsResult,
classpathResult,
)
val notYetResolvedArtifacts = artifactCoordinates.filter { artifact ->
resolvedArtifacts.add(artifact)
}

tryResolve(
notYetResolvedArtifacts,
scriptDiagnosticsResult,
classpathResult,
)
}
},
onFailure = {},
Expand All @@ -210,7 +222,7 @@ open class JupyterScriptDependenciesResolverImpl(mavenRepositories: List<Reposit
}
}

private fun resolveMpp(moduleFiles: List<File>, jvmArtifactCallback: (String) -> Unit) {
private fun resolveMpp(moduleFiles: List<File>, jvmArtifactCallback: (List<String>) -> Unit) {
val coordinates = mutableListOf<String>()

for (moduleFile in moduleFiles) {
Expand All @@ -235,7 +247,15 @@ open class JupyterScriptDependenciesResolverImpl(mavenRepositories: List<Reposit
coordinates.add(artifactCoordinates)
}
}
coordinates.forEach(jvmArtifactCallback)
jvmArtifactCallback(coordinates)
}

private fun makeResolutionResult(
classpath: List<File>,
scriptDiagnostics: List<ScriptDiagnostic>,
): ResultWithDiagnostics<List<File>> {
return if (scriptDiagnostics.isEmpty()) classpath.asSuccess()
else makeFailureResult(scriptDiagnostics)
}

private class Repo(
Expand Down