Merge pull request #315 from Kotlin/refactor-build

Refactor build, fix #306

- Use version catalogs
- Use typesafe project accessors
- Move common logic to the build plugin
- Expose configurable options as extensions
- Reduce the number of project options
- Use only lazy task configuration (register instead of create)
- Get rid of libraries submodule, load libraries on demand

.gitignore

@@ -14,11 +14,16 @@
 # Folders with build files
 out/
 /build/
-/*jupyter*/build/
+/build-plugin/**/.idea/
+/build-plugin/**/build/
+!/build-plugin/**/src/build/
 /*jupyter*/*/build/
 /api-examples/*/build/
 /teamcity-artifacts/
 
+# Folder with library descriptors
+libraries/
+
 # Gradle caches and internal files
 .gradle/
 

api-examples/getting-started/build.gradle.kts

@@ -9,19 +9,15 @@ kotlinJupyter {
     addScannerDependency()
 }
 
-project.version = rootProject.version
-
 dependencies {
-    implementation(kotlin("stdlib"))
-    implementation(kotlin("reflect"))
+    implementation(libs.kotlin.stable.stdlib)
+    implementation(libs.kotlin.stable.reflect)
 }
 
 kotlinPublications {
     publication {
-        publicationName = "example-getting-started"
-        artifactId = "kotlin-jupyter-example-getting-started"
-        description = "Basic API usage example"
-        packageName = artifactId
-        publishToSonatype = false
+        publicationName.set("example-getting-started")
+        description.set("Basic API usage example")
+        publishToSonatype.set(false)
     }
 }

build-plugin/build.gradle.kts

@@ -0,0 +1,41 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
+plugins {
+    id("build.plugins.versions")
+    `kotlin-dsl`
+}
+
+repositories {
+    mavenCentral()
+    gradlePluginPortal()
+}
+
+dependencies {
+    implementation(projects.commonDependencies)
+    api(libs.bundles.allGradlePlugins)
+}
+
+sourceSets {
+    main {
+        java.setSrcDirs(listOf("src"))
+    }
+    test {
+        allJava.setSrcDirs(emptyList<String>())
+        resources.setSrcDirs(emptyList<String>())
+    }
+}
+
+tasks.withType<KotlinCompile> {
+    kotlinOptions {
+        freeCompilerArgs = freeCompilerArgs + listOf("-Xopt-in=kotlin.RequiresOptIn")
+    }
+}
+
+gradlePlugin {
+    plugins {
+        create("dependencies") {
+            id = "build.plugins.main"
+            implementationClass = "build.KernelBuildPlugin"
+        }
+    }
+}

build-plugin/common-dependencies/build.gradle.kts

@@ -0,0 +1,45 @@
+plugins {
+    id("ru.ileasile.kotlin.publisher")
+    kotlin("jvm")
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation(libs.kotlin.gradle.stdlib)
+
+    // HTTP4K for resolving remote library dependencies
+    api(libs.bundles.http4k)
+
+    // Serialization implementation for kernel code
+    api(libs.serialization.json)
+}
+
+sourceSets {
+    main {
+        java.setSrcDirs(listOf("src"))
+    }
+    test {
+        allJava.setSrcDirs(emptyList<String>())
+        resources.setSrcDirs(emptyList<String>())
+    }
+}
+
+tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile> {
+    kotlinOptions {
+        apiVersion = "1.4"
+        languageVersion = "1.4"
+
+        @Suppress("SuspiciousCollectionReassignment")
+        freeCompilerArgs += listOf("-Xopt-in=kotlin.RequiresOptIn")
+    }
+}
+
+kotlinPublications {
+    publication {
+        publicationName.set("common-dependencies")
+        description.set("Notebook API entities used for building kernel documentation")
+    }
+}

build-plugin/common-dependencies/src/org/jetbrains/kotlinx/jupyter/common/libraries.kt

@@ -0,0 +1,178 @@
+package org.jetbrains.kotlinx.jupyter.common
+
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.jsonPrimitive
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.io.File
+
+fun interface ExceptionsHandler {
+    fun handle(logger: Logger, message: String, exception: Throwable)
+
+    object DEFAULT : ExceptionsHandler {
+        override fun handle(logger: Logger, message: String, exception: Throwable) {
+            logger.error(message)
+            throw exception
+        }
+    }
+}
+
+class LibraryDescriptorsManager private constructor(
+    user: String,
+    repo: String,
+    private val remotePath: String,
+    localPath: String,
+    private val homePath: String,
+    userPath: String,
+    private val exceptionsHandler: ExceptionsHandler,
+    userSettingsDir: File,
+    private val logger: Logger,
+) {
+    private val apiPrefix = "https://$GITHUB_API_HOST/repos/$user/$repo"
+    val userLibrariesDir = userSettingsDir.resolve(userPath)
+    val userCacheDir = userSettingsDir.resolve("cache")
+    val localLibrariesDir = File(localPath)
+    val defaultBranch = "master"
+    val latestCommitOnDefaultBranch by lazy {
+        getLatestCommitToLibraries(defaultBranch)!!.first
+    }
+
+    fun homeLibrariesDir(homeDir: File? = null) = (homeDir ?: File("")).resolve(homePath)
+
+    val localPropertiesFile = localLibrariesDir.resolve(PROPERTIES_FILE)
+    val commitHashFile by lazy {
+        localLibrariesDir.resolve(COMMIT_HASH_FILE).also { file ->
+            if (!file.exists()) {
+                file.createDirsAndWrite()
+            }
+        }
+    }
+
+    fun descriptorFileName(name: String) = "$name.$DESCRIPTOR_EXTENSION"
+
+    fun isLibraryDescriptor(file: File): Boolean {
+        return file.isFile && file.name.endsWith(".$DESCRIPTOR_EXTENSION")
+    }
+
+    fun getLatestCommitToLibraries(ref: String, sinceTimestamp: String? = null): Pair<String, String>? {
+        return catchAll {
+            var url = "$apiPrefix/commits?path=$remotePath&sha=$ref"
+            if (sinceTimestamp != null) {
+                url += "&since=$sinceTimestamp"
+            }
+            logger.info("Checking for new commits to library descriptors at $url")
+            val arr = getHttp(url).jsonArray
+            if (arr.isEmpty()) {
+                if (sinceTimestamp != null) {
+                    getLatestCommitToLibraries(ref, null)
+                } else {
+                    logger.info("Didn't find any commits to libraries at $url")
+                    null
+                }
+            } else {
+                val commit = arr[0] as JsonObject
+                val sha = (commit["sha"] as JsonPrimitive).content
+                val timestamp = (((commit["commit"] as JsonObject)["committer"] as JsonObject)["date"] as JsonPrimitive).content
+                sha to timestamp
+            }
+        }
+    }
+
+    fun downloadLibraryDescriptor(ref: String, name: String): String {
+        val url = "$apiPrefix/contents/$remotePath/$name.$DESCRIPTOR_EXTENSION?ref=$ref"
+        logger.info("Requesting library descriptor at $url")
+        return downloadSingleFile(url)
+    }
+
+    fun checkRefExistence(ref: String): Boolean {
+        val response = getHttp("$apiPrefix/contents/$remotePath?ref=$ref")
+        return response.status.successful
+    }
+
+    fun checkIfRefUpToDate(remoteRef: String): Boolean {
+        if (!commitHashFile.exists()) return false
+        val localRef = commitHashFile.readText()
+        return localRef == remoteRef
+    }
+
+    fun downloadLibraries(ref: String) {
+        localLibrariesDir.mkdirs()
+
+        val url = "$apiPrefix/contents/$remotePath?ref=$ref"
+        logger.info("Requesting library descriptors at $url")
+        val response = getHttp(url).jsonArray
+
+        for (item in response) {
+            item as JsonObject
+            if (item["type"]?.jsonPrimitive?.content != "file") continue
+
+            val fileName = item["name"]!!.jsonPrimitive.content
+            if (!fileName.endsWith(".$DESCRIPTOR_EXTENSION")) continue
+
+            val downloadUrl = item["download_url"]!!.jsonPrimitive.content
+            val descriptorResponse = getHttp(downloadUrl)
+
+            val descriptorText = descriptorResponse.text
+            val file = localLibrariesDir.resolve(fileName)
+            file.writeText(descriptorText)
+        }
+
+        saveLocalRef(ref)
+    }
+
+    fun downloadLatestPropertiesFile() {
+        val ref = latestCommitOnDefaultBranch
+        val url = "$apiPrefix/contents/$remotePath/$PROPERTIES_FILE?ref=$ref"
+        logger.info("Requesting $PROPERTIES_FILE file at $url")
+        val text = downloadSingleFile(url)
+        localPropertiesFile.createDirsAndWrite(text)
+    }
+
+    private fun downloadSingleFile(contentsApiUrl: String): String {
+        val response = getHttp(contentsApiUrl).jsonObject
+        val downloadUrl = response["download_url"]!!.jsonPrimitive.content
+        val res = getHttp(downloadUrl)
+        return res.text
+    }
+
+    private fun saveLocalRef(ref: String) {
+        commitHashFile.createDirsAndWrite(ref)
+    }
+
+    private fun File.createDirsAndWrite(text: String = "") {
+        parentFile.mkdirs()
+        writeText(text)
+    }
+
+    private fun <T> catchAll(message: String = "", body: () -> T): T? = try {
+        body()
+    } catch (e: Throwable) {
+        exceptionsHandler.handle(logger, message, e)
+        null
+    }
+
+    companion object {
+        private const val GITHUB_API_HOST = "api.github.com"
+        private const val DESCRIPTOR_EXTENSION = "json"
+        private const val PROPERTIES_FILE = ".properties"
+        private const val COMMIT_HASH_FILE = "commit_sha"
+
+        fun getInstance(
+            logger: Logger = LoggerFactory.getLogger(LibraryDescriptorsManager::class.java),
+            exceptionsHandler: ExceptionsHandler = ExceptionsHandler.DEFAULT,
+        ): LibraryDescriptorsManager {
+            return LibraryDescriptorsManager(
+                "Kotlin",
+                "kotlin-jupyter-libraries",
+                "",
+                "libraries",
+                "libraries",
+                "libraries",
+                exceptionsHandler,
+                File(System.getProperty("user.home")).resolve(".jupyter_kotlin"),
+                logger
+            )
+        }
+    }
+}

build-plugin/plugin-versions-plugin/build.gradle.kts

@@ -0,0 +1,34 @@
+plugins {
+    `java-gradle-plugin`
+    `kotlin-dsl`
+}
+
+repositories {
+    mavenCentral()
+    gradlePluginPortal()
+}
+
+dependencies {
+    implementation(libs.plugin.ktlint)
+    implementation(libs.plugin.publisher)
+    implementation(libs.plugin.serialization)
+}
+
+sourceSets {
+    main {
+        java.setSrcDirs(listOf("src"))
+    }
+    test {
+        allJava.setSrcDirs(emptyList<String>())
+        resources.setSrcDirs(emptyList<String>())
+    }
+}
+
+gradlePlugin {
+    plugins {
+        create("plugins-versions") {
+            id = "build.plugins.versions"
+            implementationClass = "build.PluginVersionsPlugin"
+        }
+    }
+}

build-plugin/plugin-versions-plugin/settings.gradle.kts

@@ -0,0 +1,11 @@
+@file:Suppress("UnstableApiUsage")
+
+rootProject.name = "plugin-versions"
+
+dependencyResolutionManagement {
+    versionCatalogs {
+        create("libs") {
+            from(files("../../gradle/libs.versions.toml"))
+        }
+    }
+}

build-plugin/plugin-versions-plugin/src/build/PluginVersionsPlugin.kt

@@ -0,0 +1,14 @@
+package build
+
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+
+class PluginVersionsPlugin: Plugin<Project> {
+    override fun apply(project: Project) {
+        with(project.plugins) {
+            apply("org.jlleitschuh.gradle.ktlint")
+            apply("org.gradle.java-gradle-plugin")
+            apply("org.jetbrains.kotlin.plugin.serialization")
+        }
+    }
+}

build-plugin/settings.gradle.kts

@@ -0,0 +1,23 @@
+@file:Suppress("UnstableApiUsage")
+
+enableFeaturePreview("VERSION_CATALOGS")
+enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
+
+rootProject.name = "build"
+
+dependencyResolutionManagement {
+    versionCatalogs {
+        create("libs") {
+            from(files("../gradle/libs.versions.toml"))
+        }
+    }
+}
+
+includeBuild("plugin-versions-plugin")
+
+subproject("common-dependencies", "")
+
+fun subproject(name: String, parentPath: String) {
+    include(name)
+    project(":$name").projectDir = file("$parentPath$name")
+}