diff --git a/README.md b/README.md index 04cd8c4..d9b3bdc 100644 --- a/README.md +++ b/README.md @@ -5,35 +5,68 @@ This ia GitHub Action for caching Gradle caches. In other words, this is [@actions/cache](https://github.com/actions/cache) customized for Gradle. -Key improvements over `@actions/cache` -- Simplified configuration -- Less space usage (there's overall 5GiB limit, so cache space matters) -- Native support for caching Gradle's local build cache +Key improvements over [@actions/cache](https://github.com/actions/cache) and [gradle-command-action](https://github.com/eskatos/gradle-command-action) +- 🚀 Gradle remote build cache backend (pulls only the needed entries from GitHub cache) +- 🎉 Support multiple remote caches via [gradle-multi-cache](https://github.com/burrunan/gradle-multi-cache) (e.g. GitHub Actions + S3) +- 👋 Simplified configuration (action name + gradle command is enough for most case) +- 👾 Less space usage (GitHub imposes overall 5GiB limit by default, so cache space matters) +- 🔗 Link to Build Scan in build results +- 💡 Gradle build failure markers added to the diff view (e.g. `compileJava` or `compileKotlin` markers right in the commit diff) ## Usage Add the following to `.github/workflows/...` +Note: like with [gradle-command-action](https://github.com/eskatos/gradle-command-action), you can +specify `gradle-version: release` to test with the current release version of Gradle, `gradle-version: nightly` for testing Gradle nightly builds, +an so on (see `gradle-version` below). + ```yaml - uses: burrunan/gradle-cache-action@v1 - name: Cache .gradle + name: Build PROJECT_NAME + # Extra environment variables for Gradle execution (regular GitHub Actions feature) + # Note: env must be outside of "with" + env: + VARIABLE: VALUE with: # If you have multiple jobs, use distinct job-id in in case you want to split caches # For instance, jobs with different JDK versions can't share caches # RUNNER_OS is added to job-id automatically job-id: jdk8 + # Specifies arguments for Gradle execution + # If arguments is missing or empty, then Gradle is not executed + arguments: build + # arguments can be multi-line for better readability + # arguments: | + # --no-paralell + # build + # -x test + # Gradle version to use for execution: + # wrapper (default), current, rc, nightly, release-nightly, or + # versions like 6.6 (see https://services.gradle.org/versions/all) + gradle-version: wrapper + # Properties are passed as -Pname=value + properties: | + kotlin.js.compiler=ir + kotlin.parallel.tasks.in.project=true ``` -You might want to enable [Gradle Build Cache](https://docs.gradle.org/current/userguide/build_cache.html) -For instance, add `--build-cache` option when running Gradle. +By default, the action enables `local` build cache, and it adds a remote build cache +that stores the data in GitHub Actions cache. +However, you might want to enable [Gradle Build Cache](https://docs.gradle.org/current/userguide/build_cache.html) +for your local builds to make them faster, or even add a remote cache instance, so your local +builds can reuse artifacts that are build on CI. + +This is how you can enable local build cache (don't forget to add `--build-cache` option or +`org.gradle.caching=true` property). ```kotlin -// build.gradle.kts +// settings.gradle.kts val isCiServer = System.getenv().containsKey("CI") // Cache build artifacts, so expensive operations do not need to be re-computed buildCache { local { - isEnabled = !isCiServer || System.getenv().containsKey("GITHUB_ACTIONS") + isEnabled = !isCiServer } } ``` @@ -45,6 +78,9 @@ The default configuration should suit for most of the cases, however, there are ```yaml - uses: burrunan/gradle-cache-action@v1 name: Cache .gradle + # Extra environment variables for Gradle execution (regular GitHub Actions feature) + env: + VARIABLE: VALUE with: # If you have multiple jobs, use distinct job-id in in case you want to split caches # For instance, jobs with different JDK versions can't share caches @@ -54,10 +90,34 @@ The default configuration should suit for most of the cases, however, there are # Disable caching of $HOME/.gradle/caches/*.*/generated-gradle-jars save-generated-gradle-jars: false + # Disable remote cache that proxies requests to GitHub Actions cache + remote-build-cache-proxy-enabled: false + # Set the cache key for Gradle version (e.g. in case multiple jobs use different versions) # By default the value is `wrapper`, so the version is determined from the gradle-wrapper.properties + # Note: this argument specifies the version for Gradle execution (if `arguments` is present) + # Supported values: + # wrapper (default), current, rc, nightly, release-nightly, or + # versions like 6.6 (see https://services.gradle.org/versions/all) gradle-version: 6.5.1-custom + # Arguments for Gradle execution + arguments: build jacocoReport + + # Properties are passed as -Pname=value + properties: | + kotlin.js.compiler=ir + kotlin.parallel.tasks.in.project=true + + # Relative path under $GITHUB_WORKSPACE where Git repository is placed + build-root-directory: sub/directory + + # Activates only the caches that are relevant for executing gradle command. + # This is helpful when build job executes multiple gradle commands sequentially. + # Then the caching is implemented in the very first one, and the subsequent should be marked + # with execution-only-caches: true + execution-only-caches: true + # Disable caching of ~/.gradle/caches/build-cache-* save-local-build-cache: false @@ -72,12 +132,17 @@ The default configuration should suit for most of the cases, however, there are # Disable caching of ~/.m2/repository/ save-maven-dependencies-cache: false + # Ignore some of the paths when caching Maven Local repository + maven-local-ignore-paths: | + org/example/ + com/example/ + # Enable concurrent cache save and restore # Default is concurrent=false for better log readability concurrent: true ``` -## How does it work? +## How does dependency caching work? The current GitHub Actions cache (both [actions/cache](https://github.com/actions/cache) action and [@actions/cache](https://github.com/actions/toolkit/tree/main/packages/cache) npm package) is immutable. @@ -89,6 +154,88 @@ If only a small fraction of files changes, then the action reuses the existing c That enables to save cache space (GitHub has a default limit of 5GiB), and it reduces upload time as only the cache receives only the updated files. +## How does GitHub Actions-based Gradle remote build cache work? + +`gradle-cache-action` launches a small proxy server that listens for Gradle requests and +then it redirects the requests to `@actions/cache` API. + +That makes Gradle believe it is talking to a regular remote cache, and the cache receives +only the relevant updates. +The increased granularity enables GitHub to evict entries better (it removes unused entries +automatically). + +The action configures the URL to the cache proxy via `~/.gradle/init.gradle` script, and +[Gradle picks it up automatically](https://docs.gradle.org/current/userguide/init_scripts.html) + +Note: saving GitHub Actions caches might take noticeable time (e.g. 100ms), so the cache uploads +in the background. In other words, build scan would show virtually zero response times for +cache save operations. + +If your build already has a remote cache declared (e.g. you are using your own cache), +then `gradle-cache-action` would configure **both** remote caches. +It would read from GitHub cache first, and it would save data to both caches. + +Multi-cache feature can be disabled via `multi-cache-enabled: false`. + +## How to enable build scans? + +1. Read and agree to the terms of service: https://gradle.com/terms-of-service +1. Add `--scan` to `arguments:`, and add the following to `settings.gradle.kts` + +```kotlin +plugins { + `gradle-enterprise` +} + +val isCiServer = System.getenv().containsKey("CI") + +if (isCiServer) { + gradleEnterprise { + buildScan { + termsOfServiceUrl = "https://gradle.com/terms-of-service" + termsOfServiceAgree = "yes" + tag("CI") + } + } +} +``` + +## Why another action instead of gradle-command-action? + +`gradle-command-action` was started as a Kotlin/JS experiment for making a customized +[@actions/cache](https://github.com/actions/cache) that would make Gradle builds faster. + +Then it turned out there's a possibility to use proxy remote cache requests to `@actions/cache` API +which is possible in case the caching action executes Gradle, so `gradle-cache-action` +got Gradle execution feature. + +Of course, the same could have been made in [gradle-command-action](https://github.com/eskatos/gradle-command-action), +however: +- The author was not familiar with TypeScript ecosystem (stdlib, typical libraries, testing libraries, etc) +- Caching logic is collections-heavy, and Kotlin stdlib shines here. + + For instance, in Kotlin `list + list` adds lists, and `array.associateWith { valueFor(it) }` converts array to map. + One writes that without StackOverflow, the code is readable, and it does not require + you [to fight with the compiler](https://blog.johnnyreilly.com/2016/06/create-es2015-map-from-array-in-typescript.html). + +- A single language helps when building connected components. + `gradle-cache-action` integrates with [gradle-multi-cache](https://github.com/burrunan/gradle-multi-cache) and + [gradle-s3-build-cache](https://github.com/burrunan/gradle-s3-build-cache), and they all are Kotlin-based. + +## Can I use the caching part of the action only? + +Yes, you can. If you omit `arguments:`, then the action runs in `cache-only` mode. +It won't launch Gradle. + +## Can I call multiple different Gradle builds in the same job? + +This might be complicated, see https://github.com/burrunan/gradle-cache-action/issues/15. + +Currently, the workaround is to configure `execution-only-caches: true` for all but one +`gradle-cache-action` executions. +Then one of the actions would do the cache save and restore, and the rest would use their own +caches only. + ## Contributing Contributions are always welcome! If you'd like to contribute (and we hope you do) please open a pull request. diff --git a/action.yml b/action.yml index 69ca6fe..ffb0f21 100644 --- a/action.yml +++ b/action.yml @@ -1,34 +1,85 @@ name: 'Gradle Cache' description: 'Caches .gradle folder (dependencies, local build cache, ...)' author: 'Vladimir Sitnikov' +outputs: + build-scan-url: + description: Link to the build scan if any inputs: job-id: description: A job identifier to avoid cache pollution from different jobs required: false - path: + build-root-directory: description: Relative path under $GITHUB_WORKSPACE where Git repository is placed required: false gradle-version: description: (wrapper | or explicit version) Caches often depend on the Gradle version, so this parameter sets the ID to use for cache keys. It does not affect the Gradle version used for build required: false + default: wrapper save-generated-gradle-jars: description: Enables caching of $HOME/.gradle/caches/*.*/generated-gradle-jars required: false + default: 'true' + save-local-build-cache: + description: Enables caching of $HOME/.gradle/caches/build-cache-1 + required: false + default: 'true' + multi-cache-enabled: + description: Adds com.github.burrunan.multi-cache plugin to settings.gradle so GitHub Actions cache can be used in parallel with Gradle remote build cache + required: false + default: 'true' + multi-cache-version: + description: Configures com.github.burrunan.multi-cache version to use + required: false + default: '1.0' + multi-cache-repository: + description: Configures repository where com.github.burrunan.multi-cache can be located + required: false + default: '' + multi-cache-group-id-filter: + description: Configures group id for selecting only com.github.burrunan.multi-cache artifacts (it enables Gradle to use custom repository for multi-cache only) + required: false + default: 'com[.]github[.]burrunan[.]multi-?cache' save-gradle-dependencies-cache: description: Enables caching of ~/.gradle/caches/modules-* required: false + default: 'true' + execution-only-caches: + description: | + Activates only the caches that are relevant for executing gradle command. + This is helpful when build job executes multiple gradle commands sequentially. + Then the caching is implemented in the very first one, and the subsequent should be marked + with execution-only-caches: true + required: false + default: 'false' + remote-build-cache-proxy-enabled: + description: Activates a remote cache that proxies requests to GitHub Actions cache + required: false + default: 'true' gradle-dependencies-cache-key: description: Extra files to take into account for ~/.gradle/caches dependencies required: false save-maven-dependencies-cache: description: Enables caching of ~/.m2/repository/ required: false + default: 'true' + maven-local-ignore-paths: + description: Specifies ignored paths in the Maven Local repository (e.g. the artifacts of the current project) + required: false + default: '' debug: description: Shows extra logging to debug the action required: false + default: 'true' concurrent: description: Enables concurent cache download and upload (disabled by default for better log output) required: false + default: 'false' + arguments: + description: Gradle arguments to pass (optionally multiline) + required: false + properties: + description: Extra Gradle properties (multiline) which would be passed as -Pname=value arguments + required: false runs: using: node12 main: dist/cache-action-entrypoint.js diff --git a/build.gradle.kts b/build.gradle.kts index af7a1af..fa81cf2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,10 +62,22 @@ allprojects { configure { js { if (project.name.endsWith("-entrypoint")) { - browser() + browser { + testTask { + useMocha { + timeout = "10000" + } + } + } binaries.executable() } else { - nodejs() + nodejs { + testTask { + useMocha { + timeout = "10000" + } + } + } } } } diff --git a/cache-action-entrypoint/build.gradle.kts b/cache-action-entrypoint/build.gradle.kts index 4e841cf..5cff07a 100644 --- a/cache-action-entrypoint/build.gradle.kts +++ b/cache-action-entrypoint/build.gradle.kts @@ -15,6 +15,12 @@ */ dependencies { + implementation(project(":cache-proxy")) + implementation(project(":gradle-launcher")) implementation(project(":layered-cache")) implementation(project(":wrappers:actions-core")) + implementation(project(":wrappers:actions-io")) + implementation(project(":wrappers:nodejs")) + implementation(project(":wrappers:octokit-webhooks")) + implementation(npm("string-argv", "0.3.1")) } diff --git a/cache-action-entrypoint/src/main/kotlin/main.kt b/cache-action-entrypoint/src/main/kotlin/main.kt index e95622f..311e813 100644 --- a/cache-action-entrypoint/src/main/kotlin/main.kt +++ b/cache-action-entrypoint/src/main/kotlin/main.kt @@ -14,36 +14,144 @@ * limitations under the License. */ +import actions.core.* import actions.core.ext.getInput -import actions.core.info +import actions.core.ext.getListInput +import actions.io.mkdirP import com.github.burrunan.gradle.GradleCacheAction import com.github.burrunan.gradle.Parameters -import com.github.burrunan.gradle.github.env.ActionsEnvironment -import com.github.burrunan.gradle.github.event.currentTrigger +import com.github.burrunan.gradle.github.stateVariable +import com.github.burrunan.gradle.proxy.CacheProxy +import com.github.burrunan.launcher.LaunchParams +import com.github.burrunan.launcher.install +import com.github.burrunan.launcher.launchGradle +import com.github.burrunan.launcher.resolveDistribution +import fs2.promises.writeFile +import octokit.currentTrigger +import path.path + +fun String.splitLines() = + split(Regex("\\s*[\r\n]+\\s*")) + .filter { !it.startsWith("#") && it.contains("=") } + .associate { + val values = it.split(Regex("\\s*=\\s*"), limit = 2) + values[0] to (values.getOrNull(1) ?: "") + } + +fun isMochaRunning() = + arrayOf("afterEach", "after", "beforeEach", "before", "describe", "it").all { + global.asDynamic()[it] is Function<*> + } suspend fun main() { - if (process.env["GITHUB_ACTIONS"].isNullOrBlank()) { - // Ignore if called outside of GitHub Actions (e.g. tests) + if (isMochaRunning()) { + // Ignore if called from tests return } + val stageVar = stateVariable("stage") { "MAIN" } + val stage = ActionStage.values().firstOrNull { it.name == stageVar.get() } + // Set next stage + stageVar.set( + when (stage) { + ActionStage.MAIN -> ActionStage.POST + null -> { + setFailed("Unable to find action stage: ${stageVar.get()}") + return + } + else -> null + }?.name ?: "FINAL", + ) + try { + mainInternal(stage) + } catch (e: ActionFailedException) { + setFailed(e.message) + } +} + +suspend fun mainInternal(stage: ActionStage) { + val gradleStartArguments = parseArgsStringToArgv(getInput("arguments")).toList() + val cacheProxyEnabled = getInput("remote-build-cache-proxy-enabled").ifBlank { "true" }.toBoolean() + + val executionOnlyCaches = getInput("execution-only-caches").ifBlank { "false" }.toBoolean() + + val buildRootDirectory = getInput("build-root-directory").trimEnd('/', '\\') + if (buildRootDirectory != "") { + info("changing working directory to $buildRootDirectory") + process.chdir(buildRootDirectory) + } + val params = Parameters( jobId = ActionsEnvironment.RUNNER_OS + "-" + getInput("job-id"), - path = getInput("path").trimEnd('/', '\\').ifBlank { "." }, + path = ".", debug = getInput("debug").toBoolean(), generatedGradleJars = getInput("save-generated-gradle-jars").ifBlank { "true" }.toBoolean(), - localBuildCache = getInput("save-local-build-cache").ifBlank { "true" }.toBoolean(), - gradleDependenciesCache = getInput("save-gradle-dependencies-cache").ifBlank { "true" }.toBoolean(), - gradleDependenciesCacheKey = getInput("gradle-dependencies-cache-key"), - mavenDependenciesCache = getInput("save-maven-dependencies-cache").ifBlank { "true" }.toBoolean(), + localBuildCache = (!cacheProxyEnabled || gradleStartArguments.isEmpty()) && getInput("save-local-build-cache").ifBlank { "true" } + .toBoolean(), + gradleDependenciesCache = !executionOnlyCaches && getInput("save-gradle-dependencies-cache").ifBlank { "true" }.toBoolean(), + gradleDependenciesCacheKey = getListInput("gradle-dependencies-cache-key"), + mavenDependenciesCache = !executionOnlyCaches && getInput("save-maven-dependencies-cache").ifBlank { "true" }.toBoolean(), + mavenLocalIgnorePaths = getListInput("maven-local-ignore-paths"), concurrent = getInput("concurrent").ifBlank { "false" }.toBoolean(), ) - if (!params.generatedGradleJars && !params.localBuildCache && - !params.gradleDependenciesCache && !params.mavenDependenciesCache - ) { - info("All the caches are disabled, skipping the action") - return + val gradleDistribution = resolveDistribution( + versionSpec = getInput("gradle-version").ifBlank { "wrapper" }, + projectPath = params.path, + distributionUrl = getInput("gradle-distribution-url").ifBlank { null }, + distributionSha256Sum = getInput("gradle-distribution-sha-256-sum").ifBlank { null }, + ) + + if (stage == ActionStage.MAIN || stage == ActionStage.POST) { + val cacheAction = GradleCacheAction(currentTrigger(), params, gradleDistribution) + + if (params.generatedGradleJars || params.localBuildCache || + params.gradleDependenciesCache || params.mavenDependenciesCache + ) { + cacheAction.execute(stage) + } } - GradleCacheAction(currentTrigger(), params).run() + if (stage == ActionStage.MAIN && gradleStartArguments.isNotEmpty()) { + val args = when (params.localBuildCache || cacheProxyEnabled) { + true -> listOf("--build-cache") + gradleStartArguments + else -> gradleStartArguments + } + val launchParams = LaunchParams( + gradle = install(gradleDistribution), + projectPath = params.path, + arguments = args, + properties = getInput("arguments").splitLines(), + ) + + val cacheProxy = CacheProxy() + + if (cacheProxyEnabled) { + info("Starting remote cache proxy, adding it via ~/.gradle/init.gradle") + cacheProxy.start() + val gradleHome = path.join(os.homedir(), ".gradle") + mkdirP(gradleHome) + writeFile( + path.join(gradleHome, "init.gradle"), + cacheProxy.getMultiCacheConfiguration( + multiCacheEnabled = getInput("multi-cache-enabled").ifBlank { "true" }.toBoolean(), + multiCacheVersion = getInput("multi-cache-version").ifBlank { "1.0" }, + multiCacheRepository = getInput("multi-cache-repository"), + multiCacheGroupIdFilter = getInput("multi-cache-group-id-filter").ifBlank { "com[.]github[.]burrunan[.]multi-?cache" }, + ), + ) + } + + try { + val result = launchGradle(launchParams) + result.buildScanUrl?.let { + warning("Gradle Build Scan: $it") + setOutput("build-scan-url", it) + } + } finally { + if (cacheProxyEnabled) { + cacheProxy.stop() + } + } + } + return } diff --git a/cache-action-entrypoint/src/main/kotlin/stringArgv.kt b/cache-action-entrypoint/src/main/kotlin/stringArgv.kt new file mode 100644 index 0000000..421174f --- /dev/null +++ b/cache-action-entrypoint/src/main/kotlin/stringArgv.kt @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:JsModule("string-argv") + +external fun parseArgsStringToArgv(value: String, env: String = definedExternally, file: String = definedExternally): Array diff --git a/cache-action-entrypoint/src/test/kotlin/com/github/burrunan/ArgumentsTest.kt b/cache-action-entrypoint/src/test/kotlin/com/github/burrunan/ArgumentsTest.kt new file mode 100644 index 0000000..d7d725b --- /dev/null +++ b/cache-action-entrypoint/src/test/kotlin/com/github/burrunan/ArgumentsTest.kt @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan + +import parseArgsStringToArgv +import kotlin.test.Test +import kotlin.test.assertEquals + +class ArgumentsTest { + private fun parse(input: String, vararg output: String) { + assertEquals(listOf(*output), parseArgsStringToArgv(input).toList(), input) + } + + @Test + fun simple() { + parse("") + parse("a b", "a", "b") + parse("a 'b'", "a", "b") + parse("a \"b\"", "a", "b") + } + + @Test + fun multiline() { + parse("a\nb", "a", "b") + parse("a\n b", "a", "b") + parse("a\n b ", "a", "b") + parse("a\n b \nc", "a", "b", "c") + } + + @Test + fun multilineWithQuotes() { + parse("'a\nb'", "a\nb") + parse("hello 'a\n b' world", "hello", "a\n b", "world") + parse("hello \"a\n b\" world", "hello", "a\n b", "world") + } + + @Test + fun withDollars() { + parse("\$HOME", "\$HOME") + } + + @Test + fun multilineWithComments() { + // TODO: "# commented" should be ignored + parse(""" + build + # commented + test + """.trimIndent(), "build", "#", "commented", "test") + } +} diff --git a/cache-action-entrypoint/src/test/kotlin/com/github/burrunan/SplitLinesTest.kt b/cache-action-entrypoint/src/test/kotlin/com/github/burrunan/SplitLinesTest.kt new file mode 100644 index 0000000..8b100f9 --- /dev/null +++ b/cache-action-entrypoint/src/test/kotlin/com/github/burrunan/SplitLinesTest.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan + +import splitLines +import kotlin.test.Test +import kotlin.test.assertEquals + +class SplitLinesTest { + @Test + fun empty() { + assertEquals(mapOf("a" to "b", "c" to ""), "a=b\nc=".splitLines()) + } + + @Test + fun withoutEquals() { + assertEquals(mapOf("a" to "b"), "a=b\nc".splitLines()) + } +} diff --git a/cache-proxy/build.gradle.kts b/cache-proxy/build.gradle.kts new file mode 100644 index 0000000..54d0d26 --- /dev/null +++ b/cache-proxy/build.gradle.kts @@ -0,0 +1,33 @@ +import org.jetbrains.kotlin.gradle.targets.js.testing.KotlinJsTest +import org.jetbrains.kotlin.gradle.targets.js.testing.mocha.KotlinMocha + +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation(project(":cache-service-mock")) + implementation(project(":wrappers:actions-cache")) + implementation(project(":wrappers:actions-core")) + implementation(project(":wrappers:actions-exec")) + implementation(project(":wrappers:actions-glob")) + implementation(project(":wrappers:nodejs")) + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime") +} + +tasks.withType().configureEach { + (testFramework as KotlinMocha).timeout = "60000" +} diff --git a/cache-proxy/src/main/kotlin/com/github/burrunan/gradle/proxy/CacheProxy.kt b/cache-proxy/src/main/kotlin/com/github/burrunan/gradle/proxy/CacheProxy.kt new file mode 100644 index 0000000..4f4c054 --- /dev/null +++ b/cache-proxy/src/main/kotlin/com/github/burrunan/gradle/proxy/CacheProxy.kt @@ -0,0 +1,200 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.gradle.proxy + +import actions.core.debug +import actions.glob.removeFiles +import com.github.burrunan.gradle.cache.* +import com.github.burrunan.wrappers.nodejs.mkdir +import com.github.burrunan.wrappers.nodejs.use +import fs.createReadStream +import fs.createWriteStream +import http.IncomingMessage +import http.ServerResponse +import kotlinext.js.jsObject +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.await +import kotlinx.coroutines.launch +import path.path +import process +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine + +class CacheProxy { + companion object { + const val GHA_CACHE_URL = "GHA_CACHE_URL" + private const val TEMP_DIR = ".cache-proxy" + } + + private var _cacheUrl: String? = null + + val cacheUrl: String? get() = _cacheUrl + + private val server = http.createServer { req, res -> + val query = url.parse(req.url, true) + val path = query.pathname ?: "" + res.handle { + val id = path.removePrefix("/") + when (req.method) { + "GET" -> getEntry(id, res) + "PUT" -> putEntry(id, req, res) + else -> HttpException.notImplemented("Not implemented: ${req.method}") + } + } + } + + private val compression = jsObject { compressionMethod = CompressionMethod.Gzip } + + private suspend fun putEntry(id: String, req: IncomingMessage, res: ServerResponse) { + val fileName = path.join(TEMP_DIR, "bc-$id") + try { + req.use { + it.pipe(createWriteStream(fileName)) + } + res.writeHead(200, "OK") + } finally { + GlobalScope.launch { + try { + val cacheIdRequest = reserveCache(id, arrayOf(id), compression) + val cacheId = cacheIdRequest.await() + saveCache(cacheId, fileName).await() + } finally { + removeFiles(listOf(fileName)) + + } + } + } + } + + private suspend fun getEntry(id: String, res: ServerResponse) { + val cacheEntry = getCacheEntry(arrayOf(id), arrayOf(id), compression).await() + ?: throw HttpException.notFound("No cache entry found for $id") + val archiveLocation = cacheEntry.archiveLocation + ?: throw HttpException.notFound("No archive location for $id") + val fileName = path.join(TEMP_DIR, "dl-$id") + debug { "Found ${cacheEntry.cacheKey}, ${cacheEntry.scope} $archiveLocation" } + try { + downloadCache(archiveLocation, fileName).await() + res.writeHead( + 200, "Ok", + jsObject { + this["content-length"] = fs2.promises.stat(fileName).size + }, + ) + createReadStream(fileName).use { + it.pipe(res) + } + } finally { + removeFiles(listOf(fileName)) + } + } + + private val pluginId = "com.github.burrunan.multi-cache" + + fun getMultiCacheConfiguration( + multiCacheEnabled: Boolean = true, + multiCacheVersion: String = "1.0", + multiCacheRepository: String = "", + multiCacheGroupIdFilter: String = "com[.]github[.]burrunan[.]multi-?cache" + ): String { + val multiCacheGroupIdFilterEscaped = multiCacheGroupIdFilter.replace("\\", "\\\\") + //language=Groovy + return """ + def pluginId = 'com.github.burrunan.multi-cache' + def multiCacheVersion = '1.0' + def multiCacheGroupIdFilter = 'com[.]github[.]burrunan[.]multi-?cache' + boolean multiCacheEnabled = $multiCacheEnabled + String multiCacheRepository = '$multiCacheRepository' + beforeSettings { settings -> + if (!multiCacheEnabled) { + return + } + def repos = settings.buildscript.repositories + if (multiCacheRepository != '') { + repos.add( + repos.maven { + url = multiCacheRepository + if ('$multiCacheGroupIdFilterEscaped' != '') { + content { + includeGroupByRegex('$multiCacheGroupIdFilterEscaped') + } + } + } + ) + } else if (repos.isEmpty()) { + repos.add(repos.gradlePluginPortal()) + } + settings.buildscript.dependencies { + classpath("$pluginId:${pluginId}.gradle.plugin:$multiCacheVersion") + } + } + + settingsEvaluated { settings -> + settings.buildCache { + boolean needMulticache = remote != null + if (needMulticache && !multiCacheEnabled) { + println("$pluginId is disabled") + return + } + + local { + enabled = true + } + if (needMulticache) { + settings.pluginManager.apply("$pluginId") + settings.multicache.push('base') + } + remote(HttpBuildCache) { + url = '$cacheUrl' + push = true + } + if (needMulticache) { + settings.multicache.pushAndConfigure('actions-cache') { + loadSequentiallyWriteConcurrently('actions-cache', 'base') + } + } + } + } + """.trimIndent() + } + + suspend fun start() { + suspendCoroutine { cont -> + server.listen(0) { + cont.resume(null) + } + } + + mkdir(TEMP_DIR) + val url = "http://localhost:${server.address().port}/" + _cacheUrl = url + process.env[GHA_CACHE_URL] = url + } + + fun stop() { + server.close() + } + + suspend inline operator fun invoke(block: () -> T): T { + start() + try { + return block() + } finally { + stop() + } + } +} diff --git a/cache-proxy/src/test/kotlin/com/github/burrunan/gradle/proxy/CacheProxyTest.kt b/cache-proxy/src/test/kotlin/com/github/burrunan/gradle/proxy/CacheProxyTest.kt new file mode 100644 index 0000000..f70be86 --- /dev/null +++ b/cache-proxy/src/test/kotlin/com/github/burrunan/gradle/proxy/CacheProxyTest.kt @@ -0,0 +1,111 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.gradle.proxy + +import actions.exec.exec +import actions.glob.removeFiles +import com.github.burrunan.gradle.cache.CacheService +import com.github.burrunan.test.runTest +import com.github.burrunan.wrappers.nodejs.mkdir +import fs2.promises.writeFile +import kotlinx.serialization.encodeToDynamic +import kotlinx.serialization.json.Json +import process +import kotlin.test.Test +import kotlin.test.assertTrue +import kotlin.test.fail + +class CacheProxyTest { + // Emulates Azure Cache Backend for @actions/cache + val cacheService = CacheService() + + // Implements Gradle HTTP Build Cache via @actions/cache + val cacheProxy = CacheProxy() + + @Test + fun abc() = runTest { + val z = mapOf("a" to 4, "b" to 6) + println("json: " + JSON.stringify(Json.encodeToDynamic(z))) + } + + @Test + fun cacheProxyWorks() = runTest { + val dir = "remote_cache_test" + mkdir(dir) + cacheService { + cacheProxy { + val outputFile = "build/out.txt" + removeFiles(listOf("$dir/$outputFile")) + writeFile( + "$dir/settings.gradle", + """ + rootProject.name = 'sample' + buildCache { + local { + // Only remote cache should be used + enabled = false + } + remote(HttpBuildCache) { + url = '${process.env["GHA_CACHE_URL"]}' + push = true + } + } + """.trimIndent(), + ) + writeFile( + "$dir/build.gradle", + """ + tasks.create('props', WriteProperties) { + outputFile = file("$outputFile") + property("hello", "world") + } + tasks.create('props2', WriteProperties) { + outputFile = file("${outputFile}2") + property("hello", "world2") + } + """.trimIndent(), + ) + writeFile( + "$dir/gradle.properties", + """ + org.gradle.caching=true + #org.gradle.caching.debug=true + """.trimIndent(), + ) + + val out = exec("gradle", "props", "-i", "--build-cache", captureOutput = true) { + cwd = dir + silent = true + } + if (out.exitCode != 0) { + fail("Unable to execute :props task: ${out.stdout}") + } + assertTrue("1 actionable task: 1 executed" in out.stdout, out.stdout) + + removeFiles(listOf("$dir/$outputFile")) + val out2 = exec("gradle", "props", "-i", "--build-cache", captureOutput = true) { + cwd = dir + silent = true + } + if (out.exitCode != 0) { + fail("Unable to execute :props task: ${out.stdout}") + } + assertTrue("1 actionable task: 1 from cache" in out2.stdout, out2.stdout) + } + } + } +} diff --git a/cache-service-mock/build.gradle.kts b/cache-service-mock/build.gradle.kts index 4a5abe5..182a0e0 100644 --- a/cache-service-mock/build.gradle.kts +++ b/cache-service-mock/build.gradle.kts @@ -15,7 +15,8 @@ */ dependencies { - api(project(":lib")) + implementation(project(":wrappers:actions-cache")) + implementation(project(":wrappers:actions-core")) implementation(project(":wrappers:nodejs")) implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core") implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime") diff --git a/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/CacheService.kt b/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/CacheService.kt index c21528f..4b7ab8a 100644 --- a/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/CacheService.kt +++ b/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/CacheService.kt @@ -15,11 +15,11 @@ */ package com.github.burrunan.gradle.cache +import actions.core.debug import com.github.burrunan.wrappers.js.suspendWithCallback import com.github.burrunan.wrappers.nodejs.exists import com.github.burrunan.wrappers.nodejs.readJson import com.github.burrunan.wrappers.nodejs.readToBuffer -import actions.core.debug import http.IncomingMessage import http.ServerResponse import kotlinext.js.jsObject diff --git a/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/HttpExtensions.kt b/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/HttpExtensions.kt index 9199e41..059640b 100644 --- a/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/HttpExtensions.kt +++ b/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/HttpExtensions.kt @@ -29,6 +29,8 @@ fun ServerResponse.handle(action: suspend CoroutineScope.() -> Unit) = } } catch (e: HttpException) { writeHead(e.code, e.message ?: "no message") + } catch (e: Throwable) { + writeHead(500, "Error processing ${e.message}") } finally { end() } diff --git a/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/httpclient.kt b/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/httpclient.kt new file mode 100644 index 0000000..c5f0a6d --- /dev/null +++ b/cache-service-mock/src/main/kotlin/com/github/burrunan/gradle/cache/httpclient.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:JsModule("@actions/cache/lib/internal/cacheHttpClient") +@file:JsNonModule +package com.github.burrunan.gradle.cache + +import actions.cache.DownloadOptions +import actions.cache.UploadOptions +import kotlin.js.Promise + +external fun getCacheEntry( + keys: Array, + paths: Array, + options: InternalCacheOptions = definedExternally +): Promise + +external fun reserveCache( + key: String, + paths: Array, + options: InternalCacheOptions = definedExternally +): Promise + +external fun saveCache( + cacheId: Number, + archivePath: String, + options: UploadOptions? = definedExternally, +): Promise + +external fun downloadCache( + archiveLocation: String, + archivePath: String, + options: DownloadOptions? = definedExternally, +): Promise diff --git a/gradle-launcher/build.gradle.kts b/gradle-launcher/build.gradle.kts new file mode 100644 index 0000000..be4c971 --- /dev/null +++ b/gradle-launcher/build.gradle.kts @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation(project(":hashing")) + implementation(project(":wrappers:actions-core")) + implementation(project(":wrappers:actions-exec")) + implementation(project(":wrappers:actions-http-client")) + implementation(project(":wrappers:actions-io")) + implementation(project(":wrappers:actions-tool-cache")) + implementation(project(":wrappers:java-properties")) + implementation(project(":wrappers:nodejs")) + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime") +} diff --git a/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleDistribution.kt b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleDistribution.kt new file mode 100644 index 0000000..24b041c --- /dev/null +++ b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleDistribution.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher + +data class GradleDistribution( + val version: String, + val distributionUrl: String, + val distributionSha256Sum: String?, +) + +suspend fun resolveDistribution( + versionSpec: String, + projectPath: String, + distributionUrl: String? = null, + distributionSha256Sum: String? = null, +): GradleDistribution { + return if (distributionUrl == null) { + when (val version = GradleVersion(versionSpec)) { + is GradleVersion.Official -> version.findUrl() + is GradleVersion.Dynamic -> version.findUrl() + is GradleVersion.Wrapper -> findVersionFromWrapper(projectPath) + } + } else { + GradleDistribution( + version = versionSpec, + distributionUrl = distributionUrl, + distributionSha256Sum = distributionSha256Sum ?: "$distributionUrl.sha256", + ) + } +} diff --git a/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleInstaller.kt b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleInstaller.kt new file mode 100644 index 0000000..ac76a55 --- /dev/null +++ b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleInstaller.kt @@ -0,0 +1,151 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher + +import actions.core.ActionFailedException +import actions.core.info +import actions.core.warning +import actions.httpclient.HttpClient +import actions.httpclient.HttpCodes +import actions.httpclient.IHeaders +import actions.io.rmRF +import actions.toolcache.cacheDir +import actions.toolcache.downloadTool +import actions.toolcache.extractZip +import com.github.burrunan.hashing.hashFiles +import com.github.burrunan.wrappers.nodejs.exists +import fs2.promises.chmod +import fs2.promises.readFile +import kotlinext.js.jsObject +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.await +import kotlinx.coroutines.launch +import path.path + +suspend fun install(distribution: GradleDistribution): String { + val cachedTool = actions.toolcache.find("gradle", distribution.version) + val gradleDir = if (cachedTool.isNotEmpty()) { + info("Detected Gradle ${distribution.version} at $cachedTool") + cachedTool + } else { + val gradleZip = downloadTool(distribution.distributionUrl) + distribution.distributionSha256Sum?.let { expectedSha256 -> + val hash = hashFiles(gradleZip, algorithm = "sha256", includeFileName = false).hash + if (hash != expectedSha256) { + throw ActionFailedException( + "Checksum mismatch for Gradle ${distribution.version} (${distribution.distributionUrl}). " + + "Expected: $expectedSha256, actual: $hash", + ) + } + } + val extractedGradleDir = extractZip(gradleZip) + cacheDir(path.join(extractedGradleDir, "gradle-${distribution.version}"), "gradle", distribution.version).also { + GlobalScope.launch { + // Remove temporary files + rmRF(gradleZip) + rmRF(extractedGradleDir) + } + } + } + return path.join(gradleDir, "bin", if (os.platform() == "win32") "gradle.bat" else "gradle").also { + if (os.platform() != "win32") { + chmod(it, "755") + } + } +} + +private val HTTP_AGENT = jsObject { set("User-Agent", "burrunan/gradle-cache-action") } + +suspend fun GradleVersion.Official.findUrl(): GradleDistribution { + val url = "https://services.gradle.org/versions/all" + val response = + HttpClient().getJson>(url, HTTP_AGENT).await() + if (response.statusCode.unsafeCast() != HttpCodes.OK) { + throw ActionFailedException("Unable to lookup $url Gradle version: ${response.statusCode}, ${JSON.stringify(response.result)}") + } + return response.result?.firstOrNull { it.version == name }?.resolveChecksum() + ?: throw ActionFailedException("Unable to find Gradle version $name") +} + +suspend fun GradleVersion.Dynamic.findUrl(): GradleDistribution { + val url = "https://services.gradle.org/versions/$apiPath" + val response = HttpClient().getJson(url, HTTP_AGENT).await() + if (response.statusCode.unsafeCast() != HttpCodes.OK) { + throw ActionFailedException("Unable to lookup $url Gradle version: ${response.statusCode}, ${JSON.stringify(response.result)}") + } + if (response.result?.version != null) { + return response.result.unsafeCast().resolveChecksum() + } + if (this is GradleVersion.ReleaseCandidate) { + return GradleVersion.Current.findUrl() + } + throw ActionFailedException("Empty result from $url: ${JSON.stringify(response.result)}") +} + +suspend fun GradleVersionResponse.resolveChecksum() = + GradleDistribution( + version = version, + distributionUrl = downloadUrl, + distributionSha256Sum = HttpClient().get(checksumUrl, HTTP_AGENT).await().readBody().await().trim(), + ) + +suspend fun findVersionFromWrapper(projectPath: String): GradleDistribution { + val gradleWrapperProperties = path.join(projectPath, "gradle", "wrapper", "gradle-wrapper.properties") + if (!exists(gradleWrapperProperties)) { + warning("Gradle wrapper configuration is not found at ${path.resolve(gradleWrapperProperties)}.\nWill use the current release Gradle version") + return GradleVersion.Current.findUrl() + } + val propString = readFile(gradleWrapperProperties) + val props = javaproperties.parseString(propString).run { + getKeys().associateWith { getFirst(it)!! } + } + + val distributionUrl = props.getValue("distributionUrl") + val distributionSha256Sum = props["distributionSha256Sum"] + + val version = distributionUrl.substringAfterLast("/") + .substringAfter("gradle-") + .removeSuffix("-all.zip") + .removeSuffix("-bin.zip") + .removeSuffix(".zip") + + if (distributionSha256Sum == null) { + warning( + "distributionSha256Sum is not set in $gradleWrapperProperties.\n" + + "Please consider adding the checksum, " + + "see https://docs.gradle.org/current/userguide/gradle_wrapper.html#configuring_checksum_verification", + ) + } + + return if (distributionUrl.removePrefix("https").removePrefix("http") + .startsWith("://services.gradle.org/") + ) { + // Official release, use shorter version + // https://services.gradle.org/distributions-snapshots/gradle-6.7-20200730220045+0000-all.zip + // https://services.gradle.org/distributions/gradle-6.6-rc-4-all.zip + // https://services.gradle.org/distributions/gradle-6.5.1-all.zip + if (distributionUrl.endsWith("-bin.zip") && distributionSha256Sum != null) { + GradleDistribution(version, distributionUrl, distributionSha256Sum) + } else { + // Resolve checksum from the official site + // This would switch to -bin distribution which is smaller + GradleVersion.Official(version).findUrl() + } + } else { + GradleDistribution(version, distributionUrl, distributionSha256Sum) + } +} diff --git a/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleLauncher.kt b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleLauncher.kt new file mode 100644 index 0000000..46c786c --- /dev/null +++ b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleLauncher.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.burrunan.launcher + +import actions.core.ExitCode +import actions.core.setFailed +import actions.core.setOutput +import actions.exec.exec +import com.github.burrunan.launcher.internal.GradleErrorCollector +import com.github.burrunan.launcher.internal.GradleOutErrorCollector +import kotlinext.js.jsObject +import path.path +import process + +class GradleResult( + val buildScanUrl: String?, +) + +suspend fun launchGradle(params: LaunchParams): GradleResult { + var buildScanUrl: String? = null + // See https://youtrack.jetbrains.com/issue/KT-41107 + var failureDetected = false + val errorCollector = GradleErrorCollector() + val outCollector = GradleOutErrorCollector() + + @Suppress("REDUNDANT_SPREAD_OPERATOR_IN_NAMED_FORM_IN_FUNCTION") + val result = exec( + params.gradle, + args = *(listOf("--no-daemon") + + params.properties.map { "-P${it.key}=${it.value}" } + + params.arguments).toTypedArray(), + ) { + cwd = params.projectPath + ignoreReturnCode = true + listeners = jsObject { + stdline = { + val str = it.trimEnd() + if (str.startsWith("https://gradle.com/s/")) { + setOutput("build-scan-url", str) + buildScanUrl = str + } + outCollector.process(str) + } + errline = { + errorCollector.process(it) + outCollector.process(it) + } + } + } + errorCollector.done() + outCollector.done() + for (error in errorCollector.errors + outCollector.errors) { + failureDetected = true + val shortFile = error.file + ?.removePrefix(process.cwd()) + actions.core.error(error.message, shortFile, error.line, error.col) + } + if (failureDetected) { + process.exitCode = ExitCode.Failure.unsafeCast() + } + if (!failureDetected && result.exitCode != 0) { + setFailed("Gradle process finished with a non-zero exit code: ${result.exitCode}") + } + return GradleResult(buildScanUrl) +} diff --git a/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleVersion.kt b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleVersion.kt new file mode 100644 index 0000000..d046069 --- /dev/null +++ b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleVersion.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher + +sealed class GradleVersion(val name: String, unused: Int = 0) { + companion object { + val DYNAMIC_VERSIONS = listOf( + Current, + ReleaseCandidate, + Nightly, + ReleaseNightly, + ) + val FIXED_VERSIONS = DYNAMIC_VERSIONS + Wrapper + } + + abstract class Dynamic(label: String, val apiPath: String) : GradleVersion(label) + class Official(label: String) : GradleVersion(label) { + override fun toString() = "Official($name)" + } + + object Current : Dynamic("current", "current") { + override fun toString() = "Current" + } + + object ReleaseCandidate : Dynamic("rc", "release-candidate") { + override fun toString() = "ReleaseCandidate" + } + + object Nightly : Dynamic("nightly", "nightly") { + override fun toString() = "Nightly" + } + + object ReleaseNightly : Dynamic("release-nightly", "release-nightly") { + override fun toString() = "ReleaseNightly" + } + + object Wrapper : GradleVersion("wrapper") { + override fun toString() = "Wrapper" + } +} + +fun GradleVersion(version: String) = + GradleVersion.FIXED_VERSIONS.firstOrNull { it.name == version } + ?: GradleVersion.Official(version) diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/formatBytes.kt b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleVersionResponse.kt similarity index 65% rename from layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/formatBytes.kt rename to gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleVersionResponse.kt index 3e3709c..ecb2c79 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/formatBytes.kt +++ b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/GradleVersionResponse.kt @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.github.burrunan.gradle.github -fun Long.formatBytes() = when { - this < 5 * 1024 -> "${this} B" - this < 5 * 1024 * 1204 -> "${(this + 512L) / (1024L)} KiB" - this < 5L * 1024 * 1204 * 1024 -> "${(this + 512L * 1024) / (1024L * 1024)} MiB" - else -> "${(this + 512L * 1024 * 1024) / (1024L * 1024 * 1024)} GiB" +package com.github.burrunan.launcher + +external interface GradleVersionResponse { + val version: String + val downloadUrl: String + val checksumUrl: String } diff --git a/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/LaunchParams.kt b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/LaunchParams.kt new file mode 100644 index 0000000..c66ebb2 --- /dev/null +++ b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/LaunchParams.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher + +class LaunchParams( + val gradle: String, + val projectPath: String, + val arguments: List, + val properties: Map, +) diff --git a/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/internal/GradleError.kt b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/internal/GradleError.kt new file mode 100644 index 0000000..599bd78 --- /dev/null +++ b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/internal/GradleError.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher.internal + +class GradleError( + val message: String, + val file: String? = null, + val line: Int? = null, + val col: Int? = null, +) { + override fun toString() = "GradleError(line=$line, col=$col, file=$file, message='$message')" +} diff --git a/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/internal/GradleErrorCollector.kt b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/internal/GradleErrorCollector.kt new file mode 100644 index 0000000..b8c647a --- /dev/null +++ b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/internal/GradleErrorCollector.kt @@ -0,0 +1,83 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher.internal + +private enum class ErrorHeader(val message: String) { + FAILURE("FAILURE: "), + WHERE("* Where:"), + WHAT_WENT_WRONG("* What went wrong:"), + TRY("* Try:"), +} + +private val errorHeaderValues = ErrorHeader.values() + +class GradleErrorCollector { + private val _errors = mutableListOf() + val errors: List = _errors + + private val sb = StringBuilder() + private var nextKey: ErrorHeader? = null + private val data = mutableMapOf() + + fun done() { + if (data.isNotEmpty()) { + val message = data[ErrorHeader.WHAT_WENT_WRONG] ?: "Unknown error" + + _errors += data[ErrorHeader.WHERE]?.let { location -> + Regex("^Build file '(.+)' line: (\\d+)$").matchEntire(location)?.let { + GradleError( + message = message, + file = it.groupValues[1], + line = it.groupValues[2].toInt(), + ) + } + } ?: GradleError(message) + } + data.clear() + sb.clear() + } + + fun process(line: String) { + val str = line.trimEnd() + if (str.startsWith(ErrorHeader.FAILURE.message)) { + done() + data[ErrorHeader.FAILURE] = str.removePrefix(ErrorHeader.FAILURE.message) + return + } + + if (str.startsWith("* Get more help") || + str.startsWith("BUILD FAILED ") + ) { + done() + nextKey = null + return + } + + errorHeaderValues.firstOrNull { str.startsWith(it.message) }?.let { + nextKey?.let { + data[it] = sb.toString().trimEnd() + } + sb.clear() + nextKey = it + return + } + + if (nextKey != null) { + sb.appendLine(line) + } + } +} diff --git a/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/internal/GradleOutErrorCollector.kt b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/internal/GradleOutErrorCollector.kt new file mode 100644 index 0000000..8060aa8 --- /dev/null +++ b/gradle-launcher/src/main/kotlin/com/github/burrunan/launcher/internal/GradleOutErrorCollector.kt @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher.internal + +// e: /../build.gradle.kts:62:1: Unresolved reference: invalid +private val KOTLIN_COMPILE_ERROR = Regex("^e: (\\S.+?):(\\d+):(?:(\\d+):)? (.+)$") +// [ant:checkstyle] [ERROR] /.../SqlHopTableFunction.java:56:35: ',' is not followed by whitespace. [WhitespaceAfter] +private val CHECKSTYLE_ERROR = Regex("^\\[ant:checkstyle] \\[ERROR] (\\S.+?):(\\d+):(?:(\\d+):)? (.+) \\[([^\\]]+)]$") +// /.../RelDataType.java:249: error: reference not found +private val JAVA_ERROR = Regex("^(\\S.+?):(\\d+): error: (.+)$") + +class GradleOutErrorCollector { + private val _errors = mutableListOf() + val errors: List = _errors + private var taskName: String = "Unknown task" + private var javaError: MatchResult? = null + private val javaErrorLines = mutableListOf() + + fun process(line: String) { + if (line.startsWith("> Task ") || + line.startsWith("> Configure") + ) { + taskName = line.removePrefix("> ").let { "[$it]" } + } + if (line.startsWith("e: ")) { + // Looks like Kotlin error + // e: /../build.gradle.kts:62:1: Unresolved reference: invalid + KOTLIN_COMPILE_ERROR.matchEntire(line)?.let { + _errors += GradleError( + message = "$taskName ${it.groupValues[4]}", + file = it.groupValues[1], + line = it.groupValues[2].toInt(), + col = it.groupValues[3].takeIf { it.isNotBlank() }?.toInt(), + ) + } + return + } + // Checkstyle error: + // [ant:checkstyle] [ERROR] /.../SqlHopTableFunction.java:56:35: ',' is not followed by whitespace. [WhitespaceAfter] + CHECKSTYLE_ERROR.matchEntire(line)?.let { + _errors += GradleError( + message = "$taskName ${"[${it.groupValues[5]}] ".removePrefix("[] ")}${it.groupValues[4]}", + file = it.groupValues[1], + line = it.groupValues[2].toInt(), + col = it.groupValues[3].takeIf { it.isNotBlank() }?.toInt(), + ) + } + processJavaError(line) + } + + private fun processJavaError(line: String) { + // /.../RelDataType.java:249: error: reference not found + JAVA_ERROR.matchEntire(line)?.let { + done() + javaError = it + return + } + if (javaError != null) { + val errorContinuation = line.startsWith(" ") + if (errorContinuation) { + javaErrorLines += line + } + if (!errorContinuation || javaErrorLines.size >= 3) { + done() + } + } + } + + fun done() { + javaError?.let { + _errors += GradleError( + message = "$taskName ${it.groupValues[3]}\n${javaErrorLines.joinToString("\n")}", + file = it.groupValues[1], + line = it.groupValues[2].toInt(), + ) + } + javaError = null + javaErrorLines.clear() + } +} diff --git a/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/PropertiesParserTest.kt b/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/PropertiesParserTest.kt new file mode 100644 index 0000000..1b0cd40 --- /dev/null +++ b/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/PropertiesParserTest.kt @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher + +import kotlin.test.Test +import kotlin.test.assertEquals + +class PropertiesParserTest { + @Test + fun simpleProperties() { + assertParse( + mapOf( + "a-b" to "1", + "c" to "34 34", + "url" to "https://example.com", + ), + """ + a-b=1 + url=https\://example.com + # asfd + c = 34 34 + + """.trimIndent(), + ) + } + + private fun assertParse(expected: Map, value: String) { + assertEquals( + expected, + javaproperties.parseString(value).run { + getKeys().associateWith { get(it) } + }, + value, + ) + } +} diff --git a/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/RetrieveGradleVersionTest.kt b/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/RetrieveGradleVersionTest.kt new file mode 100644 index 0000000..2f79dcc --- /dev/null +++ b/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/RetrieveGradleVersionTest.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher + +import actions.core.info +import actions.toolcache.findAllVersions +import com.github.burrunan.test.runTest +import kotlin.test.Test + +class RetrieveGradleVersionTest { + @Test + fun retrieve() = runTest { + for (version in GradleVersion.DYNAMIC_VERSIONS) { + val res = version.findUrl() + println("$version => $res") + } + } + + @Test + fun listTools() = runTest { + listTool("gradle") + listTool("Gradle") + listTool("mvn") + listTool("Maven") + } + + private fun listTool(tool: String) { + info("All $tool versions: ${findAllVersions(tool).joinToString(", ")}") + } +} diff --git a/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/internal/GradleErrorCollectorTest.kt b/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/internal/GradleErrorCollectorTest.kt new file mode 100644 index 0000000..b1053fe --- /dev/null +++ b/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/internal/GradleErrorCollectorTest.kt @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher.internal + +import kotlin.test.Test +import kotlin.test.assertEquals + +class GradleErrorCollectorTest { + @Test + fun buildScriptFailure() { + testCollector( + """ + GradleError(line=62, col=null, file=/home/runner/work/pgjdbc/pgjdbc/build.gradle.kts, message='Script compilation errors: + + Line 62: invalid code here + ^ Unresolved reference: invalid + + Line 62: invalid code here + ^ Unresolved reference: here + + 2 errors') + """.trimIndent(), + """ + * Where: + Build file '/home/runner/work/pgjdbc/pgjdbc/build.gradle.kts' line: 62 + + * What went wrong: + Script compilation errors: + + Line 62: invalid code here + ^ Unresolved reference: invalid + + Line 62: invalid code here + ^ Unresolved reference: here + + 2 errors + + * Try: + Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights. + + * Get more help at https://help.gradle.org + + BUILD FAILED in 44s + """.trimIndent(), + ) + } + + @Test + fun noLocation() { + testCollector( + """ + GradleError(line=null, col=null, file=null, message='Task 'asdfasdf' not found in root project 'pgjdbc'.') + """.trimIndent(), + """ + See https://docs.gradle.org/6.3/userguide/command_line_interface.html#sec:command_line_warnings + + FAILURE: Build failed with an exception. + + * What went wrong: + Task 'asdfasdf' not found in root project 'pgjdbc'. + + * Try: + Run gradle tasks to get a list of available tasks. Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights. + + * Get more help at https://help.gradle.org + + BUILD FAILED in 37s + """.trimIndent(), + ) + } + + private fun testCollector(expected: String, input: String) { + val collector = GradleErrorCollector() + input.lines().forEach { collector.process(it) } + + collector.done() + + assertEquals(expected, collector.errors.joinToString("\n")) + } +} diff --git a/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/internal/GradleOutCollectorTest.kt b/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/internal/GradleOutCollectorTest.kt new file mode 100644 index 0000000..fd1e60e --- /dev/null +++ b/gradle-launcher/src/test/kotlin/com/github/burrunan/launcher/internal/GradleOutCollectorTest.kt @@ -0,0 +1,109 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.launcher.internal + +import kotlin.test.Test +import kotlin.test.assertEquals + +class GradleOutCollectorTest { + @Test + fun koltlinCompileErrors() { + testCollector( + """ + GradleError(line=62, col=1, file=/home/runner/work/pgjdbc/pgjdbc/build.gradle.kts, message='[Configure project :] Unresolved reference: invalid') + GradleError(line=62, col=14, file=/home/runner/work/pgjdbc/pgjdbc/build.gradle.kts, message='[Configure project :] Unresolved reference: here') + """.trimIndent(), + """ + > Configure project : + Evaluating root project 'pgjdbc' using build file '/home/runner/work/pgjdbc/pgjdbc/build.gradle.kts'. + Loading cache entry 'cache/eila5i6e0q7sxpvg89w345ymz' from S3 bucket + e: /home/runner/work/pgjdbc/pgjdbc/build.gradle.kts:62:1: Unresolved reference: invalid + e: /home/runner/work/pgjdbc/pgjdbc/build.gradle.kts:62:14: Unresolved reference: here + """.trimIndent(), + ) + } + + @Test + fun checkstyleError() { + testCollector( + """ + GradleError(line=56, col=35, file=/Users/runner/work/calcite/calcite/core/src/main/java/org/apache/calcite/sql/SqlHopTableFunction.java, message='[Task :core:checkstyleMain] [WhitespaceAfter] ',' is not followed by whitespace.') + GradleError(line=32, col=null, file=/code/calcite/linq4j/src/main/java/org/apache/calcite/linq4j/AbstractEnumerable.java, message='[Task :core:checkstyleMain] [Indentation] 'method def modifier' has incorrect indentation level 0, expected level should be 2.') + """.trimIndent(), + """ + > Task :core:checkstyleMain + [ant:checkstyle] [ERROR] /Users/runner/work/calcite/calcite/core/src/main/java/org/apache/calcite/sql/SqlHopTableFunction.java:56:35: ',' is not followed by whitespace. [WhitespaceAfter] + [ant:checkstyle] [ERROR] /code/calcite/linq4j/src/main/java/org/apache/calcite/linq4j/AbstractEnumerable.java:32: 'method def modifier' has incorrect indentation level 0, expected level should be 2. [Indentation] + """.trimIndent(), + ) + } + + @Test + fun javadocError() { + testCollector( + """ + GradleError(line=249, col=null, file=/Users/../type/RelDataType.java, message='[Task :babel:javadoc] reference not found + * {@link #equals(Object)}. + ^') + """.trimIndent(), + """ + > Task :babel:javadoc + /Users/runner/runners/2.263.0/work/calcite/calcite/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java:632: warning: no @param for rel + public List getAverageColumnSizesNotNull(RelNode rel) { + ^ + /Users/runner/runners/2.263.0/work/calcite/calcite/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java:632: warning: no @return + public List getAverageColumnSizesNotNull(RelNode rel) { + ^ + /Users/../type/RelDataType.java:249: error: reference not found + * {@link #equals(Object)}. + ^ + """.trimIndent(), + ) + } + + @Test + fun javacError() { + testCollector( + """ + GradleError(line=46, col=null, file=/home/runner/../ReaderInputStreamTest.java, message='[Task :compileJava] cannot find symbol + Arrays.fill(acutal, (byte) 0x00); + ^ + symbol: variable acutal') + """.trimIndent(), + """ + > Task :compileJava + Compiling with JDK Java compiler API. + /home/runner/../ReaderInputStreamTest.java:46: error: cannot find symbol + Arrays.fill(acutal, (byte) 0x00); + ^ + symbol: variable acutal + location: class ReaderInputStreamTest + Note: Some input files use or override a deprecated API. + Note: Recompile with -Xlint:deprecation for details. + """.trimIndent(), + ) + } + + private fun testCollector(expected: String, input: String) { + val collector = GradleOutErrorCollector() + input.lines().forEach { collector.process(it) } + + collector.done() + + assertEquals(expected, collector.errors.joinToString("\n")) + } +} diff --git a/hashing/build.gradle.kts b/hashing/build.gradle.kts new file mode 100644 index 0000000..fd381b1 --- /dev/null +++ b/hashing/build.gradle.kts @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { + kotlin("plugin.serialization") +} + +dependencies { + implementation(project(":wrappers:actions-core")) + implementation(project(":wrappers:actions-glob")) + implementation(project(":wrappers:js")) + implementation(project(":wrappers:nodejs")) + implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime") +} diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/hashing/HashDetails.kt b/hashing/src/main/kotlin/com/github/burrunan/hashing/HashDetails.kt similarity index 86% rename from layered-cache/src/main/kotlin/com/github/burrunan/gradle/hashing/HashDetails.kt rename to hashing/src/main/kotlin/com/github/burrunan/hashing/HashDetails.kt index 4e998f0..26a5a8e 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/hashing/HashDetails.kt +++ b/hashing/src/main/kotlin/com/github/burrunan/hashing/HashDetails.kt @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.github.burrunan.gradle.hashing +package com.github.burrunan.hashing +import actions.core.ActionFailedException import actions.core.warning +import actions.glob.Globber +import actions.glob.glob +import com.github.burrunan.wrappers.nodejs.pipe +import com.github.burrunan.wrappers.nodejs.use import crypto.createHash import fs.createReadStream import fs2.promises.stat -import actions.glob.create -import com.github.burrunan.wrappers.nodejs.pipe -import com.github.burrunan.wrappers.nodejs.use -import kotlinx.coroutines.await import kotlinx.serialization.Serializable import process @@ -59,9 +60,13 @@ private fun sha1FromModulesFileName(key: String): String { return key.substring(hashStart, lastSlash).padStart(40, '0') } -suspend fun hashFilesDetailed(vararg paths: String, algorithm: String = "sha1"): HashDetails = try { - val globber = create(paths.joinToString("\n")).await() - val fileNames = globber.glob().await() +suspend fun hashFilesDetailed( + vararg paths: String, + algorithm: String = "sha1", + includeFileName: Boolean = true, +): HashDetails = try { + val globber = Globber(paths.joinToString("\n")) + val fileNames = globber.glob() // Sorting is needed for stable overall hash fileNames.sort() @@ -72,7 +77,7 @@ suspend fun hashFilesDetailed(vararg paths: String, algorithm: String = "sha1"): val files = mutableMapOf() val overallHash = createHash(algorithm) for (name in fileNames) { - val statSync = stat(name).await() + val statSync = stat(name) if (statSync.isDirectory()) { continue } @@ -106,7 +111,9 @@ suspend fun hashFilesDetailed(vararg paths: String, algorithm: String = "sha1"): } files[key] = FileDetails(fileSize, digest) // Add filename - overallHash.update(key) + if (includeFileName) { + overallHash.update(key) + } overallHash.update(digest) } HashDetails( @@ -114,5 +121,5 @@ suspend fun hashFilesDetailed(vararg paths: String, algorithm: String = "sha1"): HashContents(files), ) } catch (e: Throwable) { - throw IllegalArgumentException("Unable to hash ${paths.joinToString(", ")}", e) + throw ActionFailedException("Unable to hash ${paths.joinToString(", ")}: $e", e) } diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/hashing/diff.kt b/hashing/src/main/kotlin/com/github/burrunan/hashing/diff.kt similarity index 96% rename from layered-cache/src/main/kotlin/com/github/burrunan/gradle/hashing/diff.kt rename to hashing/src/main/kotlin/com/github/burrunan/hashing/diff.kt index 66392ca..747f81d 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/hashing/diff.kt +++ b/hashing/src/main/kotlin/com/github/burrunan/hashing/diff.kt @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.github.burrunan.gradle.hashing +package com.github.burrunan.hashing -import com.github.burrunan.gradle.github.formatBytes +import com.github.burrunan.formatBytes class Diff( val newFiles: Int, diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/hashFiles.kt b/hashing/src/main/kotlin/com/github/burrunan/hashing/hashFiles.kt similarity index 77% rename from layered-cache/src/main/kotlin/com/github/burrunan/gradle/hashFiles.kt rename to hashing/src/main/kotlin/com/github/burrunan/hashing/hashFiles.kt index 53b0859..263573c 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/hashFiles.kt +++ b/hashing/src/main/kotlin/com/github/burrunan/hashing/hashFiles.kt @@ -13,14 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.github.burrunan.gradle +package com.github.burrunan.hashing +import actions.core.ActionFailedException import actions.core.warning -import crypto.createHash -import actions.glob.create +import actions.glob.Globber +import actions.glob.glob import com.github.burrunan.wrappers.nodejs.pipe import com.github.burrunan.wrappers.nodejs.use -import kotlinx.coroutines.await +import crypto.createHash import process data class HashResult( @@ -29,9 +30,13 @@ data class HashResult( val totalBytes: Int, ) -suspend fun hashFiles(vararg paths: String, algorithm: String = "sha1"): HashResult = try { - val globber = create(paths.joinToString("\n")).await() - val fileNames = globber.glob().await() +suspend fun hashFiles( + vararg paths: String, + algorithm: String = "sha1", + includeFileName: Boolean = true, +): HashResult = try { + val globber = Globber(paths.joinToString("\n")) + val fileNames = globber.glob() fileNames.sort() val githubWorkspace = process.cwd() @@ -41,7 +46,7 @@ suspend fun hashFiles(vararg paths: String, algorithm: String = "sha1"): HashRes var totalBytes = 0 var numFiles = 0 for (name in fileNames) { - val statSync = fs2.promises.stat(name).await() + val statSync = fs2.promises.stat(name) if (statSync.isDirectory()) { continue } @@ -66,7 +71,9 @@ suspend fun hashFiles(vararg paths: String, algorithm: String = "sha1"): HashRes continue } - hash.update(key, "utf8") + if (includeFileName) { + hash.update(key, "utf8") + } } hash.end() HashResult( @@ -75,5 +82,5 @@ suspend fun hashFiles(vararg paths: String, algorithm: String = "sha1"): HashRes totalBytes = totalBytes, ) } catch (e: Throwable) { - throw IllegalArgumentException("Unable to hash ${paths.joinToString(", ")}", e) + throw ActionFailedException("Unable to hash ${paths.joinToString(", ")}: $e", e) } diff --git a/layered-cache/build.gradle.kts b/layered-cache/build.gradle.kts index a2f4e9d..04507f8 100644 --- a/layered-cache/build.gradle.kts +++ b/layered-cache/build.gradle.kts @@ -19,11 +19,14 @@ plugins { } dependencies { - api(project(":lib")) - implementation(project(":wrappers:nodejs")) + implementation(project(":gradle-launcher")) + implementation(project(":hashing")) implementation(project(":wrappers:actions-cache")) implementation(project(":wrappers:actions-core")) implementation(project(":wrappers:actions-exec")) + implementation(project(":wrappers:actions-glob")) + implementation(project(":wrappers:nodejs")) + implementation(project(":wrappers:octokit-webhooks")) implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core") implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime") diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/GradleCacheAction.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/GradleCacheAction.kt index a152747..35334d5 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/GradleCacheAction.kt +++ b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/GradleCacheAction.kt @@ -15,38 +15,41 @@ */ package com.github.burrunan.gradle +import actions.core.ActionFailedException +import actions.core.ActionStage +import actions.core.info +import actions.exec.exec import com.github.burrunan.gradle.cache.* -import com.github.burrunan.gradle.github.event.ActionsTrigger -import com.github.burrunan.gradle.github.stateVariable import com.github.burrunan.gradle.github.suspendingStateVariable -import com.github.burrunan.gradle.github.toBoolean -import fs2.promises.readFile -import actions.core.ext.getInput -import actions.exec.exec -import kotlinx.coroutines.await +import com.github.burrunan.launcher.GradleDistribution +import octokit.ActionsTrigger +import kotlin.js.Date +import kotlin.math.roundToInt -class GradleCacheAction(val trigger: ActionsTrigger, val params: Parameters) { +class GradleCacheAction( + val trigger: ActionsTrigger, + val params: Parameters, + val gradleDistribution: GradleDistribution, +) { companion object { const val DEFAULT_BRANCH_VAR = "defaultbranch" } private val treeId = suspendingStateVariable("tree_id") { - exec("git", "log", "-1", "--quiet", "--format=%T").stdout + exec("git", "log", "-1", "--quiet", "--format=%T", captureOutput = true).stdout } - suspend fun run() { - val gradleVersion = suspendingStateVariable("gradleVersion") { - determineGradleVersion(params.path) - } + suspend fun execute(stage: ActionStage) { + val gradleVersion = gradleDistribution.version val caches = mutableListOf() if (params.generatedGradleJars) { - caches.add(gradleGeneratedJarsCache(gradleVersion.get())) + caches.add(gradleGeneratedJarsCache(gradleVersion)) } if (params.localBuildCache) { - caches.add(localBuildCache(params.jobId, trigger, gradleVersion.get(), treeId.get())) + caches.add(localBuildCache(params.jobId, trigger, gradleVersion, treeId.get())) } if (params.gradleDependenciesCache) { @@ -54,42 +57,20 @@ class GradleCacheAction(val trigger: ActionsTrigger, val params: Parameters) { } if (params.mavenDependenciesCache) { - caches.add(mavenDependenciesCache(trigger, params.path)) + caches.add(mavenDependenciesCache(trigger, params.path, params.mavenLocalIgnorePaths)) } val cache = CompositeCache("all-caches", caches, concurrent = params.concurrent) - val post = stateVariable("POST").toBoolean() - if (post.get()) { - cache.save() - } else { - post.set(true) - cache.restore() - } - } - - private suspend fun determineGradleVersion(path: String): String { - val gradleWrapperProperties = "$path/gradle/wrapper/gradle-wrapper.properties" - val gradleVersion = getInput("gradle-version").ifBlank { "wrapper" } - if (!gradleVersion.equals("wrapper", ignoreCase = true)) { - return gradleVersion - } - val props = readFile(gradleWrapperProperties, "utf8").await() - val distributionUrlRegex = Regex("\\s*distributionUrl\\s*=\\s*([^#]+)") - val distributionUrl = props.split(Regex("[\r\n]+")) - .filter { !it.startsWith("#") && it.contains("distributionUrl") } - .mapNotNull { distributionUrlRegex.matchEntire(it)?.groupValues?.get(1) } - .firstOrNull() - return if (distributionUrl - ?.removePrefix("https")?.removePrefix("http") - ?.startsWith("\\://services.gradle.org/") == true - ) { - // Official release, use shorter version - // https://services.gradle.org/distributions-snapshots/gradle-6.7-20200730220045+0000-all.zip - // https://services.gradle.org/distributions/gradle-6.6-rc-4-all.zip - // https://services.gradle.org/distributions/gradle-6.5.1-all.zip - distributionUrl.substringAfterLast("/").substringBefore(".zip") - } else { - hashFiles(gradleWrapperProperties, algorithm = "sha1").hash + when (stage) { + ActionStage.MAIN -> { + val started = Date.now() + val restore = cache.restore() + val elapsed = Date.now() - started + info("Cache restore took ${(elapsed / 1000).roundToInt()} seconds") + } + ActionStage.POST -> cache.save() + else -> throw ActionFailedException("Cache action should be called in PRE or POST stages only. " + + "Current stage is $stage") } } } diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/Parameters.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/Parameters.kt index 5cd4530..49652ca 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/Parameters.kt +++ b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/Parameters.kt @@ -22,7 +22,8 @@ data class Parameters( val generatedGradleJars: Boolean, val localBuildCache: Boolean, val gradleDependenciesCache: Boolean, - val gradleDependenciesCacheKey: String, + val gradleDependenciesCacheKey: List, val mavenDependenciesCache: Boolean, + val mavenLocalIgnorePaths: List, val concurrent: Boolean, ) diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/ActionsTriggerExtensions.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/ActionsTriggerExtensions.kt new file mode 100644 index 0000000..7b45387 --- /dev/null +++ b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/ActionsTriggerExtensions.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2020 Vladimir Sitnikov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.burrunan.gradle.cache + +import actions.core.ActionsEnvironment +import com.github.burrunan.gradle.GradleCacheAction +import octokit.ActionsTrigger + +val ActionsTrigger.cacheKey: String + get() = when (this) { + is ActionsTrigger.PullRequest -> "PR${event.pull_request.number}" + is ActionsTrigger.BranchPush -> when (val ref = event.ref.removePrefix("refs/heads/")) { + event.repository.default_branch.removePrefix("refs/heads/") -> + GradleCacheAction.DEFAULT_BRANCH_VAR + else -> ref + } + is ActionsTrigger.Schedule, is ActionsTrigger.WorkflowDispatch -> + GradleCacheAction.DEFAULT_BRANCH_VAR + is ActionsTrigger.Other -> "$name-${ActionsEnvironment.GITHUB_WORKFLOW}-${ActionsEnvironment.GITHUB_SHA}" + } diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/DefaultCache.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/DefaultCache.kt index c194584..f0b31a9 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/DefaultCache.kt +++ b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/DefaultCache.kt @@ -16,19 +16,20 @@ package com.github.burrunan.gradle.cache import actions.cache.RestoreType -import com.github.burrunan.gradle.github.formatBytes -import com.github.burrunan.gradle.github.stateVariable -import com.github.burrunan.gradle.github.toBoolean -import com.github.burrunan.gradle.github.toInt -import com.github.burrunan.gradle.hashing.HashContents -import com.github.burrunan.gradle.hashing.HashDetails -import com.github.burrunan.gradle.hashing.HashInfo -import com.github.burrunan.gradle.hashing.hashFilesDetailed -import com.github.burrunan.wrappers.nodejs.removeFiles import actions.cache.restoreAndLog import actions.cache.saveAndLog import actions.core.debug import actions.core.info +import actions.glob.removeFiles +import com.github.burrunan.formatBytes +import com.github.burrunan.gradle.github.stateVariable +import com.github.burrunan.gradle.github.toBoolean +import com.github.burrunan.gradle.github.toInt +import com.github.burrunan.hashing.HashContents +import com.github.burrunan.hashing.HashDetails +import com.github.burrunan.hashing.HashInfo +import com.github.burrunan.hashing.hashFilesDetailed +import com.github.burrunan.wrappers.nodejs.exists import kotlin.math.absoluteValue class DefaultCache( @@ -38,6 +39,7 @@ class DefaultCache( private val paths: List, private val readOnlyMessage: String? = null, stateKey: String = "", + private val skipRestoreIfPathExists: String? = null ) : Cache { @Suppress("CanBePrimaryConstructorProperty") override val name: String = name @@ -51,6 +53,7 @@ class DefaultCache( private val saveRestorePaths = paths + cacheInfo.cachedName + cacheContents.cachedName private val isExactMatch = stateVariable("${name}_${stateKey}_exact").toBoolean() + private val isSkipped = stateVariable("${name}_${stateKey}_skip").toBoolean() private val restoredKeyIndex = stateVariable("${name}_${stateKey}_key").toInt(-1) private val restoredKey: String? @@ -83,6 +86,12 @@ class DefaultCache( } override suspend fun restore(): RestoreType { + skipRestoreIfPathExists?.let { + if (exists(it)) { + debug { "$name: $it already exists, so the cache restore and upload will be skipped" } + isSkipped.set(true) + } + } debug { "$name: restoring $primaryKey, $restoreKeys, $saveRestorePaths" } return restoreAndLog(saveRestorePaths, primaryKey, restoreKeys, version = version).also { isExactMatch.set(it is RestoreType.Exact) @@ -103,6 +112,10 @@ class DefaultCache( override suspend fun save() { debug { "$name: saving ${isExactMatch.get()} ${restoredKeyIndex.get()} $primaryKey, $restoreKeys, $saveRestorePaths" } + if (isSkipped.get()) { + debug { "$name: cache save skipped" } + return + } if (isExactMatch.get()) { info("$name loaded from exact match, no need to update the cache entry") return @@ -112,12 +125,6 @@ class DefaultCache( return } - // @actions/glob does not support negative wildcards, so we remove the files before caching - removeFiles( - paths.filter { it.startsWith("!") && it.contains("*") } - .map { it.substring(1) }, - ) - restoredKey?.let { key -> cacheInfo.prepare(key) cacheContents.prepare(key) diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/GradleGeneratedJarsCache.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/GradleGeneratedJarsCache.kt index 08a5255..4ca3e56 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/GradleGeneratedJarsCache.kt +++ b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/GradleGeneratedJarsCache.kt @@ -18,9 +18,10 @@ package com.github.burrunan.gradle.cache fun gradleGeneratedJarsCache(gradleVersion: String): Cache = DefaultCache( name = "gradle-generated-jars", - primaryKey = "generated-gradle-jars-$gradleVersion", + primaryKey = "generated-gradle-jars-gradle-$gradleVersion", paths = listOf( - "~/.gradle/caches/*.*/generated-gradle-jars", - "!~/.gradle/caches/*.*/generated-gradle-jars/*.lock", + "~/.gradle/caches/$gradleVersion/generated-gradle-jars", + "!~/.gradle/caches/$gradleVersion/generated-gradle-jars/*.lock", ), + skipRestoreIfPathExists = "~/.gradle/caches/$gradleVersion/generated-gradle-jars", ) diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/LayeredCache.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/LayeredCache.kt index 2b43198..9ff822c 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/LayeredCache.kt +++ b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/LayeredCache.kt @@ -16,13 +16,14 @@ package com.github.burrunan.gradle.cache import actions.cache.RestoreType -import com.github.burrunan.gradle.github.formatBytes -import com.github.burrunan.gradle.github.stateVariable -import com.github.burrunan.gradle.github.toBoolean -import com.github.burrunan.gradle.hashing.* -import com.github.burrunan.wrappers.nodejs.removeFiles +import actions.core.ActionFailedException import actions.core.debug import actions.core.info +import actions.glob.removeFiles +import com.github.burrunan.formatBytes +import com.github.burrunan.gradle.github.stateVariable +import com.github.burrunan.gradle.github.toBoolean +import com.github.burrunan.hashing.* import kotlinx.serialization.Serializable @Serializable @@ -51,7 +52,6 @@ class LayeredCache( private val layers = MetadataFile("layer-$name", CacheLayers.serializer()) private val isExactMatch = stateVariable("${name}_exact").toBoolean() - private val pathsWithoutNegativeGlobs = paths.filterNot { it.startsWith("!") && it.contains("*") } private val index = DefaultCache( name = "$version-index-$name", @@ -66,7 +66,7 @@ class LayeredCache( stateKey = stateKey, primaryKey = primaryKey, restoreKeys = if (paths.isNotEmpty()) listOf() else restoreKeys.map { "$version-$it" }, - paths = paths.ifEmpty { pathsWithoutNegativeGlobs }, + paths = this@toCache.paths.ifEmpty { this@LayeredCache.paths }, ) private fun Diff.toLayer(): CacheLayer { @@ -83,7 +83,7 @@ class LayeredCache( if (indexRestoreType == RestoreType.None) { return RestoreType.None } - val cacheIndex = layers.decode() ?: throw IllegalStateException("${layers.cachedName} is not found") + val cacheIndex = layers.decode() ?: throw ActionFailedException("${layers.cachedName} is not found") var restoreType: RestoreType = when (indexRestoreType) { is RestoreType.Exact -> RestoreType.Exact(indexRestoreType.path.removePrefix("index-")) @@ -117,13 +117,7 @@ class LayeredCache( return } - // @actions/glob does not support negative wildcards, so we remove the files before caching - removeFiles( - paths.filter { it.startsWith("!") && it.contains("*") } - .map { it.substring(1) }, - ) - - val cacheIndex = layers.decode() + val cacheIndex = layers.decode(warnOnMissing = false) val isBaseline = primaryKey.startsWith(baseline) if (cacheIndex == null) { if (!isBaseline) { diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/MetadataFile.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/MetadataFile.kt index ede7949..e6ac7ec 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/MetadataFile.kt +++ b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/MetadataFile.kt @@ -15,11 +15,11 @@ */ package com.github.burrunan.gradle.cache +import actions.core.warning import com.github.burrunan.wrappers.nodejs.exists import com.github.burrunan.wrappers.nodejs.normalizedPath import fs2.promises.readFile -import actions.core.warning -import kotlinx.coroutines.await +import fs2.promises.rename import kotlinx.serialization.KSerializer import kotlinx.serialization.SerializationException import kotlinx.serialization.json.Json @@ -51,21 +51,23 @@ class MetadataFile(name: String, private val serializer: KSerializer, priv val path = cachedName.normalizedPath if (exists(path)) { prepare(key) - fs2.promises.rename(path, uniqueName).await() + rename(path, uniqueName) } else { warning("$cachedName: $path does not exist") } } - suspend fun decode(): T? { + suspend fun decode(warnOnMissing: Boolean = true): T? { if (!exists(uniqueName)) { - warning("$cachedName: $uniqueName does not exist") + if (warnOnMissing) { + warning("$cachedName: $uniqueName does not exist") + } return null } return try { Json.decodeFromString( serializer, - readFile(uniqueName, "utf8").await(), + readFile(uniqueName) ) } catch (e: SerializationException) { warning("$cachedName: error deserializing $uniqueName with ${serializer.descriptor.serialName}, message: $e") @@ -78,6 +80,6 @@ class MetadataFile(name: String, private val serializer: KSerializer, priv cachedName.normalizedPath, Json.encodeToString(serializer, value), "utf8", - ).await() + ) } } diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/dependenciesCache.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/dependenciesCache.kt index 578a8ea..6a9fe3c 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/dependenciesCache.kt +++ b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/dependenciesCache.kt @@ -16,13 +16,12 @@ package com.github.burrunan.gradle.cache +import actions.core.ActionsEnvironment import actions.core.debug import com.github.burrunan.gradle.GradleCacheAction -import com.github.burrunan.gradle.github.env.ActionsEnvironment -import com.github.burrunan.gradle.github.event.ActionsTrigger -import com.github.burrunan.gradle.github.event.cacheKey import com.github.burrunan.gradle.github.suspendingStateVariable -import com.github.burrunan.gradle.hashFiles +import com.github.burrunan.hashing.hashFiles +import octokit.ActionsTrigger /** * Populate cache only when building a default branch, otherwise treat the cache as read-only. @@ -57,7 +56,7 @@ suspend fun dependenciesCache( ) } -suspend fun gradleDependenciesCache(trigger: ActionsTrigger, path: String, gradleDependenciesCacheKey: String): Cache = +suspend fun gradleDependenciesCache(trigger: ActionsTrigger, path: String, gradleDependenciesCacheKey: List): Cache = dependenciesCache( "gradle", trigger, @@ -67,26 +66,23 @@ suspend fun gradleDependenciesCache(trigger: ActionsTrigger, path: String, gradl "!~/.gradle/caches/modules-2/modules-2.lock", ), pathDependencies = listOf( - "!$path/**/.gradle/", - "$path/**/*.gradle.kts", + "$path/**/*.gradle", "$path/**/gradle/dependency-locking/**", - // We do not want .gradle folder, so we want to have at least one character before .gradle - "$path/**/?*.gradle", "$path/**/*.properties", - ) + gradleDependenciesCacheKey.split(Regex("[\r\n]+")) - .map { it.trim() } - .filterNot { it.isEmpty() } - .map { + ) + gradleDependenciesCacheKey.map { (if (it.startsWith("!")) "!" else "") + "$path/**/" + it.trim().trimStart('!') - }, + } + + // Excludes must go the last so they win + listOf("!$path/**/.gradle/"), ) -suspend fun mavenDependenciesCache(trigger: ActionsTrigger, path: String): Cache = +suspend fun mavenDependenciesCache(trigger: ActionsTrigger, path: String, mavenLocalIgnorePaths: List): Cache = dependenciesCache( "maven", trigger, - cacheLocation = listOf("~/.m2/repository"), + cacheLocation = listOf("~/.m2/repository") + + mavenLocalIgnorePaths.map { "!~/.m2/repository/$it" }, pathDependencies = listOf( "$path/**/pom.xml", ), diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/localBuildCache.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/localBuildCache.kt index 908b7c8..0201ea6 100644 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/localBuildCache.kt +++ b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/cache/localBuildCache.kt @@ -16,8 +16,7 @@ package com.github.burrunan.gradle.cache import com.github.burrunan.gradle.GradleCacheAction -import com.github.burrunan.gradle.github.event.ActionsTrigger -import com.github.burrunan.gradle.github.event.cacheKey +import octokit.ActionsTrigger fun localBuildCache(jobId: String, trigger: ActionsTrigger, gradleVersion: String, treeId: String): Cache { val buildCacheLocation = "~/.gradle/caches/build-cache-1" @@ -38,7 +37,7 @@ fun localBuildCache(jobId: String, trigger: ActionsTrigger, gradleVersion: Strin "master", "main", ) - val prefix = "gradle-build-cache-$jobId-$gradleVersion" + val prefix = "gradle-build-cache-$jobId-gradle-$gradleVersion" return LayeredCache( name = "local-build-cache", baseline = "$prefix-$defaultBranch", diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/env/ActionsEnvironment.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/env/ActionsEnvironment.kt deleted file mode 100644 index 92a9c85..0000000 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/env/ActionsEnvironment.kt +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2020 Vladimir Sitnikov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.github.burrunan.gradle.github.env - -import process -import kotlin.reflect.KProperty - -/** - * See https://docs.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables - */ -object ActionsEnvironment { - val HOME by Environment - val GITHUB_WORKFLOW by Environment - val GITHUB_RUN_ID by Environment - val GITHUB_RUN_NUMBER by Environment - val GITHUB_ACTION by Environment - val GITHUB_ACTOR by Environment - val GITHUB_REPOSITORY by Environment - val GITHUB_EVENT_NAME by Environment - val GITHUB_EVENT_PATH by Environment - val GITHUB_WORKSPACE by Environment - val GITHUB_SHA by Environment - val GITHUB_REF by Environment - val GITHUB_HEAD_REF by Environment - val GITHUB_BASE_REF by Environment - val GITHUB_SERVER_URL by Environment - val GITHUB_API_URL by Environment - val GITHUB_GRAPHQL_URL by Environment - val RUNNER_OS by Environment -} - -private object Environment { - operator fun getValue(environment: Any, property: KProperty<*>): String = - process.env[property.name] ?: throw IllegalStateException("${property.name} is not found in process.env") -} diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/event/ActionsTrigger.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/event/ActionsTrigger.kt deleted file mode 100644 index abcec12..0000000 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/event/ActionsTrigger.kt +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2020 Vladimir Sitnikov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.github.burrunan.gradle.github.event - -import com.github.burrunan.gradle.GradleCacheAction -import com.github.burrunan.gradle.github.env.ActionsEnvironment -import fs2.promises.readFile -import kotlinx.coroutines.await - -sealed class ActionsTrigger(val name: String, open val event: Event) { - class PullRequest(override val event: PullRequestEvent) : ActionsTrigger("pull_request", event) - class BranchPush(override val event: BranchPushEvent) : ActionsTrigger("push", event) - class Schedule(name: String, event: Event) : ActionsTrigger(name, event) - class WorkflowDispatch(name: String, event: Event) : ActionsTrigger(name, event) - class Other(name: String, event: Event) : ActionsTrigger(name, event) -} - -val ActionsTrigger.cacheKey: String - get() = when (this) { - is ActionsTrigger.PullRequest -> "PR${event.pull_request.number}" - is ActionsTrigger.BranchPush -> when (val ref = event.ref.removePrefix("refs/heads/")) { - event.repository.default_branch.removePrefix("refs/heads/") -> - GradleCacheAction.DEFAULT_BRANCH_VAR - else -> ref - } - is ActionsTrigger.Schedule, is ActionsTrigger.WorkflowDispatch -> - GradleCacheAction.DEFAULT_BRANCH_VAR - is ActionsTrigger.Other -> "$name-${ActionsEnvironment.GITHUB_WORKFLOW}-${ActionsEnvironment.GITHUB_SHA}" - } - -suspend fun currentTrigger(): ActionsTrigger { - val eventString = readFile(ActionsEnvironment.GITHUB_EVENT_PATH, "utf8").await() - val event = JSON.parse(eventString) - @Suppress("UNCHECKED_CAST_TO_EXTERNAL_INTERFACE") - return when (val eventName = ActionsEnvironment.GITHUB_EVENT_NAME) { - "pull_request" -> ActionsTrigger.PullRequest(event as PullRequestEvent) - "push" -> ActionsTrigger.BranchPush(event as BranchPushEvent) - "schedule" -> ActionsTrigger.Schedule(eventName, event) - "workflow_dispatch" -> ActionsTrigger.WorkflowDispatch(eventName, event) - else -> ActionsTrigger.Other(eventName, event) - } -} diff --git a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/event/Event.kt b/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/event/Event.kt deleted file mode 100644 index 1ce703d..0000000 --- a/layered-cache/src/main/kotlin/com/github/burrunan/gradle/github/event/Event.kt +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2020 Vladimir Sitnikov - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.github.burrunan.gradle.github.event - -external interface Event { -} - -/** - * See https://docs.github.com/en/developers/webhooks-and-events/webhook-events-and-payloads#push - */ -external interface BranchPushEvent : Event { - var repository: Repository - var action: String - var ref: String - var commits: List - var head_commit: Commit - var pusher: UserInfo - var compare: String - var created: Boolean - var deleted: Boolean - var forced: Boolean -} - -/** - * See https://docs.github.com/en/developers/webhooks-and-events/webhook-events-and-payloads#pull_request - */ -external interface PullRequestEvent : Event { - var repository: Repository - var url: String - var id: Number - var number: Int - var state: String - var base_ref: String - var locked: Boolean - var title: String - var body: String - var pull_request: PullRequest -} - -external interface Commit { - var author: UserInfo - var committer: UserInfo - var id: String - var message: String - var tree_id: String - var url: String -} - -external interface User : UserInfo - -external interface UserInfo { - var email: String - var name: String? - var username: String -} - -external interface Repository { - var fork: Boolean - var master_branch: String - var default_branch: String - var name: String - var organization: String -} - -external interface PullRequest { - var number: Int - var additions: Int - var deletions: Int - var commits: Int - var draft: Boolean - var changed_files: Int - var base: PullRequestBase - var head: PullRequestHead - var repo: Repository - var user: User -} - -external interface CommitRef { - var label: String - var ref: String - var sha: String -} - -external interface PullRequestBase : CommitRef - -external interface PullRequestHead : CommitRef diff --git a/layered-cache/src/test/kotlin/com/github/burrunan/gradle/CacheServerTest.kt b/layered-cache/src/test/kotlin/com/github/burrunan/gradle/CacheServerTest.kt index 41b5ae8..9078876 100644 --- a/layered-cache/src/test/kotlin/com/github/burrunan/gradle/CacheServerTest.kt +++ b/layered-cache/src/test/kotlin/com/github/burrunan/gradle/CacheServerTest.kt @@ -17,12 +17,13 @@ package com.github.burrunan.gradle import actions.cache.RestoreType +import actions.cache.restoreAndLog +import actions.cache.saveAndLog import com.github.burrunan.gradle.cache.CacheService import com.github.burrunan.test.runTest import com.github.burrunan.wrappers.nodejs.mkdir -import actions.cache.restoreAndLog -import actions.cache.saveAndLog -import kotlinx.coroutines.await +import fs2.promises.readFile +import fs2.promises.writeFile import kotlin.test.Test import kotlin.test.assertEquals @@ -35,7 +36,7 @@ class CacheServerTest { mkdir(dir) val file = "$dir/cached.txt" val contents = "hello, world" - fs2.promises.writeFile(file, contents, "utf8").await() + writeFile(file, contents) val patterns = listOf("$dir/**") val primaryKey = "linux-gradle-123123" @@ -57,7 +58,7 @@ class CacheServerTest { ) assertEquals( - fs2.promises.readFile(file, "utf8").await(), + readFile(file), contents, "Contents after restore should match", ) @@ -74,7 +75,7 @@ class CacheServerTest { ) assertEquals( - fs2.promises.readFile(file, "utf8").await(), + readFile(file), contents, "Contents after restore should match", ) diff --git a/layered-cache/src/test/kotlin/com/github/burrunan/gradle/GlobTest.kt b/layered-cache/src/test/kotlin/com/github/burrunan/gradle/GlobTest.kt index 9dd8942..2861152 100644 --- a/layered-cache/src/test/kotlin/com/github/burrunan/gradle/GlobTest.kt +++ b/layered-cache/src/test/kotlin/com/github/burrunan/gradle/GlobTest.kt @@ -16,11 +16,12 @@ */ package com.github.burrunan.gradle -import com.github.burrunan.gradle.github.formatBytes -import com.github.burrunan.gradle.hashing.hashFilesDetailed +import com.github.burrunan.formatBytes +import com.github.burrunan.hashing.hashFilesDetailed import com.github.burrunan.test.runTest import com.github.burrunan.wrappers.nodejs.mkdir -import kotlinx.coroutines.await +import fs2.promises.writeFile +import path.path import kotlin.test.Test import kotlin.test.assertEquals @@ -28,17 +29,29 @@ class GlobTest { @Test fun glob() = runTest { val dirName = "globTest" + val dotGradle = path.join(dirName, ".gradle") mkdir(dirName) - fs2.promises.writeFile("$dirName/good.txt", "a", "utf8").await() - fs2.promises.writeFile("$dirName/bad.txt", "a", "utf8").await() + mkdir(dotGradle) + writeFile(path.join(dirName, "settings.gradle"), "a") + writeFile(path.join(dirName, "good.txt"), "a") + writeFile(path.join(dirName, "bad.txt"), "a") + writeFile(path.join(dotGradle, "extra.txt"), "a") val hash = hashFilesDetailed( + "$dirName/**/*.gradle", "$dirName/**/*.txt", + "!$dirName/**/.gradle/", "!$dirName/**/*bad**", ) - val actual = hash.contents.files.entries.joinToString { (file, details) -> + val actual = hash.contents.files.entries.joinToString("\n") { (file, details) -> "${details.fileSize.formatBytes()} ${details.hash} $file" } - assertEquals("1 B 86f7e437faa5a7fce15d1ddcb9eaeaea377667b8 ws:///globTest/good.txt", actual) + assertEquals( + """ + 1 B 86f7e437faa5a7fce15d1ddcb9eaeaea377667b8 ws:///globTest/good.txt + 1 B 86f7e437faa5a7fce15d1ddcb9eaeaea377667b8 ws:///globTest/settings.gradle + """.trimIndent(), + actual, + ) } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 49404b2..8dcd7de 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -28,6 +28,9 @@ rootProject.name = "gradle-cache-action" include( "cache-service-mock", + "cache-proxy", + "gradle-launcher", + "hashing", "layered-cache", "cache-action-entrypoint", "test-library", diff --git a/test-library/build.gradle.kts b/test-library/build.gradle.kts index 0a959f5..c397f52 100644 --- a/test-library/build.gradle.kts +++ b/test-library/build.gradle.kts @@ -16,5 +16,5 @@ dependencies { api(kotlin("test-js")) - implementation(project(":lib")) + implementation(project(":wrappers:nodejs")) }