WIP: Make QuarkusBuild not pollute Gradle's build cache

Currently the `QuarkusBuild` task implementation adds even large build artifacts and unmodified dependency jars to Gradle's build cache. This pollutes the Gradle build cache quite a lot and therefore lets archived build caches become unnecessary huge.

This change updates the build logic to fix this behavior by not adding dependencies and large build artifacts, like uber-jar and native binary, to the Gradle build cache.

The `QuarkusBuild` task has been split into three tasks:
1. a new `QuarkusBuildDependencies` task that only collects the contents for `build/quarkus-app/lib`
2. a new `QuarkusBuildApp` task that builds the Quarkus app, which was previously done by the `QuarkusBuild` task
3. the `QuarkusBuild` task now combines the outputs of the above two tasks

`QuarkusBuildDependencies` (named 'quarkusDependenciesBuild`) is not cacheable, because it only collects dependencies, which come either from a repository (and are already available locally elsewhere) or are built by other Gradle tasks. This task is only executed if the configured Quarkus package type requires the "quarkus-app" directory (`fast-jar` + `native`). It's "build working directory" is `build/quarkus-build/dep`.

`QuarkusBuildApp` (named `quarkusAppBuild`) builds the Quarkus application. The contents of the "quarkus-app" directory _excluding_ the `lib/` directory are cacheable, which is the default for CI environments. Non-CI environments still cache all outputs, even uber-jars and native binaries to retain the existing behavior for developers and keep build turn-around times low. CI environments can opt-in to add even huge artifacts to Gradle's build cache by explicitly setting the `cacheUberAndNativeRunners` property in the Quarkus extension to `true`. It's "build working directory" is `build/quarkus-build/app`.

Since `QuarkusBuild` only combines the outputs of the above two tasks, the same "CI vs local" caching behavior as for the `QuarkusBuildApp` task applies. To make "up to date" checks (kind of) more reliable, all outputs are removed first. This means, that for example an existing uber-jar in `build/` will disappear, when the build's package type is "fast-jar". This behavior can be disabled by setting the `cleanupBuildOutput` property on the Quarkus extension to `false`.

The task names `quarkusDependenciesBuild` and `quarkusAppBuild` are intentionally "that way around". Letting the names of these tasks begin with `quarkusBuild...` could confuse users, who use abbreviated task names on the command line (for example `./gradlew qB` is automagically expanded to `./gradlew quarkusBuild`, which would become ambiguous with `quarkusBuildDependencies` and `quarkusBuildApp`).

Relates to: quarkusio#30852

devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/QuarkusPlugin.java

@@ -36,7 +36,9 @@
 import io.quarkus.gradle.tasks.ImagePush;
 import io.quarkus.gradle.tasks.QuarkusAddExtension;
 import io.quarkus.gradle.tasks.QuarkusBuild;
+import io.quarkus.gradle.tasks.QuarkusBuildApp;
 import io.quarkus.gradle.tasks.QuarkusBuildConfiguration;
+import io.quarkus.gradle.tasks.QuarkusBuildDependencies;
 import io.quarkus.gradle.tasks.QuarkusDev;
 import io.quarkus.gradle.tasks.QuarkusGenerateCode;
 import io.quarkus.gradle.tasks.QuarkusGoOffline;
@@ -56,6 +58,11 @@ public class QuarkusPlugin implements Plugin<Project> {
 
     public static final String ID = "io.quarkus";
     public static final String QUARKUS_PACKAGE_TYPE = "quarkus.package.type";
+    public static final String OUTPUT_DIRECTORY = "quarkus.package.output-directory";
+    public static final String CONTAINER_BUILD = "quarkus.native.container-build";
+    public static final String BUILDER_IMAGE = "quarkus.native.builder-image";
+    public static final String CLASS_LOADING_REMOVED_ARTIFACTS = "quarkus.class-loading.removed-artifacts";
+    public static final String CLASS_LOADING_PARENT_FIRST_ARTIFACTS = "quarkus.class-loading.parent-first-artifacts";
 
     public static final String EXTENSION_NAME = "quarkus";
     public static final String LIST_EXTENSIONS_TASK_NAME = "listExtensions";
@@ -66,6 +73,8 @@ public class QuarkusPlugin implements Plugin<Project> {
     public static final String QUARKUS_GENERATE_CODE_TASK_NAME = "quarkusGenerateCode";
     public static final String QUARKUS_GENERATE_CODE_DEV_TASK_NAME = "quarkusGenerateCodeDev";
     public static final String QUARKUS_GENERATE_CODE_TESTS_TASK_NAME = "quarkusGenerateCodeTests";
+    public static final String QUARKUS_BUILD_DEP_TASK_NAME = "quarkusDependenciesBuild";
+    public static final String QUARKUS_BUILD_APP_TASK_NAME = "quarkusAppBuild";
     public static final String QUARKUS_BUILD_TASK_NAME = "quarkusBuild";
     public static final String QUARKUS_DEV_TASK_NAME = "quarkusDev";
     public static final String QUARKUS_REMOTE_DEV_TASK_NAME = "quarkusRemoteDev";
@@ -150,9 +159,28 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
 
         QuarkusBuildConfiguration buildConfig = new QuarkusBuildConfiguration(project);
 
+        TaskProvider<QuarkusBuildDependencies> quarkusBuildLibs = tasks.register(QUARKUS_BUILD_DEP_TASK_NAME,
+                QuarkusBuildDependencies.class);
+        quarkusBuildLibs.configure(task -> {
+            task.getOutputs().doNotCacheIf("Dependencies are never cached", t -> true);
+        });
+
+        TaskProvider<QuarkusBuildApp> quarkusBuildApp = tasks.register(QUARKUS_BUILD_APP_TASK_NAME,
+                QuarkusBuildApp.class);
+        quarkusBuildApp.configure(task -> {
+            task.getOutputs().doNotCacheIf(
+                    "Not adding uber-jars and native binaries to Gradle build cache by default. " +
+                            "To allow caching of uber-jars and native binaries set 'cacheUberAndNativeRunners' to 'true' in the 'quarkus' extension.",
+                    t -> !quarkusExt.getBuildInternal().get().isFastJarLike()
+                            && !quarkusExt.getCacheUberAndNativeRunners().get());
+        });
+
         TaskProvider<QuarkusBuild> quarkusBuild = tasks.register(QUARKUS_BUILD_TASK_NAME, QuarkusBuild.class, build -> {
-            build.dependsOn(quarkusGenerateCode);
+            build.dependsOn(quarkusGenerateCode, quarkusBuildLibs, quarkusBuildApp);
             build.getForcedProperties().set(buildConfig.getForcedProperties());
+            build.getOutputs().doNotCacheIf(
+                    "Only collects outputs of " + QUARKUS_BUILD_APP_TASK_NAME + " and " + QUARKUS_BUILD_DEP_TASK_NAME,
+                    t -> !quarkusExt.getCacheUberAndNativeRunners().get());
         });
 
         TaskProvider<ImageBuild> imageBuild = tasks.register(IMAGE_BUILD_TASK_NAME, ImageBuild.class, buildConfig);

devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/extension/QuarkusPluginExtension.java

@@ -1,8 +1,12 @@
 package io.quarkus.gradle.extension;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.EnumMap;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Objects;
@@ -30,6 +34,7 @@
 import io.quarkus.bootstrap.model.ApplicationModel;
 import io.quarkus.bootstrap.resolver.AppModelResolver;
 import io.quarkus.gradle.AppModelGradleResolver;
+import io.quarkus.gradle.tasks.QuarkusBuildInternal;
 import io.quarkus.gradle.tasks.QuarkusGradleUtils;
 import io.quarkus.gradle.tooling.ToolingUtils;
 import io.quarkus.runtime.LaunchMode;
@@ -39,17 +44,37 @@ public class QuarkusPluginExtension {
 
     private final Property<String> finalName;
 
+    private final MapProperty<String, String> applicationProperties;
     private final MapProperty<String, String> quarkusBuildProperties;
+    private final MapProperty<String, String> buildSystemProperties;
     private final SourceSetExtension sourceSetExtension;
 
+    private final Property<Boolean> cacheUberAndNativeRunners;
+    private final Property<Boolean> cleanupBuildOutput;
+    private final Property<QuarkusBuildInternal> buildInternal;
+
+    private final Map<LaunchMode, ApplicationModel> applicationModels = new EnumMap<>(LaunchMode.class);
+
     public QuarkusPluginExtension(Project project) {
         this.project = project;
 
         finalName = project.getObjects().property(String.class);
         finalName.convention(project.provider(() -> String.format("%s-%s", project.getName(), project.getVersion())));
 
+        this.cleanupBuildOutput = project.getObjects().property(Boolean.class)
+                .convention(true);
+        this.cacheUberAndNativeRunners = project.getObjects().property(Boolean.class)
+                .convention(!System.getenv().containsKey("CI"));
+
         this.sourceSetExtension = new SourceSetExtension();
+        this.applicationProperties = project.getObjects().mapProperty(String.class, String.class)
+                .convention(project.provider(() -> loadApplicationProperties(project)));
         this.quarkusBuildProperties = project.getObjects().mapProperty(String.class, String.class);
+        this.buildSystemProperties = project.getObjects().mapProperty(String.class, String.class)
+                .convention(project.provider(() -> collectBuildSystemProperties()));
+
+        this.buildInternal = project.getObjects().property(QuarkusBuildInternal.class)
+                .convention(project.provider(() -> new QuarkusBuildInternal(project, this)));
     }
 
     public void beforeTest(Test task) {
@@ -124,6 +149,30 @@ public Property<String> getFinalName() {
         return finalName;
     }
 
+    /**
+     * Whether the build output, build/*-runner[.jar] and build/quarkus-app, for other package types than the
+     * currently configured one are removed, default is 'true'.
+     */
+    public Property<Boolean> getCleanupBuildOutput() {
+        return cleanupBuildOutput;
+    }
+
+    public void setCleanupBuildOutput(boolean cleanupBuildOutput) {
+        this.cleanupBuildOutput.set(cleanupBuildOutput);
+    }
+
+    /**
+     * Whether large build artifacts, uber-jar and native, are cached. Defaults to 'false' if the 'CI' environment
+     * variable is set, otherwise defaults to 'true'.
+     */
+    public Property<Boolean> getCacheUberAndNativeRunners() {
+        return cacheUberAndNativeRunners;
+    }
+
+    public void setCacheUberAndNativeRunners(boolean cacheUberAndNativeRunners) {
+        this.cacheUberAndNativeRunners.set(cacheUberAndNativeRunners);
+    }
+
     public String finalName() {
         return getFinalName().get();
     }
@@ -164,7 +213,8 @@ public ApplicationModel getApplicationModel() {
     }
 
     public ApplicationModel getApplicationModel(LaunchMode mode) {
-        return ToolingUtils.create(project, mode);
+        // Prevent duplicate computation of ApplicationModel(s), same model's needed by multiple tasks.
+        return applicationModels.computeIfAbsent(mode, m -> ToolingUtils.create(project, m));
     }
 
     /**
@@ -215,10 +265,22 @@ public Path appJarOrClasses() {
         return classesDir;
     }
 
+    public MapProperty<String, String> getApplicationProperties() {
+        return applicationProperties;
+    }
+
     public MapProperty<String, String> getQuarkusBuildProperties() {
         return quarkusBuildProperties;
     }
 
+    public MapProperty<String, String> getBuildSystemProperties() {
+        return buildSystemProperties;
+    }
+
+    public Property<QuarkusBuildInternal> getBuildInternal() {
+        return buildInternal;
+    }
+
     public void set(String name, @Nullable String value) {
         quarkusBuildProperties.put(String.format("quarkus.%s", name), value);
     }
@@ -227,4 +289,49 @@ public void set(String name, Property<String> value) {
         quarkusBuildProperties.put(String.format("quarkus.%s", name), value);
     }
 
+    private Map<String, String> collectBuildSystemProperties() {
+        final Map<String, ?> properties = project.getProperties();
+        final Map<String, String> realProperties = new HashMap<>();
+        for (Map.Entry<String, ?> entry : properties.entrySet()) {
+            final String key = entry.getKey();
+            final Object value = entry.getValue();
+            if (key != null && value instanceof String && key.startsWith("quarkus.")) {
+                realProperties.put(key, (String) value);
+            }
+        }
+        Map<String, String> quarkusBuildProperties = getQuarkusBuildProperties().get();
+        if (!quarkusBuildProperties.isEmpty()) {
+            quarkusBuildProperties.entrySet().stream().filter(entry -> entry.getKey().startsWith("quarkus."))
+                    .forEach(entry -> {
+                        realProperties.put(entry.getKey(), entry.getValue());
+                    });
+        }
+        return realProperties;
+    }
+
+    private static Map<String, String> loadApplicationProperties(Project project) {
+        SourceSet mainSourceSet = QuarkusGradleUtils.getSourceSet(project, SourceSet.MAIN_SOURCE_SET_NAME);
+        FileCollection configFiles = mainSourceSet.getResources()
+                .filter(file -> "application.properties".equalsIgnoreCase(file.getName()));
+        Properties properties = new Properties();
+        configFiles.forEach(file -> {
+            FileInputStream appPropsIS = null;
+            try {
+                appPropsIS = new FileInputStream(file.getAbsoluteFile());
+                properties.load(appPropsIS);
+                appPropsIS.close();
+            } catch (IOException e) {
+                if (appPropsIS != null) {
+                    try {
+                        appPropsIS.close();
+                    } catch (IOException ex) {
+                        // Ignore exception closing.
+                    }
+                }
+            }
+        });
+        Map<String, String> result = new HashMap<>();
+        properties.forEach((k, v) -> result.put(k.toString(), v.toString()));
+        return result;
+    }
 }