Skip to content

Commit

Permalink
Gradle-Plugin: Do not let Gradle build fail w/ configuration cache
Browse files Browse the repository at this point in the history 
Gradle's configuration cache aims to prevent running (expensive) code to configure the build (tasks, et al). Preventing configuration can save a (comparatively) huge amount of time, especially on small/tiny CI machines (looking at you, GH standard hosted runner).

But to be able to use the configuration cache, the build itself and all plugins must respect a couple of rules. Gradle types that must not be referenced during task _execution_ are for example: `Project`, `Configuration`, `Task` and some more. Those need to be replaced with either more specialized types or with different mechanisms.

See [Gradle build cache requirements](https://docs.gradle.org/8.0.2/userguide/configuration_cache.html#config_cache:requirements) for details about the Gradle feature.

This change basically prevents that a Gradle build w/ enabled configuration cache fails, by telling Gradle that the Quarkus tasks do not work with the configuration cache, so it can fall back to some best effort (for the rest of the build).

Other changes lift or lower object references and store values in fields to prevent hard build time errors (despite the "incompatible w/ configuration cache setting"). Most of it is to remove the usage of `Project` in the relevant code paths.

Gradle tests in the Quarkus source tree got the `--configuration-cache` setting, so that the tests run with an enabled configuration cache. I was able to successfully run tests, checkstyle, spotless in a local Nessie build w/ the updated Quarkus plugin and Gradle's configuration cache enabled.

Fully adopting the Quarkus plugin to Gradle's configuration cache is a very big effort, and even for a "normal build" (so no dev mode) there are a lot of quite dynamic things happening and it will at least take quite a while and multiple PRs. Not sure, whether it's achievable at a reasonable cost.

One commit in this PR lets the Gradle integrations tests use the Gradle daemon, which at least feels to improve integration test runtime behavior. The daemon is started with an idle timeout of 10 seconds. Local tests and CI runs did not "complain".

Also "added support" for the VLSI Gradle Jandex plugin in addition to the "existing support" for the Kordamp Gradle Jandex plugin. ITs adopted accordingly.
  • Loading branch information
@snazy
snazy committed Apr 8, 2023
1 parent f686168 commit ef1c253
Show file tree
Hide file tree
Showing 36 changed files with 393 additions and 169 deletions.
134 changes: 87 additions & 47 deletions devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/QuarkusPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.io.File;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

Expand All @@ -18,8 +19,11 @@
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.ConfigurationContainer;
import org.gradle.api.artifacts.ProjectDependency;
import org.gradle.api.file.FileCollection;
import org.gradle.api.plugins.BasePlugin;
import org.gradle.api.plugins.JavaPlugin;
import org.gradle.api.provider.Property;
import org.gradle.api.specs.Spec;
import org.gradle.api.tasks.SourceSet;
import org.gradle.api.tasks.SourceSetContainer;
import org.gradle.api.tasks.TaskContainer;
Expand All @@ -42,6 +46,7 @@
import io.quarkus.gradle.tasks.QuarkusDev;
import io.quarkus.gradle.tasks.QuarkusGenerateCode;
import io.quarkus.gradle.tasks.QuarkusGoOffline;
import io.quarkus.gradle.tasks.QuarkusGradleUtils;
import io.quarkus.gradle.tasks.QuarkusInfo;
import io.quarkus.gradle.tasks.QuarkusListCategories;
import io.quarkus.gradle.tasks.QuarkusListExtensions;
Expand Down Expand Up @@ -151,12 +156,24 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
.getByName(ApplicationDeploymentClasspathBuilder.getBaseRuntimeConfigName(LaunchMode.DEVELOPMENT)));
});

// quarkusGenerateCode
TaskProvider<QuarkusGenerateCode> quarkusGenerateCode = tasks.register(QUARKUS_GENERATE_CODE_TASK_NAME,
QuarkusGenerateCode.class);
QuarkusGenerateCode.class, LaunchMode.NORMAL, SourceSet.MAIN_SOURCE_SET_NAME);
quarkusGenerateCode.configure(task -> configureGenerateCodeTask(task, QuarkusGenerateCode.QUARKUS_GENERATED_SOURCES));
// quarkusGenerateCodeDev
TaskProvider<QuarkusGenerateCode> quarkusGenerateCodeDev = tasks.register(QUARKUS_GENERATE_CODE_DEV_TASK_NAME,
QuarkusGenerateCode.class, task -> task.setDevMode(true));
QuarkusGenerateCode.class, LaunchMode.DEVELOPMENT, SourceSet.MAIN_SOURCE_SET_NAME);
quarkusGenerateCodeDev.configure(task -> {
task.dependsOn(quarkusGenerateCode);
configureGenerateCodeTask(task, QuarkusGenerateCode.QUARKUS_GENERATED_SOURCES);
});
// quarkusGenerateCodeTests
TaskProvider<QuarkusGenerateCode> quarkusGenerateCodeTests = tasks.register(QUARKUS_GENERATE_CODE_TESTS_TASK_NAME,
QuarkusGenerateCode.class, task -> task.setTest(true));
QuarkusGenerateCode.class, LaunchMode.TEST, SourceSet.TEST_SOURCE_SET_NAME);
quarkusGenerateCodeTests.configure(task -> {
task.dependsOn("compileQuarkusTestGeneratedSourcesJava");
configureGenerateCodeTask(task, QuarkusGenerateCode.QUARKUS_TEST_GENERATED_SOURCES);
});

tasks.register(QUARKUS_SHOW_EFFECTIVE_CONFIG_TASK_NAME,
QuarkusShowEffectiveConfig.class, task -> {
Expand All @@ -167,6 +184,8 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
QuarkusBuildDependencies.class,
task -> task.getOutputs().doNotCacheIf("Dependencies are never cached", t -> true));

Property<Boolean> cacheLargeArtifacts = quarkusExt.getCacheLargeArtifacts();

TaskProvider<QuarkusBuildCacheableAppParts> quarkusBuildCacheableAppParts = tasks.register(
QUARKUS_BUILD_APP_PARTS_TASK_NAME,
QuarkusBuildCacheableAppParts.class, task -> {
Expand All @@ -175,7 +194,13 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
"Not adding uber-jars, native binaries and mutable-jar package type to Gradle " +
"build cache by default. To allow caching of uber-jars, native binaries and mutable-jar " +
"package type, set 'cacheUberAndNativeRunners' in the 'quarkus' Gradle extension to 'true'.",
t -> !task.isCachedByDefault() && !quarkusExt.getCacheLargeArtifacts().get());
new Spec<Task>() {
@Override
public boolean isSatisfiedBy(Task t) {
QuarkusBuildCacheableAppParts q = (QuarkusBuildCacheableAppParts) t;
return !q.isCachedByDefault() && !cacheLargeArtifacts.get();
}
});
});

TaskProvider<QuarkusBuild> quarkusBuild = tasks.register(QUARKUS_BUILD_TASK_NAME, QuarkusBuild.class, build -> {
Expand All @@ -184,7 +209,12 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
"Only collects and combines the outputs of " + QUARKUS_BUILD_APP_PARTS_TASK_NAME + " and "
+ QUARKUS_BUILD_DEP_TASK_NAME + ", see 'cacheLargeArtifacts' in the 'quarkus' Gradle extension " +
"for details.",
t -> !quarkusExt.getCacheLargeArtifacts().get());
new Spec<Task>() {
@Override
public boolean isSatisfiedBy(Task t) {
return !cacheLargeArtifacts.get();
}
});
});

tasks.register(IMAGE_BUILD_TASK_NAME, ImageBuild.class, task -> task.finalizedBy(quarkusBuild));
Expand Down Expand Up @@ -295,12 +325,16 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
.plus(mainSourceSet.getOutput())
.plus(testSourceSet.getOutput()));

TaskProvider<Test> testTask = tasks.named(JavaPlugin.TEST_TASK_NAME, Test.class);
FileCollection intTestSourceOutputClasses = intTestSourceSet.getOutput().getClassesDirs();
FileCollection intTestClasspath = intTestSourceSet.getRuntimeClasspath();

tasks.register(INTEGRATION_TEST_TASK_NAME, Test.class, intTestTask -> {
intTestTask.setGroup("verification");
intTestTask.setDescription("Runs Quarkus integration tests");
intTestTask.dependsOn(quarkusBuild, tasks.named(JavaPlugin.TEST_TASK_NAME));
intTestTask.setClasspath(intTestSourceSet.getRuntimeClasspath());
intTestTask.setTestClassesDirs(intTestSourceSet.getOutput().getClassesDirs());
intTestTask.dependsOn(quarkusBuild, testTask);
intTestTask.setClasspath(intTestClasspath);
intTestTask.setTestClassesDirs(intTestSourceOutputClasses);
});

SourceSet nativeTestSourceSet = sourceSets.create(NATIVE_TEST_SOURCE_SET_NAME);
Expand All @@ -315,23 +349,32 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {
.plus(intTestSourceSet.getOutput())
.plus(testSourceSet.getOutput()));

FileCollection nativeTestClassesDirs = project.files(
nativeTestSourceSet.getOutput().getClassesDirs(),
intTestSourceOutputClasses);
FileCollection nativeTestClasspath = nativeTestSourceSet.getRuntimeClasspath();

tasks.register(TEST_NATIVE_TASK_NAME, Test.class, testNative -> {
testNative.setDescription("Runs native image tests");
testNative.setGroup("verification");
testNative.dependsOn(quarkusBuild, tasks.named(JavaPlugin.TEST_TASK_NAME));

testNative.setTestClassesDirs(project.files(nativeTestSourceSet.getOutput().getClassesDirs(),
intTestSourceSet.getOutput().getClassesDirs()));
testNative.setClasspath(nativeTestSourceSet.getRuntimeClasspath());
testNative.dependsOn(quarkusBuild, testTask);
testNative.setClasspath(nativeTestClasspath);
testNative.setTestClassesDirs(nativeTestClassesDirs);
});

tasks.withType(Test.class).configureEach(t -> {
// Calling this method tells Gradle that it should not fail the build.
// Side effect is that the configuration cache will be at least degraded,
// but the build will not fail.
t.notCompatibleWithConfigurationCache(
"The quarkus-plugin isn't compatible with the configuration cache");

// Quarkus test configuration action which should be executed before any Quarkus test
// Use anonymous classes in order to leverage task avoidance.
t.doFirst(new Action<Task>() {
@Override
public void execute(Task task) {
quarkusExt.beforeTest(t);
quarkusExt.beforeTest((Test) task);
}
});
// also make each task use the JUnit platform since it's the only supported test environment
Expand Down Expand Up @@ -362,6 +405,16 @@ public void execute(Task task) {
});
}

private static void configureGenerateCodeTask(QuarkusGenerateCode task, String generateSourcesDir) {
SourceSet generatedSources = QuarkusGradleUtils.getSourceSet(task.getProject(), generateSourcesDir);
Set<File> sourceSetOutput = generatedSources.getOutput().filter(f -> f.getName().equals(generateSourcesDir)).getFiles();
if (sourceSetOutput.isEmpty()) {
throw new GradleException("Failed to configure " + task.getPath() + ": sourceSet " + generateSourcesDir
+ " has no output");
}
task.getGeneratedOutputDirectory().set(generatedSources.getJava().getClassesDirectory().get().getAsFile());
}

private void createConfigurations(Project project) {

final ConfigurationContainer configContainer = project.getConfigurations();
Expand Down Expand Up @@ -465,38 +518,25 @@ private void setupQuarkusBuildTaskDeps(Project project, Project dep, Set<String>
}
project.getLogger().debug("Configuring {} task dependencies on {} tasks", project, dep);

final TaskProvider<Task> quarkusBuild = getLazyTaskOrNull(project, QUARKUS_BUILD_TASK_NAME);
if (quarkusBuild != null) {
final TaskProvider<Task> jarTask = getLazyTaskOrNull(dep, JavaPlugin.JAR_TASK_NAME);
if (jarTask != null) {
final TaskProvider<Task> quarkusPrepare = getLazyTaskOrNull(project, QUARKUS_GENERATE_CODE_TASK_NAME);
final TaskProvider<Task> quarkusPrepareDev = getLazyTaskOrNull(project, QUARKUS_GENERATE_CODE_DEV_TASK_NAME);
final TaskProvider<Task> quarkusPrepareTests = getLazyTaskOrNull(project,
QUARKUS_GENERATE_CODE_TESTS_TASK_NAME);
quarkusBuild.configure(task -> task.dependsOn(jarTask));
if (quarkusPrepare != null) {
quarkusPrepare.configure(task -> task.dependsOn(jarTask));
}
if (quarkusPrepareDev != null) {
quarkusPrepareDev.configure(task -> task.dependsOn(jarTask));
}
if (quarkusPrepareTests != null) {
quarkusPrepareTests.configure(task -> task.dependsOn(jarTask));
}
}
}
getLazyTask(project, QUARKUS_BUILD_TASK_NAME)
.flatMap(quarkusBuild -> getLazyTask(dep, JavaPlugin.JAR_TASK_NAME))
.ifPresent(jarTask -> {
for (String taskName : new String[] { QUARKUS_GENERATE_CODE_TASK_NAME, QUARKUS_GENERATE_CODE_DEV_TASK_NAME,
QUARKUS_GENERATE_CODE_TESTS_TASK_NAME }) {
getLazyTask(project, taskName)
.ifPresent(quarkusTask -> quarkusTask.configure(t -> t.dependsOn(jarTask)));
}
});

final TaskProvider<Task> quarkusDev = getLazyTaskOrNull(project, QUARKUS_DEV_TASK_NAME);
if (quarkusDev != null) {
final TaskProvider<Task> resourcesTask = getLazyTaskOrNull(dep, JavaPlugin.PROCESS_RESOURCES_TASK_NAME);
if (resourcesTask != null) {
quarkusDev.configure(task -> task.dependsOn(resourcesTask));
}
final TaskProvider<Task> resourcesTaskJandex = getLazyTaskOrNull(dep, "jandex");
if (resourcesTaskJandex != null) {
quarkusDev.configure(task -> task.dependsOn(resourcesTaskJandex));
getLazyTask(project, QUARKUS_DEV_TASK_NAME).ifPresent(quarkusDev -> {
for (String taskName : new String[] { JavaPlugin.PROCESS_RESOURCES_TASK_NAME,
// This is the task of the 'org.kordamp.gradle.jandex' Gradle plugin
"jandex",
// This is the task of the 'com.github.vlsi.jandex' Gradle plugin
"processJandexIndex" }) {
getLazyTask(dep, taskName).ifPresent(t -> quarkusDev.configure(qd -> qd.dependsOn(t)));
}
}
});

visitProjectDependencies(project, dep, visited);
}
Expand All @@ -520,11 +560,11 @@ protected void visitProjectDependencies(Project project, Project dep, Set<String
}
}

private TaskProvider<Task> getLazyTaskOrNull(Project project, String name) {
private Optional<TaskProvider<Task>> getLazyTask(Project project, String name) {
try {
return project.getTasks().named(name);
return Optional.of(project.getTasks().named(name));
} catch (UnknownTaskException e) {
return null;
return Optional.empty();
}
}
}
17 changes: 10 additions & 7 deletions ...-application-plugin/src/main/java/io/quarkus/gradle/extension/QuarkusPluginExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,25 +78,27 @@ public void beforeTest(Test task) {
}
props.put(BootstrapConstants.OUTPUT_SOURCES_DIR, outputSourcesDir.toString());

final SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
final SourceSetContainer sourceSets = getSourceSets();
final SourceSet mainSourceSet = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME);

final File outputDirectoryAsFile = getLastFile(mainSourceSet.getOutput().getClassesDirs());

Path projectDirPath = projectDir.toPath();

// Identify the folder containing the sources associated with this test task
String fileList = getSourceSets().stream()
String fileList = sourceSets.stream()
.filter(sourceSet -> Objects.equals(
task.getTestClassesDirs().getAsPath(),
sourceSet.getOutput().getClassesDirs().getAsPath()))
.flatMap(sourceSet -> sourceSet.getOutput().getClassesDirs().getFiles().stream())
.filter(File::exists)
.distinct()
.map(testSrcDir -> String.format("%s:%s",
project.relativePath(testSrcDir),
project.relativePath(outputDirectoryAsFile)))
projectDirPath.relativize(testSrcDir.toPath()),
projectDirPath.relativize(outputDirectoryAsFile.toPath())))
.collect(Collectors.joining(","));
task.environment(BootstrapConstants.TEST_TO_MAIN_MAPPINGS, fileList);
project.getLogger().debug("test dir mapping - {}", fileList);
task.getLogger().debug("test dir mapping - {}", fileList);

QuarkusBuild quarkusBuild = project.getTasks().named(QuarkusPlugin.QUARKUS_BUILD_TASK_NAME, QuarkusBuild.class)
.get();
Expand Down Expand Up @@ -157,8 +159,9 @@ public Set<File> resourcesDir() {

public Set<File> combinedOutputSourceDirs() {
Set<File> sourcesDirs = new LinkedHashSet<>();
sourcesDirs.addAll(getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME).getOutput().getClassesDirs().getFiles());
sourcesDirs.addAll(getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME).getOutput().getClassesDirs().getFiles());
SourceSetContainer sourceSets = getSourceSets();
sourcesDirs.addAll(sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getOutput().getClassesDirs().getFiles());
sourcesDirs.addAll(sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getOutput().getClassesDirs().getFiles());
return sourcesDirs;
}

Expand Down
7 changes: 4 additions & 3 deletions ...le-application-plugin/src/main/java/io/quarkus/gradle/tasks/AbstractQuarkusExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,19 @@ public abstract class AbstractQuarkusExtension {

private static final String QUARKUS_PROFILE = "quarkus.profile";
protected final Project project;
protected final File projectDir;
protected final Property<String> finalName;
private final MapProperty<String, String> forcedPropertiesProperty;
protected final MapProperty<String, String> quarkusBuildProperties;
private final ListProperty<String> ignoredEntries;
private final SourceSet mainSourceSet;
private final FileCollection classpath;
private final Property<BaseConfig> baseConfig;
protected final List<Action<? super JavaForkOptions>> codeGenForkOptions;
protected final List<Action<? super JavaForkOptions>> buildForkOptions;

protected AbstractQuarkusExtension(Project project) {
this.project = project;
this.projectDir = project.getProjectDir();
this.finalName = project.getObjects().property(String.class);
this.finalName.convention(project.provider(() -> String.format("%s-%s", project.getName(), project.getVersion())));
this.forcedPropertiesProperty = project.getObjects().mapProperty(String.class, String.class);
Expand All @@ -55,7 +56,7 @@ protected AbstractQuarkusExtension(Project project) {
this.ignoredEntries.convention(
project.provider(() -> baseConfig().packageConfig().userConfiguredIgnoredEntries.orElse(emptyList())));
this.baseConfig = project.getObjects().property(BaseConfig.class).value(project.provider(this::buildBaseConfig));
this.mainSourceSet = getSourceSet(project, SourceSet.MAIN_SOURCE_SET_NAME);
SourceSet mainSourceSet = getSourceSet(project, SourceSet.MAIN_SOURCE_SET_NAME);
this.classpath = dependencyClasspath(mainSourceSet);
this.codeGenForkOptions = new ArrayList<>();
this.buildForkOptions = new ArrayList<>();
Expand Down Expand Up @@ -127,7 +128,7 @@ private EffectiveConfig buildEffectiveConfiguration(Map<String, Object> properti
.build();
}

String quarkusProfile() {
private String quarkusProfile() {
String profile = System.getProperty(QUARKUS_PROFILE);
if (profile == null) {
profile = System.getenv("QUARKUS_PROFILE");
Expand Down
Loading

0 comments on commit ef1c253

Please sign in to comment.