Skip to content

Commit

Permalink
Merge parent-first jars into single jar
Browse files Browse the repository at this point in the history 
Relates to: quarkusio#36099
  • Loading branch information
@geoand
geoand committed Sep 25, 2023
1 parent 383c126 commit e9af6db
Showing 1 changed file with 97 additions and 14 deletions.
111 changes: 97 additions & 14 deletions core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ public boolean test(String path) {
public static final String DEFAULT_FAST_JAR_DIRECTORY_NAME = "quarkus-app";

public static final String MP_CONFIG_FILE = "META-INF/microprofile-config.properties";
private static final String MERGED_BOOT_LIB = "quarkus-boot-lib";

@BuildStep
OutputTargetBuildItem outputTarget(BuildSystemTargetBuildItem bst, PackageConfig packageConfig) {
Expand Down Expand Up @@ -571,8 +572,8 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
Path libDir = buildDir.resolve(LIB);
Path mainLib = libDir.resolve(MAIN);
//parent first entries
Path baseLib = libDir.resolve(BOOT_LIB);
Files.createDirectories(baseLib);
Path bootLib = libDir.resolve(BOOT_LIB);
Files.createDirectories(bootLib);

Path appDir = buildDir.resolve(APP);
Path quarkus = buildDir.resolve(QUARKUS);
Expand All @@ -583,7 +584,7 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
if (!rebuild) {
IoUtils.createOrEmptyDir(buildDir);
Files.createDirectories(mainLib);
Files.createDirectories(baseLib);
Files.createDirectories(bootLib);
Files.createDirectories(appDir);
Files.createDirectories(quarkus);
if (userProviders != null) {
Expand Down Expand Up @@ -620,6 +621,7 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,

FastJarJars.FastJarJarsBuilder fastJarJarsBuilder = new FastJarJars.FastJarJarsBuilder();
List<Path> parentFirst = new ArrayList<>();
Map<Path, Dependency> parentFirstTargetPathToArtifact = new HashMap<>();
//we process in order of priority
//transformed classes first
if (!transformedClasses.getTransformedClassesByJar().isEmpty()) {
Expand Down Expand Up @@ -684,14 +686,20 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
}
}
final Set<ArtifactKey> parentFirstKeys = getParentFirstKeys(curateOutcomeBuildItem, classLoadingConfig);
final StringBuilder classPath = new StringBuilder();
final Set<ArtifactKey> removed = getRemovedKeys(classLoadingConfig);
final Map<ArtifactKey, List<Path>> copiedArtifacts = new HashMap<>();
for (ResolvedDependency appDep : curateOutcomeBuildItem.getApplicationModel().getRuntimeDependencies()) {
if (!rebuild) {
copyDependency(parentFirstKeys, outputTargetBuildItem, copiedArtifacts, mainLib, baseLib,
fastJarJarsBuilder::addDep, true,
classPath, appDep, transformedClasses, removed);
copyDependency(parentFirstKeys, outputTargetBuildItem, copiedArtifacts, mainLib, bootLib,
(p) -> {
if (parentFirstKeys.contains(appDep.getKey())) {
parentFirstTargetPathToArtifact.put(p, appDep);
} else {
fastJarJarsBuilder.addDep(p);
}
},
true,
appDep, transformedClasses, removed);
} else if (includeAppDep(appDep, outputTargetBuildItem.getIncludedOptionalDependencies(), removed)) {
appDep.getResolvedPaths().forEach(fastJarJarsBuilder::addDep);
}
Expand Down Expand Up @@ -727,6 +735,17 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
}

Path appInfo = buildDir.resolve(QuarkusEntryPoint.QUARKUS_APPLICATION_DAT);

// merge all the jars in boot into a single one
// the idea is to cut down on the memory needed to load them

List<Path> copiedBootJars = Files.list(bootLib)
.filter(p -> p.getFileName().toString().endsWith(".jar"))
.collect(Collectors.toList());

mergeBootJars(packageConfig, bootLib, copiedBootJars,
parentFirstTargetPathToArtifact);

try (OutputStream out = Files.newOutputStream(appInfo)) {
FastJarJars fastJarJars = fastJarJarsBuilder.build();
List<Path> allJars = new ArrayList<>();
Expand All @@ -746,6 +765,20 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
sortedNonExistentResources);
}

// remove the boot jars since they are no longer needed
copiedBootJars.forEach(p -> {
try {
Files.delete(p);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
});

StringBuilder classPath = new StringBuilder(LIB + "/" + BOOT_LIB + "/" + MERGED_BOOT_LIB + ".jar");
Files.list(bootLib).filter(p -> !p.getFileName().toString().contains(MERGED_BOOT_LIB)).forEach(p -> {
classPath.append(" " + LIB + "/" + BOOT_LIB + "/" + p.getFileName().toString());
});

runnerJar.toFile().setReadable(true, false);
Path initJar = buildDir.resolve(QUARKUS_RUN_JAR);
boolean mutableJar = packageConfig.type.equalsIgnoreCase(PackageConfig.BuiltInType.MUTABLE_JAR.getValue());
Expand All @@ -766,7 +799,9 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
if (!rebuild) {
try (FileSystem runnerZipFs = ZipUtils.newZip(initJar)) {
ResolvedDependency appArtifact = curateOutcomeBuildItem.getApplicationModel().getAppArtifact();
generateManifest(runnerZipFs, classPath.toString(), packageConfig, appArtifact,
generateManifest(runnerZipFs,
classPath.toString(),
packageConfig, appArtifact,
QuarkusEntryPoint.class.getName(),
applicationInfo);
}
Expand All @@ -776,9 +811,9 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem,
Path deploymentLib = libDir.resolve(DEPLOYMENT_LIB);
Files.createDirectories(deploymentLib);
for (ResolvedDependency appDep : curateOutcomeBuildItem.getApplicationModel().getDependencies()) {
copyDependency(parentFirstKeys, outputTargetBuildItem, copiedArtifacts, deploymentLib, baseLib, (p) -> {
copyDependency(parentFirstKeys, outputTargetBuildItem, copiedArtifacts, deploymentLib, bootLib, (p) -> {
},
false, classPath,
false,
appDep, new TransformedClassesBuildItem(Map.of()), removed); //we don't care about transformation here, so just pass in an empty item
}
Map<ArtifactKey, List<String>> relativePaths = new HashMap<>();
Expand Down Expand Up @@ -843,6 +878,54 @@ public void accept(Path path) {
return new JarBuildItem(initJar, null, libDir, packageConfig.type, null);
}

private void mergeBootJars(PackageConfig packageConfig,
Path bootLib, List<Path> copiedBootJars,
Map<Path, Dependency> parentFirstTargetPathToArtifact) throws IOException {
final Map<String, String> seen = new HashMap<>();
final Map<String, Set<Dependency>> duplicateCatcher = new HashMap<>();
final Map<String, List<byte[]>> concatenatedEntries = new HashMap<>();
Set<String> ignoredEntries = new HashSet<>();
packageConfig.userConfiguredIgnoredEntries.ifPresent(ignoredEntries::addAll);
Predicate<String> allIgnoredEntriesPredicate = new Predicate<>() {
@Override
public boolean test(String path) {
return UBER_JAR_IGNORED_ENTRIES_PREDICATE.test(path)
|| ignoredEntries.contains(path);
}
};

Path mergedBootLibJar = bootLib.resolve(MERGED_BOOT_LIB + ".jar");
try (FileSystem runnerZipFs = ZipUtils.newZip(mergedBootLibJar)) {
Path manifestPath = runnerZipFs.getPath("META-INF", "MANIFEST.MF");
Manifest manifest = new Manifest();
Files.createDirectories(manifestPath.getParent());
Attributes attributes = manifest.getMainAttributes();
attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0");

for (Path bootJar : copiedBootJars) {
Dependency dependency = parentFirstTargetPathToArtifact.get(bootJar);
try (FileSystem artifactFs = ZipUtils.newZip(bootJar)) {
for (Path root : artifactFs.getRootDirectories()) {
walkFileDependencyForDependency(root, runnerZipFs, seen, duplicateCatcher, concatenatedEntries,
allIgnoredEntriesPredicate, dependency, new HashSet<>(), new HashSet<>());
}
}
}

for (Map.Entry<String, List<byte[]>> entry : concatenatedEntries.entrySet()) {
try (final OutputStream os = wrapForJDK8232879(
Files.newOutputStream(runnerZipFs.getPath(entry.getKey())))) {
// TODO: Handle merging of XMLs
for (byte[] i : entry.getValue()) {
os.write(i);
os.write('\n');
}
}
}
}

}

/**
* @return a {@code Set} containing the key of the artifacts to load from the parent ClassLoader first.
*/
Expand Down Expand Up @@ -877,9 +960,10 @@ private Set<ArtifactKey> getRemovedKeys(ClassLoadingConfig classLoadingConfig) {
return removed;
}

private void copyDependency(Set<ArtifactKey> parentFirstArtifacts, OutputTargetBuildItem outputTargetBuildItem,
private void copyDependency(Set<ArtifactKey> parentFirstArtifacts,
OutputTargetBuildItem outputTargetBuildItem,
Map<ArtifactKey, List<Path>> runtimeArtifacts, Path libDir, Path baseLib, Consumer<Path> targetPathConsumer,
boolean allowParentFirst, StringBuilder classPath, ResolvedDependency appDep,
boolean allowParentFirst, ResolvedDependency appDep,
TransformedClassesBuildItem transformedClasses, Set<ArtifactKey> removedDeps)
throws IOException {

Expand All @@ -897,11 +981,10 @@ private void copyDependency(Set<ArtifactKey> parentFirstArtifacts, OutputTargetB

if (allowParentFirst && parentFirstArtifacts.contains(appDep.getKey())) {
targetPath = baseLib.resolve(fileName);
classPath.append(" ").append(LIB).append("/").append(BOOT_LIB).append("/").append(fileName);
} else {
targetPath = libDir.resolve(fileName);
targetPathConsumer.accept(targetPath);
}
targetPathConsumer.accept(targetPath);
runtimeArtifacts.computeIfAbsent(appDep.getKey(), (s) -> new ArrayList<>(1)).add(targetPath);

if (Files.isDirectory(resolvedDep)) {
Expand Down

0 comments on commit e9af6db

Please sign in to comment.