From e9af6dba5f36c2b77b95ac3e0cc402e5bb938ebd Mon Sep 17 00:00:00 2001 From: Georgios Andrianakis Date: Fri, 22 Sep 2023 16:04:38 +0300 Subject: [PATCH] Merge parent-first jars into single jar Relates to: #36099 --- .../pkg/steps/JarResultBuildStep.java | 111 +++++++++++++++--- 1 file changed, 97 insertions(+), 14 deletions(-) diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java index 06dc22675293c..b1f90b4b0dd7a 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java @@ -154,6 +154,7 @@ public boolean test(String path) { public static final String DEFAULT_FAST_JAR_DIRECTORY_NAME = "quarkus-app"; public static final String MP_CONFIG_FILE = "META-INF/microprofile-config.properties"; + private static final String MERGED_BOOT_LIB = "quarkus-boot-lib"; @BuildStep OutputTargetBuildItem outputTarget(BuildSystemTargetBuildItem bst, PackageConfig packageConfig) { @@ -571,8 +572,8 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem, Path libDir = buildDir.resolve(LIB); Path mainLib = libDir.resolve(MAIN); //parent first entries - Path baseLib = libDir.resolve(BOOT_LIB); - Files.createDirectories(baseLib); + Path bootLib = libDir.resolve(BOOT_LIB); + Files.createDirectories(bootLib); Path appDir = buildDir.resolve(APP); Path quarkus = buildDir.resolve(QUARKUS); @@ -583,7 +584,7 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem, if (!rebuild) { IoUtils.createOrEmptyDir(buildDir); Files.createDirectories(mainLib); - Files.createDirectories(baseLib); + Files.createDirectories(bootLib); Files.createDirectories(appDir); Files.createDirectories(quarkus); if (userProviders != null) { @@ -620,6 +621,7 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem, FastJarJars.FastJarJarsBuilder fastJarJarsBuilder = new FastJarJars.FastJarJarsBuilder(); List parentFirst = new ArrayList<>(); + Map parentFirstTargetPathToArtifact = new HashMap<>(); //we process in order of priority //transformed classes first if (!transformedClasses.getTransformedClassesByJar().isEmpty()) { @@ -684,14 +686,20 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem, } } final Set parentFirstKeys = getParentFirstKeys(curateOutcomeBuildItem, classLoadingConfig); - final StringBuilder classPath = new StringBuilder(); final Set removed = getRemovedKeys(classLoadingConfig); final Map> copiedArtifacts = new HashMap<>(); for (ResolvedDependency appDep : curateOutcomeBuildItem.getApplicationModel().getRuntimeDependencies()) { if (!rebuild) { - copyDependency(parentFirstKeys, outputTargetBuildItem, copiedArtifacts, mainLib, baseLib, - fastJarJarsBuilder::addDep, true, - classPath, appDep, transformedClasses, removed); + copyDependency(parentFirstKeys, outputTargetBuildItem, copiedArtifacts, mainLib, bootLib, + (p) -> { + if (parentFirstKeys.contains(appDep.getKey())) { + parentFirstTargetPathToArtifact.put(p, appDep); + } else { + fastJarJarsBuilder.addDep(p); + } + }, + true, + appDep, transformedClasses, removed); } else if (includeAppDep(appDep, outputTargetBuildItem.getIncludedOptionalDependencies(), removed)) { appDep.getResolvedPaths().forEach(fastJarJarsBuilder::addDep); } @@ -727,6 +735,17 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem, } Path appInfo = buildDir.resolve(QuarkusEntryPoint.QUARKUS_APPLICATION_DAT); + + // merge all the jars in boot into a single one + // the idea is to cut down on the memory needed to load them + + List copiedBootJars = Files.list(bootLib) + .filter(p -> p.getFileName().toString().endsWith(".jar")) + .collect(Collectors.toList()); + + mergeBootJars(packageConfig, bootLib, copiedBootJars, + parentFirstTargetPathToArtifact); + try (OutputStream out = Files.newOutputStream(appInfo)) { FastJarJars fastJarJars = fastJarJarsBuilder.build(); List allJars = new ArrayList<>(); @@ -746,6 +765,20 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem, sortedNonExistentResources); } + // remove the boot jars since they are no longer needed + copiedBootJars.forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + + StringBuilder classPath = new StringBuilder(LIB + "/" + BOOT_LIB + "/" + MERGED_BOOT_LIB + ".jar"); + Files.list(bootLib).filter(p -> !p.getFileName().toString().contains(MERGED_BOOT_LIB)).forEach(p -> { + classPath.append(" " + LIB + "/" + BOOT_LIB + "/" + p.getFileName().toString()); + }); + runnerJar.toFile().setReadable(true, false); Path initJar = buildDir.resolve(QUARKUS_RUN_JAR); boolean mutableJar = packageConfig.type.equalsIgnoreCase(PackageConfig.BuiltInType.MUTABLE_JAR.getValue()); @@ -766,7 +799,9 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem, if (!rebuild) { try (FileSystem runnerZipFs = ZipUtils.newZip(initJar)) { ResolvedDependency appArtifact = curateOutcomeBuildItem.getApplicationModel().getAppArtifact(); - generateManifest(runnerZipFs, classPath.toString(), packageConfig, appArtifact, + generateManifest(runnerZipFs, + classPath.toString(), + packageConfig, appArtifact, QuarkusEntryPoint.class.getName(), applicationInfo); } @@ -776,9 +811,9 @@ private JarBuildItem buildThinJar(CurateOutcomeBuildItem curateOutcomeBuildItem, Path deploymentLib = libDir.resolve(DEPLOYMENT_LIB); Files.createDirectories(deploymentLib); for (ResolvedDependency appDep : curateOutcomeBuildItem.getApplicationModel().getDependencies()) { - copyDependency(parentFirstKeys, outputTargetBuildItem, copiedArtifacts, deploymentLib, baseLib, (p) -> { + copyDependency(parentFirstKeys, outputTargetBuildItem, copiedArtifacts, deploymentLib, bootLib, (p) -> { }, - false, classPath, + false, appDep, new TransformedClassesBuildItem(Map.of()), removed); //we don't care about transformation here, so just pass in an empty item } Map> relativePaths = new HashMap<>(); @@ -843,6 +878,54 @@ public void accept(Path path) { return new JarBuildItem(initJar, null, libDir, packageConfig.type, null); } + private void mergeBootJars(PackageConfig packageConfig, + Path bootLib, List copiedBootJars, + Map parentFirstTargetPathToArtifact) throws IOException { + final Map seen = new HashMap<>(); + final Map> duplicateCatcher = new HashMap<>(); + final Map> concatenatedEntries = new HashMap<>(); + Set ignoredEntries = new HashSet<>(); + packageConfig.userConfiguredIgnoredEntries.ifPresent(ignoredEntries::addAll); + Predicate allIgnoredEntriesPredicate = new Predicate<>() { + @Override + public boolean test(String path) { + return UBER_JAR_IGNORED_ENTRIES_PREDICATE.test(path) + || ignoredEntries.contains(path); + } + }; + + Path mergedBootLibJar = bootLib.resolve(MERGED_BOOT_LIB + ".jar"); + try (FileSystem runnerZipFs = ZipUtils.newZip(mergedBootLibJar)) { + Path manifestPath = runnerZipFs.getPath("META-INF", "MANIFEST.MF"); + Manifest manifest = new Manifest(); + Files.createDirectories(manifestPath.getParent()); + Attributes attributes = manifest.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + + for (Path bootJar : copiedBootJars) { + Dependency dependency = parentFirstTargetPathToArtifact.get(bootJar); + try (FileSystem artifactFs = ZipUtils.newZip(bootJar)) { + for (Path root : artifactFs.getRootDirectories()) { + walkFileDependencyForDependency(root, runnerZipFs, seen, duplicateCatcher, concatenatedEntries, + allIgnoredEntriesPredicate, dependency, new HashSet<>(), new HashSet<>()); + } + } + } + + for (Map.Entry> entry : concatenatedEntries.entrySet()) { + try (final OutputStream os = wrapForJDK8232879( + Files.newOutputStream(runnerZipFs.getPath(entry.getKey())))) { + // TODO: Handle merging of XMLs + for (byte[] i : entry.getValue()) { + os.write(i); + os.write('\n'); + } + } + } + } + + } + /** * @return a {@code Set} containing the key of the artifacts to load from the parent ClassLoader first. */ @@ -877,9 +960,10 @@ private Set getRemovedKeys(ClassLoadingConfig classLoadingConfig) { return removed; } - private void copyDependency(Set parentFirstArtifacts, OutputTargetBuildItem outputTargetBuildItem, + private void copyDependency(Set parentFirstArtifacts, + OutputTargetBuildItem outputTargetBuildItem, Map> runtimeArtifacts, Path libDir, Path baseLib, Consumer targetPathConsumer, - boolean allowParentFirst, StringBuilder classPath, ResolvedDependency appDep, + boolean allowParentFirst, ResolvedDependency appDep, TransformedClassesBuildItem transformedClasses, Set removedDeps) throws IOException { @@ -897,11 +981,10 @@ private void copyDependency(Set parentFirstArtifacts, OutputTargetB if (allowParentFirst && parentFirstArtifacts.contains(appDep.getKey())) { targetPath = baseLib.resolve(fileName); - classPath.append(" ").append(LIB).append("/").append(BOOT_LIB).append("/").append(fileName); } else { targetPath = libDir.resolve(fileName); - targetPathConsumer.accept(targetPath); } + targetPathConsumer.accept(targetPath); runtimeArtifacts.computeIfAbsent(appDep.getKey(), (s) -> new ArrayList<>(1)).add(targetPath); if (Files.isDirectory(resolvedDep)) {