diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java
index 4266d04727a6e..eab173ac613a8 100644
--- a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java
+++ b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/JarResultBuildStep.java
@@ -926,9 +926,11 @@ private void copyDependency(Set<ArtifactKey> parentFirstArtifacts, OutputTargetB
                     }
                 }
                 if (removedFromThisArchive.isEmpty()) {
-                    Files.copy(resolvedDep, targetPath, StandardCopyOption.REPLACE_EXISTING);
+                    Files.copy(resolvedDep, targetPath, StandardCopyOption.REPLACE_EXISTING,
+                            StandardCopyOption.COPY_ATTRIBUTES);
                 } else {
-                    //we have removed classes, we need to handle them correctly
+                    // we copy jars for which we remove entries to the same directory
+                    // which seems a bit odd to me
                     filterZipFile(resolvedDep, targetPath, removedFromThisArchive);
                 }
             }
@@ -1251,6 +1253,8 @@ private void filterZipFile(Path resolvedDep, Path targetPath, Set<String> transf
                         }
                     }
                 }
+                // let's make sure we keep the original timestamp
+                Files.setLastModifiedTime(targetPath, Files.getLastModifiedTime(resolvedDep));
             }
         } catch (IOException e) {
             throw new RuntimeException(e);
diff --git a/extensions/container-image/container-image-jib/deployment/src/main/java/io/quarkus/container/image/jib/deployment/JibProcessor.java b/extensions/container-image/container-image-jib/deployment/src/main/java/io/quarkus/container/image/jib/deployment/JibProcessor.java
index 79c40bf24ea13..90c49d2339780 100644
--- a/extensions/container-image/container-image-jib/deployment/src/main/java/io/quarkus/container/image/jib/deployment/JibProcessor.java
+++ b/extensions/container-image/container-image-jib/deployment/src/main/java/io/quarkus/container/image/jib/deployment/JibProcessor.java
@@ -529,13 +529,14 @@ private JibContainerBuilder createContainerBuilderFromFastJar(String baseJvmImag
 
         try {
             Instant now = Instant.now();
+            boolean enforceModificationTime = !jibConfig.useCurrentTimestampFileModification;
             Instant modificationTime = jibConfig.useCurrentTimestampFileModification ? now : Instant.EPOCH;
 
             JibContainerBuilder jibContainerBuilder = toJibContainerBuilder(baseJvmImage, jibConfig);
             if (fastChangingLibPaths.isEmpty()) {
                 // just create a layer with the entire lib structure intact
                 addLayer(jibContainerBuilder, Collections.singletonList(componentsPath.resolve(JarResultBuildStep.LIB)),
-                        workDirInContainer, "fast-jar-lib", isMutableJar, modificationTime);
+                        workDirInContainer, "fast-jar-lib", isMutableJar, enforceModificationTime, modificationTime);
             } else {
                 // we need to manually create each layer
                 // the idea here is that the fast changing libraries are created in a later layer, thus when they do change,
@@ -549,14 +550,9 @@ private JibContainerBuilder createContainerBuilderFromFastJar(String baseJvmImag
                             AbsoluteUnixPath libPathInContainer = workDirInContainer.resolve(JarResultBuildStep.LIB)
                                     .resolve(JarResultBuildStep.BOOT_LIB)
                                     .resolve(lib.getFileName());
-                            if (appCDSResult.isPresent()) {
-                                // the boot lib jars need to preserve the modification time because otherwise AppCDS won't work
-                                bootLibsLayerBuilder.addEntry(lib, libPathInContainer,
-                                        Files.getLastModifiedTime(lib).toInstant());
-                            } else {
-                                bootLibsLayerBuilder.addEntry(lib, libPathInContainer);
-                            }
-
+                            // the boot lib jars need to preserve the modification time because otherwise AppCDS won't work
+                            bootLibsLayerBuilder.addEntry(lib, libPathInContainer,
+                                    Files.getLastModifiedTime(lib).toInstant());
                         } catch (IOException e) {
                             throw new UncheckedIOException(e);
                         }
@@ -569,15 +565,15 @@ private JibContainerBuilder createContainerBuilderFromFastJar(String baseJvmImag
                             .resolve(JarResultBuildStep.DEPLOYMENT_LIB);
                     addLayer(jibContainerBuilder, Collections.singletonList(deploymentPath),
                             workDirInContainer.resolve(JarResultBuildStep.LIB),
-                            "fast-jar-deployment-libs", true, modificationTime);
+                            "fast-jar-deployment-libs", true, enforceModificationTime, modificationTime);
                 }
 
                 AbsoluteUnixPath libsMainPath = workDirInContainer.resolve(JarResultBuildStep.LIB)
                         .resolve(JarResultBuildStep.MAIN);
                 addLayer(jibContainerBuilder, nonFastChangingLibPaths, libsMainPath, "fast-jar-normal-libs",
-                        isMutableJar, modificationTime);
+                        isMutableJar, enforceModificationTime, modificationTime);
                 addLayer(jibContainerBuilder, new ArrayList<>(fastChangingLibPaths), libsMainPath, "fast-jar-changing-libs",
-                        isMutableJar, modificationTime);
+                        isMutableJar, enforceModificationTime, modificationTime);
             }
 
             if (appCDSResult.isPresent()) {
@@ -601,9 +597,9 @@ private JibContainerBuilder createContainerBuilderFromFastJar(String baseJvmImag
             }
 
             addLayer(jibContainerBuilder, Collections.singletonList(componentsPath.resolve(JarResultBuildStep.APP)),
-                    workDirInContainer, "fast-jar-quarkus-app", isMutableJar, modificationTime);
+                    workDirInContainer, "fast-jar-quarkus-app", isMutableJar, enforceModificationTime, modificationTime);
             addLayer(jibContainerBuilder, Collections.singletonList(componentsPath.resolve(JarResultBuildStep.QUARKUS)),
-                    workDirInContainer, "fast-jar-quarkus", isMutableJar, modificationTime);
+                    workDirInContainer, "fast-jar-quarkus", isMutableJar, enforceModificationTime, modificationTime);
             if (ContainerImageJibConfig.DEFAULT_WORKING_DIR.equals(jibConfig.workingDirectory)) {
                 // this layer ensures that the working directory is writeable
                 // see https://github.com/GoogleContainerTools/jib/issues/1270
@@ -667,7 +663,7 @@ private boolean containsRunJava(String baseJvmImage) {
 
     public JibContainerBuilder addLayer(JibContainerBuilder jibContainerBuilder, List<Path> files,
             AbsoluteUnixPath pathInContainer, String name, boolean isMutableJar,
-            Instant now)
+            boolean enforceModificationTime, Instant forcedModificationTime)
             throws IOException {
         FileEntriesLayer.Builder layerConfigurationBuilder = FileEntriesLayer.builder().setName(name);
 
@@ -675,7 +671,17 @@ public JibContainerBuilder addLayer(JibContainerBuilder jibContainerBuilder, Lis
             layerConfigurationBuilder.addEntryRecursive(
                     file, pathInContainer.resolve(file.getFileName()),
                     isMutableJar ? REMOTE_DEV_FOLDER_PERMISSIONS_PROVIDER : DEFAULT_FILE_PERMISSIONS_PROVIDER,
-                    (sourcePath, destinationPath) -> now,
+                    (sourcePath, destinationPath) -> {
+                        if (enforceModificationTime) {
+                            return forcedModificationTime;
+                        }
+
+                        try {
+                            return Files.getLastModifiedTime(sourcePath).toInstant();
+                        } catch (IOException e) {
+                            throw new RuntimeException("Unable to get last modified time for " + sourcePath, e);
+                        }
+                    },
                     isMutableJar ? REMOTE_DEV_OWNERSHIP_PROVIDER : DEFAULT_OWNERSHIP_PROVIDER);
         }