#3 Store Compression optimisation

oleg-cherednik · Oct 31, 2024 · be0f2d2 · be0f2d2
1 parent 746a0f3
commit be0f2d2
Show file tree

Hide file tree

Showing 7 changed files with 81 additions and 24 deletions.
diff --git a/src/main/java/ru/olegcherednik/zip4jvm/crypto/pkware/PkwareEncoder.java b/src/main/java/ru/olegcherednik/zip4jvm/crypto/pkware/PkwareEncoder.java
@@ -22,10 +22,10 @@
 import ru.olegcherednik.zip4jvm.io.out.data.DataOutput;
 import ru.olegcherednik.zip4jvm.model.entry.ZipEntry;
 
-import com.sun.org.apache.bcel.internal.generic.PUSH;
 import lombok.RequiredArgsConstructor;
 
 import java.io.IOException;
+import java.util.function.IntSupplier;
 
 import static ru.olegcherednik.zip4jvm.utils.ValidationUtils.requireNotEmpty;
 
@@ -37,23 +37,22 @@
 public final class PkwareEncoder implements Encoder {
 
     private final PkwareEngine engine;
-    private final PkwareHeader header;
+    private final IntSupplier key;
 
     public static PkwareEncoder create(ZipEntry entry) {
         requireNotEmpty(entry.getPassword(), entry.getFileName() + ".password");
 
         PkwareEngine engine = new PkwareEngine(entry.getPassword());
-        int key = entry.isDataDescriptorAvailable() ? entry.getLastModifiedTime()
-                                                    : (int) entry.getChecksum() >> 16;
-        PkwareHeader header = PkwareHeader.create(engine, key);
-        return new PkwareEncoder(engine, header);
+
+        return new PkwareEncoder(engine, () -> entry.isDataDescriptorAvailable() ? entry.getLastModifiedTime()
+                                                                                 : (int) entry.getChecksum() >> 16);
     }
 
     // ---------- Encoder ----------
 
     @Override
     public void writeEncryptionHeader(DataOutput out) throws IOException {
-        header.write(out);
+        PkwareHeader.create(engine, key.getAsInt()).write(out);
     }
 
     // ---------- Encrypt ----------

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/engine/ZipEngine.java b/src/main/java/ru/olegcherednik/zip4jvm/engine/ZipEngine.java
@@ -38,6 +38,7 @@
 import ru.olegcherednik.zip4jvm.utils.PathUtils;
 import ru.olegcherednik.zip4jvm.utils.ZipUtils;
 import ru.olegcherednik.zip4jvm.utils.function.Writer;
+import ru.olegcherednik.zip4jvm.utils.quitely.Quietly;
 
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -50,6 +51,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import java.util.stream.Collectors;
 
 import static ru.olegcherednik.zip4jvm.utils.ValidationUtils.requireNotBlank;
@@ -138,14 +140,20 @@ public void add(ZipFile.Entry entry) {
         add(zipEntry);
     }
 
-    private void add(ZipEntry zipEntry) {
-        if (fileNameWriter.containsKey(zipEntry.getFileName()))
-            throw new EntryDuplicationException(zipEntry.getFileName());
+    private void add(ZipEntry entry) {
+        if (fileNameWriter.containsKey(entry.getFileName()))
+            throw new EntryDuplicationException(entry.getFileName());
 
-        Writer writer = zipEntry.isDataDescriptorAvailable() ? new ZipEntryWriter(zipEntry)
-                                                             : new ZipEntryNoDataDescriptorWriter(zipEntry);
-        tempZipModel.addEntry(zipEntry);
-        fileNameWriter.put(zipEntry.getFileName(), writer);
+        tempZipModel.addEntry(entry);
+        fileNameWriter.put(entry.getFileName(), createWriter(entry));
+    }
+
+    private Writer createWriter(ZipEntry entry) {
+        if (entry.isDataDescriptorAvailable())
+            return new ZipEntryWriter(entry);
+
+        Path dir = tempZipModel.getTempDir().resolve(UUID.randomUUID().toString());
+        return new ZipEntryNoDataDescriptorWriter(entry, dir);
     }
 
     @Override
@@ -233,10 +241,10 @@ private void moveTempZipFiles() throws IOException {
         Files.deleteIfExists(tempZipModel.getSrcZip().getPath().getParent());
     }
 
-    private static ZipModel createTempZipModel(Path zip, ZipSettings settings, Map<String, Writer> fileNameWriter)
-            throws IOException {
+    private static ZipModel createTempZipModel(Path zip, ZipSettings settings, Map<String, Writer> fileNameWriter) {
         Path tempZip = createTempZip(zip);
         ZipModel tempZipModel = ZipModelBuilder.build(tempZip, settings);
+        tempZipModel.setTempDir(tempZip.getParent());
 
         if (Files.exists(zip)) {
             ZipModel zipModel = ZipModelBuilder.read(SrcZip.of(zip));
@@ -257,10 +265,12 @@ private static ZipModel createTempZipModel(Path zip, ZipSettings settings, Map<S
         return tempZipModel;
     }
 
-    private static Path createTempZip(Path zip) throws IOException {
-        Path dir = zip.getParent().resolve("tmp");
-        Files.createDirectories(dir);
-        return dir.resolve(zip.getFileName());
+    private static Path createTempZip(Path zip) {
+        return Quietly.doQuietly(() -> {
+            Path dir = zip.getParent().resolve("tmp");
+            Files.createDirectories(dir);
+            return dir.resolve(zip.getFileName());
+        });
     }
 
     private static DataOutput creatDataOutput(ZipModel zipModel) throws IOException {

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/io/out/data/EncryptedDataOutput.java b/src/main/java/ru/olegcherednik/zip4jvm/io/out/data/EncryptedDataOutput.java
@@ -48,6 +48,10 @@ public void writeEncryptionHeader() throws IOException {
         encoder.writeEncryptionHeader(out);
     }
 
+    public void writeEncryptionHeader(DataOutput out) throws IOException {
+        encoder.writeEncryptionHeader(out);
+    }
+
     public void encodingAccomplished() throws IOException {
         encoder.close(out);
     }

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/io/writers/ZipEntryNoDataDescriptorWriter.java b/src/main/java/ru/olegcherednik/zip4jvm/io/writers/ZipEntryNoDataDescriptorWriter.java
@@ -31,6 +31,7 @@
 
 import lombok.RequiredArgsConstructor;
 import org.apache.commons.codec.digest.PureJavaCrc32;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 
 import java.io.IOException;
@@ -52,6 +53,7 @@ public final class ZipEntryNoDataDescriptorWriter implements Writer {
             ZipEntryNoDataDescriptorWriter.class.getSimpleName() + ".entryCompressedDataOffs";
 
     private final ZipEntry entry;
+    private final Path tempDir;
 
     @Override
     public void write(DataOutput out) throws IOException {
@@ -67,13 +69,13 @@ public void write(DataOutput out) throws IOException {
         [data descriptor]
          */
 
-        Path tmpFile = Paths.get("d:/zip4jvm/foo/bar/" + entry.getFileName());
+        Path tmpFile = tempDir.resolve(entry.getFileName());
         Files.deleteIfExists(tmpFile);
 
         byte[] data = IOUtils.toByteArray(entry.getInputStream());
         Checksum checksum = new PureJavaCrc32();
         checksum.update(data, 0, data.length);
-        System.out.println("CRC32 Checksum: "+ checksum.getValue());
+        System.out.println("CRC32 Checksum: " + checksum.getValue());
         entry.setChecksum(checksum.getValue());
 
         try (WriteFileDataOutput tmpOut = new WriteFileDataOutput()) {
@@ -90,6 +92,7 @@ public void write(DataOutput out) throws IOException {
         }
 
         new UpdateZip64().update(entry);
+        FileUtils.deleteQuietly(tempDir.toFile());
     }
 
     private void foo(DataOutput out) throws IOException {

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/model/ZipModel.java b/src/main/java/ru/olegcherednik/zip4jvm/model/ZipModel.java
@@ -57,6 +57,7 @@ public final class ZipModel {
     public static final int MAX_COMMENT_SIZE = Zip64.LIMIT_WORD;
 
     private final SrcZip srcZip;
+    private Path tempDir;
     private long splitSize = NO_SPLIT;
 
     private String comment;

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/model/builders/ZipModelBuilder.java b/src/main/java/ru/olegcherednik/zip4jvm/model/builders/ZipModelBuilder.java
@@ -32,7 +32,6 @@
 
 import lombok.RequiredArgsConstructor;
 
-import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -51,7 +50,7 @@ public final class ZipModelBuilder {
     private final CentralDirectory centralDirectory;
     private final Function<Charset, Charset> charsetCustomizer;
 
-    public static ZipModel read(SrcZip srcZip) throws IOException {
+    public static ZipModel read(SrcZip srcZip) {
         return read(srcZip, Charsets.UNMODIFIED, null);
     }
 

diff --git a/src/main/java/ru/olegcherednik/zip4jvm/utils/ChecksumUtils.java b/src/main/java/ru/olegcherednik/zip4jvm/utils/ChecksumUtils.java
@@ -0,0 +1,41 @@
+package ru.olegcherednik.zip4jvm.utils;
+
+import ru.olegcherednik.zip4jvm.utils.quitely.Quietly;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+import org.apache.commons.codec.digest.PureJavaCrc32;
+import org.apache.commons.io.IOUtils;
+
+import java.io.BufferedInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.Checksum;
+
+/**
+ * @author Oleg Cherednik
+ * @since 31.10.2024
+ */
+@NoArgsConstructor(access = AccessLevel.PACKAGE)
+public final class ChecksumUtils {
+
+    @SuppressWarnings("NewMethodNamingConvention")
+    public static long crc32(Path file) {
+        return Quietly.doQuietly(() -> {
+            Checksum crc32 = new PureJavaCrc32();
+
+            try (CheckedInputStream in = new CheckedInputStream(
+                    new BufferedInputStream(Files.newInputStream(file)), crc32)) {
+                byte[] buf = new byte[1024];
+
+                while (in.read(buf) != IOUtils.EOF) {
+                    // read file in completely
+                }
+            }
+
+            return crc32.getValue();
+        });
+    }
+
+}