From cd3cbd2b3272f002693aad84a79da145604e9618 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 9 Mar 2017 23:18:57 +0100 Subject: [PATCH 1/6] Support for writing zip64 headers in the unused code --- .../Zip/Headers/DirectoryEntryHeader.cs | 32 +++++++++++++---- .../Common/Zip/Headers/LocalEntryHeader.cs | 35 ++++++++++++++++--- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs index 3b50d1189..b7d3d5c77 100644 --- a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs @@ -63,6 +63,10 @@ internal override void Read(BinaryReader reader) internal override void Write(BinaryWriter writer) { + var zip64 = CompressedSize >= uint.MaxValue || UncompressedSize >= uint.MaxValue || RelativeOffsetOfEntryHeader >= uint.MaxValue; + if (zip64) + Version = (ushort)(Version > 45 ? Version : 45); + writer.Write(Version); writer.Write(VersionNeededToExtract); writer.Write((ushort)Flags); @@ -70,24 +74,40 @@ internal override void Write(BinaryWriter writer) writer.Write(LastModifiedTime); writer.Write(LastModifiedDate); writer.Write(Crc); - writer.Write((uint)CompressedSize); - writer.Write((uint)UncompressedSize); + writer.Write(zip64 ? uint.MaxValue : CompressedSize); + writer.Write(zip64 ? uint.MaxValue : UncompressedSize); byte[] nameBytes = EncodeString(Name); writer.Write((ushort)nameBytes.Length); - //writer.Write((ushort)Extra.Length); - writer.Write((ushort)0); + if (zip64) + { + writer.Write((ushort)(2 + 2 + 8 + 8 + 8 + 4)); + } + else + { + //writer.Write((ushort)Extra.Length); + writer.Write((ushort)0); + } writer.Write((ushort)Comment.Length); writer.Write(DiskNumberStart); writer.Write(InternalFileAttributes); writer.Write(ExternalFileAttributes); - writer.Write(RelativeOffsetOfEntryHeader); + writer.Write(zip64 ? uint.MaxValue : RelativeOffsetOfEntryHeader); writer.Write(nameBytes); - // writer.Write(Extra); + if (zip64) + { + writer.Write((ushort)0x0001); + writer.Write((ushort)((8 + 8 + 8 + 4))); + + writer.Write((ulong)UncompressedSize); + writer.Write((ulong)CompressedSize); + writer.Write((ulong)RelativeOffsetOfEntryHeader); + writer.Write((uint)0); // VolumeNumber = 0 + } writer.Write(Comment); } diff --git a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs index 2e311cc86..fe5394069 100644 --- a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs @@ -49,25 +49,52 @@ internal override void Read(BinaryReader reader) internal override void Write(BinaryWriter writer) { - writer.Write(Version); + if (IsZip64) + Version = (ushort)(Version > 45 ? Version : 45); + + writer.Write(Version); + writer.Write((ushort)Flags); writer.Write((ushort)CompressionMethod); writer.Write(LastModifiedTime); writer.Write(LastModifiedDate); writer.Write(Crc); - writer.Write((uint)CompressedSize); - writer.Write((uint)UncompressedSize); + + if (IsZip64) + { + writer.Write(uint.MaxValue); + writer.Write(uint.MaxValue); + } + else + { + writer.Write(CompressedSize); + writer.Write(UncompressedSize); + } byte[] nameBytes = EncodeString(Name); writer.Write((ushort)nameBytes.Length); - writer.Write((ushort)0); + if (IsZip64) + { + writer.Write((ushort)(2 + 2 + (2 * 8))); + } + else + { + writer.Write((ushort)0); + } //if (Extra != null) //{ // writer.Write(Extra); //} writer.Write(nameBytes); + if (IsZip64) + { + writer.Write((ushort)0x0001); + writer.Write((ushort)(2 * 8)); + writer.Write((ulong)CompressedSize); + writer.Write((ulong)UncompressedSize); + } } internal ushort Version { get; private set; } From 1263c0d97601e509c1aab0bfe31ba089d74b5a9e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 9 Mar 2017 23:19:41 +0100 Subject: [PATCH 2/6] Added support for writing zip64 headers --- .../Zip/Headers/DirectoryEntryHeader.cs | 44 +++--- .../Common/Zip/Headers/LocalEntryHeader.cs | 58 ++++---- .../IO/CountingWritableSubStream.cs | 2 +- .../Writers/Zip/ZipCentralDirectoryEntry.cs | 48 ++++-- src/SharpCompress/Writers/Zip/ZipWriter.cs | 137 +++++++++++++++--- .../Writers/Zip/ZipWriterEntryOptions.cs | 5 + .../Writers/Zip/ZipWriterOptions.cs | 7 + 7 files changed, 219 insertions(+), 82 deletions(-) diff --git a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs index b7d3d5c77..acca3bb12 100644 --- a/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/DirectoryEntryHeader.cs @@ -63,9 +63,9 @@ internal override void Read(BinaryReader reader) internal override void Write(BinaryWriter writer) { - var zip64 = CompressedSize >= uint.MaxValue || UncompressedSize >= uint.MaxValue || RelativeOffsetOfEntryHeader >= uint.MaxValue; - if (zip64) - Version = (ushort)(Version > 45 ? Version : 45); + var zip64 = CompressedSize >= uint.MaxValue || UncompressedSize >= uint.MaxValue || RelativeOffsetOfEntryHeader >= uint.MaxValue; + if (zip64) + Version = (ushort)(Version > 45 ? Version : 45); writer.Write(Version); writer.Write(VersionNeededToExtract); @@ -74,21 +74,21 @@ internal override void Write(BinaryWriter writer) writer.Write(LastModifiedTime); writer.Write(LastModifiedDate); writer.Write(Crc); - writer.Write(zip64 ? uint.MaxValue : CompressedSize); + writer.Write(zip64 ? uint.MaxValue : CompressedSize); writer.Write(zip64 ? uint.MaxValue : UncompressedSize); byte[] nameBytes = EncodeString(Name); writer.Write((ushort)nameBytes.Length); - if (zip64) - { - writer.Write((ushort)(2 + 2 + 8 + 8 + 8 + 4)); - } - else - { - //writer.Write((ushort)Extra.Length); - writer.Write((ushort)0); - } + if (zip64) + { + writer.Write((ushort)(2 + 2 + 8 + 8 + 8 + 4)); + } + else + { + //writer.Write((ushort)Extra.Length); + writer.Write((ushort)0); + } writer.Write((ushort)Comment.Length); writer.Write(DiskNumberStart); @@ -98,16 +98,16 @@ internal override void Write(BinaryWriter writer) writer.Write(nameBytes); - if (zip64) - { - writer.Write((ushort)0x0001); - writer.Write((ushort)((8 + 8 + 8 + 4))); + if (zip64) + { + writer.Write((ushort)0x0001); + writer.Write((ushort)((8 + 8 + 8 + 4))); - writer.Write((ulong)UncompressedSize); - writer.Write((ulong)CompressedSize); - writer.Write((ulong)RelativeOffsetOfEntryHeader); - writer.Write((uint)0); // VolumeNumber = 0 - } + writer.Write((ulong)UncompressedSize); + writer.Write((ulong)CompressedSize); + writer.Write((ulong)RelativeOffsetOfEntryHeader); + writer.Write((uint)0); // VolumeNumber = 0 + } writer.Write(Comment); } diff --git a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs index fe5394069..40e2a6629 100644 --- a/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs +++ b/src/SharpCompress/Common/Zip/Headers/LocalEntryHeader.cs @@ -49,52 +49,52 @@ internal override void Read(BinaryReader reader) internal override void Write(BinaryWriter writer) { - if (IsZip64) - Version = (ushort)(Version > 45 ? Version : 45); + if (IsZip64) + Version = (ushort)(Version > 45 ? Version : 45); - writer.Write(Version); - + writer.Write(Version); + writer.Write((ushort)Flags); writer.Write((ushort)CompressionMethod); writer.Write(LastModifiedTime); writer.Write(LastModifiedDate); writer.Write(Crc); - if (IsZip64) - { - writer.Write(uint.MaxValue); - writer.Write(uint.MaxValue); - } - else - { - writer.Write(CompressedSize); - writer.Write(UncompressedSize); - } + if (IsZip64) + { + writer.Write(uint.MaxValue); + writer.Write(uint.MaxValue); + } + else + { + writer.Write(CompressedSize); + writer.Write(UncompressedSize); + } byte[] nameBytes = EncodeString(Name); writer.Write((ushort)nameBytes.Length); - if (IsZip64) - { - writer.Write((ushort)(2 + 2 + (2 * 8))); - } - else - { - writer.Write((ushort)0); - } + if (IsZip64) + { + writer.Write((ushort)(2 + 2 + (2 * 8))); + } + else + { + writer.Write((ushort)0); + } //if (Extra != null) //{ // writer.Write(Extra); //} writer.Write(nameBytes); - if (IsZip64) - { - writer.Write((ushort)0x0001); - writer.Write((ushort)(2 * 8)); - writer.Write((ulong)CompressedSize); - writer.Write((ulong)UncompressedSize); - } + if (IsZip64) + { + writer.Write((ushort)0x0001); + writer.Write((ushort)(2 * 8)); + writer.Write((ulong)CompressedSize); + writer.Write((ulong)UncompressedSize); + } } internal ushort Version { get; private set; } diff --git a/src/SharpCompress/IO/CountingWritableSubStream.cs b/src/SharpCompress/IO/CountingWritableSubStream.cs index 17313612f..51989b75c 100644 --- a/src/SharpCompress/IO/CountingWritableSubStream.cs +++ b/src/SharpCompress/IO/CountingWritableSubStream.cs @@ -12,7 +12,7 @@ internal CountingWritableSubStream(Stream stream) writableStream = stream; } - public uint Count { get; private set; } + public ulong Count { get; private set; } public override bool CanRead { get { return false; } } diff --git a/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs b/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs index 28b9aa638..03f1ee5a6 100644 --- a/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs +++ b/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs @@ -13,36 +13,51 @@ internal class ZipCentralDirectoryEntry internal DateTime? ModificationTime { get; set; } internal string Comment { get; set; } internal uint Crc { get; set; } - internal uint HeaderOffset { get; set; } - internal uint Compressed { get; set; } - internal uint Decompressed { get; set; } + internal ulong HeaderOffset { get; set; } + internal ulong Compressed { get; set; } + internal ulong Decompressed { get; set; } + internal ushort Zip64HeaderOffset { get; set; } internal uint Write(Stream outputStream, ZipCompressionMethod compression) { byte[] encodedFilename = Encoding.UTF8.GetBytes(FileName); byte[] encodedComment = Encoding.UTF8.GetBytes(Comment); - //constant sig, then version made by, compabitility, then version to extract - outputStream.Write(new byte[] {80, 75, 1, 2, 0x14, 0, 0x0A, 0}, 0, 8); + var zip64 = Compressed >= uint.MaxValue || Decompressed >= uint.MaxValue || HeaderOffset >= uint.MaxValue || Zip64HeaderOffset != 0; + + var compressedvalue = zip64 ? uint.MaxValue : (uint)Compressed; + var decompressedvalue = zip64 ? uint.MaxValue : (uint)Decompressed; + var headeroffsetvalue = zip64 ? uint.MaxValue : (uint)HeaderOffset; + var extralength = zip64 ? (2 + 2 + 8 + 8 + 8 + 4) : 0; + var version = (byte)(zip64 ? 45 : 10); + HeaderFlags flags = HeaderFlags.UTF8; if (!outputStream.CanSeek) { - flags |= HeaderFlags.UsePostDataDescriptor; + // Cannot use data descriptors with zip64: + // https://blogs.oracle.com/xuemingshen/entry/is_zipinput_outputstream_handling_of + if (!zip64) + flags |= HeaderFlags.UsePostDataDescriptor; + if (compression == ZipCompressionMethod.LZMA) { flags |= HeaderFlags.Bit1; // eos marker } } + + //constant sig, then version made by, compabitility, then version to extract + outputStream.Write(new byte[] { 80, 75, 1, 2, 0x14, 0, version, 0 }, 0, 8); + outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)flags), 0, 2); outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)compression), 0, 2); // zipping method outputStream.Write(DataConverter.LittleEndian.GetBytes(ModificationTime.DateTimeToDosTime()), 0, 4); // zipping date and time outputStream.Write(DataConverter.LittleEndian.GetBytes(Crc), 0, 4); // file CRC - outputStream.Write(DataConverter.LittleEndian.GetBytes(Compressed), 0, 4); // compressed file size - outputStream.Write(DataConverter.LittleEndian.GetBytes(Decompressed), 0, 4); // uncompressed file size + outputStream.Write(DataConverter.LittleEndian.GetBytes(compressedvalue), 0, 4); // compressed file size + outputStream.Write(DataConverter.LittleEndian.GetBytes(decompressedvalue), 0, 4); // uncompressed file size outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedFilename.Length), 0, 2); // Filename in zip - outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // extra length + outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)extralength), 0, 2); // extra length outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedComment.Length), 0, 2); outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // disk=0 @@ -51,13 +66,24 @@ internal uint Write(Stream outputStream, ZipCompressionMethod compression) outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x8100), 0, 2); // External file attributes (normal/readable) - outputStream.Write(DataConverter.LittleEndian.GetBytes(HeaderOffset), 0, 4); // Offset of header + outputStream.Write(DataConverter.LittleEndian.GetBytes(headeroffsetvalue), 0, 4); // Offset of header outputStream.Write(encodedFilename, 0, encodedFilename.Length); + if (zip64) + { + outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x0001), 0, 2); + outputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)(extralength - 4)), 0, 2); + + outputStream.Write(DataConverter.LittleEndian.GetBytes(Decompressed), 0, 8); + outputStream.Write(DataConverter.LittleEndian.GetBytes(Compressed), 0, 8); + outputStream.Write(DataConverter.LittleEndian.GetBytes(HeaderOffset), 0, 8); + outputStream.Write(DataConverter.LittleEndian.GetBytes(0), 0, 4); // VolumeNumber = 0 + } + outputStream.Write(encodedComment, 0, encodedComment.Length); return (uint)(8 + 2 + 2 + 4 + 4 + 4 + 4 + 2 + 2 + 2 - + 2 + 2 + 2 + 2 + 4 + encodedFilename.Length + encodedComment.Length); + + 2 + 2 + 2 + 2 + 4 + encodedFilename.Length + extralength + encodedComment.Length); } } } \ No newline at end of file diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index d89294b54..f40726c62 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -23,11 +23,13 @@ public class ZipWriter : AbstractWriter private readonly string zipComment; private long streamPosition; private PpmdProperties ppmdProps; + private bool isZip64; public ZipWriter(Stream destination, ZipWriterOptions zipWriterOptions) : base(ArchiveType.Zip) { zipComment = zipWriterOptions.ArchiveComment ?? string.Empty; + isZip64 = zipWriterOptions.UseZip64; compressionType = zipWriterOptions.CompressionType; compressionLevel = zipWriterOptions.DeflateCompressionLevel; @@ -50,7 +52,7 @@ protected override void Dispose(bool isDisposing) { if (isDisposing) { - uint size = 0; + ulong size = 0; foreach (ZipCentralDirectoryEntry entry in entries) { size += entry.Write(OutputStream, ToZipCompressionMethod(compressionType)); @@ -117,7 +119,14 @@ public Stream WriteToStream(string entryPath, ZipWriterEntryOptions options) HeaderOffset = (uint)streamPosition }; - var headersize = (uint)WriteHeader(entryPath, options); + // Switch to allocating space for zip64, if the archive is larger than 2GB + var useZip64 = (OutputStream.CanSeek && OutputStream.Length > int.MaxValue) || isZip64; + + // Allow direct disabling + if (options.EnableZip64.HasValue) + useZip64 = options.EnableZip64.Value; + + var headersize = (uint)WriteHeader(entryPath, options, entry, useZip64); streamPosition += headersize; return new ZipWritingStream(this, OutputStream, entry, ToZipCompressionMethod(options.CompressionType ?? compressionType), @@ -137,7 +146,7 @@ private string NormalizeFilename(string filename) return filename.Trim('/'); } - private int WriteHeader(string filename, ZipWriterEntryOptions zipWriterEntryOptions) + private int WriteHeader(string filename, ZipWriterEntryOptions zipWriterEntryOptions, ZipCentralDirectoryEntry entry, bool useZip64) { var explicitZipCompressionInfo = ToZipCompressionMethod(zipWriterEntryOptions.CompressionType ?? compressionType); byte[] encodedFilename = ArchiveEncoding.Default.GetBytes(filename); @@ -145,16 +154,21 @@ private int WriteHeader(string filename, ZipWriterEntryOptions zipWriterEntryOpt OutputStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.ENTRY_HEADER_BYTES), 0, 4); if (explicitZipCompressionInfo == ZipCompressionMethod.Deflate) { - OutputStream.Write(new byte[] {20, 0}, 0, 2); //older version which is more compatible + if (OutputStream.CanSeek && useZip64) + OutputStream.Write(new byte[] { 45, 0 }, 0, 2); //smallest allowed version for zip64 + else + OutputStream.Write(new byte[] { 20, 0 }, 0, 2); //older version which is more compatible } else { - OutputStream.Write(new byte[] {63, 0}, 0, 2); //version says we used PPMd or LZMA + OutputStream.Write(new byte[] { 63, 0 }, 0, 2); //version says we used PPMd or LZMA } HeaderFlags flags = ArchiveEncoding.Default == Encoding.UTF8 ? HeaderFlags.UTF8 : 0; if (!OutputStream.CanSeek) { + // We cannot really use post data with zip64, but we have nothing else flags |= HeaderFlags.UsePostDataDescriptor; + if (explicitZipCompressionInfo == ZipCompressionMethod.LZMA) { flags |= HeaderFlags.Bit1; // eos marker @@ -165,14 +179,25 @@ private int WriteHeader(string filename, ZipWriterEntryOptions zipWriterEntryOpt OutputStream.Write(DataConverter.LittleEndian.GetBytes(zipWriterEntryOptions.ModificationDateTime.DateTimeToDosTime()), 0, 4); // zipping date and time - OutputStream.Write(new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 0, 12); + OutputStream.Write(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, 0, 12); // unused CRC, un/compressed size, updated later OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedFilename.Length), 0, 2); // filename length - OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // extra length + + var extralength = 0; + if (OutputStream.CanSeek && useZip64) + extralength = 2 + 2 + 8 + 8; + + OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)extralength), 0, 2); // extra length OutputStream.Write(encodedFilename, 0, encodedFilename.Length); - return 6 + 2 + 2 + 4 + 12 + 2 + 2 + encodedFilename.Length; + if (extralength != 0) + { + OutputStream.Write(new byte[extralength], 0, extralength); // reserve space for zip64 data + entry.Zip64HeaderOffset = (ushort)(6 + 2 + 2 + 4 + 12 + 2 + 2 + encodedFilename.Length); + } + + return 6 + 2 + 2 + 4 + 12 + 2 + 2 + encodedFilename.Length + extralength; } private void WriteFooter(uint crc, uint compressed, uint uncompressed) @@ -182,15 +207,58 @@ private void WriteFooter(uint crc, uint compressed, uint uncompressed) OutputStream.Write(DataConverter.LittleEndian.GetBytes(uncompressed), 0, 4); } - private void WriteEndRecord(uint size) + private void WritePostdataDescriptor(uint crc, ulong compressed, ulong uncompressed) + { + OutputStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.POST_DATA_DESCRIPTOR), 0, 4); + OutputStream.Write(DataConverter.LittleEndian.GetBytes(crc), 0, 4); + OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)compressed), 0, 4); + OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)uncompressed), 0, 4); + } + + private void WriteEndRecord(ulong size) { byte[] encodedComment = ArchiveEncoding.Default.GetBytes(zipComment); + var zip64 = isZip64 || entries.Count > ushort.MaxValue || streamPosition >= uint.MaxValue || size >= uint.MaxValue; + + var sizevalue = size >= uint.MaxValue ? uint.MaxValue : (uint)size; + var streampositionvalue = streamPosition >= uint.MaxValue ? uint.MaxValue : (uint)streamPosition; + + if (zip64) + { + var recordlen = 4 + 8 + 2 + 2 + 4 + 4 + 8 + 8 + 8 + 8; + + // Write zip64 end of central directory record + OutputStream.Write(new byte[] { 80, 75, 6, 6 }, 0, 4); + OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)recordlen), 0, 8); // Size of zip64 end of central directory record + OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0), 0, 2); // Made by + OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)45), 0, 2); // Version needed + + OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)0), 0, 4); // Disk number + OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)0), 0, 4); // Central dir disk + // TODO: entries.Count is int, so max 2^31 files + OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)entries.Count), 0, 8); // Entries in this disk + OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)entries.Count), 0, 8); // Total entries + OutputStream.Write(DataConverter.LittleEndian.GetBytes(size), 0, 8); // Central Directory size + OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)streamPosition), 0, 8); // Disk offset + + // Write zip64 end of central directory locator + OutputStream.Write(new byte[] { 80, 75, 6, 7 }, 0, 4); + + OutputStream.Write(DataConverter.LittleEndian.GetBytes(0uL), 0, 4); // Entry disk + OutputStream.Write(DataConverter.LittleEndian.GetBytes((ulong)streamPosition + size), 0, 8); // Offset to the zip64 central directory + OutputStream.Write(DataConverter.LittleEndian.GetBytes(0u), 0, 4); // Number of disks + + streamPosition += recordlen + (4 + 4 + 8 + 4); + streampositionvalue = streamPosition >= uint.MaxValue ? uint.MaxValue : (uint)streampositionvalue; + } + + // Write normal end of central directory record OutputStream.Write(new byte[] {80, 75, 5, 6, 0, 0, 0, 0}, 0, 8); OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)entries.Count), 0, 2); OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)entries.Count), 0, 2); - OutputStream.Write(DataConverter.LittleEndian.GetBytes(size), 0, 4); - OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)streamPosition), 0, 4); + OutputStream.Write(DataConverter.LittleEndian.GetBytes(sizevalue), 0, 4); + OutputStream.Write(DataConverter.LittleEndian.GetBytes((uint)streampositionvalue), 0, 4); OutputStream.Write(DataConverter.LittleEndian.GetBytes((ushort)encodedComment.Length), 0, 2); OutputStream.Write(encodedComment, 0, encodedComment.Length); } @@ -207,7 +275,7 @@ internal class ZipWritingStream : Stream private readonly ZipCompressionMethod zipCompressionMethod; private readonly CompressionLevel compressionLevel; private CountingWritableSubStream counting; - private uint decompressed; + private ulong decompressed; internal ZipWritingStream(ZipWriter writer, Stream originalStream, ZipCentralDirectoryEntry entry, ZipCompressionMethod zipCompressionMethod, CompressionLevel compressionLevel) @@ -283,20 +351,51 @@ protected override void Dispose(bool disposing) entry.Crc = (uint)crc.Crc32Result; entry.Compressed = counting.Count; entry.Decompressed = decompressed; + + var zip64 = entry.Compressed >= uint.MaxValue || entry.Decompressed >= uint.MaxValue || entry.HeaderOffset >= uint.MaxValue; + + writer.isZip64 |= zip64; + + var compressedvalue = zip64 ? uint.MaxValue : (uint)counting.Count; + var decompressedvalue = zip64 ? uint.MaxValue : (uint)entry.Decompressed; + if (originalStream.CanSeek) { - originalStream.Position = entry.HeaderOffset + 6; + originalStream.Position = (long)(entry.HeaderOffset + 6); originalStream.WriteByte(0); - originalStream.Position = entry.HeaderOffset + 14; - writer.WriteFooter(entry.Crc, counting.Count, decompressed); - originalStream.Position = writer.streamPosition + entry.Compressed; - writer.streamPosition += entry.Compressed; + + originalStream.Position = (long)(entry.HeaderOffset + 14); + + writer.WriteFooter(entry.Crc, compressedvalue, decompressedvalue); + + // If we have pre-allocated space for zip64 data, fill it out + if (entry.Zip64HeaderOffset != 0) + { + originalStream.Position = (long)(entry.HeaderOffset + entry.Zip64HeaderOffset); + originalStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x0001), 0, 2); + originalStream.Write(DataConverter.LittleEndian.GetBytes((ushort)(8 + 8)), 0, 2); + + originalStream.Write(DataConverter.LittleEndian.GetBytes(entry.Decompressed), 0, 8); + originalStream.Write(DataConverter.LittleEndian.GetBytes(entry.Compressed), 0, 8); + } + + originalStream.Position = writer.streamPosition + (long)entry.Compressed; + writer.streamPosition += (long)entry.Compressed; + } else { + // Bit unclear what happens here, with zip64 + // We have a streaming archive, so we should add a post-data-descriptor, + // but we cannot as it does not hold the zip64 values + + // The current implementation writes 0xffffffff in the fields here, and the + // the central directory has the extra data required if the fields are overflown originalStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.POST_DATA_DESCRIPTOR), 0, 4); - writer.WriteFooter(entry.Crc, counting.Count, decompressed); - writer.streamPosition += entry.Compressed + 16; + writer.WriteFooter(entry.Crc, + (uint)(counting.Count >= uint.MaxValue ? uint.MaxValue : counting.Count), + (uint)(entry.Decompressed >= uint.MaxValue ? uint.MaxValue : entry.Decompressed)); + writer.streamPosition += (long)entry.Compressed + 16; } writer.entries.Add(entry); } diff --git a/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs b/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs index 81f1d1c06..406090989 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs @@ -15,5 +15,10 @@ public class ZipWriterEntryOptions public string EntryComment { get; set; } public DateTime? ModificationDateTime { get; set; } + + /// + /// Allocate space for storing values if the file is larger than 4GiB + /// + public bool? EnableZip64 { get; set; } } } \ No newline at end of file diff --git a/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs b/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs index 662a50d45..1cad53d43 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs @@ -15,6 +15,8 @@ internal ZipWriterOptions(WriterOptions options) : base(options.CompressionType) { LeaveStreamOpen = options.LeaveStreamOpen; + if (options is ZipWriterOptions) + UseZip64 = ((ZipWriterOptions)options).UseZip64; } /// /// When CompressionType.Deflate is used, this property is referenced. Defaults to CompressionLevel.Default. @@ -22,5 +24,10 @@ internal ZipWriterOptions(WriterOptions options) public CompressionLevel DeflateCompressionLevel { get; set; } = CompressionLevel.Default; public string ArchiveComment { get; set; } + + /// + /// Sets a value indicating if zip64 support is enabled. If this is not set, zip64 will be enabled once the file is larger than 2GB + /// + public bool UseZip64 { get; set; } } } \ No newline at end of file From d7f4c0ee323b60cf7845e19e611cbfdd4381e79b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 10 Mar 2017 23:10:06 +0100 Subject: [PATCH 3/6] Fixed an error in the zip64 central end of header: the signature + length (12 bytes) are not included in the reported length. --- src/SharpCompress/Writers/Zip/ZipWriter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index f40726c62..6f38a28d4 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -225,7 +225,7 @@ private void WriteEndRecord(ulong size) if (zip64) { - var recordlen = 4 + 8 + 2 + 2 + 4 + 4 + 8 + 8 + 8 + 8; + var recordlen = 2 + 2 + 4 + 4 + 8 + 8 + 8 + 8; // Write zip64 end of central directory record OutputStream.Write(new byte[] { 80, 75, 6, 6 }, 0, 4); From 85280f6f4fcf7e845330755f5f12b567f83eb3f1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 10 Mar 2017 23:18:26 +0100 Subject: [PATCH 4/6] Changed the logic to throw exceptions when sizes exceed the zip archive limits, and zip64 is not enabled. This changes the logic, such that archives larger than 4GiB are still automatically written correct (only the central header is special). Archives with individual streams larger than 4 GiB must set the zip64 flag, either on the archive or the individual streams. --- .../Writers/Zip/ZipCentralDirectoryEntry.cs | 9 +- src/SharpCompress/Writers/Zip/ZipWriter.cs | 105 ++++++++++++------ .../Writers/Zip/ZipWriterEntryOptions.cs | 5 +- .../Writers/Zip/ZipWriterOptions.cs | 6 +- 4 files changed, 90 insertions(+), 35 deletions(-) diff --git a/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs b/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs index 03f1ee5a6..3e29a0f5a 100644 --- a/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs +++ b/src/SharpCompress/Writers/Zip/ZipCentralDirectoryEntry.cs @@ -23,7 +23,8 @@ internal uint Write(Stream outputStream, ZipCompressionMethod compression) byte[] encodedFilename = Encoding.UTF8.GetBytes(FileName); byte[] encodedComment = Encoding.UTF8.GetBytes(Comment); - var zip64 = Compressed >= uint.MaxValue || Decompressed >= uint.MaxValue || HeaderOffset >= uint.MaxValue || Zip64HeaderOffset != 0; + var zip64_stream = Compressed >= uint.MaxValue || Decompressed >= uint.MaxValue; + var zip64 = zip64_stream || HeaderOffset >= uint.MaxValue || Zip64HeaderOffset != 0; var compressedvalue = zip64 ? uint.MaxValue : (uint)Compressed; var decompressedvalue = zip64 ? uint.MaxValue : (uint)Decompressed; @@ -36,7 +37,11 @@ internal uint Write(Stream outputStream, ZipCompressionMethod compression) { // Cannot use data descriptors with zip64: // https://blogs.oracle.com/xuemingshen/entry/is_zipinput_outputstream_handling_of - if (!zip64) + + // We check that streams are not written too large in the ZipWritingStream, + // so this extra guard is not required, but kept to simplify changing the code + // once the zip64 post-data issue is resolved + if (!zip64_stream) flags |= HeaderFlags.UsePostDataDescriptor; if (compression == ZipCompressionMethod.LZMA) diff --git a/src/SharpCompress/Writers/Zip/ZipWriter.cs b/src/SharpCompress/Writers/Zip/ZipWriter.cs index 6f38a28d4..e1a001b5e 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriter.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriter.cs @@ -116,13 +116,11 @@ public Stream WriteToStream(string entryPath, ZipWriterEntryOptions options) Comment = options.EntryComment, FileName = entryPath, ModificationTime = options.ModificationDateTime, - HeaderOffset = (uint)streamPosition - }; + HeaderOffset = (ulong)streamPosition + }; - // Switch to allocating space for zip64, if the archive is larger than 2GB - var useZip64 = (OutputStream.CanSeek && OutputStream.Length > int.MaxValue) || isZip64; - - // Allow direct disabling + // Use the archive default setting for zip64 and allow overrides + var useZip64 = isZip64; if (options.EnableZip64.HasValue) useZip64 = options.EnableZip64.Value; @@ -148,6 +146,10 @@ private string NormalizeFilename(string filename) private int WriteHeader(string filename, ZipWriterEntryOptions zipWriterEntryOptions, ZipCentralDirectoryEntry entry, bool useZip64) { + // We err on the side of caution until the zip specification clarifies how to support this + if (!OutputStream.CanSeek && useZip64) + throw new NotSupportedException("Zip64 extensions are not supported on non-seekable streams"); + var explicitZipCompressionInfo = ToZipCompressionMethod(zipWriterEntryOptions.CompressionType ?? compressionType); byte[] encodedFilename = ArchiveEncoding.Default.GetBytes(filename); @@ -166,7 +168,6 @@ private int WriteHeader(string filename, ZipWriterEntryOptions zipWriterEntryOpt HeaderFlags flags = ArchiveEncoding.Default == Encoding.UTF8 ? HeaderFlags.UTF8 : 0; if (!OutputStream.CanSeek) { - // We cannot really use post data with zip64, but we have nothing else flags |= HeaderFlags.UsePostDataDescriptor; if (explicitZipCompressionInfo == ZipCompressionMethod.LZMA) @@ -277,6 +278,9 @@ internal class ZipWritingStream : Stream private CountingWritableSubStream counting; private ulong decompressed; + // Flag to prevent throwing exceptions on Dispose + private bool limitsExceeded; + internal ZipWritingStream(ZipWriter writer, Stream originalStream, ZipCentralDirectoryEntry entry, ZipCompressionMethod zipCompressionMethod, CompressionLevel compressionLevel) { @@ -348,16 +352,23 @@ protected override void Dispose(bool disposing) if (disposing) { writeStream.Dispose(); + + if (limitsExceeded) + { + // We have written invalid data into the archive, + // so we destroy it now, instead of allowing the user to continue + // with a defunct archive + originalStream.Dispose(); + return; + } + entry.Crc = (uint)crc.Crc32Result; entry.Compressed = counting.Count; entry.Decompressed = decompressed; - var zip64 = entry.Compressed >= uint.MaxValue || entry.Decompressed >= uint.MaxValue || entry.HeaderOffset >= uint.MaxValue; - - writer.isZip64 |= zip64; - - var compressedvalue = zip64 ? uint.MaxValue : (uint)counting.Count; - var decompressedvalue = zip64 ? uint.MaxValue : (uint)entry.Decompressed; + var zip64 = entry.Compressed >= uint.MaxValue || entry.Decompressed >= uint.MaxValue; + var compressedvalue = zip64 ? uint.MaxValue : (uint)counting.Count; + var decompressedvalue = zip64 ? uint.MaxValue : (uint)entry.Decompressed; if (originalStream.CanSeek) { @@ -368,33 +379,41 @@ protected override void Dispose(bool disposing) writer.WriteFooter(entry.Crc, compressedvalue, decompressedvalue); - // If we have pre-allocated space for zip64 data, fill it out - if (entry.Zip64HeaderOffset != 0) - { - originalStream.Position = (long)(entry.HeaderOffset + entry.Zip64HeaderOffset); - originalStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x0001), 0, 2); - originalStream.Write(DataConverter.LittleEndian.GetBytes((ushort)(8 + 8)), 0, 2); + // Ideally, we should not throw from Dispose() + // We should not get here as the Write call checks the limits + if (zip64 && entry.Zip64HeaderOffset == 0) + throw new NotSupportedException("Attempted to write a stream that is larger than 4GiB without setting the zip64 option"); + + // If we have pre-allocated space for zip64 data, + // fill it out, even if it is not required + if (entry.Zip64HeaderOffset != 0) + { + originalStream.Position = (long)(entry.HeaderOffset + entry.Zip64HeaderOffset); + originalStream.Write(DataConverter.LittleEndian.GetBytes((ushort)0x0001), 0, 2); + originalStream.Write(DataConverter.LittleEndian.GetBytes((ushort)(8 + 8)), 0, 2); - originalStream.Write(DataConverter.LittleEndian.GetBytes(entry.Decompressed), 0, 8); - originalStream.Write(DataConverter.LittleEndian.GetBytes(entry.Compressed), 0, 8); - } + originalStream.Write(DataConverter.LittleEndian.GetBytes(entry.Decompressed), 0, 8); + originalStream.Write(DataConverter.LittleEndian.GetBytes(entry.Compressed), 0, 8); + } originalStream.Position = writer.streamPosition + (long)entry.Compressed; writer.streamPosition += (long)entry.Compressed; - } else { - // Bit unclear what happens here, with zip64 - // We have a streaming archive, so we should add a post-data-descriptor, - // but we cannot as it does not hold the zip64 values + // We have a streaming archive, so we should add a post-data-descriptor, + // but we cannot as it does not hold the zip64 values + // Throwing an exception until the zip specification is clarified + + // Ideally, we should not throw from Dispose() + // We should not get here as the Write call checks the limits + if (zip64) + throw new NotSupportedException("Streams larger than 4GiB are not supported for non-seekable streams"); - // The current implementation writes 0xffffffff in the fields here, and the - // the central directory has the extra data required if the fields are overflown - originalStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.POST_DATA_DESCRIPTOR), 0, 4); + originalStream.Write(DataConverter.LittleEndian.GetBytes(ZipHeaderFactory.POST_DATA_DESCRIPTOR), 0, 4); writer.WriteFooter(entry.Crc, - (uint)(counting.Count >= uint.MaxValue ? uint.MaxValue : counting.Count), - (uint)(entry.Decompressed >= uint.MaxValue ? uint.MaxValue : entry.Decompressed)); + (uint)compressedvalue, + (uint)decompressedvalue); writer.streamPosition += (long)entry.Compressed + 16; } writer.entries.Add(entry); @@ -423,9 +442,33 @@ public override void SetLength(long value) public override void Write(byte[] buffer, int offset, int count) { + // We check the limits first, because we can keep the archive consistent + // if we can prevent the writes from happening + if (entry.Zip64HeaderOffset == 0) + { + // Pre-check, the counting.Count is not exact, as we do not know the size before having actually compressed it + if (limitsExceeded || ((decompressed + (uint)count) > uint.MaxValue) || (counting.Count + (uint)count) > uint.MaxValue) + throw new NotSupportedException("Attempted to write a stream that is larger than 4GiB without setting the zip64 option"); + } + decompressed += (uint)count; crc.SlurpBlock(buffer, offset, count); writeStream.Write(buffer, offset, count); + + if (entry.Zip64HeaderOffset == 0) + { + // Post-check, this is accurate + if ((decompressed > uint.MaxValue) || counting.Count > uint.MaxValue) + { + // We have written the data, so the archive is now broken + // Throwing the exception here, allows us to avoid + // throwing an exception in Dispose() which is discouraged + // as it can mask other errors + limitsExceeded = true; + throw new NotSupportedException("Attempted to write a stream that is larger than 4GiB without setting the zip64 option"); + } + } + } } diff --git a/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs b/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs index 406090989..5f1d8152d 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriterEntryOptions.cs @@ -17,7 +17,10 @@ public class ZipWriterEntryOptions public DateTime? ModificationDateTime { get; set; } /// - /// Allocate space for storing values if the file is larger than 4GiB + /// Allocate an extra 20 bytes for this entry to store, + /// 64 bit length values, thus enabling streams + /// larger than 4GiB. + /// This option is not supported with non-seekable streams. /// public bool? EnableZip64 { get; set; } } diff --git a/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs b/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs index 1cad53d43..81c2afbee 100644 --- a/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs +++ b/src/SharpCompress/Writers/Zip/ZipWriterOptions.cs @@ -26,7 +26,11 @@ internal ZipWriterOptions(WriterOptions options) public string ArchiveComment { get; set; } /// - /// Sets a value indicating if zip64 support is enabled. If this is not set, zip64 will be enabled once the file is larger than 2GB + /// Sets a value indicating if zip64 support is enabled. + /// If this is not set, individual stream lengths cannot exceed 4 GiB. + /// This option is not supported for non-seekable streams. + /// Archives larger than 4GiB are supported as long as all streams + /// are less than 4GiB in length. /// public bool UseZip64 { get; set; } } From 2894711c5128d16a80e70f7e551fe6c16e892b1d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 11 Mar 2017 00:54:06 +0100 Subject: [PATCH 5/6] Added a test suite to verify zip64 write support is working, and can be read in both Archive and Stream mode --- test/SharpCompress.Test/Zip/Zip64Tests.cs | 213 ++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 test/SharpCompress.Test/Zip/Zip64Tests.cs diff --git a/test/SharpCompress.Test/Zip/Zip64Tests.cs b/test/SharpCompress.Test/Zip/Zip64Tests.cs new file mode 100644 index 000000000..fb39468d4 --- /dev/null +++ b/test/SharpCompress.Test/Zip/Zip64Tests.cs @@ -0,0 +1,213 @@ +using System; +using System.IO; +using System.Linq; +using SharpCompress.Archives; +using SharpCompress.Common; +using SharpCompress.Readers; +using SharpCompress.Readers.Zip; +using SharpCompress.Writers; +using SharpCompress.Writers.Zip; +using Xunit; + +namespace SharpCompress.Test +{ + public class Zip64Tests : WriterTests + { + // 4GiB + 1 + const long FOUR_GB_LIMIT = ((long)uint.MaxValue) + 1; + + [Fact] + public void Zip64_Single_Large_File() + { + // One single file, requires zip64 + RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: true, forward_only: false); + } + + [Fact] + public void Zip64_Two_Large_Files() + { + // One single file, requires zip64 + RunSingleTest(2, FOUR_GB_LIMIT, set_zip64: true, forward_only: false); + } + + [Fact] + public void Zip64_Two_Small_files() + { + // Multiple files, does not require zip64 + RunSingleTest(2, FOUR_GB_LIMIT / 2, set_zip64: false, forward_only: false); + } + + [Fact] + public void Zip64_Two_Small_files_stream() + { + // Multiple files, does not require zip64, and works with streams + RunSingleTest(2, FOUR_GB_LIMIT / 2, set_zip64: false, forward_only: true); + } + + [Fact] + public void Zip64_Two_Small_Files_Zip64() + { + // Multiple files, use zip64 even though it is not required + RunSingleTest(2, FOUR_GB_LIMIT / 2, set_zip64: true, forward_only: false); + } + + [Fact] + public void Zip64_Single_Large_File_Fail() + { + try + { + // One single file, should fail + RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: false, forward_only: false); + throw new Exception("Test did not fail?"); + } + catch (NotSupportedException) + { + } + } + + [Fact] + public void Zip64_Single_Large_File_Zip64_Streaming_Fail() + { + try + { + // One single file, should fail (fast) with zip64 + RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: true, forward_only: true); + throw new Exception("Test did not fail?"); + } + catch (NotSupportedException) + { + } + } + + [Fact] + public void Zip64_Single_Large_File_Streaming_Fail() + { + try + { + // One single file, should fail once the write discovers the problem + RunSingleTest(1, FOUR_GB_LIMIT, set_zip64: false, forward_only: true); + throw new Exception("Test did not fail?"); + } + catch (NotSupportedException) + { + } + } + + public void RunSingleTest(long files, long filesize, bool set_zip64, bool forward_only, long write_chunk_size = 1024 * 1024, string filename = "zip64-test.zip") + { + ResetScratch(); + filename = Path.Combine(SCRATCH2_FILES_PATH, filename); + + if (File.Exists(filename)) + File.Delete(filename); + + if (!File.Exists(filename)) + CreateZipArchive(filename, files, filesize, write_chunk_size, set_zip64, forward_only); + + var resForward = ReadForwardOnly(filename); + if (resForward.Item1 != files) + throw new Exception($"Incorrect number of items reported: {resForward.Item1}, should have been {files}"); + + if (resForward.Item2 != files * filesize) + throw new Exception($"Incorrect combined size reported: {resForward.Item2}, should have been {files * filesize}"); + + var resArchive = ReadArchive(filename); + if (resArchive.Item1 != files) + throw new Exception($"Incorrect number of items reported: {resArchive.Item1}, should have been {files}"); + if (resArchive.Item2 != files * filesize) + throw new Exception($"Incorrect number of items reported: {resArchive.Item2}, should have been {files * filesize}"); + } + + public void CreateZipArchive(string filename, long files, long filesize, long chunksize, bool set_zip64, bool forward_only) + { + var data = new byte[chunksize]; + + // Use deflate for speed + var opts = new ZipWriterOptions(CompressionType.Deflate) { UseZip64 = set_zip64 }; + + // Use no compression to ensure we hit the limits (actually inflates a bit, but seems better than using method==Store) + var eo = new ZipWriterEntryOptions() { DeflateCompressionLevel = SharpCompress.Compressors.Deflate.CompressionLevel.None }; + + using (var zip = File.OpenWrite(filename)) + using(var st = forward_only ? (Stream)new NonSeekableStream(zip) : zip) + using (var zipWriter = (ZipWriter)WriterFactory.Open(st, ArchiveType.Zip, opts)) + { + + for (var i = 0; i < files; i++) + using (var str = zipWriter.WriteToStream(i.ToString(), eo)) + { + var left = filesize; + while (left > 0) + { + var b = (int)Math.Min(left, data.Length); + str.Write(data, 0, b); + left -= b; + } + } + } + } + + public Tuple ReadForwardOnly(string filename) + { + long count = 0; + long size = 0; + Common.Zip.ZipEntry prev = null; + using (var fs = File.OpenRead(filename)) + using (var rd = ZipReader.Open(fs, new ReaderOptions() { LookForHeader = false })) + while (rd.MoveToNextEntry()) + { + using (rd.OpenEntryStream()) + { } + + count++; + if (prev != null) + size += prev.Size; + + prev = rd.Entry; + } + + if (prev != null) + size += prev.Size; + + return new Tuple(count, size); + } + + public Tuple ReadArchive(string filename) + { + using (var archive = ArchiveFactory.Open(filename)) + { + return new Tuple( + archive.Entries.Count(), + archive.Entries.Select(x => x.Size).Sum() + ); + } + } + + /// + /// Helper to create non-seekable streams from filestream + /// + private class NonSeekableStream : Stream + { + private readonly Stream stream; + public NonSeekableStream(Stream s) { stream = s; } + public override bool CanRead { get { return stream.CanRead; } } + public override bool CanSeek { get { return false; } } + public override bool CanWrite { get { return stream.CanWrite; } } + public override long Length { get { throw new NotImplementedException(); } } + public override long Position { get { throw new NotImplementedException(); } set { throw new NotImplementedException(); } } + public override void Flush() { stream.Flush(); } + + public override int Read(byte[] buffer, int offset, int count) + { return stream.Read(buffer, offset, count); } + + public override long Seek(long offset, SeekOrigin origin) + { throw new NotImplementedException(); } + + public override void SetLength(long value) + { throw new NotImplementedException(); } + + public override void Write(byte[] buffer, int offset, int count) + { stream.Write(buffer, offset, count); } + } + } +} \ No newline at end of file From 726b9c80f6cfed1d677b602209860b076f450d8a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 11 Mar 2017 01:05:58 +0100 Subject: [PATCH 6/6] Fixed compiling the unittest --- test/SharpCompress.Test/Zip/Zip64Tests.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/SharpCompress.Test/Zip/Zip64Tests.cs b/test/SharpCompress.Test/Zip/Zip64Tests.cs index fb39468d4..4a4bf6392 100644 --- a/test/SharpCompress.Test/Zip/Zip64Tests.cs +++ b/test/SharpCompress.Test/Zip/Zip64Tests.cs @@ -13,8 +13,13 @@ namespace SharpCompress.Test { public class Zip64Tests : WriterTests { - // 4GiB + 1 - const long FOUR_GB_LIMIT = ((long)uint.MaxValue) + 1; + public Zip64Tests() + : base(ArchiveType.Zip) + { + } + + // 4GiB + 1 + const long FOUR_GB_LIMIT = ((long)uint.MaxValue) + 1; [Fact] public void Zip64_Single_Large_File()