From d3c3deda1655430fbae23d62cda985ce2f3ab2ec Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Sat, 7 Apr 2018 22:33:32 +0700 Subject: [PATCH 01/28] Fix improper gzip end stream This patch adds additional call of Z_FINISH to properly ends gzip stream. Signed-off-by: Dhi Aurrahman --- .../common/compressor/zlib_compressor_impl.cc | 23 +++++++++++++++++++ .../common/compressor/zlib_compressor_impl.h | 12 ++++++++++ .../decompressor/zlib_decompressor_impl.cc | 7 ++++++ source/common/http/filter/gzip_filter.cc | 4 ++++ .../compressor/zlib_compressor_impl_test.cc | 18 +++++++++++++++ 5 files changed, 64 insertions(+) diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index 0e26d8c202ea..f64a515ae247 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -82,5 +82,28 @@ void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer) { zstream_ptr_->next_out = chunk_char_ptr_.get(); } +void ZlibCompressorImpl::reset() { + const bool result = deflateReset(zstream_ptr_.get()); + RELEASE_ASSERT(result == Z_OK); +} + +void ZlibCompressorImpl::finish(Buffer::Instance& output_buffer) { + // Once the Z_FINISH parameter is provided, deflate() will begin to complete the compressed output + // stream. However depending on how much output space is provided, deflate() may have to be called + // several times until it has provided the complete compressed stream, even after it has consumed + // all of the input. The flush parameter must continue to be Z_FINISH for those subsequent calls. + // Ref: https://zlib.net/zlib_how.html. + const int result = deflate(zstream_ptr_.get(), Z_FINISH); + updateOutput(output_buffer); + if (result == Z_OK || result == Z_BUF_ERROR) { + finish(output_buffer); + } else { + // While Z_OK, Z_BUF_ERROR (covered in above case) and Z_STREAM_END are the acceptable values, + // the Z_STREAM_ERROR is only possible if the stream is not initialized properly. Ref: + // https://zlib.net/zlib_how.html. + RELEASE_ASSERT(result == Z_STREAM_END); + } +} + } // namespace Compressor } // namespace Envoy diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 3da8590618d8..1e47100ad2d2 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -81,6 +81,18 @@ class ZlibCompressorImpl : public Compressor { */ uint64_t checksum(); + /** + * This is required to re-init the stream after calling finish(_). + */ + void reset(); + + /** + * Completes the compressed output stream. To indicate that the currently pointed data is the last + * chunk of input data to compress. + * @param output_buffer supplies the buffer to output compressed data. + */ + void finish(Buffer::Instance& output_buffer); + // Compressor void compress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) override; diff --git a/source/common/decompressor/zlib_decompressor_impl.cc b/source/common/decompressor/zlib_decompressor_impl.cc index 442d13985f2c..b18270e48d6f 100644 --- a/source/common/decompressor/zlib_decompressor_impl.cc +++ b/source/common/decompressor/zlib_decompressor_impl.cc @@ -59,6 +59,13 @@ void ZlibDecompressorImpl::decompress(const Buffer::Instance& input_buffer, bool ZlibDecompressorImpl::inflateNext() { const int result = inflate(zstream_ptr_.get(), Z_NO_FLUSH); + if (result == Z_STREAM_END) { + // Z_FINISH informs inflate to not maintain a sliding window if the stream completes, which + // reduces inflate's memory footprint. Ref: https://www.zlib.net/manual.html. + inflate(zstream_ptr_.get(), Z_FINISH); + return false; + } + if (result == Z_BUF_ERROR && zstream_ptr_->avail_in == 0) { return false; // This means that zlib needs more input, so stop here. } diff --git a/source/common/http/filter/gzip_filter.cc b/source/common/http/filter/gzip_filter.cc index 4288f5666093..694b2d686f33 100644 --- a/source/common/http/filter/gzip_filter.cc +++ b/source/common/http/filter/gzip_filter.cc @@ -139,11 +139,15 @@ FilterDataStatus GzipFilter::encodeData(Buffer::Instance& data, bool end_stream) if (end_stream) { compressor_.flush(compressed_data_); + compressor_.finish(compressed_data_); } if (compressed_data_.length()) { data.drain(n_data); data.move(compressed_data_); + + // Reset is required to re-init the compressor's z_stream. + compressor_.reset(); return Http::FilterDataStatus::Continue; } diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index fea592bb5088..e7ec2d0e9ff6 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -103,6 +103,9 @@ TEST_F(ZlibCompressorImplTest, CompressWithReducedInternalMemory) { compressed_slices[num_comp_slices - 1].len_); // FOOTER four-byte sequence (sync flush) EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + compressor.finish(output_buffer); + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); } /** @@ -142,6 +145,12 @@ TEST_F(ZlibCompressorImplTest, CompressWithContinuesFlush) { compressed_slices[num_comp_slices - 1].len_); // FOOTER four-byte sequence (sync flush) EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + compressor.finish(output_buffer); + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + // Re-init the compressor. + compressor.reset(); } } @@ -181,6 +190,9 @@ TEST_F(ZlibCompressorImplTest, CompressSmallInputMemory) { compressed_slices[num_comp_slices - 1].len_); // FOOTER four-byte sequence (sync flush) EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + compressor.finish(output_buffer); + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); } /** @@ -241,6 +253,9 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { compressed_slices[num_comp_slices - 1].len_); // FOOTER four-byte sequence (sync flush) EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + compressor.finish(output_buffer); + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); } /** @@ -279,6 +294,9 @@ TEST_F(ZlibCompressorImplTest, CompressWithNotCommonParams) { compressed_slices[num_comp_slices - 1].len_); // FOOTER four-byte sequence (sync flush) EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + compressor.finish(output_buffer); + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); } } // namespace From 86e4b868370b072416bbcc355b86fbefc8313dda Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Mon, 9 Apr 2018 07:22:29 +0700 Subject: [PATCH 02/28] Review comments: remove finish() and prefer loop Signed-off-by: Dhi Aurrahman --- .../common/compressor/zlib_compressor_impl.cc | 36 +++-- .../common/compressor/zlib_compressor_impl.h | 11 +- source/common/http/filter/gzip_filter.cc | 1 - .../compressor/zlib_compressor_impl_test.cc | 131 +++++------------- 4 files changed, 51 insertions(+), 128 deletions(-) diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index f64a515ae247..534e105157ad 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -68,11 +68,11 @@ void ZlibCompressorImpl::process(Buffer::Instance& output_buffer, int64_t flush_ } if (flush_state == Z_SYNC_FLUSH) { - updateOutput(output_buffer); + updateOutput(output_buffer, true); } } -void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer) { +void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer, bool require_finish) { const uint64_t n_output = chunk_size_ - zstream_ptr_->avail_out; if (n_output > 0) { output_buffer.add(static_cast(chunk_char_ptr_.get()), n_output); @@ -80,6 +80,20 @@ void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer) { chunk_char_ptr_.reset(new unsigned char[chunk_size_]); zstream_ptr_->avail_out = chunk_size_; zstream_ptr_->next_out = chunk_char_ptr_.get(); + + if (require_finish) { + // Once the Z_FINISH parameter is provided, deflate() will begin to complete the compressed + // output stream. However depending on how much output space is provided, deflate() may have to + // be called several times until it has provided the complete compressed stream, even after it + // has consumed all of the input. The flush parameter must continue to be Z_FINISH for those + // subsequent calls. Ref: https://zlib.net/zlib_how.html. + int result = Z_OK; + while (result != Z_STREAM_END) { + result = deflate(zstream_ptr_.get(), Z_FINISH); + RELEASE_ASSERT(result != Z_STREAM_ERROR); + updateOutput(output_buffer); + } + } } void ZlibCompressorImpl::reset() { @@ -87,23 +101,5 @@ void ZlibCompressorImpl::reset() { RELEASE_ASSERT(result == Z_OK); } -void ZlibCompressorImpl::finish(Buffer::Instance& output_buffer) { - // Once the Z_FINISH parameter is provided, deflate() will begin to complete the compressed output - // stream. However depending on how much output space is provided, deflate() may have to be called - // several times until it has provided the complete compressed stream, even after it has consumed - // all of the input. The flush parameter must continue to be Z_FINISH for those subsequent calls. - // Ref: https://zlib.net/zlib_how.html. - const int result = deflate(zstream_ptr_.get(), Z_FINISH); - updateOutput(output_buffer); - if (result == Z_OK || result == Z_BUF_ERROR) { - finish(output_buffer); - } else { - // While Z_OK, Z_BUF_ERROR (covered in above case) and Z_STREAM_END are the acceptable values, - // the Z_STREAM_ERROR is only possible if the stream is not initialized properly. Ref: - // https://zlib.net/zlib_how.html. - RELEASE_ASSERT(result == Z_STREAM_END); - } -} - } // namespace Compressor } // namespace Envoy diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 1e47100ad2d2..d1959c45e858 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -82,24 +82,17 @@ class ZlibCompressorImpl : public Compressor { uint64_t checksum(); /** - * This is required to re-init the stream after calling finish(_). + * This is required to re-init the stream after calling flush(). */ void reset(); - /** - * Completes the compressed output stream. To indicate that the currently pointed data is the last - * chunk of input data to compress. - * @param output_buffer supplies the buffer to output compressed data. - */ - void finish(Buffer::Instance& output_buffer); - // Compressor void compress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) override; private: bool deflateNext(int64_t flush_state); void process(Buffer::Instance& output_buffer, int64_t flush_state); - void updateOutput(Buffer::Instance& output_buffer); + void updateOutput(Buffer::Instance& output_buffer, bool require_finish = false); const uint64_t chunk_size_; bool initialized_; diff --git a/source/common/http/filter/gzip_filter.cc b/source/common/http/filter/gzip_filter.cc index 694b2d686f33..34e385d956ca 100644 --- a/source/common/http/filter/gzip_filter.cc +++ b/source/common/http/filter/gzip_filter.cc @@ -139,7 +139,6 @@ FilterDataStatus GzipFilter::encodeData(Buffer::Instance& data, bool end_stream) if (end_stream) { compressor_.flush(compressed_data_); - compressor_.finish(compressed_data_); } if (compressed_data_.length()) { diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index e7ec2d0e9ff6..5636bbbcf0d6 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -12,6 +12,26 @@ namespace { class ZlibCompressorImplTest : public testing::Test { protected: + void expectValidCompressedBuffer(const Buffer::OwnedImpl& output_buffer) { + uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); + Buffer::RawSlice compressed_slices[num_comp_slices]; + output_buffer.getRawSlices(compressed_slices, num_comp_slices); + + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); + + const std::string footer_hex_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + + // FOOTER four-byte sequence (sync flush) + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 28, 8)); + } + static const int64_t gzip_window_bits{31}; static const int64_t memory_level{8}; static const uint64_t default_input_size{796}; @@ -86,26 +106,7 @@ TEST_F(ZlibCompressorImplTest, CompressWithReducedInternalMemory) { } compressor.flush(output_buffer); - - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - compressor.finish(output_buffer); - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + expectValidCompressedBuffer(output_buffer); } /** @@ -128,28 +129,10 @@ TEST_F(ZlibCompressorImplTest, CompressWithContinuesFlush) { ASSERT_EQ(0, input_buffer.length()); compressor.flush(output_buffer); + expectValidCompressedBuffer(output_buffer); - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - compressor.finish(output_buffer); - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - // Re-init the compressor. + // Re-init the compressor. If not, the stream would be failed because of initialization error + // (Z_STREAM_ERROR). compressor.reset(); } } @@ -174,25 +157,7 @@ TEST_F(ZlibCompressorImplTest, CompressSmallInputMemory) { compressor.flush(output_buffer); EXPECT_LE(10, output_buffer.length()); - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - compressor.finish(output_buffer); - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + expectValidCompressedBuffer(output_buffer); } /** @@ -216,6 +181,10 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { ASSERT_EQ(0, input_buffer.length()); output_buffer.move(temp_buffer); ASSERT_EQ(0, temp_buffer.length()); + + // Re-init the compressor. If not, the stream would be failed because of initialization error + // (Z_STREAM_ERROR). + compressor.reset(); } compressor.flush(temp_buffer); @@ -230,6 +199,8 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { ASSERT_EQ(0, input_buffer.length()); output_buffer.move(temp_buffer); ASSERT_EQ(0, temp_buffer.length()); + + compressor.reset(); } compressor.flush(temp_buffer); @@ -237,25 +208,7 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { ASSERT_EQ(0, temp_buffer.length()); EXPECT_GE(output_buffer.length(), first_n_compressed_bytes); - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - compressor.finish(output_buffer); - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + expectValidCompressedBuffer(output_buffer); } /** @@ -278,25 +231,7 @@ TEST_F(ZlibCompressorImplTest, CompressWithNotCommonParams) { compressor.flush(output_buffer); - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - compressor.finish(output_buffer); - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + expectValidCompressedBuffer(output_buffer); } } // namespace From 9a0d52941bac6db3b12b74bd328d50edf8efe640 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Mon, 9 Apr 2018 11:28:21 +0700 Subject: [PATCH 03/28] Move reset Confirmed to be working with streaming file > 100MB. Signed-off-by: Dhi Aurrahman --- .../common/compressor/zlib_compressor_impl.cc | 36 ++--- .../common/compressor/zlib_compressor_impl.h | 11 +- source/common/http/filter/gzip_filter.cc | 5 +- .../compressor/zlib_compressor_impl_test.cc | 131 +++++++++++++----- 4 files changed, 130 insertions(+), 53 deletions(-) diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index 534e105157ad..f64a515ae247 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -68,11 +68,11 @@ void ZlibCompressorImpl::process(Buffer::Instance& output_buffer, int64_t flush_ } if (flush_state == Z_SYNC_FLUSH) { - updateOutput(output_buffer, true); + updateOutput(output_buffer); } } -void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer, bool require_finish) { +void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer) { const uint64_t n_output = chunk_size_ - zstream_ptr_->avail_out; if (n_output > 0) { output_buffer.add(static_cast(chunk_char_ptr_.get()), n_output); @@ -80,20 +80,6 @@ void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer, bool requ chunk_char_ptr_.reset(new unsigned char[chunk_size_]); zstream_ptr_->avail_out = chunk_size_; zstream_ptr_->next_out = chunk_char_ptr_.get(); - - if (require_finish) { - // Once the Z_FINISH parameter is provided, deflate() will begin to complete the compressed - // output stream. However depending on how much output space is provided, deflate() may have to - // be called several times until it has provided the complete compressed stream, even after it - // has consumed all of the input. The flush parameter must continue to be Z_FINISH for those - // subsequent calls. Ref: https://zlib.net/zlib_how.html. - int result = Z_OK; - while (result != Z_STREAM_END) { - result = deflate(zstream_ptr_.get(), Z_FINISH); - RELEASE_ASSERT(result != Z_STREAM_ERROR); - updateOutput(output_buffer); - } - } } void ZlibCompressorImpl::reset() { @@ -101,5 +87,23 @@ void ZlibCompressorImpl::reset() { RELEASE_ASSERT(result == Z_OK); } +void ZlibCompressorImpl::finish(Buffer::Instance& output_buffer) { + // Once the Z_FINISH parameter is provided, deflate() will begin to complete the compressed output + // stream. However depending on how much output space is provided, deflate() may have to be called + // several times until it has provided the complete compressed stream, even after it has consumed + // all of the input. The flush parameter must continue to be Z_FINISH for those subsequent calls. + // Ref: https://zlib.net/zlib_how.html. + const int result = deflate(zstream_ptr_.get(), Z_FINISH); + updateOutput(output_buffer); + if (result == Z_OK || result == Z_BUF_ERROR) { + finish(output_buffer); + } else { + // While Z_OK, Z_BUF_ERROR (covered in above case) and Z_STREAM_END are the acceptable values, + // the Z_STREAM_ERROR is only possible if the stream is not initialized properly. Ref: + // https://zlib.net/zlib_how.html. + RELEASE_ASSERT(result == Z_STREAM_END); + } +} + } // namespace Compressor } // namespace Envoy diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index d1959c45e858..1e47100ad2d2 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -82,17 +82,24 @@ class ZlibCompressorImpl : public Compressor { uint64_t checksum(); /** - * This is required to re-init the stream after calling flush(). + * This is required to re-init the stream after calling finish(_). */ void reset(); + /** + * Completes the compressed output stream. To indicate that the currently pointed data is the last + * chunk of input data to compress. + * @param output_buffer supplies the buffer to output compressed data. + */ + void finish(Buffer::Instance& output_buffer); + // Compressor void compress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) override; private: bool deflateNext(int64_t flush_state); void process(Buffer::Instance& output_buffer, int64_t flush_state); - void updateOutput(Buffer::Instance& output_buffer, bool require_finish = false); + void updateOutput(Buffer::Instance& output_buffer); const uint64_t chunk_size_; bool initialized_; diff --git a/source/common/http/filter/gzip_filter.cc b/source/common/http/filter/gzip_filter.cc index 34e385d956ca..6c15a699bce8 100644 --- a/source/common/http/filter/gzip_filter.cc +++ b/source/common/http/filter/gzip_filter.cc @@ -139,14 +139,15 @@ FilterDataStatus GzipFilter::encodeData(Buffer::Instance& data, bool end_stream) if (end_stream) { compressor_.flush(compressed_data_); + compressor_.finish(compressed_data_); + // Reset is required to re-init the compressor's z_stream. + compressor_.reset(); } if (compressed_data_.length()) { data.drain(n_data); data.move(compressed_data_); - // Reset is required to re-init the compressor's z_stream. - compressor_.reset(); return Http::FilterDataStatus::Continue; } diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index 5636bbbcf0d6..e7ec2d0e9ff6 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -12,26 +12,6 @@ namespace { class ZlibCompressorImplTest : public testing::Test { protected: - void expectValidCompressedBuffer(const Buffer::OwnedImpl& output_buffer) { - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 28, 8)); - } - static const int64_t gzip_window_bits{31}; static const int64_t memory_level{8}; static const uint64_t default_input_size{796}; @@ -106,7 +86,26 @@ TEST_F(ZlibCompressorImplTest, CompressWithReducedInternalMemory) { } compressor.flush(output_buffer); - expectValidCompressedBuffer(output_buffer); + + uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); + Buffer::RawSlice compressed_slices[num_comp_slices]; + output_buffer.getRawSlices(compressed_slices, num_comp_slices); + + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); + + const std::string footer_hex_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + // FOOTER four-byte sequence (sync flush) + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + compressor.finish(output_buffer); + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); } /** @@ -129,10 +128,28 @@ TEST_F(ZlibCompressorImplTest, CompressWithContinuesFlush) { ASSERT_EQ(0, input_buffer.length()); compressor.flush(output_buffer); - expectValidCompressedBuffer(output_buffer); - // Re-init the compressor. If not, the stream would be failed because of initialization error - // (Z_STREAM_ERROR). + uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); + Buffer::RawSlice compressed_slices[num_comp_slices]; + output_buffer.getRawSlices(compressed_slices, num_comp_slices); + + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); + + const std::string footer_hex_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + // FOOTER four-byte sequence (sync flush) + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + compressor.finish(output_buffer); + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + // Re-init the compressor. compressor.reset(); } } @@ -157,7 +174,25 @@ TEST_F(ZlibCompressorImplTest, CompressSmallInputMemory) { compressor.flush(output_buffer); EXPECT_LE(10, output_buffer.length()); - expectValidCompressedBuffer(output_buffer); + uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); + Buffer::RawSlice compressed_slices[num_comp_slices]; + output_buffer.getRawSlices(compressed_slices, num_comp_slices); + + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); + + const std::string footer_hex_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + // FOOTER four-byte sequence (sync flush) + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + compressor.finish(output_buffer); + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); } /** @@ -181,10 +216,6 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { ASSERT_EQ(0, input_buffer.length()); output_buffer.move(temp_buffer); ASSERT_EQ(0, temp_buffer.length()); - - // Re-init the compressor. If not, the stream would be failed because of initialization error - // (Z_STREAM_ERROR). - compressor.reset(); } compressor.flush(temp_buffer); @@ -199,8 +230,6 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { ASSERT_EQ(0, input_buffer.length()); output_buffer.move(temp_buffer); ASSERT_EQ(0, temp_buffer.length()); - - compressor.reset(); } compressor.flush(temp_buffer); @@ -208,7 +237,25 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { ASSERT_EQ(0, temp_buffer.length()); EXPECT_GE(output_buffer.length(), first_n_compressed_bytes); - expectValidCompressedBuffer(output_buffer); + uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); + Buffer::RawSlice compressed_slices[num_comp_slices]; + output_buffer.getRawSlices(compressed_slices, num_comp_slices); + + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); + + const std::string footer_hex_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + // FOOTER four-byte sequence (sync flush) + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + compressor.finish(output_buffer); + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); } /** @@ -231,7 +278,25 @@ TEST_F(ZlibCompressorImplTest, CompressWithNotCommonParams) { compressor.flush(output_buffer); - expectValidCompressedBuffer(output_buffer); + uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); + Buffer::RawSlice compressed_slices[num_comp_slices]; + output_buffer.getRawSlices(compressed_slices, num_comp_slices); + + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); + + const std::string footer_hex_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + // FOOTER four-byte sequence (sync flush) + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + + compressor.finish(output_buffer); + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); } } // namespace From d0c4239e5aa78ea98680a221c1aa3fa827ebdf1a Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Mon, 9 Apr 2018 12:07:25 +0700 Subject: [PATCH 04/28] Fix tests Signed-off-by: Dhi Aurrahman --- .../common/compressor/zlib_compressor_impl.cc | 22 +-- .../common/compressor/zlib_compressor_impl.h | 17 ++- source/common/http/filter/gzip_filter.cc | 2 +- .../compressor/zlib_compressor_impl_test.cc | 126 ++++-------------- 4 files changed, 47 insertions(+), 120 deletions(-) diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index f64a515ae247..eb9224eb1d3a 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -68,11 +68,11 @@ void ZlibCompressorImpl::process(Buffer::Instance& output_buffer, int64_t flush_ } if (flush_state == Z_SYNC_FLUSH) { - updateOutput(output_buffer); + updateOutput(output_buffer, true); } } -void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer) { +void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer, const bool require_finish) { const uint64_t n_output = chunk_size_ - zstream_ptr_->avail_out; if (n_output > 0) { output_buffer.add(static_cast(chunk_char_ptr_.get()), n_output); @@ -80,6 +80,10 @@ void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer) { chunk_char_ptr_.reset(new unsigned char[chunk_size_]); zstream_ptr_->avail_out = chunk_size_; zstream_ptr_->next_out = chunk_char_ptr_.get(); + + if (require_finish) { + finish(output_buffer); + } } void ZlibCompressorImpl::reset() { @@ -93,16 +97,12 @@ void ZlibCompressorImpl::finish(Buffer::Instance& output_buffer) { // several times until it has provided the complete compressed stream, even after it has consumed // all of the input. The flush parameter must continue to be Z_FINISH for those subsequent calls. // Ref: https://zlib.net/zlib_how.html. - const int result = deflate(zstream_ptr_.get(), Z_FINISH); - updateOutput(output_buffer); - if (result == Z_OK || result == Z_BUF_ERROR) { - finish(output_buffer); - } else { - // While Z_OK, Z_BUF_ERROR (covered in above case) and Z_STREAM_END are the acceptable values, - // the Z_STREAM_ERROR is only possible if the stream is not initialized properly. Ref: - // https://zlib.net/zlib_how.html. - RELEASE_ASSERT(result == Z_STREAM_END); + int result = Z_OK; + while (result == Z_OK || result == Z_BUF_ERROR) { + result = deflate(zstream_ptr_.get(), Z_FINISH); + updateOutput(output_buffer); } + RELEASE_ASSERT(result == Z_STREAM_END); } } // namespace Compressor diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 1e47100ad2d2..7895cf1c83ce 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -82,24 +82,23 @@ class ZlibCompressorImpl : public Compressor { uint64_t checksum(); /** - * This is required to re-init the stream after calling finish(_). + * This function is equivalent to deflateEnd followed by deflateInit, but does not free and + * reallocate the internal compression state. The stream will leave the compression level and any + * other attributes that may have been set unchanged. */ void reset(); - /** - * Completes the compressed output stream. To indicate that the currently pointed data is the last - * chunk of input data to compress. - * @param output_buffer supplies the buffer to output compressed data. - */ - void finish(Buffer::Instance& output_buffer); - // Compressor void compress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) override; private: bool deflateNext(int64_t flush_state); void process(Buffer::Instance& output_buffer, int64_t flush_state); - void updateOutput(Buffer::Instance& output_buffer); + void updateOutput(Buffer::Instance& output_buffer, const bool require_finish = false); + + // Completes the compressed output stream. To indicate that the currently pointed data is the last + // chunk of input data to compress. + void finish(Buffer::Instance& output_buffer); const uint64_t chunk_size_; bool initialized_; diff --git a/source/common/http/filter/gzip_filter.cc b/source/common/http/filter/gzip_filter.cc index 6c15a699bce8..1edee5b2576a 100644 --- a/source/common/http/filter/gzip_filter.cc +++ b/source/common/http/filter/gzip_filter.cc @@ -139,7 +139,7 @@ FilterDataStatus GzipFilter::encodeData(Buffer::Instance& data, bool end_stream) if (end_stream) { compressor_.flush(compressed_data_); - compressor_.finish(compressed_data_); + // Reset is required to re-init the compressor's z_stream. compressor_.reset(); } diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index e7ec2d0e9ff6..f8a909626ff3 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -12,6 +12,26 @@ namespace { class ZlibCompressorImplTest : public testing::Test { protected: + void expectValidCompressedBuffer(const Buffer::OwnedImpl& output_buffer) { + uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); + Buffer::RawSlice compressed_slices[num_comp_slices]; + output_buffer.getRawSlices(compressed_slices, num_comp_slices); + + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); + + const std::string footer_hex_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + + // FOOTER four-byte sequence (sync flush) + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 28, 8)); + } + static const int64_t gzip_window_bits{31}; static const int64_t memory_level{8}; static const uint64_t default_input_size{796}; @@ -86,26 +106,7 @@ TEST_F(ZlibCompressorImplTest, CompressWithReducedInternalMemory) { } compressor.flush(output_buffer); - - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - compressor.finish(output_buffer); - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + expectValidCompressedBuffer(output_buffer); } /** @@ -128,28 +129,8 @@ TEST_F(ZlibCompressorImplTest, CompressWithContinuesFlush) { ASSERT_EQ(0, input_buffer.length()); compressor.flush(output_buffer); + expectValidCompressedBuffer(output_buffer); - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - compressor.finish(output_buffer); - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - // Re-init the compressor. compressor.reset(); } } @@ -174,25 +155,7 @@ TEST_F(ZlibCompressorImplTest, CompressSmallInputMemory) { compressor.flush(output_buffer); EXPECT_LE(10, output_buffer.length()); - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - compressor.finish(output_buffer); - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + expectValidCompressedBuffer(output_buffer); } /** @@ -223,6 +186,8 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { output_buffer.move(temp_buffer); const uint64_t first_n_compressed_bytes = output_buffer.length(); + compressor.reset(); + for (uint64_t i = 0; i < 15; i++) { TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i * 10); compressor.compress(input_buffer, temp_buffer); @@ -237,25 +202,7 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { ASSERT_EQ(0, temp_buffer.length()); EXPECT_GE(output_buffer.length(), first_n_compressed_bytes); - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - compressor.finish(output_buffer); - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + expectValidCompressedBuffer(output_buffer); } /** @@ -277,26 +224,7 @@ TEST_F(ZlibCompressorImplTest, CompressWithNotCommonParams) { } compressor.flush(output_buffer); - - uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); - Buffer::RawSlice compressed_slices[num_comp_slices]; - output_buffer.getRawSlices(compressed_slices, num_comp_slices); - - const std::string header_hex_str = Hex::encode( - reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); - // HEADER 0x1f = 31 (window_bits) - EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); - // CM 0x8 = deflate (compression method) - EXPECT_EQ("08", header_hex_str.substr(4, 2)); - - const std::string footer_hex_str = - Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), - compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); - - compressor.finish(output_buffer); - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + expectValidCompressedBuffer(output_buffer); } } // namespace From 74063b9622ca75b2d2d6df1f841b87037bd1ab2e Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Mon, 9 Apr 2018 12:44:53 +0700 Subject: [PATCH 05/28] Add death tests Signed-off-by: Dhi Aurrahman --- .../common/compressor/zlib_compressor_impl.h | 12 +++++--- .../compressor/zlib_compressor_impl_test.cc | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 7895cf1c83ce..59371ec71ae5 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -91,15 +91,19 @@ class ZlibCompressorImpl : public Compressor { // Compressor void compress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) override; +protected: + /** + * Completes the compressed output stream. To indicate that the currently pointed data is the last + * chunk of input data to compress. + * @param output_buffer supplies the buffer to output compressed data. + */ + void finish(Buffer::Instance& output_buffer); + private: bool deflateNext(int64_t flush_state); void process(Buffer::Instance& output_buffer, int64_t flush_state); void updateOutput(Buffer::Instance& output_buffer, const bool require_finish = false); - // Completes the compressed output stream. To indicate that the currently pointed data is the last - // chunk of input data to compress. - void finish(Buffer::Instance& output_buffer); - const uint64_t chunk_size_; bool initialized_; diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index f8a909626ff3..383c4b33ff47 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -37,6 +37,13 @@ class ZlibCompressorImplTest : public testing::Test { static const uint64_t default_input_size{796}; }; +struct ZlibCompressorImplTestHelper : public ZlibCompressorImpl { + void unitializedFinish() { + Buffer::OwnedImpl output_buffer; + ZlibCompressorImpl::finish(output_buffer); + } +}; + class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { protected: static void compressorBadInitTestHelper(int64_t window_bits, int64_t mem_level) { @@ -52,6 +59,22 @@ class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { TestUtility::feedBufferWithRandomCharacters(input_buffer, 100); compressor.compress(input_buffer, output_buffer); } + + static void resetUnitializedCompressorTestHelper() { + ZlibCompressorImpl compressor; + compressor.reset(); + } + + static void flushUnitializedCompressorTestHelper() { + ZlibCompressorImpl compressor; + Buffer::OwnedImpl output_buffer; + compressor.flush(output_buffer); + } + + static void finishUnitializedCompressorTestHelper() { + ZlibCompressorImplTestHelper compressor; + compressor.unitializedFinish(); + } }; /** @@ -62,6 +85,13 @@ TEST_F(ZlibCompressorImplDeathTest, CompressorTestDeath) { EXPECT_DEATH(compressorBadInitTestHelper(100, 8), std::string{"assert failure: result >= 0"}); EXPECT_DEATH(compressorBadInitTestHelper(31, 10), std::string{"assert failure: result >= 0"}); EXPECT_DEATH(unitializedCompressorTestHelper(), std::string{"assert failure: result == Z_OK"}); + + EXPECT_DEATH(resetUnitializedCompressorTestHelper(), + std::string{"assert failure: result == Z_OK"}); + EXPECT_DEATH(flushUnitializedCompressorTestHelper(), + std::string{"assert failure: result == Z_OK"}); + EXPECT_DEATH(finishUnitializedCompressorTestHelper(), + std::string{"assert failure: result == Z_STREAM_END"}); } /** From d8239b3b9e0c3afa373fe33a79b488d4475c7060 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Mon, 9 Apr 2018 12:55:38 +0700 Subject: [PATCH 06/28] Remove new lines Signed-off-by: Dhi Aurrahman --- source/common/http/filter/gzip_filter.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/common/http/filter/gzip_filter.cc b/source/common/http/filter/gzip_filter.cc index 1edee5b2576a..de38664fbed6 100644 --- a/source/common/http/filter/gzip_filter.cc +++ b/source/common/http/filter/gzip_filter.cc @@ -139,15 +139,12 @@ FilterDataStatus GzipFilter::encodeData(Buffer::Instance& data, bool end_stream) if (end_stream) { compressor_.flush(compressed_data_); - - // Reset is required to re-init the compressor's z_stream. compressor_.reset(); } if (compressed_data_.length()) { data.drain(n_data); data.move(compressed_data_); - return Http::FilterDataStatus::Continue; } From 56b5e88f5b8f5303ba0a20f70f59801a16d1713e Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Mon, 9 Apr 2018 14:23:49 +0700 Subject: [PATCH 07/28] Fix typo Signed-off-by: Dhi Aurrahman --- .../compressor/zlib_compressor_impl_test.cc | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index 383c4b33ff47..7cda6699b8ec 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -38,7 +38,7 @@ class ZlibCompressorImplTest : public testing::Test { }; struct ZlibCompressorImplTestHelper : public ZlibCompressorImpl { - void unitializedFinish() { + void uninitializedFinish() { Buffer::OwnedImpl output_buffer; ZlibCompressorImpl::finish(output_buffer); } @@ -52,7 +52,7 @@ class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { ZlibCompressorImpl::CompressionStrategy::Standard, window_bits, mem_level); } - static void unitializedCompressorTestHelper() { + static void uninitializedCompressorTestHelper() { Buffer::OwnedImpl input_buffer; Buffer::OwnedImpl output_buffer; ZlibCompressorImpl compressor; @@ -60,20 +60,20 @@ class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { compressor.compress(input_buffer, output_buffer); } - static void resetUnitializedCompressorTestHelper() { + static void resetUninitializedCompressorTestHelper() { ZlibCompressorImpl compressor; compressor.reset(); } - static void flushUnitializedCompressorTestHelper() { + static void flushUninitializedCompressorTestHelper() { ZlibCompressorImpl compressor; Buffer::OwnedImpl output_buffer; compressor.flush(output_buffer); } - static void finishUnitializedCompressorTestHelper() { + static void finishUninitializedCompressorTestHelper() { ZlibCompressorImplTestHelper compressor; - compressor.unitializedFinish(); + compressor.uninitializedFinish(); } }; @@ -84,13 +84,13 @@ class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { TEST_F(ZlibCompressorImplDeathTest, CompressorTestDeath) { EXPECT_DEATH(compressorBadInitTestHelper(100, 8), std::string{"assert failure: result >= 0"}); EXPECT_DEATH(compressorBadInitTestHelper(31, 10), std::string{"assert failure: result >= 0"}); - EXPECT_DEATH(unitializedCompressorTestHelper(), std::string{"assert failure: result == Z_OK"}); + EXPECT_DEATH(uninitializedCompressorTestHelper(), std::string{"assert failure: result == Z_OK"}); - EXPECT_DEATH(resetUnitializedCompressorTestHelper(), + EXPECT_DEATH(resetUninitializedCompressorTestHelper(), std::string{"assert failure: result == Z_OK"}); - EXPECT_DEATH(flushUnitializedCompressorTestHelper(), + EXPECT_DEATH(flushUninitializedCompressorTestHelper(), std::string{"assert failure: result == Z_OK"}); - EXPECT_DEATH(finishUnitializedCompressorTestHelper(), + EXPECT_DEATH(finishUninitializedCompressorTestHelper(), std::string{"assert failure: result == Z_STREAM_END"}); } From 6552468ec98e6f58112b63a94ab0b7e308df9bb3 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Wed, 11 Apr 2018 10:53:44 +0700 Subject: [PATCH 08/28] Remove flush() Signed-off-by: Dhi Aurrahman --- .../common/compressor/zlib_compressor_impl.cc | 8 -- .../common/compressor/zlib_compressor_impl.h | 16 +--- source/common/http/filter/gzip_filter.cc | 2 +- .../compressor/zlib_compressor_impl_test.cc | 78 ++++++++++--------- 4 files changed, 44 insertions(+), 60 deletions(-) diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index eb9224eb1d3a..9bb2f95a6340 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -31,10 +31,6 @@ void ZlibCompressorImpl::init(CompressionLevel comp_level, CompressionStrategy c initialized_ = true; } -void ZlibCompressorImpl::flush(Buffer::Instance& output_buffer) { - process(output_buffer, Z_SYNC_FLUSH); -} - uint64_t ZlibCompressorImpl::checksum() { return zstream_ptr_->adler; } void ZlibCompressorImpl::compress(const Buffer::Instance& input_buffer, @@ -66,10 +62,6 @@ void ZlibCompressorImpl::process(Buffer::Instance& output_buffer, int64_t flush_ updateOutput(output_buffer); } } - - if (flush_state == Z_SYNC_FLUSH) { - updateOutput(output_buffer, true); - } } void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer, const bool require_finish) { diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 59371ec71ae5..6efd0eb24dd9 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -64,15 +64,6 @@ class ZlibCompressorImpl : public Compressor { void init(CompressionLevel level, CompressionStrategy strategy, int64_t window_bits, uint64_t memory_level); - /** - * Flush should be called when no more data needs to be compressed. It will compress - * any remaining input available in the compressor and flush the compressed data to the output - * buffer. Note that forcing flush frequently degrades the compression ratio, so this should only - * be called when necessary. - * @param output_buffer supplies the buffer to output compressed data. - */ - void flush(Buffer::Instance& output_buffer); - /** * It returns the checksum of all output produced so far. Compressor's checksum at the end of the * stream has to match decompressor's checksum produced at the end of the decompression. @@ -88,10 +79,6 @@ class ZlibCompressorImpl : public Compressor { */ void reset(); - // Compressor - void compress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) override; - -protected: /** * Completes the compressed output stream. To indicate that the currently pointed data is the last * chunk of input data to compress. @@ -99,6 +86,9 @@ class ZlibCompressorImpl : public Compressor { */ void finish(Buffer::Instance& output_buffer); + // Compressor + void compress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) override; + private: bool deflateNext(int64_t flush_state); void process(Buffer::Instance& output_buffer, int64_t flush_state); diff --git a/source/common/http/filter/gzip_filter.cc b/source/common/http/filter/gzip_filter.cc index de38664fbed6..e239ffcd4c4f 100644 --- a/source/common/http/filter/gzip_filter.cc +++ b/source/common/http/filter/gzip_filter.cc @@ -138,7 +138,7 @@ FilterDataStatus GzipFilter::encodeData(Buffer::Instance& data, bool end_stream) } if (end_stream) { - compressor_.flush(compressed_data_); + compressor_.finish(compressed_data_); compressor_.reset(); } diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index 7cda6699b8ec..2a61b8b18a20 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -12,7 +12,8 @@ namespace { class ZlibCompressorImplTest : public testing::Test { protected: - void expectValidCompressedBuffer(const Buffer::OwnedImpl& output_buffer) { + void expectValidCompressedBuffer(const Buffer::OwnedImpl& output_buffer, + const uint32_t input_size) { uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); Buffer::RawSlice compressed_slices[num_comp_slices]; output_buffer.getRawSlices(compressed_slices, num_comp_slices); @@ -24,12 +25,18 @@ class ZlibCompressorImplTest : public testing::Test { // CM 0x8 = deflate (compression method) EXPECT_EQ("08", header_hex_str.substr(4, 2)); - const std::string footer_hex_str = + const std::string footer_bytes_str = Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), compressed_slices[num_comp_slices - 1].len_); - // FOOTER four-byte sequence (sync flush) - EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 28, 8)); + expectEqualInputSize(footer_bytes_str, input_size); + } + + void expectEqualInputSize(const std::string& footer_bytes, const uint32_t input_size) { + const std::string size_bytes = footer_bytes.substr(footer_bytes.size() - 8, 8); + uint64_t size; + StringUtil::atoul(size_bytes.c_str(), size, 16); + EXPECT_EQ(ntohl(size), input_size); } static const int64_t gzip_window_bits{31}; @@ -37,13 +44,6 @@ class ZlibCompressorImplTest : public testing::Test { static const uint64_t default_input_size{796}; }; -struct ZlibCompressorImplTestHelper : public ZlibCompressorImpl { - void uninitializedFinish() { - Buffer::OwnedImpl output_buffer; - ZlibCompressorImpl::finish(output_buffer); - } -}; - class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { protected: static void compressorBadInitTestHelper(int64_t window_bits, int64_t mem_level) { @@ -65,15 +65,10 @@ class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { compressor.reset(); } - static void flushUninitializedCompressorTestHelper() { + static void finishUninitializedCompressorTestHelper() { ZlibCompressorImpl compressor; Buffer::OwnedImpl output_buffer; - compressor.flush(output_buffer); - } - - static void finishUninitializedCompressorTestHelper() { - ZlibCompressorImplTestHelper compressor; - compressor.uninitializedFinish(); + compressor.finish(output_buffer); } }; @@ -85,11 +80,8 @@ TEST_F(ZlibCompressorImplDeathTest, CompressorTestDeath) { EXPECT_DEATH(compressorBadInitTestHelper(100, 8), std::string{"assert failure: result >= 0"}); EXPECT_DEATH(compressorBadInitTestHelper(31, 10), std::string{"assert failure: result >= 0"}); EXPECT_DEATH(uninitializedCompressorTestHelper(), std::string{"assert failure: result == Z_OK"}); - EXPECT_DEATH(resetUninitializedCompressorTestHelper(), std::string{"assert failure: result == Z_OK"}); - EXPECT_DEATH(flushUninitializedCompressorTestHelper(), - std::string{"assert failure: result == Z_OK"}); EXPECT_DEATH(finishUninitializedCompressorTestHelper(), std::string{"assert failure: result == Z_STREAM_END"}); } @@ -109,11 +101,14 @@ TEST_F(ZlibCompressorImplTest, CallingChecksum) { gzip_window_bits, memory_level); EXPECT_EQ(0, compressor.checksum()); - TestUtility::feedBufferWithRandomCharacters(compressor_input_buffer, 4096); + const uint32_t input_size = 4096; + TestUtility::feedBufferWithRandomCharacters(compressor_input_buffer, input_size); compressor.compress(compressor_input_buffer, compressor_output_buffer); - compressor.flush(compressor_output_buffer); - compressor_input_buffer.drain(4096); + compressor.finish(compressor_output_buffer); + compressor_input_buffer.drain(input_size); EXPECT_TRUE(compressor.checksum() > 0); + + expectValidCompressedBuffer(compressor_output_buffer, input_size); } /** @@ -128,19 +123,21 @@ TEST_F(ZlibCompressorImplTest, CompressWithReducedInternalMemory) { ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, memory_level); + uint64_t input_size = 0; for (uint64_t i = 0; i < 10; i++) { TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); compressor.compress(input_buffer, output_buffer); + input_size += input_buffer.length(); input_buffer.drain(default_input_size * i); ASSERT_EQ(0, input_buffer.length()); } - compressor.flush(output_buffer); - expectValidCompressedBuffer(output_buffer); + compressor.finish(output_buffer); + expectValidCompressedBuffer(output_buffer, input_size); } /** - * Exercises compression with a flush call on each received buffer. + * Exercises compression with a finish call on each received buffer. */ TEST_F(ZlibCompressorImplTest, CompressWithContinuesFlush) { Buffer::OwnedImpl input_buffer; @@ -158,8 +155,8 @@ TEST_F(ZlibCompressorImplTest, CompressWithContinuesFlush) { input_buffer.drain(default_input_size * i); ASSERT_EQ(0, input_buffer.length()); - compressor.flush(output_buffer); - expectValidCompressedBuffer(output_buffer); + compressor.finish(output_buffer); + expectValidCompressedBuffer(output_buffer, default_input_size * i); compressor.reset(); } @@ -177,15 +174,16 @@ TEST_F(ZlibCompressorImplTest, CompressSmallInputMemory) { ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, memory_level); - TestUtility::feedBufferWithRandomCharacters(input_buffer, 10); + const uint32_t input_size = 10; + TestUtility::feedBufferWithRandomCharacters(input_buffer, input_size); compressor.compress(input_buffer, output_buffer); EXPECT_EQ(0, output_buffer.length()); - compressor.flush(output_buffer); - EXPECT_LE(10, output_buffer.length()); + compressor.finish(output_buffer); + EXPECT_LE(input_size, output_buffer.length()); - expectValidCompressedBuffer(output_buffer); + expectValidCompressedBuffer(output_buffer, input_size); } /** @@ -211,28 +209,30 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { ASSERT_EQ(0, temp_buffer.length()); } - compressor.flush(temp_buffer); + compressor.finish(temp_buffer); ASSERT_TRUE(temp_buffer.length() > 0); output_buffer.move(temp_buffer); const uint64_t first_n_compressed_bytes = output_buffer.length(); compressor.reset(); + uint64_t input_size = 0; for (uint64_t i = 0; i < 15; i++) { TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i * 10); compressor.compress(input_buffer, temp_buffer); + input_size += input_buffer.length(); input_buffer.drain(default_input_size * i); ASSERT_EQ(0, input_buffer.length()); output_buffer.move(temp_buffer); ASSERT_EQ(0, temp_buffer.length()); } - compressor.flush(temp_buffer); + compressor.finish(temp_buffer); output_buffer.move(temp_buffer); ASSERT_EQ(0, temp_buffer.length()); EXPECT_GE(output_buffer.length(), first_n_compressed_bytes); - expectValidCompressedBuffer(output_buffer); + expectValidCompressedBuffer(output_buffer, input_size); } /** @@ -246,15 +246,17 @@ TEST_F(ZlibCompressorImplTest, CompressWithNotCommonParams) { compressor.init(ZlibCompressorImpl::CompressionLevel::Speed, ZlibCompressorImpl::CompressionStrategy::Rle, gzip_window_bits, 1); + uint64_t input_size = 0; for (uint64_t i = 0; i < 10; i++) { TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); compressor.compress(input_buffer, output_buffer); + input_size += input_buffer.length(); input_buffer.drain(default_input_size * i); ASSERT_EQ(0, input_buffer.length()); } - compressor.flush(output_buffer); - expectValidCompressedBuffer(output_buffer); + compressor.finish(output_buffer); + expectValidCompressedBuffer(output_buffer, input_size); } } // namespace From 767fb60f285f4e199f944c98e23fad2979d8e22b Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Wed, 11 Apr 2018 10:56:56 +0700 Subject: [PATCH 09/28] Remove require_finish flag Signed-off-by: Dhi Aurrahman --- source/common/compressor/zlib_compressor_impl.cc | 6 +----- source/common/compressor/zlib_compressor_impl.h | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index 9bb2f95a6340..2c97fbfa29bd 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -64,7 +64,7 @@ void ZlibCompressorImpl::process(Buffer::Instance& output_buffer, int64_t flush_ } } -void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer, const bool require_finish) { +void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer) { const uint64_t n_output = chunk_size_ - zstream_ptr_->avail_out; if (n_output > 0) { output_buffer.add(static_cast(chunk_char_ptr_.get()), n_output); @@ -72,10 +72,6 @@ void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer, const boo chunk_char_ptr_.reset(new unsigned char[chunk_size_]); zstream_ptr_->avail_out = chunk_size_; zstream_ptr_->next_out = chunk_char_ptr_.get(); - - if (require_finish) { - finish(output_buffer); - } } void ZlibCompressorImpl::reset() { diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 6efd0eb24dd9..4a7013a68fe2 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -92,7 +92,7 @@ class ZlibCompressorImpl : public Compressor { private: bool deflateNext(int64_t flush_state); void process(Buffer::Instance& output_buffer, int64_t flush_state); - void updateOutput(Buffer::Instance& output_buffer, const bool require_finish = false); + void updateOutput(Buffer::Instance& output_buffer); const uint64_t chunk_size_; bool initialized_; From 2b568b9aafdf0e089c8482b0c0c7d78203fd4432 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Wed, 11 Apr 2018 13:11:28 +0700 Subject: [PATCH 10/28] Add flipOrder helper Signed-off-by: Dhi Aurrahman --- .../compressor/zlib_compressor_impl_test.cc | 2 +- test/test_common/utility.h | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index 2a61b8b18a20..f3103d5be78c 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -36,7 +36,7 @@ class ZlibCompressorImplTest : public testing::Test { const std::string size_bytes = footer_bytes.substr(footer_bytes.size() - 8, 8); uint64_t size; StringUtil::atoul(size_bytes.c_str(), size, 16); - EXPECT_EQ(ntohl(size), input_size); + EXPECT_EQ(TestUtility::flipOrder(size, 0x000000FF), input_size); } static const int64_t gzip_window_bits{31}; diff --git a/test/test_common/utility.h b/test/test_common/utility.h index a5c8072a88f9..19c5b833af93 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -239,6 +239,25 @@ class TestUtility { ipTestParamsToString(const testing::TestParamInfo& params) { return params.param == Network::Address::IpVersion::v4 ? "IPv4" : "IPv6"; } + + /** + * Return flip-ordered bytes. + * @param bytes input bytes. + * @param mask bit mask + * @return Type flip-ordered bytes. + */ + template static Type flipOrder(const Type& bytes, const Type mask) { + Type result{0}; + Type data = bytes; + for (Type i = 0; i < sizeof(Type); i++) { + result <<= 8; + // TODO(dio): infer mask from Type. + result |= (data & mask); + data >>= 8; + } + return result; +} + }; /** From dff37df9ae91098d204c3bda0b30b930ac5c78b6 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Wed, 11 Apr 2018 13:14:55 +0700 Subject: [PATCH 11/28] Fix format Signed-off-by: Dhi Aurrahman --- test/test_common/utility.h | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 19c5b833af93..4bcfd6b1c18e 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -247,17 +247,16 @@ class TestUtility { * @return Type flip-ordered bytes. */ template static Type flipOrder(const Type& bytes, const Type mask) { - Type result{0}; - Type data = bytes; - for (Type i = 0; i < sizeof(Type); i++) { - result <<= 8; - // TODO(dio): infer mask from Type. - result |= (data & mask); - data >>= 8; + Type result{0}; + Type data = bytes; + for (Type i = 0; i < sizeof(Type); i++) { + result <<= 8; + // TODO(dio): infer mask from Type. + result |= (data & mask); + data >>= 8; + } + return result; } - return result; -} - }; /** From d44d929d63d6f0821231d8341cfac2242f860a0c Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Wed, 11 Apr 2018 13:51:47 +0700 Subject: [PATCH 12/28] Use finish() Signed-off-by: Dhi Aurrahman --- test/common/decompressor/zlib_decompressor_impl_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index f32b2a0365f2..e0a580126418 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -60,7 +60,7 @@ TEST_F(ZlibDecompressorImplTest, CallingChecksum) { TestUtility::feedBufferWithRandomCharacters(compressor_input_buffer, 4096); compressor.compress(compressor_input_buffer, compressor_output_buffer); - compressor.flush(compressor_output_buffer); + compressor.finish(compressor_output_buffer); compressor_input_buffer.drain(4096); ASSERT_TRUE(compressor.checksum() > 0); @@ -95,7 +95,7 @@ TEST_F(ZlibDecompressorImplTest, CompressAndDecompress) { compressor_input_buffer.drain(default_input_size * i); } - compressor.flush(compressor_output_buffer); + compressor.finish(compressor_output_buffer); ZlibDecompressorImpl decompressor; decompressor.init(gzip_window_bits); @@ -131,7 +131,7 @@ TEST_F(ZlibDecompressorImplTest, DecompressWithSmallOutputBuffer) { original_text.append(TestUtility::bufferToString(input_buffer)); input_buffer.drain(default_input_size * i); } - compressor.flush(output_buffer); + compressor.finish(output_buffer); ZlibDecompressorImpl decompressor(16); decompressor.init(gzip_window_bits); @@ -163,7 +163,7 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressWithUncommonParams) { original_text.append(TestUtility::bufferToString(input_buffer)); input_buffer.drain(default_input_size * i); } - compressor.flush(output_buffer); + compressor.finish(output_buffer); ZlibDecompressorImpl decompressor; decompressor.init(15); From 5400b19265f651d42ccfd0b7ea8936476efc65b7 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Sat, 14 Apr 2018 19:12:13 +0700 Subject: [PATCH 13/28] Call finish on end_stream Signed-off-by: Dhi Aurrahman --- .../common/compressor/zlib_compressor_impl.cc | 31 ++-- .../common/compressor/zlib_compressor_impl.h | 12 +- .../filters/http/gzip/gzip_filter.cc | 8 +- .../compressor/zlib_compressor_impl_test.cc | 150 +++++++++++------- .../zlib_decompressor_impl_test.cc | 8 +- test/test_common/utility.h | 12 +- 6 files changed, 133 insertions(+), 88 deletions(-) diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index 2c97fbfa29bd..499f506bb5b1 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -31,6 +31,19 @@ void ZlibCompressorImpl::init(CompressionLevel comp_level, CompressionStrategy c initialized_ = true; } +void ZlibCompressorImpl::flush(Buffer::Instance& output_buffer) { + process(output_buffer, Z_SYNC_FLUSH); +} + +void ZlibCompressorImpl::finish(Buffer::Instance& output_buffer) { + int result = Z_OK; + do { + result = deflate(zstream_ptr_.get(), Z_FINISH); + updateOutput(output_buffer); + } while (result == Z_OK); + RELEASE_ASSERT(result == Z_STREAM_END); +} + uint64_t ZlibCompressorImpl::checksum() { return zstream_ptr_->adler; } void ZlibCompressorImpl::compress(const Buffer::Instance& input_buffer, @@ -62,6 +75,10 @@ void ZlibCompressorImpl::process(Buffer::Instance& output_buffer, int64_t flush_ updateOutput(output_buffer); } } + + if (flush_state == Z_SYNC_FLUSH) { + updateOutput(output_buffer); + } } void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer) { @@ -79,19 +96,5 @@ void ZlibCompressorImpl::reset() { RELEASE_ASSERT(result == Z_OK); } -void ZlibCompressorImpl::finish(Buffer::Instance& output_buffer) { - // Once the Z_FINISH parameter is provided, deflate() will begin to complete the compressed output - // stream. However depending on how much output space is provided, deflate() may have to be called - // several times until it has provided the complete compressed stream, even after it has consumed - // all of the input. The flush parameter must continue to be Z_FINISH for those subsequent calls. - // Ref: https://zlib.net/zlib_how.html. - int result = Z_OK; - while (result == Z_OK || result == Z_BUF_ERROR) { - result = deflate(zstream_ptr_.get(), Z_FINISH); - updateOutput(output_buffer); - } - RELEASE_ASSERT(result == Z_STREAM_END); -} - } // namespace Compressor } // namespace Envoy diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 4a7013a68fe2..37b0f233a153 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -80,12 +80,20 @@ class ZlibCompressorImpl : public Compressor { void reset(); /** - * Completes the compressed output stream. To indicate that the currently pointed data is the last - * chunk of input data to compress. + * It writes gzip trailer to the end of output buffer. * @param output_buffer supplies the buffer to output compressed data. */ void finish(Buffer::Instance& output_buffer); + /** + * Flush should be called when no more data needs to be compressed. It will compress + * any remaining input available in the compressor and flush the compressed data to the output + * buffer. Note that forcing flush frequently degrades the compression ratio, so this should only + * be called when necessary. + * @param output_buffer supplies the buffer to output compressed data. + */ + void flush(Buffer::Instance& output_buffer); + // Compressor void compress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) override; diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index ede71ce3933f..b34a81b6c8cd 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -134,17 +134,17 @@ Http::FilterDataStatus GzipFilter::encodeData(Buffer::Instance& data, bool end_s } const uint64_t n_data = data.length(); - - if (n_data) { + if (n_data > 0) { compressor_.compress(data, compressed_data_); + } else { + compressor_.flush(compressed_data_); } if (end_stream) { compressor_.finish(compressed_data_); - compressor_.reset(); } - if (compressed_data_.length()) { + if (compressed_data_.length() > 0) { data.drain(n_data); data.move(compressed_data_); return Http::FilterDataStatus::Continue; diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index f3103d5be78c..52b11be70a54 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -12,8 +12,27 @@ namespace { class ZlibCompressorImplTest : public testing::Test { protected: - void expectValidCompressedBuffer(const Buffer::OwnedImpl& output_buffer, - const uint32_t input_size) { + void expectValidFlushedBuffer(const Buffer::OwnedImpl& output_buffer) { + uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); + Buffer::RawSlice compressed_slices[num_comp_slices]; + output_buffer.getRawSlices(compressed_slices, num_comp_slices); + + const std::string header_hex_str = Hex::encode( + reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) + EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); + // CM 0x8 = deflate (compression method) + EXPECT_EQ("08", header_hex_str.substr(4, 2)); + + const std::string footer_hex_str = + Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), + compressed_slices[num_comp_slices - 1].len_); + // FOOTER four-byte sequence (sync flush) + EXPECT_EQ("0000ffff", footer_hex_str.substr(footer_hex_str.size() - 8, 10)); + } + + void expectValidFinishedBuffer(const Buffer::OwnedImpl& output_buffer, + const uint32_t input_size) { uint64_t num_comp_slices = output_buffer.getRawSlices(nullptr, 0); Buffer::RawSlice compressed_slices[num_comp_slices]; output_buffer.getRawSlices(compressed_slices, num_comp_slices); @@ -36,7 +55,7 @@ class ZlibCompressorImplTest : public testing::Test { const std::string size_bytes = footer_bytes.substr(footer_bytes.size() - 8, 8); uint64_t size; StringUtil::atoul(size_bytes.c_str(), size, 16); - EXPECT_EQ(TestUtility::flipOrder(size, 0x000000FF), input_size); + EXPECT_EQ(TestUtility::flipOrder(size), input_size); } static const int64_t gzip_window_bits{31}; @@ -60,35 +79,39 @@ class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { compressor.compress(input_buffer, output_buffer); } - static void resetUninitializedCompressorTestHelper() { + static void uninitializedCompressorFlushTestHelper() { + Buffer::OwnedImpl output_buffer; ZlibCompressorImpl compressor; - compressor.reset(); + compressor.flush(output_buffer); } - static void finishUninitializedCompressorTestHelper() { - ZlibCompressorImpl compressor; + static void uninitializedCompressorFinishTestHelper() { Buffer::OwnedImpl output_buffer; + ZlibCompressorImpl compressor; compressor.finish(output_buffer); } + + static void uninitializedCompressorResetTestHelper() { + ZlibCompressorImpl compressor; + compressor.reset(); + } }; -/** - * Exercises death by passing bad initialization params or by calling - * compress before init. - */ +// Exercises death by passing bad initialization params or by calling +// compress before init. TEST_F(ZlibCompressorImplDeathTest, CompressorTestDeath) { EXPECT_DEATH(compressorBadInitTestHelper(100, 8), std::string{"assert failure: result >= 0"}); EXPECT_DEATH(compressorBadInitTestHelper(31, 10), std::string{"assert failure: result >= 0"}); EXPECT_DEATH(uninitializedCompressorTestHelper(), std::string{"assert failure: result == Z_OK"}); - EXPECT_DEATH(resetUninitializedCompressorTestHelper(), + EXPECT_DEATH(uninitializedCompressorFlushTestHelper(), std::string{"assert failure: result == Z_OK"}); - EXPECT_DEATH(finishUninitializedCompressorTestHelper(), + EXPECT_DEATH(uninitializedCompressorFinishTestHelper(), std::string{"assert failure: result == Z_STREAM_END"}); + EXPECT_DEATH(uninitializedCompressorResetTestHelper(), + std::string{"assert failure: result == Z_OK"}); } -/** - * Exercises compressor's checksum by calling it before init or compress. - */ +// Exercises compressor's checksum by calling it before init or compress. TEST_F(ZlibCompressorImplTest, CallingChecksum) { Buffer::OwnedImpl compressor_input_buffer; Buffer::OwnedImpl compressor_output_buffer; @@ -101,19 +124,14 @@ TEST_F(ZlibCompressorImplTest, CallingChecksum) { gzip_window_bits, memory_level); EXPECT_EQ(0, compressor.checksum()); - const uint32_t input_size = 4096; - TestUtility::feedBufferWithRandomCharacters(compressor_input_buffer, input_size); + TestUtility::feedBufferWithRandomCharacters(compressor_input_buffer, 4096); compressor.compress(compressor_input_buffer, compressor_output_buffer); - compressor.finish(compressor_output_buffer); - compressor_input_buffer.drain(input_size); + compressor.flush(compressor_output_buffer); + compressor_input_buffer.drain(4096); EXPECT_TRUE(compressor.checksum() > 0); - - expectValidCompressedBuffer(compressor_output_buffer, input_size); } -/** - * Exercises compression with a very small output buffer. - */ +// Exercises compression with a very small output buffer. TEST_F(ZlibCompressorImplTest, CompressWithReducedInternalMemory) { Buffer::OwnedImpl input_buffer; Buffer::OwnedImpl output_buffer; @@ -132,13 +150,39 @@ TEST_F(ZlibCompressorImplTest, CompressWithReducedInternalMemory) { ASSERT_EQ(0, input_buffer.length()); } + compressor.flush(output_buffer); + expectValidFlushedBuffer(output_buffer); + compressor.finish(output_buffer); - expectValidCompressedBuffer(output_buffer, input_size); + + // A valid finished buffer should have trailer with input size in it. + expectValidFinishedBuffer(output_buffer, input_size); } -/** - * Exercises compression with a finish call on each received buffer. - */ +// Exercises compression with a very small output buffer then calls finish(). +TEST_F(ZlibCompressorImplTest, CompressWithReducedInternalMemoryThenFinishIt) { + Buffer::OwnedImpl input_buffer; + Buffer::OwnedImpl output_buffer; + + Envoy::Compressor::ZlibCompressorImpl compressor(8); + compressor.init(ZlibCompressorImpl::CompressionLevel::Standard, + ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, + memory_level); + + for (uint64_t i = 0; i < 10; i++) { + TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); + compressor.compress(input_buffer, output_buffer); + input_buffer.drain(default_input_size * i); + ASSERT_EQ(0, input_buffer.length()); + } + + compressor.flush(output_buffer); + expectValidFlushedBuffer(output_buffer); + + compressor.finish(output_buffer); +} + +// Exercises compression with a flush call on each received buffer. TEST_F(ZlibCompressorImplTest, CompressWithContinuesFlush) { Buffer::OwnedImpl input_buffer; Buffer::OwnedImpl output_buffer; @@ -155,16 +199,12 @@ TEST_F(ZlibCompressorImplTest, CompressWithContinuesFlush) { input_buffer.drain(default_input_size * i); ASSERT_EQ(0, input_buffer.length()); - compressor.finish(output_buffer); - expectValidCompressedBuffer(output_buffer, default_input_size * i); - - compressor.reset(); + compressor.flush(output_buffer); + expectValidFlushedBuffer(output_buffer); } } -/** - * Exercises compression with very small input buffer. - */ +// Exercises compression with very small input buffer. TEST_F(ZlibCompressorImplTest, CompressSmallInputMemory) { Buffer::OwnedImpl input_buffer; Buffer::OwnedImpl output_buffer; @@ -174,22 +214,19 @@ TEST_F(ZlibCompressorImplTest, CompressSmallInputMemory) { ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, memory_level); - const uint32_t input_size = 10; - TestUtility::feedBufferWithRandomCharacters(input_buffer, input_size); + TestUtility::feedBufferWithRandomCharacters(input_buffer, 10); compressor.compress(input_buffer, output_buffer); EXPECT_EQ(0, output_buffer.length()); - compressor.finish(output_buffer); - EXPECT_LE(input_size, output_buffer.length()); + compressor.flush(output_buffer); + EXPECT_LE(10, output_buffer.length()); - expectValidCompressedBuffer(output_buffer, input_size); + expectValidFlushedBuffer(output_buffer); } -/** - * Exercises common flow of compressing some data, making it available to output buffer, - * then moving output buffer to another buffer and so on. - */ +// Exercises common flow of compressing some data, making it available to output buffer, +// then moving output buffer to another buffer and so on. TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { Buffer::OwnedImpl input_buffer; Buffer::OwnedImpl temp_buffer; @@ -209,35 +246,29 @@ TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { ASSERT_EQ(0, temp_buffer.length()); } - compressor.finish(temp_buffer); + compressor.flush(temp_buffer); ASSERT_TRUE(temp_buffer.length() > 0); output_buffer.move(temp_buffer); const uint64_t first_n_compressed_bytes = output_buffer.length(); - compressor.reset(); - - uint64_t input_size = 0; for (uint64_t i = 0; i < 15; i++) { TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i * 10); compressor.compress(input_buffer, temp_buffer); - input_size += input_buffer.length(); input_buffer.drain(default_input_size * i); ASSERT_EQ(0, input_buffer.length()); output_buffer.move(temp_buffer); ASSERT_EQ(0, temp_buffer.length()); } - compressor.finish(temp_buffer); + compressor.flush(temp_buffer); output_buffer.move(temp_buffer); ASSERT_EQ(0, temp_buffer.length()); EXPECT_GE(output_buffer.length(), first_n_compressed_bytes); - expectValidCompressedBuffer(output_buffer, input_size); + expectValidFlushedBuffer(output_buffer); } -/** - * Exercises compression with other supported zlib initialization params. - */ +// Exercises compression with other supported zlib initialization params. TEST_F(ZlibCompressorImplTest, CompressWithNotCommonParams) { Buffer::OwnedImpl input_buffer; Buffer::OwnedImpl output_buffer; @@ -246,19 +277,18 @@ TEST_F(ZlibCompressorImplTest, CompressWithNotCommonParams) { compressor.init(ZlibCompressorImpl::CompressionLevel::Speed, ZlibCompressorImpl::CompressionStrategy::Rle, gzip_window_bits, 1); - uint64_t input_size = 0; for (uint64_t i = 0; i < 10; i++) { TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); compressor.compress(input_buffer, output_buffer); - input_size += input_buffer.length(); input_buffer.drain(default_input_size * i); ASSERT_EQ(0, input_buffer.length()); } - compressor.finish(output_buffer); - expectValidCompressedBuffer(output_buffer, input_size); + compressor.flush(output_buffer); + + expectValidFlushedBuffer(output_buffer); } } // namespace } // namespace Compressor -} // namespace Envoy +} // namespace Envoy \ No newline at end of file diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index e0a580126418..f32b2a0365f2 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -60,7 +60,7 @@ TEST_F(ZlibDecompressorImplTest, CallingChecksum) { TestUtility::feedBufferWithRandomCharacters(compressor_input_buffer, 4096); compressor.compress(compressor_input_buffer, compressor_output_buffer); - compressor.finish(compressor_output_buffer); + compressor.flush(compressor_output_buffer); compressor_input_buffer.drain(4096); ASSERT_TRUE(compressor.checksum() > 0); @@ -95,7 +95,7 @@ TEST_F(ZlibDecompressorImplTest, CompressAndDecompress) { compressor_input_buffer.drain(default_input_size * i); } - compressor.finish(compressor_output_buffer); + compressor.flush(compressor_output_buffer); ZlibDecompressorImpl decompressor; decompressor.init(gzip_window_bits); @@ -131,7 +131,7 @@ TEST_F(ZlibDecompressorImplTest, DecompressWithSmallOutputBuffer) { original_text.append(TestUtility::bufferToString(input_buffer)); input_buffer.drain(default_input_size * i); } - compressor.finish(output_buffer); + compressor.flush(output_buffer); ZlibDecompressorImpl decompressor(16); decompressor.init(gzip_window_bits); @@ -163,7 +163,7 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressWithUncommonParams) { original_text.append(TestUtility::bufferToString(input_buffer)); input_buffer.drain(default_input_size * i); } - compressor.finish(output_buffer); + compressor.flush(output_buffer); ZlibDecompressorImpl decompressor; decompressor.init(15); diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 4bcfd6b1c18e..e32c0ad1918e 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -71,6 +71,12 @@ class TestRandomGenerator { std::ranlux48 generator_; }; +template class Mask {}; +template <> class Mask { +public: + static const uint32_t value = 0x000000FF; +}; + class TestUtility { public: /** @@ -243,16 +249,14 @@ class TestUtility { /** * Return flip-ordered bytes. * @param bytes input bytes. - * @param mask bit mask * @return Type flip-ordered bytes. */ - template static Type flipOrder(const Type& bytes, const Type mask) { + template static Type flipOrder(const Type& bytes) { Type result{0}; Type data = bytes; for (Type i = 0; i < sizeof(Type); i++) { result <<= 8; - // TODO(dio): infer mask from Type. - result |= (data & mask); + result |= (data & Mask::value); data >>= 8; } return result; From 5a486f2cca06feb003202e425e8cf41d799537e0 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Sat, 14 Apr 2018 19:57:07 +0700 Subject: [PATCH 14/28] Simplify flipOrder Signed-off-by: Dhi Aurrahman --- test/test_common/utility.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/test_common/utility.h b/test/test_common/utility.h index e32c0ad1918e..def0cbefe0e3 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -71,12 +71,6 @@ class TestRandomGenerator { std::ranlux48 generator_; }; -template class Mask {}; -template <> class Mask { -public: - static const uint32_t value = 0x000000FF; -}; - class TestUtility { public: /** @@ -256,7 +250,7 @@ class TestUtility { Type data = bytes; for (Type i = 0; i < sizeof(Type); i++) { result <<= 8; - result |= (data & Mask::value); + result |= (data & Type(0xFF)); data >>= 8; } return result; From 8a9cccea8460e1a0c7276b236a56ded478b04cf8 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Sun, 15 Apr 2018 07:02:28 +0700 Subject: [PATCH 15/28] Flush and compress each frame Signed-off-by: Dhi Aurrahman --- source/extensions/filters/http/gzip/gzip_filter.cc | 1 - test/extensions/filters/http/gzip/gzip_filter_test.cc | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index b34a81b6c8cd..bdaa18e2ef5a 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -136,7 +136,6 @@ Http::FilterDataStatus GzipFilter::encodeData(Buffer::Instance& data, bool end_s const uint64_t n_data = data.length(); if (n_data > 0) { compressor_.compress(data, compressed_data_); - } else { compressor_.flush(compressed_data_); } diff --git a/test/extensions/filters/http/gzip/gzip_filter_test.cc b/test/extensions/filters/http/gzip/gzip_filter_test.cc index 98f169028fba..26f945173540 100644 --- a/test/extensions/filters/http/gzip/gzip_filter_test.cc +++ b/test/extensions/filters/http/gzip/gzip_filter_test.cc @@ -86,10 +86,9 @@ class GzipFilterTest : public testing::Test { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(headers, false)); EXPECT_EQ("", headers.get_("content-length")); EXPECT_EQ(Http::Headers::get().ContentEncodingValues.Gzip, headers.get_("content-encoding")); - EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data_, false)); - drainBuffer(); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data_, true)); verifyCompressedData(); + drainBuffer(); } void doResponseNoCompression(Http::TestHeaderMapImpl&& headers) { From 059441ed47baf44b9702c47971a12c56f990a2f7 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Sun, 15 Apr 2018 16:03:51 +0700 Subject: [PATCH 16/28] Use one buffer only Signed-off-by: Dhi Aurrahman --- include/envoy/compressor/compressor.h | 8 +- .../common/compressor/zlib_compressor_impl.cc | 41 ++-- .../common/compressor/zlib_compressor_impl.h | 17 +- .../filters/http/gzip/gzip_filter.cc | 23 +- .../compressor/zlib_compressor_impl_test.cc | 228 ++++++------------ .../zlib_decompressor_impl_test.cc | 136 ++++++----- 6 files changed, 173 insertions(+), 280 deletions(-) diff --git a/include/envoy/compressor/compressor.h b/include/envoy/compressor/compressor.h index 4f9567a96ab2..8ac75b7e14bd 100644 --- a/include/envoy/compressor/compressor.h +++ b/include/envoy/compressor/compressor.h @@ -13,11 +13,11 @@ class Compressor { virtual ~Compressor() {} /** - * Compresses data from one buffer into another buffer. - * @param input_buffer supplies the buffer with data to be compressed. - * @param output_buffer supplies the buffer to output compressed data. + * Compresses data buffer. + * @param buffer supplies the reference to data to be compressed. + * @param trailer supplies the indication to write trailer at the end of compressed buffer. */ - virtual void compress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) PURE; + virtual void compress(Buffer::Instance& buffer, bool trailer) PURE; }; } // namespace Compressor diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index 499f506bb5b1..6aefa6a3b215 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -31,41 +31,38 @@ void ZlibCompressorImpl::init(CompressionLevel comp_level, CompressionStrategy c initialized_ = true; } -void ZlibCompressorImpl::flush(Buffer::Instance& output_buffer) { - process(output_buffer, Z_SYNC_FLUSH); -} - -void ZlibCompressorImpl::finish(Buffer::Instance& output_buffer) { - int result = Z_OK; - do { - result = deflate(zstream_ptr_.get(), Z_FINISH); - updateOutput(output_buffer); - } while (result == Z_OK); - RELEASE_ASSERT(result == Z_STREAM_END); -} - uint64_t ZlibCompressorImpl::checksum() { return zstream_ptr_->adler; } -void ZlibCompressorImpl::compress(const Buffer::Instance& input_buffer, - Buffer::Instance& output_buffer) { - const uint64_t num_slices = input_buffer.getRawSlices(nullptr, 0); +void ZlibCompressorImpl::compress(Buffer::Instance& buffer, bool trailer) { + const uint64_t num_slices = buffer.getRawSlices(nullptr, 0); Buffer::RawSlice slices[num_slices]; - input_buffer.getRawSlices(slices, num_slices); + buffer.getRawSlices(slices, num_slices); for (const Buffer::RawSlice& input_slice : slices) { zstream_ptr_->avail_in = input_slice.len_; zstream_ptr_->next_in = static_cast(input_slice.mem_); - process(output_buffer, Z_NO_FLUSH); + process(buffer, Z_NO_FLUSH); } + + buffer.drain(buffer.length()); + process(buffer, trailer ? Z_FINISH : Z_SYNC_FLUSH); } bool ZlibCompressorImpl::deflateNext(int64_t flush_state) { const int result = deflate(zstream_ptr_.get(), flush_state); - if (result == Z_BUF_ERROR && zstream_ptr_->avail_in == 0) { - return false; // This means that zlib needs more input, so stop here. + switch (flush_state) { + case Z_FINISH: + if (result != Z_OK && result != Z_BUF_ERROR) { + RELEASE_ASSERT(result == Z_STREAM_END); + return false; + } + default: + if (result == Z_BUF_ERROR && zstream_ptr_->avail_in == 0) { + return false; // This means that zlib needs more input, so stop here. + } + RELEASE_ASSERT(result == Z_OK); } - RELEASE_ASSERT(result == Z_OK); return true; } @@ -76,7 +73,7 @@ void ZlibCompressorImpl::process(Buffer::Instance& output_buffer, int64_t flush_ } } - if (flush_state == Z_SYNC_FLUSH) { + if (flush_state == Z_SYNC_FLUSH || flush_state == Z_FINISH) { updateOutput(output_buffer); } } diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 37b0f233a153..f877ef330e14 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -79,23 +79,8 @@ class ZlibCompressorImpl : public Compressor { */ void reset(); - /** - * It writes gzip trailer to the end of output buffer. - * @param output_buffer supplies the buffer to output compressed data. - */ - void finish(Buffer::Instance& output_buffer); - - /** - * Flush should be called when no more data needs to be compressed. It will compress - * any remaining input available in the compressor and flush the compressed data to the output - * buffer. Note that forcing flush frequently degrades the compression ratio, so this should only - * be called when necessary. - * @param output_buffer supplies the buffer to output compressed data. - */ - void flush(Buffer::Instance& output_buffer); - // Compressor - void compress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) override; + void compress(Buffer::Instance& buffer, bool trailer) override; private: bool deflateNext(int64_t flush_state); diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index bdaa18e2ef5a..aa1c8c8ccaaa 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -129,27 +129,10 @@ Http::FilterHeadersStatus GzipFilter::encodeHeaders(Http::HeaderMap& headers, bo } Http::FilterDataStatus GzipFilter::encodeData(Buffer::Instance& data, bool end_stream) { - if (skip_compression_) { - return Http::FilterDataStatus::Continue; + if (!skip_compression_) { + compressor_.compress(data, end_stream); } - - const uint64_t n_data = data.length(); - if (n_data > 0) { - compressor_.compress(data, compressed_data_); - compressor_.flush(compressed_data_); - } - - if (end_stream) { - compressor_.finish(compressed_data_); - } - - if (compressed_data_.length() > 0) { - data.drain(n_data); - data.move(compressed_data_); - return Http::FilterDataStatus::Continue; - } - - return Http::FilterDataStatus::StopIterationNoBuffer; + return Http::FilterDataStatus::Continue; } bool GzipFilter::hasCacheControlNoTransform(Http::HeaderMap& headers) const { diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index 52b11be70a54..f1615d0fd377 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -19,6 +19,7 @@ class ZlibCompressorImplTest : public testing::Test { const std::string header_hex_str = Hex::encode( reinterpret_cast(compressed_slices[0].mem_), compressed_slices[0].len_); + // HEADER 0x1f = 31 (window_bits) EXPECT_EQ("1f8b", header_hex_str.substr(0, 4)); // CM 0x8 = deflate (compression method) @@ -48,6 +49,7 @@ class ZlibCompressorImplTest : public testing::Test { Hex::encode(reinterpret_cast(compressed_slices[num_comp_slices - 1].mem_), compressed_slices[num_comp_slices - 1].len_); + // A valid finished compressed buffer should have trailer with input size in it. expectEqualInputSize(footer_bytes_str, input_size); } @@ -58,11 +60,21 @@ class ZlibCompressorImplTest : public testing::Test { EXPECT_EQ(TestUtility::flipOrder(size), input_size); } + void drainBuffer(Buffer::OwnedImpl& buffer) { buffer.drain(buffer.length()); } + static const int64_t gzip_window_bits{31}; static const int64_t memory_level{8}; static const uint64_t default_input_size{796}; }; +class ZlibCompressorImplTester : public ZlibCompressorImpl { +public: + ZlibCompressorImplTester() : ZlibCompressorImpl() {} + ZlibCompressorImplTester(uint64_t chunk_size) : ZlibCompressorImpl(chunk_size) {} + void compressThenFlush(Buffer::OwnedImpl& buffer) { compress(buffer, false); } + void finish(Buffer::OwnedImpl& buffer) { compress(buffer, true); } +}; + class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { protected: static void compressorBadInitTestHelper(int64_t window_bits, int64_t mem_level) { @@ -72,28 +84,22 @@ class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { } static void uninitializedCompressorTestHelper() { - Buffer::OwnedImpl input_buffer; - Buffer::OwnedImpl output_buffer; - ZlibCompressorImpl compressor; - TestUtility::feedBufferWithRandomCharacters(input_buffer, 100); - compressor.compress(input_buffer, output_buffer); + Buffer::OwnedImpl buffer; + ZlibCompressorImplTester compressor; + TestUtility::feedBufferWithRandomCharacters(buffer, 100); + compressor.finish(buffer); } static void uninitializedCompressorFlushTestHelper() { - Buffer::OwnedImpl output_buffer; - ZlibCompressorImpl compressor; - compressor.flush(output_buffer); + Buffer::OwnedImpl buffer; + ZlibCompressorImplTester compressor; + compressor.compressThenFlush(buffer); } static void uninitializedCompressorFinishTestHelper() { - Buffer::OwnedImpl output_buffer; - ZlibCompressorImpl compressor; - compressor.finish(output_buffer); - } - - static void uninitializedCompressorResetTestHelper() { - ZlibCompressorImpl compressor; - compressor.reset(); + Buffer::OwnedImpl buffer; + ZlibCompressorImplTester compressor; + compressor.finish(buffer); } }; @@ -107,16 +113,13 @@ TEST_F(ZlibCompressorImplDeathTest, CompressorTestDeath) { std::string{"assert failure: result == Z_OK"}); EXPECT_DEATH(uninitializedCompressorFinishTestHelper(), std::string{"assert failure: result == Z_STREAM_END"}); - EXPECT_DEATH(uninitializedCompressorResetTestHelper(), - std::string{"assert failure: result == Z_OK"}); } // Exercises compressor's checksum by calling it before init or compress. TEST_F(ZlibCompressorImplTest, CallingChecksum) { - Buffer::OwnedImpl compressor_input_buffer; - Buffer::OwnedImpl compressor_output_buffer; + Buffer::OwnedImpl buffer; - ZlibCompressorImpl compressor; + ZlibCompressorImplTester compressor; EXPECT_EQ(0, compressor.checksum()); compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, @@ -124,169 +127,80 @@ TEST_F(ZlibCompressorImplTest, CallingChecksum) { gzip_window_bits, memory_level); EXPECT_EQ(0, compressor.checksum()); - TestUtility::feedBufferWithRandomCharacters(compressor_input_buffer, 4096); - compressor.compress(compressor_input_buffer, compressor_output_buffer); - compressor.flush(compressor_output_buffer); - compressor_input_buffer.drain(4096); - EXPECT_TRUE(compressor.checksum() > 0); -} - -// Exercises compression with a very small output buffer. -TEST_F(ZlibCompressorImplTest, CompressWithReducedInternalMemory) { - Buffer::OwnedImpl input_buffer; - Buffer::OwnedImpl output_buffer; + TestUtility::feedBufferWithRandomCharacters(buffer, 4096); + compressor.compressThenFlush(buffer); + expectValidFlushedBuffer(buffer); - Envoy::Compressor::ZlibCompressorImpl compressor(8); - compressor.init(ZlibCompressorImpl::CompressionLevel::Standard, - ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, - memory_level); - - uint64_t input_size = 0; - for (uint64_t i = 0; i < 10; i++) { - TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); - compressor.compress(input_buffer, output_buffer); - input_size += input_buffer.length(); - input_buffer.drain(default_input_size * i); - ASSERT_EQ(0, input_buffer.length()); - } - - compressor.flush(output_buffer); - expectValidFlushedBuffer(output_buffer); - - compressor.finish(output_buffer); - - // A valid finished buffer should have trailer with input size in it. - expectValidFinishedBuffer(output_buffer, input_size); + drainBuffer(buffer); + EXPECT_TRUE(compressor.checksum() > 0); } -// Exercises compression with a very small output buffer then calls finish(). -TEST_F(ZlibCompressorImplTest, CompressWithReducedInternalMemoryThenFinishIt) { - Buffer::OwnedImpl input_buffer; - Buffer::OwnedImpl output_buffer; - - Envoy::Compressor::ZlibCompressorImpl compressor(8); - compressor.init(ZlibCompressorImpl::CompressionLevel::Standard, - ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, - memory_level); - - for (uint64_t i = 0; i < 10; i++) { - TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); - compressor.compress(input_buffer, output_buffer); - input_buffer.drain(default_input_size * i); - ASSERT_EQ(0, input_buffer.length()); - } +// Exercises compressor's checksum by calling it before init or compress. +TEST_F(ZlibCompressorImplTest, CallingFinishOnly) { + Buffer::OwnedImpl buffer; - compressor.flush(output_buffer); - expectValidFlushedBuffer(output_buffer); + ZlibCompressorImplTester compressor; + compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, + gzip_window_bits, memory_level); + EXPECT_EQ(0, compressor.checksum()); - compressor.finish(output_buffer); + TestUtility::feedBufferWithRandomCharacters(buffer, 4096); + compressor.finish(buffer); + expectValidFinishedBuffer(buffer, 4096); } -// Exercises compression with a flush call on each received buffer. -TEST_F(ZlibCompressorImplTest, CompressWithContinuesFlush) { - Buffer::OwnedImpl input_buffer; - Buffer::OwnedImpl output_buffer; +TEST_F(ZlibCompressorImplTest, CompressWithSmallChunkSize) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; - Envoy::Compressor::ZlibCompressorImpl compressor; + ZlibCompressorImplTester compressor(8); compressor.init(ZlibCompressorImpl::CompressionLevel::Standard, ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, memory_level); + uint64_t input_size = 0; for (uint64_t i = 0; i < 10; i++) { - TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); - compressor.compress(input_buffer, output_buffer); - - input_buffer.drain(default_input_size * i); - ASSERT_EQ(0, input_buffer.length()); - - compressor.flush(output_buffer); - expectValidFlushedBuffer(output_buffer); + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + ASSERT_EQ(default_input_size * i, buffer.length()); + input_size += buffer.length(); + compressor.compressThenFlush(buffer); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); } -} - -// Exercises compression with very small input buffer. -TEST_F(ZlibCompressorImplTest, CompressSmallInputMemory) { - Buffer::OwnedImpl input_buffer; - Buffer::OwnedImpl output_buffer; - - Envoy::Compressor::ZlibCompressorImpl compressor; - compressor.init(ZlibCompressorImpl::CompressionLevel::Standard, - ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, - memory_level); - - TestUtility::feedBufferWithRandomCharacters(input_buffer, 10); + expectValidFlushedBuffer(accumulation_buffer); - compressor.compress(input_buffer, output_buffer); - EXPECT_EQ(0, output_buffer.length()); - - compressor.flush(output_buffer); - EXPECT_LE(10, output_buffer.length()); - - expectValidFlushedBuffer(output_buffer); -} - -// Exercises common flow of compressing some data, making it available to output buffer, -// then moving output buffer to another buffer and so on. -TEST_F(ZlibCompressorImplTest, CompressMoveFlushAndRepeat) { - Buffer::OwnedImpl input_buffer; - Buffer::OwnedImpl temp_buffer; - Buffer::OwnedImpl output_buffer; - - Envoy::Compressor::ZlibCompressorImpl compressor; - compressor.init(ZlibCompressorImpl::CompressionLevel::Standard, - ZlibCompressorImpl::CompressionStrategy::Standard, gzip_window_bits, - memory_level); - - for (uint64_t i = 0; i < 20; i++) { - TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); - compressor.compress(input_buffer, temp_buffer); - input_buffer.drain(default_input_size * i); - ASSERT_EQ(0, input_buffer.length()); - output_buffer.move(temp_buffer); - ASSERT_EQ(0, temp_buffer.length()); - } - - compressor.flush(temp_buffer); - ASSERT_TRUE(temp_buffer.length() > 0); - output_buffer.move(temp_buffer); - const uint64_t first_n_compressed_bytes = output_buffer.length(); - - for (uint64_t i = 0; i < 15; i++) { - TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i * 10); - compressor.compress(input_buffer, temp_buffer); - input_buffer.drain(default_input_size * i); - ASSERT_EQ(0, input_buffer.length()); - output_buffer.move(temp_buffer); - ASSERT_EQ(0, temp_buffer.length()); - } - - compressor.flush(temp_buffer); - output_buffer.move(temp_buffer); - ASSERT_EQ(0, temp_buffer.length()); - EXPECT_GE(output_buffer.length(), first_n_compressed_bytes); - - expectValidFlushedBuffer(output_buffer); + compressor.finish(buffer); + accumulation_buffer.add(buffer); + expectValidFinishedBuffer(accumulation_buffer, input_size); } // Exercises compression with other supported zlib initialization params. TEST_F(ZlibCompressorImplTest, CompressWithNotCommonParams) { - Buffer::OwnedImpl input_buffer; - Buffer::OwnedImpl output_buffer; + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; - Envoy::Compressor::ZlibCompressorImpl compressor; + ZlibCompressorImplTester compressor; compressor.init(ZlibCompressorImpl::CompressionLevel::Speed, ZlibCompressorImpl::CompressionStrategy::Rle, gzip_window_bits, 1); + uint64_t input_size = 0; for (uint64_t i = 0; i < 10; i++) { - TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); - compressor.compress(input_buffer, output_buffer); - input_buffer.drain(default_input_size * i); - ASSERT_EQ(0, input_buffer.length()); + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + ASSERT_EQ(default_input_size * i, buffer.length()); + input_size += buffer.length(); + compressor.compressThenFlush(buffer); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); } - compressor.flush(output_buffer); + expectValidFlushedBuffer(accumulation_buffer); - expectValidFlushedBuffer(output_buffer); + compressor.finish(buffer); + accumulation_buffer.add(buffer); + expectValidFinishedBuffer(accumulation_buffer, input_size); } } // namespace diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index f32b2a0365f2..821fd0557aa1 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -13,6 +13,8 @@ namespace { class ZlibDecompressorImplTest : public testing::Test { protected: + void drainBuffer(Buffer::OwnedImpl& buffer) { buffer.drain(buffer.length()); } + static const int64_t gzip_window_bits{31}; static const int64_t memory_level{8}; static const uint64_t default_input_size{796}; @@ -34,21 +36,16 @@ class ZlibDecompressorImplDeathTest : public ZlibDecompressorImplTest { } }; -/** - * Exercises death by passing bad initialization params or by calling - * decompress before init. - */ +// Exercises death by passing bad initialization params or by calling decompress before init. TEST_F(ZlibDecompressorImplDeathTest, DecompressorTestDeath) { EXPECT_DEATH(decompressorBadInitTestHelper(100), std::string{"assert failure: result >= 0"}); EXPECT_DEATH(unitializedDecompressorTestHelper(), std::string{"assert failure: result == Z_OK"}); } -/** - * Exercises decompressor's checksum by calling it before init or decompress. - */ +// Exercises decompressor's checksum by calling it before init or decompress. TEST_F(ZlibDecompressorImplTest, CallingChecksum) { - Buffer::OwnedImpl compressor_input_buffer; - Buffer::OwnedImpl compressor_output_buffer; + Buffer::OwnedImpl compressor_buffer; + Buffer::OwnedImpl decompressor_output_buffer; Envoy::Compressor::ZlibCompressorImpl compressor; ASSERT_EQ(0, compressor.checksum()); @@ -58,29 +55,27 @@ TEST_F(ZlibDecompressorImplTest, CallingChecksum) { gzip_window_bits, memory_level); ASSERT_EQ(0, compressor.checksum()); - TestUtility::feedBufferWithRandomCharacters(compressor_input_buffer, 4096); - compressor.compress(compressor_input_buffer, compressor_output_buffer); - compressor.flush(compressor_output_buffer); - compressor_input_buffer.drain(4096); + TestUtility::feedBufferWithRandomCharacters(compressor_buffer, 4096); + compressor.compress(compressor_buffer, false); ASSERT_TRUE(compressor.checksum() > 0); ZlibDecompressorImpl decompressor; decompressor.init(gzip_window_bits); EXPECT_EQ(0, decompressor.checksum()); - // compressor_output_buffer becomes decompressor input param. - // compressor_input_buffer is re-used as decompressor output since it is empty. - decompressor.decompress(compressor_output_buffer, compressor_input_buffer); + decompressor.decompress(compressor_buffer, decompressor_output_buffer); + + drainBuffer(compressor_buffer); + drainBuffer(decompressor_output_buffer); + EXPECT_EQ(compressor.checksum(), decompressor.checksum()); } -/** - * Exercises compression and decompression by compressing some data, decompressing it and then - * comparing compressor's input/checksum with decompressor's output/checksum. - */ +// Exercises compression and decompression by compressing some data, decompressing it and then +// comparing compressor's input/checksum with decompressor's output/checksum. TEST_F(ZlibDecompressorImplTest, CompressAndDecompress) { - Buffer::OwnedImpl compressor_input_buffer; - Buffer::OwnedImpl compressor_output_buffer; + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; Envoy::Compressor::ZlibCompressorImpl compressor; compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, @@ -89,35 +84,38 @@ TEST_F(ZlibDecompressorImplTest, CompressAndDecompress) { std::string original_text{}; for (uint64_t i = 0; i < 20; ++i) { - TestUtility::feedBufferWithRandomCharacters(compressor_input_buffer, default_input_size * i, i); - compressor.compress(compressor_input_buffer, compressor_output_buffer); - original_text.append(TestUtility::bufferToString(compressor_input_buffer)); - compressor_input_buffer.drain(default_input_size * i); + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + original_text.append(TestUtility::bufferToString(buffer)); + compressor.compress(buffer, false); + accumulation_buffer.add(buffer); + drainBuffer(buffer); } - compressor.flush(compressor_output_buffer); + ASSERT_EQ(0, buffer.length()); + + compressor.compress(buffer, true); + ASSERT_GE(10, buffer.length()); + + accumulation_buffer.add(buffer); + + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); ZlibDecompressorImpl decompressor; decompressor.init(gzip_window_bits); - ASSERT_EQ(0, compressor_input_buffer.length()); - // compressor_output_buffer becomes decompressor input param. - // compressor_input_buffer is re-used as decompressor output since it is empty. - decompressor.decompress(compressor_output_buffer, compressor_input_buffer); - - std::string decompressed_text{TestUtility::bufferToString(compressor_input_buffer)}; + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{TestUtility::bufferToString(buffer)}; ASSERT_EQ(compressor.checksum(), decompressor.checksum()); ASSERT_EQ(original_text.length(), decompressed_text.length()); EXPECT_EQ(original_text, decompressed_text); } -/** - * Exercises decompression with a very small output buffer. - */ +// Exercises decompression with a very small output buffer. TEST_F(ZlibDecompressorImplTest, DecompressWithSmallOutputBuffer) { - Buffer::OwnedImpl input_buffer; - Buffer::OwnedImpl output_buffer; + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; Envoy::Compressor::ZlibCompressorImpl compressor; compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, @@ -126,51 +124,67 @@ TEST_F(ZlibDecompressorImplTest, DecompressWithSmallOutputBuffer) { std::string original_text{}; for (uint64_t i = 0; i < 20; ++i) { - TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); - compressor.compress(input_buffer, output_buffer); - original_text.append(TestUtility::bufferToString(input_buffer)); - input_buffer.drain(default_input_size * i); + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + original_text.append(TestUtility::bufferToString(buffer)); + compressor.compress(buffer, false); + accumulation_buffer.add(buffer); + drainBuffer(buffer); } - compressor.flush(output_buffer); + + ASSERT_EQ(0, buffer.length()); + + compressor.compress(buffer, true); + ASSERT_GE(10, buffer.length()); + + accumulation_buffer.add(buffer); + + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); ZlibDecompressorImpl decompressor(16); decompressor.init(gzip_window_bits); - ASSERT_EQ(0, input_buffer.length()); - decompressor.decompress(output_buffer, input_buffer); - std::string decompressed_text{TestUtility::bufferToString(input_buffer)}; + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{TestUtility::bufferToString(buffer)}; ASSERT_EQ(compressor.checksum(), decompressor.checksum()); ASSERT_EQ(original_text.length(), decompressed_text.length()); EXPECT_EQ(original_text, decompressed_text); } -/** - * Exercises decompression with other supported zlib initialization params. - */ +// Exercises decompression with other supported zlib initialization params. TEST_F(ZlibDecompressorImplTest, CompressDecompressWithUncommonParams) { - Buffer::OwnedImpl input_buffer; - Buffer::OwnedImpl output_buffer; + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; Envoy::Compressor::ZlibCompressorImpl compressor; compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, 2); + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, 1); std::string original_text{}; - for (uint64_t i = 0; i < 20; ++i) { - TestUtility::feedBufferWithRandomCharacters(input_buffer, default_input_size * i, i); - compressor.compress(input_buffer, output_buffer); - original_text.append(TestUtility::bufferToString(input_buffer)); - input_buffer.drain(default_input_size * i); + for (uint64_t i = 0; i < 5; ++i) { + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + original_text.append(TestUtility::bufferToString(buffer)); + compressor.compress(buffer, false); + accumulation_buffer.add(buffer); + drainBuffer(buffer); } - compressor.flush(output_buffer); + + ASSERT_EQ(0, buffer.length()); + + compressor.compress(buffer, true); + ASSERT_GE(10, buffer.length()); + + accumulation_buffer.add(buffer); + + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); ZlibDecompressorImpl decompressor; decompressor.init(15); - ASSERT_EQ(0, input_buffer.length()); - decompressor.decompress(output_buffer, input_buffer); - std::string decompressed_text{TestUtility::bufferToString(input_buffer)}; + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{TestUtility::bufferToString(buffer)}; ASSERT_EQ(compressor.checksum(), decompressor.checksum()); ASSERT_EQ(original_text.length(), decompressed_text.length()); From 2fd487682f1542c6cbe75c2eaefcf3f0485411b0 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Sun, 15 Apr 2018 16:32:57 +0700 Subject: [PATCH 17/28] Remove reset Signed-off-by: Dhi Aurrahman --- source/common/compressor/zlib_compressor_impl.cc | 5 ----- source/common/compressor/zlib_compressor_impl.h | 7 ------- 2 files changed, 12 deletions(-) diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index 6aefa6a3b215..cabd3641c48a 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -88,10 +88,5 @@ void ZlibCompressorImpl::updateOutput(Buffer::Instance& output_buffer) { zstream_ptr_->next_out = chunk_char_ptr_.get(); } -void ZlibCompressorImpl::reset() { - const bool result = deflateReset(zstream_ptr_.get()); - RELEASE_ASSERT(result == Z_OK); -} - } // namespace Compressor } // namespace Envoy diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index f877ef330e14..91a86d391d25 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -72,13 +72,6 @@ class ZlibCompressorImpl : public Compressor { */ uint64_t checksum(); - /** - * This function is equivalent to deflateEnd followed by deflateInit, but does not free and - * reallocate the internal compression state. The stream will leave the compression level and any - * other attributes that may have been set unchanged. - */ - void reset(); - // Compressor void compress(Buffer::Instance& buffer, bool trailer) override; From 3713794d1ae4f481e41140ab04110aaf3758760b Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Tue, 17 Apr 2018 14:23:39 +0700 Subject: [PATCH 18/28] Use enum instead of boolean Signed-off-by: Dhi Aurrahman --- include/envoy/compressor/compressor.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/include/envoy/compressor/compressor.h b/include/envoy/compressor/compressor.h index 8ac75b7e14bd..94660138cc21 100644 --- a/include/envoy/compressor/compressor.h +++ b/include/envoy/compressor/compressor.h @@ -5,6 +5,11 @@ namespace Envoy { namespace Compressor { +/** + * Compressor state whether to flush the compressor or to finish the compression stream. + */ +enum class State { Flush, Finish }; + /** * Allows compressing data. */ @@ -15,9 +20,9 @@ class Compressor { /** * Compresses data buffer. * @param buffer supplies the reference to data to be compressed. - * @param trailer supplies the indication to write trailer at the end of compressed buffer. + * @param state supplies the compressor state. */ - virtual void compress(Buffer::Instance& buffer, bool trailer) PURE; + virtual void compress(Buffer::Instance& buffer, State state) PURE; }; } // namespace Compressor From 467df231e70d6a8c7678093c0bdb3061cb5b1590 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Tue, 17 Apr 2018 14:24:54 +0700 Subject: [PATCH 19/28] Implement the new compressor interface Signed-off-by: Dhi Aurrahman --- source/common/compressor/zlib_compressor_impl.cc | 6 +++--- source/common/compressor/zlib_compressor_impl.h | 2 +- source/extensions/filters/http/gzip/gzip_filter.cc | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index cabd3641c48a..b4e48d67cb32 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -33,7 +33,7 @@ void ZlibCompressorImpl::init(CompressionLevel comp_level, CompressionStrategy c uint64_t ZlibCompressorImpl::checksum() { return zstream_ptr_->adler; } -void ZlibCompressorImpl::compress(Buffer::Instance& buffer, bool trailer) { +void ZlibCompressorImpl::compress(Buffer::Instance& buffer, State state) { const uint64_t num_slices = buffer.getRawSlices(nullptr, 0); Buffer::RawSlice slices[num_slices]; buffer.getRawSlices(slices, num_slices); @@ -42,10 +42,10 @@ void ZlibCompressorImpl::compress(Buffer::Instance& buffer, bool trailer) { zstream_ptr_->avail_in = input_slice.len_; zstream_ptr_->next_in = static_cast(input_slice.mem_); process(buffer, Z_NO_FLUSH); + buffer.drain(input_slice.len_); } - buffer.drain(buffer.length()); - process(buffer, trailer ? Z_FINISH : Z_SYNC_FLUSH); + process(buffer, state == State::Finish ? Z_FINISH : Z_SYNC_FLUSH); } bool ZlibCompressorImpl::deflateNext(int64_t flush_state) { diff --git a/source/common/compressor/zlib_compressor_impl.h b/source/common/compressor/zlib_compressor_impl.h index 91a86d391d25..ecfdc8aa4163 100644 --- a/source/common/compressor/zlib_compressor_impl.h +++ b/source/common/compressor/zlib_compressor_impl.h @@ -73,7 +73,7 @@ class ZlibCompressorImpl : public Compressor { uint64_t checksum(); // Compressor - void compress(Buffer::Instance& buffer, bool trailer) override; + void compress(Buffer::Instance& buffer, State state) override; private: bool deflateNext(int64_t flush_state); diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index aa1c8c8ccaaa..922aebe49326 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -130,7 +130,7 @@ Http::FilterHeadersStatus GzipFilter::encodeHeaders(Http::HeaderMap& headers, bo Http::FilterDataStatus GzipFilter::encodeData(Buffer::Instance& data, bool end_stream) { if (!skip_compression_) { - compressor_.compress(data, end_stream); + compressor_.compress(data, end_stream ? Compressor::State::Finish : Compressor::State::Flush); } return Http::FilterDataStatus::Continue; } From 467af66ac9a2e1f9932ba2f946a5df8b54fd5c64 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Tue, 17 Apr 2018 14:25:27 +0700 Subject: [PATCH 20/28] Update tests Signed-off-by: Dhi Aurrahman --- .../compressor/zlib_compressor_impl_test.cc | 4 ++-- .../decompressor/zlib_decompressor_impl_test.cc | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index f1615d0fd377..86593ab40b4a 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -71,8 +71,8 @@ class ZlibCompressorImplTester : public ZlibCompressorImpl { public: ZlibCompressorImplTester() : ZlibCompressorImpl() {} ZlibCompressorImplTester(uint64_t chunk_size) : ZlibCompressorImpl(chunk_size) {} - void compressThenFlush(Buffer::OwnedImpl& buffer) { compress(buffer, false); } - void finish(Buffer::OwnedImpl& buffer) { compress(buffer, true); } + void compressThenFlush(Buffer::OwnedImpl& buffer) { compress(buffer, State::Flush); } + void finish(Buffer::OwnedImpl& buffer) { compress(buffer, State::Finish); } }; class ZlibCompressorImplDeathTest : public ZlibCompressorImplTest { diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index 821fd0557aa1..525a52785522 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -56,7 +56,7 @@ TEST_F(ZlibDecompressorImplTest, CallingChecksum) { ASSERT_EQ(0, compressor.checksum()); TestUtility::feedBufferWithRandomCharacters(compressor_buffer, 4096); - compressor.compress(compressor_buffer, false); + compressor.compress(compressor_buffer, Compressor::State::Flush); ASSERT_TRUE(compressor.checksum() > 0); ZlibDecompressorImpl decompressor; @@ -86,14 +86,14 @@ TEST_F(ZlibDecompressorImplTest, CompressAndDecompress) { for (uint64_t i = 0; i < 20; ++i) { TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); original_text.append(TestUtility::bufferToString(buffer)); - compressor.compress(buffer, false); + compressor.compress(buffer, Compressor::State::Flush); accumulation_buffer.add(buffer); drainBuffer(buffer); } ASSERT_EQ(0, buffer.length()); - compressor.compress(buffer, true); + compressor.compress(buffer, Compressor::State::Finish); ASSERT_GE(10, buffer.length()); accumulation_buffer.add(buffer); @@ -126,14 +126,14 @@ TEST_F(ZlibDecompressorImplTest, DecompressWithSmallOutputBuffer) { for (uint64_t i = 0; i < 20; ++i) { TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); original_text.append(TestUtility::bufferToString(buffer)); - compressor.compress(buffer, false); + compressor.compress(buffer, Compressor::State::Flush); accumulation_buffer.add(buffer); drainBuffer(buffer); } ASSERT_EQ(0, buffer.length()); - compressor.compress(buffer, true); + compressor.compress(buffer, Compressor::State::Finish); ASSERT_GE(10, buffer.length()); accumulation_buffer.add(buffer); @@ -162,17 +162,17 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressWithUncommonParams) { Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, 1); std::string original_text{}; - for (uint64_t i = 0; i < 5; ++i) { + for (uint64_t i = 0; i < 20; ++i) { TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); original_text.append(TestUtility::bufferToString(buffer)); - compressor.compress(buffer, false); + compressor.compress(buffer, Compressor::State::Flush); accumulation_buffer.add(buffer); drainBuffer(buffer); } ASSERT_EQ(0, buffer.length()); - compressor.compress(buffer, true); + compressor.compress(buffer, Compressor::State::Finish); ASSERT_GE(10, buffer.length()); accumulation_buffer.add(buffer); From ebd3e6d6615d6541a66219c60be83d9176cffd5a Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Tue, 17 Apr 2018 16:25:46 +0700 Subject: [PATCH 21/28] Add test of compress-decompress with different memory levels Signed-off-by: Dhi Aurrahman --- .../zlib_decompressor_impl_test.cc | 91 ++++++++++++------- 1 file changed, 57 insertions(+), 34 deletions(-) diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index 525a52785522..5c579563bc4f 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -15,6 +15,46 @@ class ZlibDecompressorImplTest : public testing::Test { protected: void drainBuffer(Buffer::OwnedImpl& buffer) { buffer.drain(buffer.length()); } + void testcompressDecompressWithUncommonParams( + Compressor::ZlibCompressorImpl::CompressionLevel comp_level, + Compressor::ZlibCompressorImpl::CompressionStrategy comp_strategy, int64_t window_bits, + uint64_t memory_level) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; + + Envoy::Compressor::ZlibCompressorImpl compressor; + compressor.init(comp_level, comp_strategy, window_bits, memory_level); + + std::string original_text{}; + for (uint64_t i = 0; i < 30; ++i) { + TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); + original_text.append(TestUtility::bufferToString(buffer)); + compressor.compress(buffer, Compressor::State::Flush); + accumulation_buffer.add(buffer); + drainBuffer(buffer); + } + + ASSERT_EQ(0, buffer.length()); + + compressor.compress(buffer, Compressor::State::Finish); + ASSERT_GE(10, buffer.length()); + + accumulation_buffer.add(buffer); + + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); + + ZlibDecompressorImpl decompressor; + decompressor.init(15); + + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{TestUtility::bufferToString(buffer)}; + + ASSERT_EQ(compressor.checksum(), decompressor.checksum()); + ASSERT_EQ(original_text.length(), decompressed_text.length()); + EXPECT_EQ(original_text, decompressed_text); + } + static const int64_t gzip_window_bits{31}; static const int64_t memory_level{8}; static const uint64_t default_input_size{796}; @@ -154,41 +194,24 @@ TEST_F(ZlibDecompressorImplTest, DecompressWithSmallOutputBuffer) { // Exercises decompression with other supported zlib initialization params. TEST_F(ZlibDecompressorImplTest, CompressDecompressWithUncommonParams) { - Buffer::OwnedImpl buffer; - Buffer::OwnedImpl accumulation_buffer; - - Envoy::Compressor::ZlibCompressorImpl compressor; - compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Best, - Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, 1); - - std::string original_text{}; - for (uint64_t i = 0; i < 20; ++i) { - TestUtility::feedBufferWithRandomCharacters(buffer, default_input_size * i, i); - original_text.append(TestUtility::bufferToString(buffer)); - compressor.compress(buffer, Compressor::State::Flush); - accumulation_buffer.add(buffer); - drainBuffer(buffer); + // Test with different memory levels. + for (uint64_t i = 1; i < 10; ++i) { + testcompressDecompressWithUncommonParams( + Compressor::ZlibCompressorImpl::CompressionLevel::Best, + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, i); + + testcompressDecompressWithUncommonParams( + Compressor::ZlibCompressorImpl::CompressionLevel::Best, + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Rle, 15, i); + + testcompressDecompressWithUncommonParams( + Compressor::ZlibCompressorImpl::CompressionLevel::Speed, + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Huffman, 15, i); + + testcompressDecompressWithUncommonParams( + Compressor::ZlibCompressorImpl::CompressionLevel::Speed, + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Filtered, 15, i); } - - ASSERT_EQ(0, buffer.length()); - - compressor.compress(buffer, Compressor::State::Finish); - ASSERT_GE(10, buffer.length()); - - accumulation_buffer.add(buffer); - - drainBuffer(buffer); - ASSERT_EQ(0, buffer.length()); - - ZlibDecompressorImpl decompressor; - decompressor.init(15); - - decompressor.decompress(accumulation_buffer, buffer); - std::string decompressed_text{TestUtility::bufferToString(buffer)}; - - ASSERT_EQ(compressor.checksum(), decompressor.checksum()); - ASSERT_EQ(original_text.length(), decompressed_text.length()); - EXPECT_EQ(original_text, decompressed_text); } } // namespace From a291b5c02f0f5d0bab2c0145c0da338f6588de92 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Wed, 18 Apr 2018 05:16:34 +0700 Subject: [PATCH 22/28] Add test with multiple slices of buffer Signed-off-by: Dhi Aurrahman --- .../zlib_decompressor_impl_test.cc | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index 5c579563bc4f..1db867e86923 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -45,7 +45,7 @@ class ZlibDecompressorImplTest : public testing::Test { ASSERT_EQ(0, buffer.length()); ZlibDecompressorImpl decompressor; - decompressor.init(15); + decompressor.init(window_bits); decompressor.decompress(accumulation_buffer, buffer); std::string decompressed_text{TestUtility::bufferToString(buffer)}; @@ -214,6 +214,43 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressWithUncommonParams) { } } +TEST_F(ZlibDecompressorImplTest, CompressDecompressOfMultipleSlices) { + Buffer::OwnedImpl buffer; + Buffer::OwnedImpl accumulation_buffer; + + const std::string sample{"slice, slice, slice, slice, slice, "}; + std::string original_text{}; + for (uint64_t i = 0; i < 20; ++i) { + Buffer::BufferFragmentImpl frag(sample.c_str(), sample.size(), nullptr); + buffer.addBufferFragment(frag); + original_text.append(sample); + } + + const uint64_t num_slices = buffer.getRawSlices(nullptr, 0); + EXPECT_EQ(num_slices, 20); + + Envoy::Compressor::ZlibCompressorImpl compressor; + compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Envoy::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, + gzip_window_bits, memory_level); + + compressor.compress(buffer, Compressor::State::Flush); + accumulation_buffer.add(buffer); + + ZlibDecompressorImpl decompressor; + decompressor.init(gzip_window_bits); + + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); + + decompressor.decompress(accumulation_buffer, buffer); + std::string decompressed_text{TestUtility::bufferToString(buffer)}; + + ASSERT_EQ(compressor.checksum(), decompressor.checksum()); + ASSERT_EQ(original_text.length(), decompressed_text.length()); + EXPECT_EQ(original_text, decompressed_text); +} + } // namespace } // namespace Decompressor } // namespace Envoy From 4a1dd67c94839b4b9e2c69cb0d93724d81440ba0 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Wed, 18 Apr 2018 14:42:45 +0700 Subject: [PATCH 23/28] Dynamically allocate the fragments Signed-off-by: Dhi Aurrahman --- test/common/decompressor/zlib_decompressor_impl_test.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index 1db867e86923..a6380a83b463 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -219,10 +219,13 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressOfMultipleSlices) { Buffer::OwnedImpl accumulation_buffer; const std::string sample{"slice, slice, slice, slice, slice, "}; - std::string original_text{}; + std::string original_text; for (uint64_t i = 0; i < 20; ++i) { - Buffer::BufferFragmentImpl frag(sample.c_str(), sample.size(), nullptr); - buffer.addBufferFragment(frag); + Buffer::BufferFragmentImpl* frag = new Buffer::BufferFragmentImpl( + sample.c_str(), sample.size(), + [this](const void*, size_t, const Buffer::BufferFragmentImpl* frag) { delete frag; }); + + buffer.addBufferFragment(*frag); original_text.append(sample); } From 3f3a1757b08128dcd7f21c9a6980c5d4c9f33e69 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Wed, 18 Apr 2018 16:27:19 +0700 Subject: [PATCH 24/28] Lambda capture 'this' is not used Signed-off-by: Dhi Aurrahman --- test/common/decompressor/zlib_decompressor_impl_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index a6380a83b463..207be21313b7 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -223,7 +223,7 @@ TEST_F(ZlibDecompressorImplTest, CompressDecompressOfMultipleSlices) { for (uint64_t i = 0; i < 20; ++i) { Buffer::BufferFragmentImpl* frag = new Buffer::BufferFragmentImpl( sample.c_str(), sample.size(), - [this](const void*, size_t, const Buffer::BufferFragmentImpl* frag) { delete frag; }); + [](const void*, size_t, const Buffer::BufferFragmentImpl* frag) { delete frag; }); buffer.addBufferFragment(*frag); original_text.append(sample); From 9ef74ad551dd8dbb2120a4b4662e16383245c57e Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Thu, 19 Apr 2018 14:31:14 +0700 Subject: [PATCH 25/28] Remove debugging assertion Signed-off-by: Dhi Aurrahman --- test/common/decompressor/zlib_decompressor_impl_test.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index 207be21313b7..3bf5fbb72b44 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -33,12 +33,9 @@ class ZlibDecompressorImplTest : public testing::Test { accumulation_buffer.add(buffer); drainBuffer(buffer); } - ASSERT_EQ(0, buffer.length()); compressor.compress(buffer, Compressor::State::Finish); - ASSERT_GE(10, buffer.length()); - accumulation_buffer.add(buffer); drainBuffer(buffer); From e10d6aa83ac61670b15f7b9ef4e6316fc3259415 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Fri, 20 Apr 2018 06:53:45 +0700 Subject: [PATCH 26/28] Add comments Signed-off-by: Dhi Aurrahman --- include/envoy/compressor/compressor.h | 3 ++- source/common/compressor/zlib_compressor_impl.cc | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/envoy/compressor/compressor.h b/include/envoy/compressor/compressor.h index 94660138cc21..a755c513115a 100644 --- a/include/envoy/compressor/compressor.h +++ b/include/envoy/compressor/compressor.h @@ -19,7 +19,8 @@ class Compressor { /** * Compresses data buffer. - * @param buffer supplies the reference to data to be compressed. + * @param buffer supplies the reference to data to be compressed. The content of the buffer will + * be replaced inline with the compressed data. * @param state supplies the compressor state. */ virtual void compress(Buffer::Instance& buffer, State state) PURE; diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index b4e48d67cb32..80642dfef8f3 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -41,6 +41,10 @@ void ZlibCompressorImpl::compress(Buffer::Instance& buffer, State state) { for (const Buffer::RawSlice& input_slice : slices) { zstream_ptr_->avail_in = input_slice.len_; zstream_ptr_->next_in = static_cast(input_slice.mem_); + // Z_NO_FLUSH tells the compressor to take the data in and compresses it as much as possible + // without flushing it out. However, if the data output is greater or equal to the allocated + // chunk size, process() outputs it to the front section of the buffer. This is fine, since at + // the next step, the buffer is drained by the size of input. process(buffer, Z_NO_FLUSH); buffer.drain(input_slice.len_); } From b36f2fc95a46da7e723b18c16066d7164c3e4974 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Fri, 20 Apr 2018 06:57:36 +0700 Subject: [PATCH 27/28] Remove superfluous string constructions Signed-off-by: Dhi Aurrahman --- test/common/compressor/zlib_compressor_impl_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index 2d0c8a0a4b22..861f8b5c20e7 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -110,9 +110,9 @@ TEST_F(ZlibCompressorImplDeathTest, CompressorTestDeath) { EXPECT_DEATH_LOG_TO_STDERR(compressorBadInitTestHelper(31, 10), "assert failure: result >= 0"); EXPECT_DEATH_LOG_TO_STDERR(uninitializedCompressorTestHelper(), "assert failure: result == Z_OK"); EXPECT_DEATH_LOG_TO_STDERR(uninitializedCompressorFlushTestHelper(), - std::string{"assert failure: result == Z_OK"}); + "assert failure: result == Z_OK"); EXPECT_DEATH_LOG_TO_STDERR(uninitializedCompressorFinishTestHelper(), - std::string{"assert failure: result == Z_STREAM_END"}); + "assert failure: result == Z_STREAM_END"); } // Exercises compressor's checksum by calling it before init or compress. From ac6ec7619b017becd75f3c9b0d48c2642424d28b Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Fri, 20 Apr 2018 07:32:54 +0700 Subject: [PATCH 28/28] Comment revision Signed-off-by: Dhi Aurrahman --- source/common/compressor/zlib_compressor_impl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/compressor/zlib_compressor_impl.cc b/source/common/compressor/zlib_compressor_impl.cc index 80642dfef8f3..648fac8f6185 100644 --- a/source/common/compressor/zlib_compressor_impl.cc +++ b/source/common/compressor/zlib_compressor_impl.cc @@ -43,8 +43,8 @@ void ZlibCompressorImpl::compress(Buffer::Instance& buffer, State state) { zstream_ptr_->next_in = static_cast(input_slice.mem_); // Z_NO_FLUSH tells the compressor to take the data in and compresses it as much as possible // without flushing it out. However, if the data output is greater or equal to the allocated - // chunk size, process() outputs it to the front section of the buffer. This is fine, since at - // the next step, the buffer is drained by the size of input. + // chunk size, process() outputs it to the end of the buffer. This is fine, since at the next + // step, the buffer is drained from the beginning of the buffer by the size of input. process(buffer, Z_NO_FLUSH); buffer.drain(input_slice.len_); }