From 5ccb7a4c6867573b41d4016a05489ec5b8b18a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=BE=D1=80=D0=B5=D0=BD=D0=B1=D0=B5=D1=80=D0=B3=20?= =?UTF-8?q?=E2=98=A2=EF=B8=8F=20=20=D0=9C=D0=B0=D1=80=D0=BA?= Date: Fri, 17 Jan 2020 19:26:20 +0500 Subject: [PATCH] Fix deflate compression (#4506) --- CHANGES/4506.bugfix | 1 + aiohttp/http_parser.py | 6 ++++-- aiohttp/http_writer.py | 2 +- aiohttp/web_response.py | 3 ++- tests/test_http_parser.py | 4 ++-- tests/test_http_writer.py | 9 +++++++-- tests/test_web_functional.py | 2 +- tests/test_web_sendfile_functional.py | 2 +- 8 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 CHANGES/4506.bugfix diff --git a/CHANGES/4506.bugfix b/CHANGES/4506.bugfix new file mode 100644 index 00000000000..eaf4bb88aac --- /dev/null +++ b/CHANGES/4506.bugfix @@ -0,0 +1 @@ +Fixed 'deflate' compressions. According to RFC 2616 now. diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index f2881c3bf11..6c5a4673e5c 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -727,7 +727,7 @@ def flush(self) -> bytes: self.decompressor = BrotliDecoder() # type: Any else: zlib_mode = (16 + zlib.MAX_WBITS - if encoding == 'gzip' else -zlib.MAX_WBITS) + if encoding == 'gzip' else zlib.MAX_WBITS) self.decompressor = zlib.decompressobj(wbits=zlib_mode) def set_exception(self, exc: BaseException) -> None: @@ -739,7 +739,9 @@ def feed_data(self, chunk: bytes, size: int) -> None: chunk = self.decompressor.decompress(chunk) except Exception: if not self._started_decoding and self.encoding == 'deflate': - self.decompressor = zlib.decompressobj() + # Try to change the decoder to decompress incorrectly + # compressed data + self.decompressor = zlib.decompressobj(wbits=-zlib.MAX_WBITS) try: chunk = self.decompressor.decompress(chunk) except Exception: diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index 7e27fbf6a43..102fb3ef2f4 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -55,7 +55,7 @@ def enable_chunking(self) -> None: def enable_compression(self, encoding: str='deflate') -> None: zlib_mode = (16 + zlib.MAX_WBITS - if encoding == 'gzip' else -zlib.MAX_WBITS) + if encoding == 'gzip' else zlib.MAX_WBITS) self._compress = zlib.compressobj(wbits=zlib_mode) def _write(self, chunk: bytes) -> None: diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index 4dc64976839..fdb7aa4d359 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -669,6 +669,7 @@ async def _start(self, request: 'BaseRequest') -> AbstractStreamWriter: return await super()._start(request) def _compress_body(self, zlib_mode: int) -> None: + assert zlib_mode > 0 compressobj = zlib.compressobj(wbits=zlib_mode) body_in = self._body assert body_in is not None @@ -683,7 +684,7 @@ async def _do_start_compression(self, coding: ContentCoding) -> None: # Instead of using _payload_writer.enable_compression, # compress the whole body zlib_mode = (16 + zlib.MAX_WBITS - if coding == ContentCoding.gzip else -zlib.MAX_WBITS) + if coding == ContentCoding.gzip else zlib.MAX_WBITS) body_in = self._body assert body_in is not None if self._zlib_executor_size is not None and \ diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 19fe9be7a3c..8a8044ec021 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -837,7 +837,7 @@ async def test_http_payload_parser_length(self, stream) -> None: assert b'12' == b''.join(d for d, _ in out._buffer) assert b'45' == tail - _comp = zlib.compressobj(wbits=-zlib.MAX_WBITS) + _comp = zlib.compressobj(wbits=zlib.MAX_WBITS) _COMPRESSED = b''.join([_comp.compress(b'data'), _comp.flush()]) async def test_http_payload_parser_deflate(self, stream) -> None: @@ -905,7 +905,7 @@ async def test_feed_data_err(self, stream) -> None: dbuf.decompressor.decompress.side_effect = exc with pytest.raises(http_exceptions.ContentEncodingError): - dbuf.feed_data(b'data', 4) + dbuf.feed_data(b'data1', 5) async def test_feed_eof(self, stream) -> None: buf = aiohttp.FlowControlDataQueue(stream, diff --git a/tests/test_http_writer.py b/tests/test_http_writer.py index 2f8085f8a85..50efacb58c5 100644 --- a/tests/test_http_writer.py +++ b/tests/test_http_writer.py @@ -117,7 +117,7 @@ async def test_write_payload_chunked_filter_mutiple_chunks( b'2\r\na2\r\n0\r\n\r\n') -compressor = zlib.compressobj(wbits=-zlib.MAX_WBITS) +compressor = zlib.compressobj(wbits=zlib.MAX_WBITS) COMPRESSED = b''.join([compressor.compress(b'data'), compressor.flush()]) @@ -148,7 +148,12 @@ async def test_write_payload_deflate_and_chunked( await msg.write(b'ta') await msg.write_eof() - assert b'6\r\nKI,I\x04\x00\r\n0\r\n\r\n' == buf + thing = ( + b'2\r\nx\x9c\r\n' + b'a\r\nKI,I\x04\x00\x04\x00\x01\x9b\r\n' + b'0\r\n\r\n' + ) + assert thing == buf async def test_write_drain(protocol, transport, loop) -> None: diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 4be7a962303..896c092189b 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -953,7 +953,7 @@ async def test_response_with_precompressed_body_deflate( async def handler(request): headers = {'Content-Encoding': 'deflate'} - zcomp = zlib.compressobj(wbits=-zlib.MAX_WBITS) + zcomp = zlib.compressobj(wbits=zlib.MAX_WBITS) data = zcomp.compress(b'mydata') + zcomp.flush() return web.Response(body=data, headers=headers) diff --git a/tests/test_web_sendfile_functional.py b/tests/test_web_sendfile_functional.py index c2be5dbff0d..02aceb69f7b 100644 --- a/tests/test_web_sendfile_functional.py +++ b/tests/test_web_sendfile_functional.py @@ -742,7 +742,7 @@ async def handler(request): resp = await client.get('/') assert resp.status == 200 - zcomp = zlib.compressobj(wbits=-zlib.MAX_WBITS) + zcomp = zlib.compressobj(wbits=zlib.MAX_WBITS) expected_body = zcomp.compress(b'file content\n') + zcomp.flush() assert expected_body == await resp.read() assert 'application/octet-stream' == resp.headers['Content-Type']