diff --git a/CHANGES/4506.bugfix b/CHANGES/4506.bugfix new file mode 100644 index 00000000000..eaf4bb88aac --- /dev/null +++ b/CHANGES/4506.bugfix @@ -0,0 +1 @@ +Fixed 'deflate' compressions. According to RFC 2616 now. diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index f2881c3bf11..93fbfc66a05 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -727,7 +727,7 @@ def flush(self) -> bytes: self.decompressor = BrotliDecoder() # type: Any else: zlib_mode = (16 + zlib.MAX_WBITS - if encoding == 'gzip' else -zlib.MAX_WBITS) + if encoding == 'gzip' else zlib.MAX_WBITS) self.decompressor = zlib.decompressobj(wbits=zlib_mode) def set_exception(self, exc: BaseException) -> None: @@ -739,7 +739,8 @@ def feed_data(self, chunk: bytes, size: int) -> None: chunk = self.decompressor.decompress(chunk) except Exception: if not self._started_decoding and self.encoding == 'deflate': - self.decompressor = zlib.decompressobj() + # Try to change the decoder to decompress incorrectly compressed data + self.decompressor = zlib.decompressobj(wbits=-zlib.MAX_WBITS) try: chunk = self.decompressor.decompress(chunk) except Exception: diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index 7e27fbf6a43..102fb3ef2f4 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -55,7 +55,7 @@ def enable_chunking(self) -> None: def enable_compression(self, encoding: str='deflate') -> None: zlib_mode = (16 + zlib.MAX_WBITS - if encoding == 'gzip' else -zlib.MAX_WBITS) + if encoding == 'gzip' else zlib.MAX_WBITS) self._compress = zlib.compressobj(wbits=zlib_mode) def _write(self, chunk: bytes) -> None: diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 7b01bf1f742..3fd5a96dae6 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -479,7 +479,10 @@ def _decode_content(self, data: bytes) -> bytes: encoding = self.headers.get(CONTENT_ENCODING, '').lower() if encoding == 'deflate': - return zlib.decompress(data, -zlib.MAX_WBITS) + try: + return zlib.decompress(data, zlib.MAX_WBITS) + except zlib.error: + return zlib.decompress(data, -zlib.MAX_WBITS) elif encoding == 'gzip': return zlib.decompress(data, 16 + zlib.MAX_WBITS) elif encoding == 'identity': @@ -978,7 +981,7 @@ def enable_encoding(self, encoding: str) -> None: def enable_compression(self, encoding: str='deflate') -> None: zlib_mode = (16 + zlib.MAX_WBITS - if encoding == 'gzip' else -zlib.MAX_WBITS) + if encoding == 'gzip' else zlib.MAX_WBITS) self._compress = zlib.compressobj(wbits=zlib_mode) async def write_eof(self) -> None: diff --git a/aiohttp/web_response.py b/aiohttp/web_response.py index 4dc64976839..fdb7aa4d359 100644 --- a/aiohttp/web_response.py +++ b/aiohttp/web_response.py @@ -669,6 +669,7 @@ async def _start(self, request: 'BaseRequest') -> AbstractStreamWriter: return await super()._start(request) def _compress_body(self, zlib_mode: int) -> None: + assert zlib_mode > 0 compressobj = zlib.compressobj(wbits=zlib_mode) body_in = self._body assert body_in is not None @@ -683,7 +684,7 @@ async def _do_start_compression(self, coding: ContentCoding) -> None: # Instead of using _payload_writer.enable_compression, # compress the whole body zlib_mode = (16 + zlib.MAX_WBITS - if coding == ContentCoding.gzip else -zlib.MAX_WBITS) + if coding == ContentCoding.gzip else zlib.MAX_WBITS) body_in = self._body assert body_in is not None if self._zlib_executor_size is not None and \ diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 19fe9be7a3c..8a8044ec021 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -837,7 +837,7 @@ async def test_http_payload_parser_length(self, stream) -> None: assert b'12' == b''.join(d for d, _ in out._buffer) assert b'45' == tail - _comp = zlib.compressobj(wbits=-zlib.MAX_WBITS) + _comp = zlib.compressobj(wbits=zlib.MAX_WBITS) _COMPRESSED = b''.join([_comp.compress(b'data'), _comp.flush()]) async def test_http_payload_parser_deflate(self, stream) -> None: @@ -905,7 +905,7 @@ async def test_feed_data_err(self, stream) -> None: dbuf.decompressor.decompress.side_effect = exc with pytest.raises(http_exceptions.ContentEncodingError): - dbuf.feed_data(b'data', 4) + dbuf.feed_data(b'data1', 5) async def test_feed_eof(self, stream) -> None: buf = aiohttp.FlowControlDataQueue(stream, diff --git a/tests/test_http_writer.py b/tests/test_http_writer.py index 2f8085f8a85..4328990faa9 100644 --- a/tests/test_http_writer.py +++ b/tests/test_http_writer.py @@ -117,7 +117,7 @@ async def test_write_payload_chunked_filter_mutiple_chunks( b'2\r\na2\r\n0\r\n\r\n') -compressor = zlib.compressobj(wbits=-zlib.MAX_WBITS) +compressor = zlib.compressobj(wbits=zlib.MAX_WBITS) COMPRESSED = b''.join([compressor.compress(b'data'), compressor.flush()]) @@ -148,7 +148,7 @@ async def test_write_payload_deflate_and_chunked( await msg.write(b'ta') await msg.write_eof() - assert b'6\r\nKI,I\x04\x00\r\n0\r\n\r\n' == buf + assert b'2\r\nx\x9c\r\na\r\nKI,I\x04\x00\x04\x00\x01\x9b\r\n0\r\n\r\n' == buf async def test_write_drain(protocol, transport, loop) -> None: diff --git a/tests/test_multipart.py b/tests/test_multipart.py index e491e82d16e..346e7cde31e 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -328,7 +328,8 @@ async def test_read_with_content_encoding_gzip(self, newline) -> None: result = await obj.read(decode=True) assert b'Time to Relax!' == result - async def test_read_with_content_encoding_deflate(self, newline) -> None: + async def test_read_with_content_encoding_deflate_no_headers(self, newline) -> None: + """ Test parsing of wrongly encoded deflate data (without zlib headers) """ data = b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00' tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( @@ -339,6 +340,17 @@ async def test_read_with_content_encoding_deflate(self, newline) -> None: result = await obj.read(decode=True) assert b'Time to Relax!' == result + async def test_read_with_content_encoding_deflate(self, newline) -> None: + data = b'x\x9c\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00$\xe3\x04\xd0' + tail = b'%s--:--' % newline + obj = aiohttp.BodyPartReader( + BOUNDARY, {CONTENT_ENCODING: 'deflate'}, + Stream(data + tail), + _newline=newline, + ) + result = await obj.read(decode=True) + assert b'Time to Relax!' == result + async def test_read_with_content_encoding_identity(self, newline) -> None: thing = (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x0b\xc9\xccMU' b'(\xc9W\x08J\xcdI\xacP\x04\x00$\xfb\x9eV\x0e\x00\x00\x00') @@ -1086,7 +1098,7 @@ async def test_writer_serialize_with_content_encoding_deflate(buf, stream, assert (b'--:\r\nContent-Type: text/plain; charset=utf-8\r\n' b'Content-Encoding: deflate' == headers) - thing = b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00\r\n--:--\r\n' + thing = b'x\x9c\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00$\xe3\x04\xd0\r\n--:--\r\n' assert thing == message diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 4be7a962303..896c092189b 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -953,7 +953,7 @@ async def test_response_with_precompressed_body_deflate( async def handler(request): headers = {'Content-Encoding': 'deflate'} - zcomp = zlib.compressobj(wbits=-zlib.MAX_WBITS) + zcomp = zlib.compressobj(wbits=zlib.MAX_WBITS) data = zcomp.compress(b'mydata') + zcomp.flush() return web.Response(body=data, headers=headers) diff --git a/tests/test_web_sendfile_functional.py b/tests/test_web_sendfile_functional.py index c2be5dbff0d..02aceb69f7b 100644 --- a/tests/test_web_sendfile_functional.py +++ b/tests/test_web_sendfile_functional.py @@ -742,7 +742,7 @@ async def handler(request): resp = await client.get('/') assert resp.status == 200 - zcomp = zlib.compressobj(wbits=-zlib.MAX_WBITS) + zcomp = zlib.compressobj(wbits=zlib.MAX_WBITS) expected_body = zcomp.compress(b'file content\n') + zcomp.flush() assert expected_body == await resp.read() assert 'application/octet-stream' == resp.headers['Content-Type']