Fix deflate compression (#4506)

CHANGES/4506.bugfix

@@ -0,0 +1 @@
+Fixed 'deflate' compressions. According to RFC 2616 now.

aiohttp/http_parser.py

@@ -727,7 +727,7 @@ def flush(self) -> bytes:
             self.decompressor = BrotliDecoder()  # type: Any
         else:
             zlib_mode = (16 + zlib.MAX_WBITS
-                         if encoding == 'gzip' else -zlib.MAX_WBITS)
+                         if encoding == 'gzip' else zlib.MAX_WBITS)
             self.decompressor = zlib.decompressobj(wbits=zlib_mode)
 
     def set_exception(self, exc: BaseException) -> None:
@@ -739,7 +739,8 @@ def feed_data(self, chunk: bytes, size: int) -> None:
             chunk = self.decompressor.decompress(chunk)
         except Exception:
             if not self._started_decoding and self.encoding == 'deflate':
-                self.decompressor = zlib.decompressobj()
+                # Try to change the decoder to decompress incorrectly compressed data
+                self.decompressor = zlib.decompressobj(wbits=-zlib.MAX_WBITS)
                 try:
                     chunk = self.decompressor.decompress(chunk)
                 except Exception:

aiohttp/http_writer.py

@@ -55,7 +55,7 @@ def enable_chunking(self) -> None:
 
     def enable_compression(self, encoding: str='deflate') -> None:
         zlib_mode = (16 + zlib.MAX_WBITS
-                     if encoding == 'gzip' else -zlib.MAX_WBITS)
+                     if encoding == 'gzip' else zlib.MAX_WBITS)
         self._compress = zlib.compressobj(wbits=zlib_mode)
 
     def _write(self, chunk: bytes) -> None:

aiohttp/multipart.py

@@ -479,7 +479,10 @@ def _decode_content(self, data: bytes) -> bytes:
         encoding = self.headers.get(CONTENT_ENCODING, '').lower()
 
         if encoding == 'deflate':
-            return zlib.decompress(data, -zlib.MAX_WBITS)
+            try:
+                return zlib.decompress(data, zlib.MAX_WBITS)
+            except zlib.error:
+                return zlib.decompress(data, -zlib.MAX_WBITS)
         elif encoding == 'gzip':
             return zlib.decompress(data, 16 + zlib.MAX_WBITS)
         elif encoding == 'identity':
@@ -978,7 +981,7 @@ def enable_encoding(self, encoding: str) -> None:
 
     def enable_compression(self, encoding: str='deflate') -> None:
         zlib_mode = (16 + zlib.MAX_WBITS
-                     if encoding == 'gzip' else -zlib.MAX_WBITS)
+                     if encoding == 'gzip' else zlib.MAX_WBITS)
         self._compress = zlib.compressobj(wbits=zlib_mode)
 
     async def write_eof(self) -> None:

aiohttp/web_response.py

@@ -669,6 +669,7 @@ async def _start(self, request: 'BaseRequest') -> AbstractStreamWriter:
         return await super()._start(request)
 
     def _compress_body(self, zlib_mode: int) -> None:
+        assert zlib_mode > 0
         compressobj = zlib.compressobj(wbits=zlib_mode)
         body_in = self._body
         assert body_in is not None
@@ -683,7 +684,7 @@ async def _do_start_compression(self, coding: ContentCoding) -> None:
             # Instead of using _payload_writer.enable_compression,
             # compress the whole body
             zlib_mode = (16 + zlib.MAX_WBITS
-                         if coding == ContentCoding.gzip else -zlib.MAX_WBITS)
+                         if coding == ContentCoding.gzip else zlib.MAX_WBITS)
             body_in = self._body
             assert body_in is not None
             if self._zlib_executor_size is not None and \

tests/test_http_parser.py

@@ -837,7 +837,7 @@ async def test_http_payload_parser_length(self, stream) -> None:
         assert b'12' == b''.join(d for d, _ in out._buffer)
         assert b'45' == tail
 
-    _comp = zlib.compressobj(wbits=-zlib.MAX_WBITS)
+    _comp = zlib.compressobj(wbits=zlib.MAX_WBITS)
     _COMPRESSED = b''.join([_comp.compress(b'data'), _comp.flush()])
 
     async def test_http_payload_parser_deflate(self, stream) -> None:
@@ -905,7 +905,7 @@ async def test_feed_data_err(self, stream) -> None:
         dbuf.decompressor.decompress.side_effect = exc
 
         with pytest.raises(http_exceptions.ContentEncodingError):
-            dbuf.feed_data(b'data', 4)
+            dbuf.feed_data(b'data1', 5)
 
     async def test_feed_eof(self, stream) -> None:
         buf = aiohttp.FlowControlDataQueue(stream,

tests/test_http_writer.py

@@ -117,7 +117,7 @@ async def test_write_payload_chunked_filter_mutiple_chunks(
         b'2\r\na2\r\n0\r\n\r\n')
 
 
-compressor = zlib.compressobj(wbits=-zlib.MAX_WBITS)
+compressor = zlib.compressobj(wbits=zlib.MAX_WBITS)
 COMPRESSED = b''.join([compressor.compress(b'data'), compressor.flush()])
 
 
@@ -148,7 +148,7 @@ async def test_write_payload_deflate_and_chunked(
     await msg.write(b'ta')
     await msg.write_eof()
 
-    assert b'6\r\nKI,I\x04\x00\r\n0\r\n\r\n' == buf
+    assert b'2\r\nx\x9c\r\na\r\nKI,I\x04\x00\x04\x00\x01\x9b\r\n0\r\n\r\n' == buf
 
 
 async def test_write_drain(protocol, transport, loop) -> None:

tests/test_multipart.py

@@ -328,7 +328,8 @@ async def test_read_with_content_encoding_gzip(self, newline) -> None:
         result = await obj.read(decode=True)
         assert b'Time to Relax!' == result
 
-    async def test_read_with_content_encoding_deflate(self, newline) -> None:
+    async def test_read_with_content_encoding_deflate_no_headers(self, newline) -> None:
+        """ Test parsing of wrongly encoded deflate data (without zlib headers) """
         data = b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00'
         tail = b'%s--:--' % newline
         obj = aiohttp.BodyPartReader(
@@ -339,6 +340,17 @@ async def test_read_with_content_encoding_deflate(self, newline) -> None:
         result = await obj.read(decode=True)
         assert b'Time to Relax!' == result
 
+    async def test_read_with_content_encoding_deflate(self, newline) -> None:
+        data = b'x\x9c\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00$\xe3\x04\xd0'
+        tail = b'%s--:--' % newline
+        obj = aiohttp.BodyPartReader(
+            BOUNDARY, {CONTENT_ENCODING: 'deflate'},
+            Stream(data + tail),
+            _newline=newline,
+        )
+        result = await obj.read(decode=True)
+        assert b'Time to Relax!' == result
+
     async def test_read_with_content_encoding_identity(self, newline) -> None:
         thing = (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x0b\xc9\xccMU'
                  b'(\xc9W\x08J\xcdI\xacP\x04\x00$\xfb\x9eV\x0e\x00\x00\x00')
@@ -1086,7 +1098,7 @@ async def test_writer_serialize_with_content_encoding_deflate(buf, stream,
     assert (b'--:\r\nContent-Type: text/plain; charset=utf-8\r\n'
             b'Content-Encoding: deflate' == headers)
 
-    thing = b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00\r\n--:--\r\n'
+    thing = b'x\x9c\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00$\xe3\x04\xd0\r\n--:--\r\n'
     assert thing == message
 
 

tests/test_web_functional.py

@@ -953,7 +953,7 @@ async def test_response_with_precompressed_body_deflate(
 
     async def handler(request):
         headers = {'Content-Encoding': 'deflate'}
-        zcomp = zlib.compressobj(wbits=-zlib.MAX_WBITS)
+        zcomp = zlib.compressobj(wbits=zlib.MAX_WBITS)
         data = zcomp.compress(b'mydata') + zcomp.flush()
         return web.Response(body=data, headers=headers)
 

tests/test_web_sendfile_functional.py

@@ -742,7 +742,7 @@ async def handler(request):
 
     resp = await client.get('/')
     assert resp.status == 200
-    zcomp = zlib.compressobj(wbits=-zlib.MAX_WBITS)
+    zcomp = zlib.compressobj(wbits=zlib.MAX_WBITS)
     expected_body = zcomp.compress(b'file content\n') + zcomp.flush()
     assert expected_body == await resp.read()
     assert 'application/octet-stream' == resp.headers['Content-Type']