From fcedc6603c18c96219bc3ac48484475d8e1cfc25 Mon Sep 17 00:00:00 2001 From: Alexander Shorin Date: Tue, 15 Jan 2019 10:01:13 +0300 Subject: [PATCH] Support reading multipart data with `\n` (`LF`) lines (#3492) * Support reading multipart data with `\n` (`LF`) lines While RFC clearly says about `CRLF` newlines, there quite a lot of implementations which uses just `LF`. Even Python's stdlib produces multiparts with `\n` newlines by default for compatibility reasons. We wouldn't change how we produce multipart content - here we follow RFC. However, we can detect `\n` lines quite easily which makes their support quite cheap. * Add test about mixed newlines Just for case. That's a strange case, but it seems we pass it. * Make newline argument as keyword one and explicitly private one This argument is not designed to be defined by users. Depending on parsing multipart newline format it will be chosen automatically and due to recursive native of multipart format it have to be passed around for nested readers. --- CHANGES/2302.feature | 1 + aiohttp/multipart.py | 53 +++- tests/test_multipart.py | 583 ++++++++++++++++++++++++++-------------- 3 files changed, 429 insertions(+), 208 deletions(-) create mode 100644 CHANGES/2302.feature diff --git a/CHANGES/2302.feature b/CHANGES/2302.feature new file mode 100644 index 00000000000..697d8c45a6f --- /dev/null +++ b/CHANGES/2302.feature @@ -0,0 +1 @@ +Support reading multipart data with `\n` (`LF`) lines diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index b16598eb77b..02e9f0065a8 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -237,11 +237,17 @@ class BodyPartReader: chunk_size = 8192 - def __init__(self, boundary: bytes, - headers: Mapping[str, Optional[str]], - content: StreamReader) -> None: + def __init__( + self, + boundary: bytes, + headers: Mapping[str, Optional[str]], + content: StreamReader, + *, + _newline: bytes = b'\r\n' + ) -> None: self.headers = headers self._boundary = boundary + self._newline = _newline self._content = content self._at_eof = False length = self.headers.get(CONTENT_LENGTH, None) @@ -300,8 +306,8 @@ async def read_chunk(self, size: int=chunk_size) -> bytes: if self._read_bytes == self._length: self._at_eof = True if self._at_eof: - clrf = await self._content.readline() - assert b'\r\n' == clrf, \ + newline = await self._content.readline() + assert newline == self._newline, \ 'reader did not read all the data or it is malformed' return chunk @@ -328,11 +334,15 @@ async def _read_chunk_from_stream(self, size: int) -> bytes: assert self._content_eof < 3, "Reading after EOF" assert self._prev_chunk is not None window = self._prev_chunk + chunk - sub = b'\r\n' + self._boundary + + intermeditate_boundary = self._newline + self._boundary + if first_chunk: - idx = window.find(sub) + pos = 0 else: - idx = window.find(sub, max(0, len(self._prev_chunk) - len(sub))) + pos = max(0, len(self._prev_chunk) - len(intermeditate_boundary)) + + idx = window.find(intermeditate_boundary, pos) if idx >= 0: # pushing boundary back to content with warnings.catch_warnings(): @@ -344,6 +354,7 @@ async def _read_chunk_from_stream(self, size: int) -> bytes: chunk = window[len(self._prev_chunk):idx] if not chunk: self._at_eof = True + result = self._prev_chunk self._prev_chunk = chunk return result @@ -372,7 +383,8 @@ async def readline(self) -> bytes: else: next_line = await self._content.readline() if next_line.startswith(self._boundary): - line = line[:-2] # strip CRLF but only once + # strip newline but only once + line = line[:-len(self._newline)] self._unread.append(next_line) return line @@ -516,10 +528,16 @@ class MultipartReader: #: Body part reader class for non multipart/* content types. part_reader_cls = BodyPartReader - def __init__(self, headers: Mapping[str, str], - content: StreamReader) -> None: + def __init__( + self, + headers: Mapping[str, str], + content: StreamReader, + *, + _newline: bytes = b'\r\n' + ) -> None: self.headers = headers self._boundary = ('--' + self._get_boundary()).encode() + self._newline = _newline self._content = content self._last_part = None self._at_eof = False @@ -592,9 +610,13 @@ def _get_part_reader(self, headers: 'CIMultiDictProxy[str]') -> Any: if mimetype.type == 'multipart': if self.multipart_reader_cls is None: return type(self)(headers, self._content) - return self.multipart_reader_cls(headers, self._content) + return self.multipart_reader_cls( + headers, self._content, _newline=self._newline + ) else: - return self.part_reader_cls(self._boundary, headers, self._content) + return self.part_reader_cls( + self._boundary, headers, self._content, _newline=self._newline + ) def _get_boundary(self) -> str: mimetype = parse_mimetype(self.headers[CONTENT_TYPE]) @@ -625,6 +647,11 @@ async def _read_until_first_boundary(self) -> None: if chunk == b'': raise ValueError("Could not find starting boundary %r" % (self._boundary)) + if chunk.startswith(self._boundary): + _, newline = chunk.split(self._boundary, 1) + assert newline in (b'\r\n', b'\n') + self._newline = newline + chunk = chunk.rstrip() if chunk == self._boundary: return diff --git a/tests/test_multipart.py b/tests/test_multipart.py index fd7c4b16283..8518a082715 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -23,6 +23,11 @@ BOUNDARY = b'--:' +def pytest_generate_tests(metafunc): # pragma: no cover + if "newline" in metafunc.fixturenames: + metafunc.parametrize("newline", [b'\r\n', b'\n'], ids=str) + + @pytest.fixture def buf(): return bytearray() @@ -118,46 +123,53 @@ async def test_release_when_stream_at_eof(self) -> None: class TestPartReader: - async def test_next(self) -> None: + async def test_next(self, newline) -> None: + data = b'Hello, world!%s--:' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello, world!\r\n--:')) + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.next() assert b'Hello, world!' == result assert obj.at_eof() - async def test_next_next(self) -> None: + async def test_next_next(self, newline) -> None: + data = b'Hello, world!%s--:' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello, world!\r\n--:')) + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.next() assert b'Hello, world!' == result assert obj.at_eof() result = await obj.next() assert result is None - async def test_read(self) -> None: + async def test_read(self, newline) -> None: + data = b'Hello, world!%s--:' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello, world!\r\n--:')) + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.read() assert b'Hello, world!' == result assert obj.at_eof() async def test_read_chunk_at_eof(self) -> None: - obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'--:')) + obj = aiohttp.BodyPartReader(BOUNDARY, {}, Stream(b'--:')) obj._at_eof = True result = await obj.read_chunk() assert b'' == result - async def test_read_chunk_without_content_length(self) -> None: + async def test_read_chunk_without_content_length(self, newline) -> None: + data = b'Hello, world!%s--:' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello, world!\r\n--:')) + BOUNDARY, {}, Stream(data), _newline=newline + ) c1 = await obj.read_chunk(8) c2 = await obj.read_chunk(8) c3 = await obj.read_chunk(8) assert c1 + c2 == b'Hello, world!' assert c3 == b'' - async def test_read_incomplete_chunk(self) -> None: + async def test_read_incomplete_chunk(self, newline) -> None: loop = asyncio.get_event_loop() stream = Stream(b'') @@ -169,11 +181,12 @@ def prepare(data): with mock.patch.object(stream, 'read', side_effect=[ prepare(b'Hello, '), prepare(b'World'), - prepare(b'!\r\n--:'), + prepare(b'!%s--:' % newline), prepare(b'') ]): obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + BOUNDARY, {}, stream, _newline=newline + ) c1 = await obj.read_chunk(8) assert c1 == b'Hello, ' c2 = await obj.read_chunk(8) @@ -181,25 +194,29 @@ def prepare(data): c3 = await obj.read_chunk(8) assert c3 == b'!' - async def test_read_all_at_once(self) -> None: - stream = Stream(b'Hello, World!\r\n--:--\r\n') - obj = aiohttp.BodyPartReader(BOUNDARY, {}, stream) + async def test_read_all_at_once(self, newline) -> None: + data = b'Hello, World!%s--:--%s' % (newline, newline) + obj = aiohttp.BodyPartReader( + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.read_chunk() assert b'Hello, World!' == result result = await obj.read_chunk() assert b'' == result assert obj.at_eof() - async def test_read_incomplete_body_chunked(self) -> None: - stream = Stream(b'Hello, World!\r\n-') - obj = aiohttp.BodyPartReader(BOUNDARY, {}, stream) + async def test_read_incomplete_body_chunked(self, newline) -> None: + data = b'Hello, World!%s--' % newline + obj = aiohttp.BodyPartReader( + BOUNDARY, {}, Stream(data), _newline=newline + ) result = b'' with pytest.raises(AssertionError): for _ in range(4): result += await obj.read_chunk(7) - assert b'Hello, World!\r\n-' == result + assert data == result - async def test_read_boundary_with_incomplete_chunk(self) -> None: + async def test_read_boundary_with_incomplete_chunk(self, newline) -> None: loop = asyncio.get_event_loop() stream = Stream(b'') @@ -210,12 +227,13 @@ def prepare(data): with mock.patch.object(stream, 'read', side_effect=[ prepare(b'Hello, World'), - prepare(b'!\r\n'), + prepare(b'!%s' % newline), prepare(b'--:'), prepare(b'') ]): obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + BOUNDARY, {}, stream, _newline=newline + ) c1 = await obj.read_chunk(12) assert c1 == b'Hello, World' c2 = await obj.read_chunk(8) @@ -223,21 +241,28 @@ def prepare(data): c3 = await obj.read_chunk(8) assert c3 == b'' - async def test_multi_read_chunk(self) -> None: - stream = Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--') - obj = aiohttp.BodyPartReader(BOUNDARY, {}, stream) + async def test_multi_read_chunk(self, newline) -> None: + data = b'Hello,%s--:%s%sworld!%s--:--' % ((newline,) * 4) + obj = aiohttp.BodyPartReader( + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.read_chunk(8) assert b'Hello,' == result result = await obj.read_chunk(8) assert b'' == result assert obj.at_eof() - async def test_read_chunk_properly_counts_read_bytes(self) -> None: + async def test_read_chunk_properly_counts_read_bytes( + self, newline + ) -> None: expected = b'.' * 10 + tail = b'%s--:--' % newline size = len(expected) obj = aiohttp.BodyPartReader( BOUNDARY, {'CONTENT-LENGTH': size}, - StreamWithShortenRead(expected + b'\r\n--:--')) + StreamWithShortenRead(expected + tail), + _newline=newline, + ) result = bytearray() while True: chunk = await obj.read_chunk() @@ -248,86 +273,111 @@ async def test_read_chunk_properly_counts_read_bytes(self) -> None: assert b'.' * size == result assert obj.at_eof() - async def test_read_does_not_read_boundary(self) -> None: - stream = Stream(b'Hello, world!\r\n--:') - obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + async def test_read_does_not_read_boundary(self, newline) -> None: + data = b'Hello, world!%s--:' % newline + stream = Stream(data) + obj = aiohttp.BodyPartReader(BOUNDARY, {}, stream, _newline=newline) result = await obj.read() assert b'Hello, world!' == result assert b'--:' == (await stream.read()) - async def test_multiread(self) -> None: + async def test_multiread(self, newline) -> None: + data = b'Hello,%s--:%s%sworld!%s--:--' % ((newline,) * 4) obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--')) + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.read() assert b'Hello,' == result result = await obj.read() assert b'' == result assert obj.at_eof() - async def test_read_multiline(self) -> None: + async def test_read_multiline(self, newline) -> None: + data = b'Hello\n,\r\nworld!%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello\n,\r\nworld!\r\n--:--')) + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.read() assert b'Hello\n,\r\nworld!' == result result = await obj.read() assert b'' == result assert obj.at_eof() - async def test_read_respects_content_length(self) -> None: + async def test_read_respects_content_length(self, newline) -> None: + data = b'.' * 100500 + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {'CONTENT-LENGTH': 100500}, - Stream(b'.' * 100500 + b'\r\n--:--')) + BOUNDARY, + {'CONTENT-LENGTH': 100500}, + Stream(data + tail), + _newline=newline, + ) result = await obj.read() - assert b'.' * 100500 == result + assert data == result assert obj.at_eof() - async def test_read_with_content_encoding_gzip(self) -> None: + async def test_read_with_content_encoding_gzip(self, newline) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_ENCODING: 'gzip'}, Stream(b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x0b\xc9\xccMU' b'(\xc9W\x08J\xcdI\xacP\x04\x00$\xfb\x9eV\x0e\x00\x00\x00' - b'\r\n--:--')) + b'%s--:--' % newline), + _newline=newline, + ) result = await obj.read(decode=True) assert b'Time to Relax!' == result - async def test_read_with_content_encoding_deflate(self) -> None: + async def test_read_with_content_encoding_deflate(self, newline) -> None: + data = b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00' + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_ENCODING: 'deflate'}, - Stream(b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00\r\n--:--')) + Stream(data + tail), + _newline=newline, + ) result = await obj.read(decode=True) assert b'Time to Relax!' == result - async def test_read_with_content_encoding_identity(self) -> None: + async def test_read_with_content_encoding_identity(self, newline) -> None: thing = (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x0b\xc9\xccMU' - b'(\xc9W\x08J\xcdI\xacP\x04\x00$\xfb\x9eV\x0e\x00\x00\x00' - b'\r\n') + b'(\xc9W\x08J\xcdI\xacP\x04\x00$\xfb\x9eV\x0e\x00\x00\x00') obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_ENCODING: 'identity'}, - Stream(thing + b'--:--')) + Stream(thing + b'%s--:--' % newline), + _newline=newline, + ) result = await obj.read(decode=True) - assert thing[:-2] == result + assert thing == result - async def test_read_with_content_encoding_unknown(self) -> None: + async def test_read_with_content_encoding_unknown(self, newline) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_ENCODING: 'snappy'}, - Stream(b'\x0e4Time to Relax!\r\n--:--')) + Stream(b'\x0e4Time to Relax!%s--:--' % newline), + _newline=newline, + ) with pytest.raises(RuntimeError): await obj.read(decode=True) - async def test_read_with_content_transfer_encoding_base64(self) -> None: + async def test_read_with_content_transfer_encoding_base64( + self, newline + ) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TRANSFER_ENCODING: 'base64'}, - Stream(b'VGltZSB0byBSZWxheCE=\r\n--:--')) + Stream(b'VGltZSB0byBSZWxheCE=%s--:--' % newline), + _newline=newline, + ) result = await obj.read(decode=True) assert b'Time to Relax!' == result async def test_read_with_content_transfer_encoding_quoted_printable( - self) -> None: + self, newline + ) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TRANSFER_ENCODING: 'quoted-printable'}, Stream(b'=D0=9F=D1=80=D0=B8=D0=B2=D0=B5=D1=82,' - b' =D0=BC=D0=B8=D1=80!\r\n--:--')) + b' =D0=BC=D0=B8=D1=80!%s--:--' % newline), + _newline=newline, + ) result = await obj.read(decode=True) expected = (b'\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82,' b' \xd0\xbc\xd0\xb8\xd1\x80!') @@ -335,54 +385,86 @@ async def test_read_with_content_transfer_encoding_quoted_printable( @pytest.mark.parametrize('encoding', ('binary', '8bit', '7bit')) async def test_read_with_content_transfer_encoding_binary( - self, encoding) -> None: + self, encoding, newline + ) -> None: data = b'\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82,' \ b' \xd0\xbc\xd0\xb8\xd1\x80!' obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TRANSFER_ENCODING: encoding}, - Stream(data + b'\r\n--:--')) + Stream(data + b'%s--:--' % newline), + _newline=newline, + ) result = await obj.read(decode=True) assert data == result - async def test_read_with_content_transfer_encoding_unknown(self) -> None: + async def test_read_with_content_transfer_encoding_unknown( + self, newline + ) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TRANSFER_ENCODING: 'unknown'}, - Stream(b'\x0e4Time to Relax!\r\n--:--')) + Stream(b'\x0e4Time to Relax!%s--:--' % newline), + _newline=newline, + ) with pytest.raises(RuntimeError): await obj.read(decode=True) - async def test_read_text(self) -> None: + async def test_read_text(self, newline) -> None: obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello, world!\r\n--:--')) + BOUNDARY, + {}, + Stream(b'Hello, world!%s--:--' % newline), + _newline=newline, + ) result = await obj.text() assert 'Hello, world!' == result - async def test_read_text_default_encoding(self) -> None: + async def test_read_text_default_encoding(self, newline) -> None: + data = 'Привет, Мир!' + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, - Stream('Привет, Мир!\r\n--:--'.encode('utf-8'))) + BOUNDARY, + {}, + Stream(data.encode('utf-8') + tail), + _newline=newline, + ) result = await obj.text() - assert 'Привет, Мир!' == result + assert data == result - async def test_read_text_encoding(self) -> None: + async def test_read_text_encoding(self, newline) -> None: + data = 'Привет, Мир!' + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, - Stream('Привет, Мир!\r\n--:--'.encode('cp1251'))) + BOUNDARY, + {}, + Stream(data.encode('cp1251') + tail), + _newline=newline, + ) result = await obj.text(encoding='cp1251') - assert 'Привет, Мир!' == result + assert data == result - async def test_read_text_guess_encoding(self) -> None: + async def test_read_text_guess_encoding(self, newline) -> None: + data = 'Привет, Мир!' + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {CONTENT_TYPE: 'text/plain;charset=cp1251'}, - Stream('Привет, Мир!\r\n--:--'.encode('cp1251'))) + BOUNDARY, + {CONTENT_TYPE: 'text/plain;charset=cp1251'}, + Stream(data.encode('cp1251') + tail), + _newline=newline, + ) result = await obj.text() - assert 'Привет, Мир!' == result + assert data == result - async def test_read_text_compressed(self) -> None: + async def test_read_text_compressed(self, newline) -> None: + data = ( + b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00' + b'%s--:--' % newline + ) obj = aiohttp.BodyPartReader( - BOUNDARY, {CONTENT_ENCODING: 'deflate', - CONTENT_TYPE: 'text/plain'}, - Stream(b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00\r\n--:--')) + BOUNDARY, + {CONTENT_ENCODING: 'deflate', CONTENT_TYPE: 'text/plain'}, + Stream(data), + _newline=newline, + ) result = await obj.text() assert 'Time to Relax!' == result @@ -393,32 +475,45 @@ async def test_read_text_while_closed(self) -> None: result = await obj.text() assert '' == result - async def test_read_json(self) -> None: + async def test_read_json(self, newline) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/json'}, - Stream(b'{"test": "passed"}\r\n--:--')) + Stream(b'{"test": "passed"}%s--:--' % newline), + _newline=newline, + ) result = await obj.json() assert {'test': 'passed'} == result - async def test_read_json_encoding(self) -> None: + async def test_read_json_encoding(self, newline) -> None: + data = '{"тест": "пассед"}'.encode('cp1251') + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/json'}, - Stream('{"тест": "пассед"}\r\n--:--'.encode('cp1251'))) + Stream(data + tail), + _newline=newline, + ) result = await obj.json(encoding='cp1251') assert {'тест': 'пассед'} == result - async def test_read_json_guess_encoding(self) -> None: + async def test_read_json_guess_encoding(self, newline) -> None: + data = '{"тест": "пассед"}'.encode('cp1251') + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/json; charset=cp1251'}, - Stream('{"тест": "пассед"}\r\n--:--'.encode('cp1251'))) + Stream(data + tail), + _newline=newline, + ) result = await obj.json() assert {'тест': 'пассед'} == result - async def test_read_json_compressed(self) -> None: + async def test_read_json_compressed(self, newline) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_ENCODING: 'deflate', CONTENT_TYPE: 'application/json'}, - Stream(b'\xabV*I-.Q\xb2RP*H,.NMQ\xaa\x05\x00\r\n--:--')) + Stream(b'\xabV*I-.Q\xb2RP*H,.NMQ\xaa\x05\x00' + b'%s--:--' % newline), + _newline=newline, + ) result = await obj.json() assert {'test': 'passed'} == result @@ -430,25 +525,34 @@ async def test_read_json_while_closed(self) -> None: result = await obj.json() assert result is None - async def test_read_form(self) -> None: + async def test_read_form(self, newline) -> None: + data = b'foo=bar&foo=baz&boo=%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/x-www-form-urlencoded'}, - Stream(b'foo=bar&foo=baz&boo=\r\n--:--')) + Stream(data), + _newline=newline, + ) result = await obj.form() assert [('foo', 'bar'), ('foo', 'baz'), ('boo', '')] == result - async def test_read_form_encoding(self) -> None: + async def test_read_form_encoding(self, newline) -> None: + data = b'foo=bar&foo=baz&boo=%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/x-www-form-urlencoded'}, - Stream('foo=bar&foo=baz&boo=\r\n--:--'.encode('cp1251'))) + Stream(data), + _newline=newline, + ) result = await obj.form(encoding='cp1251') assert [('foo', 'bar'), ('foo', 'baz'), ('boo', '')] == result - async def test_read_form_guess_encoding(self) -> None: + async def test_read_form_guess_encoding(self, newline) -> None: + data = b'foo=bar&foo=baz&boo=%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/x-www-form-urlencoded; charset=utf-8'}, - Stream('foo=bar&foo=baz&boo=\r\n--:--'.encode('utf-8'))) + Stream(data), + _newline=newline, + ) result = await obj.form() assert [('foo', 'bar'), ('foo', 'baz'), ('boo', '')] == result @@ -456,14 +560,21 @@ async def test_read_form_while_closed(self) -> None: stream = Stream(b'') obj = aiohttp.BodyPartReader( BOUNDARY, - {CONTENT_TYPE: 'application/x-www-form-urlencoded'}, stream) + {CONTENT_TYPE: 'application/x-www-form-urlencoded'}, + stream, + ) obj._at_eof = True result = await obj.form() assert not result - async def test_readline(self) -> None: + async def test_readline(self, newline) -> None: + data = b'Hello\n,\r\nworld!%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello\n,\r\nworld!\r\n--:--')) + BOUNDARY, + {}, + Stream(data), + _newline=newline, + ) result = await obj.readline() assert b'Hello\n' == result result = await obj.readline() @@ -474,29 +585,43 @@ async def test_readline(self) -> None: assert b'' == result assert obj.at_eof() - async def test_release(self) -> None: - stream = Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--') + async def test_release(self, newline) -> None: + data = b'Hello,%s--:\r\n\r\nworld!%s--:--' % (newline, newline) + stream = Stream(data) obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + BOUNDARY, + {}, + stream, + _newline=newline, + ) + remained = b'--:\r\n\r\nworld!%s--:--' % newline await obj.release() assert obj.at_eof() - assert b'--:\r\n\r\nworld!\r\n--:--' == stream.content.read() + assert remained == stream.content.read() - async def test_release_respects_content_length(self) -> None: + async def test_release_respects_content_length(self, newline) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {'CONTENT-LENGTH': 100500}, - Stream(b'.' * 100500 + b'\r\n--:--')) + Stream(b'.' * 100500 + b'%s--:--' % newline), + _newline=newline, + ) result = await obj.release() assert result is None assert obj.at_eof() - async def test_release_release(self) -> None: - stream = Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--') + async def test_release_release(self, newline) -> None: + data = b'Hello,%s--:\r\n\r\nworld!%s--:--' % (newline, newline) + remained = b'--:\r\n\r\nworld!%s--:--' % newline + stream = Stream(data) obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + BOUNDARY, + {}, + stream, + _newline=newline, + ) await obj.release() await obj.release() - assert b'--:\r\n\r\nworld!\r\n--:--' == stream.content.read() + assert remained == stream.content.read() async def test_filename(self) -> None: part = aiohttp.BodyPartReader( @@ -505,23 +630,22 @@ async def test_filename(self) -> None: None) assert 'foo.html' == part.filename - async def test_reading_long_part(self) -> None: + async def test_reading_long_part(self, newline) -> None: size = 2 * stream_reader_default_limit protocol = mock.Mock(_reading_paused=False) stream = StreamReader(protocol) - stream.feed_data(b'0' * size + b'\r\n--:--') + stream.feed_data(b'0' * size + b'%s--:--' % newline) stream.feed_eof() - obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + obj = aiohttp.BodyPartReader(BOUNDARY, {}, stream, _newline=newline) data = await obj.read() assert len(data) == size class TestMultipartReader: - def test_from_response(self) -> None: + def test_from_response(self, newline) -> None: resp = Response({CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\nhello\r\n--:--')) + Stream(b'--:%s\r\nhello%s--:--' % (newline, newline))) res = aiohttp.MultipartReader.from_response(resp) assert isinstance(res, MultipartResponseWrapper) @@ -535,129 +659,154 @@ def test_bad_boundary(self) -> None: with pytest.raises(ValueError): aiohttp.MultipartReader.from_response(resp) - def test_dispatch(self) -> None: + def test_dispatch(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\necho\r\n--:--')) + Stream(b'--:%s\r\necho%s--:--' % (newline, newline))) res = reader._get_part_reader({CONTENT_TYPE: 'text/plain'}) assert isinstance(res, reader.part_reader_cls) - def test_dispatch_bodypart(self) -> None: + def test_dispatch_bodypart(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\necho\r\n--:--')) + Stream(b'--:%s\r\necho%s--:--' % (newline, newline))) res = reader._get_part_reader({CONTENT_TYPE: 'text/plain'}) assert isinstance(res, reader.part_reader_cls) - def test_dispatch_multipart(self) -> None: + def test_dispatch_multipart(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'----:--\r\n' - b'\r\n' - b'test\r\n' - b'----:--\r\n' - b'\r\n' - b'passed\r\n' - b'----:----\r\n' - b'--:--')) + Stream( + newline.join([ + b'----:--', + b'', + b'test', + b'----:--', + b'', + b'passed', + b'----:----' + b'--:--', + ]) + ) + ) res = reader._get_part_reader( {CONTENT_TYPE: 'multipart/related;boundary=--:--'}) assert isinstance(res, reader.__class__) - def test_dispatch_custom_multipart_reader(self) -> None: + def test_dispatch_custom_multipart_reader(self, newline) -> None: class CustomReader(aiohttp.MultipartReader): pass reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'----:--\r\n' - b'\r\n' - b'test\r\n' - b'----:--\r\n' - b'\r\n' - b'passed\r\n' - b'----:----\r\n' - b'--:--')) + Stream( + newline.join([ + b'----:--', + b'', + b'test', + b'----:--', + b'', + b'passed', + b'----:----', + b'--:--', + ]) + ) + ) reader.multipart_reader_cls = CustomReader res = reader._get_part_reader( {CONTENT_TYPE: 'multipart/related;boundary=--:--'}) assert isinstance(res, CustomReader) - async def test_emit_next(self) -> None: + async def test_emit_next(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\necho\r\n--:--')) + Stream(b'--:%s\r\necho%s--:--' % (newline, newline))) res = await reader.next() assert isinstance(res, reader.part_reader_cls) - async def test_invalid_boundary(self) -> None: + async def test_invalid_boundary(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'---:\r\n\r\necho\r\n---:--')) + Stream(b'---:%s\r\necho%s---:--' % (newline, newline))) with pytest.raises(ValueError): await reader.next() - async def test_release(self) -> None: + async def test_release(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/mixed;boundary=":"'}, - Stream(b'--:\r\n' - b'Content-Type: multipart/related;boundary=--:--\r\n' - b'\r\n' - b'----:--\r\n' - b'\r\n' - b'test\r\n' - b'----:--\r\n' - b'\r\n' - b'passed\r\n' - b'----:----\r\n' - b'\r\n' - b'--:--')) + Stream( + newline.join([ + b'--:', + b'Content-Type: multipart/related;boundary=--:--', + b'', + b'----:--', + b'', + b'test', + b'----:--', + b'', + b'passed', + b'----:----', + b'', + b'--:--', + ]) + ) + ) await reader.release() assert reader.at_eof() - async def test_release_release(self) -> None: + async def test_release_release(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\necho\r\n--:--')) + Stream(b'--:%s\r\necho%s--:--' % (newline, newline))) await reader.release() assert reader.at_eof() await reader.release() assert reader.at_eof() - async def test_release_next(self) -> None: + async def test_release_next(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\necho\r\n--:--')) + Stream(b'--:%s\r\necho%s--:--' % (newline, newline))) await reader.release() assert reader.at_eof() res = await reader.next() assert res is None - async def test_second_next_releases_previous_object(self) -> None: + async def test_second_next_releases_previous_object(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n' - b'\r\n' - b'test\r\n' - b'--:\r\n' - b'\r\n' - b'passed\r\n' - b'--:--')) + Stream( + newline.join([ + b'--:', + b'', + b'test', + b'--:', + b'', + b'passed', + b'--:--', + ]) + ) + ) first = await reader.next() assert isinstance(first, aiohttp.BodyPartReader) second = await reader.next() assert first.at_eof() assert not second.at_eof() - async def test_release_without_read_the_last_object(self) -> None: + async def test_release_without_read_the_last_object(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n' - b'\r\n' - b'test\r\n' - b'--:\r\n' - b'\r\n' - b'passed\r\n' - b'--:--')) + Stream( + newline.join([ + b'--:', + b'', + b'test', + b'--:', + b'', + b'passed', + b'--:--', + ]) + ) + ) first = await reader.next() second = await reader.next() third = await reader.next() @@ -666,16 +815,23 @@ async def test_release_without_read_the_last_object(self) -> None: assert second.at_eof() assert third is None - async def test_read_chunk_by_length_doesnt_breaks_reader(self) -> None: + async def test_read_chunk_by_length_doesnt_breaks_reader( + self, newline + ) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n' - b'Content-Length: 4\r\n\r\n' - b'test' - b'\r\n--:\r\n' - b'Content-Length: 6\r\n\r\n' - b'passed' - b'\r\n--:--')) + Stream(newline.join([ + b'--:', + b'Content-Length: 4', + b'', + b'test', + b'--:', + b'Content-Length: 6', + b'', + b'passed', + b'--:--', + ])) + ) body_parts = [] while True: read_part = b'' @@ -687,16 +843,21 @@ async def test_read_chunk_by_length_doesnt_breaks_reader(self) -> None: body_parts.append(read_part) assert body_parts == [b'test', b'passed'] - async def test_read_chunk_from_stream_doesnt_breaks_reader(self) -> None: + async def test_read_chunk_from_stream_doesnt_breaks_reader( + self, newline + ) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n' - b'\r\n' - b'chunk' - b'\r\n--:\r\n' - b'\r\n' - b'two_chunks' - b'\r\n--:--')) + Stream(newline.join([ + b'--:', + b'', + b'chunk', + b'--:', + b'', + b'two_chunks', + b'--:--', + ])) + ) body_parts = [] while True: read_part = b'' @@ -710,24 +871,56 @@ async def test_read_chunk_from_stream_doesnt_breaks_reader(self) -> None: body_parts.append(read_part) assert body_parts == [b'chunk', b'two_chunks'] - async def test_reading_skips_prelude(self) -> None: + async def test_reading_skips_prelude(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'Multi-part data is not supported.\r\n' - b'\r\n' - b'--:\r\n' - b'\r\n' - b'test\r\n' - b'--:\r\n' - b'\r\n' - b'passed\r\n' - b'--:--')) + Stream(newline.join([ + b'Multi-part data is not supported.', + b'', + b'--:', + b'', + b'test', + b'--:', + b'', + b'passed', + b'--:--' + ])) + ) first = await reader.next() assert isinstance(first, aiohttp.BodyPartReader) second = await reader.next() assert first.at_eof() assert not second.at_eof() + async def test_read_mixed_newlines(self) -> None: + reader = aiohttp.MultipartReader( + {CONTENT_TYPE: 'multipart/mixed;boundary=":"'}, + Stream( + b''.join([ + b'--:\n', + b'Content-Type: multipart/related;boundary=--:--\n', + b'\n', + b'----:--\r\n', + b'\r\n', + b'test\r\n', + b'----:--\r\n', + b'\r\n', + b'passed\r\n', + b'----:----\r\n', + b'\n', + b'--:--', + ]) + ) + ) + while True: + part = await reader.next() + if part is None: + break + while True: + subpart = await part.next() + if subpart is None: + break + async def test_writer(writer) -> None: assert writer.size == 0