diff --git a/CHANGES/2302.feature b/CHANGES/2302.feature new file mode 100644 index 00000000000..697d8c45a6f --- /dev/null +++ b/CHANGES/2302.feature @@ -0,0 +1 @@ +Support reading multipart data with `\n` (`LF`) lines diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index b16598eb77b..02e9f0065a8 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -237,11 +237,17 @@ class BodyPartReader: chunk_size = 8192 - def __init__(self, boundary: bytes, - headers: Mapping[str, Optional[str]], - content: StreamReader) -> None: + def __init__( + self, + boundary: bytes, + headers: Mapping[str, Optional[str]], + content: StreamReader, + *, + _newline: bytes = b'\r\n' + ) -> None: self.headers = headers self._boundary = boundary + self._newline = _newline self._content = content self._at_eof = False length = self.headers.get(CONTENT_LENGTH, None) @@ -300,8 +306,8 @@ async def read_chunk(self, size: int=chunk_size) -> bytes: if self._read_bytes == self._length: self._at_eof = True if self._at_eof: - clrf = await self._content.readline() - assert b'\r\n' == clrf, \ + newline = await self._content.readline() + assert newline == self._newline, \ 'reader did not read all the data or it is malformed' return chunk @@ -328,11 +334,15 @@ async def _read_chunk_from_stream(self, size: int) -> bytes: assert self._content_eof < 3, "Reading after EOF" assert self._prev_chunk is not None window = self._prev_chunk + chunk - sub = b'\r\n' + self._boundary + + intermeditate_boundary = self._newline + self._boundary + if first_chunk: - idx = window.find(sub) + pos = 0 else: - idx = window.find(sub, max(0, len(self._prev_chunk) - len(sub))) + pos = max(0, len(self._prev_chunk) - len(intermeditate_boundary)) + + idx = window.find(intermeditate_boundary, pos) if idx >= 0: # pushing boundary back to content with warnings.catch_warnings(): @@ -344,6 +354,7 @@ async def _read_chunk_from_stream(self, size: int) -> bytes: chunk = window[len(self._prev_chunk):idx] if not chunk: self._at_eof = True + result = self._prev_chunk self._prev_chunk = chunk return result @@ -372,7 +383,8 @@ async def readline(self) -> bytes: else: next_line = await self._content.readline() if next_line.startswith(self._boundary): - line = line[:-2] # strip CRLF but only once + # strip newline but only once + line = line[:-len(self._newline)] self._unread.append(next_line) return line @@ -516,10 +528,16 @@ class MultipartReader: #: Body part reader class for non multipart/* content types. part_reader_cls = BodyPartReader - def __init__(self, headers: Mapping[str, str], - content: StreamReader) -> None: + def __init__( + self, + headers: Mapping[str, str], + content: StreamReader, + *, + _newline: bytes = b'\r\n' + ) -> None: self.headers = headers self._boundary = ('--' + self._get_boundary()).encode() + self._newline = _newline self._content = content self._last_part = None self._at_eof = False @@ -592,9 +610,13 @@ def _get_part_reader(self, headers: 'CIMultiDictProxy[str]') -> Any: if mimetype.type == 'multipart': if self.multipart_reader_cls is None: return type(self)(headers, self._content) - return self.multipart_reader_cls(headers, self._content) + return self.multipart_reader_cls( + headers, self._content, _newline=self._newline + ) else: - return self.part_reader_cls(self._boundary, headers, self._content) + return self.part_reader_cls( + self._boundary, headers, self._content, _newline=self._newline + ) def _get_boundary(self) -> str: mimetype = parse_mimetype(self.headers[CONTENT_TYPE]) @@ -625,6 +647,11 @@ async def _read_until_first_boundary(self) -> None: if chunk == b'': raise ValueError("Could not find starting boundary %r" % (self._boundary)) + if chunk.startswith(self._boundary): + _, newline = chunk.split(self._boundary, 1) + assert newline in (b'\r\n', b'\n') + self._newline = newline + chunk = chunk.rstrip() if chunk == self._boundary: return diff --git a/tests/test_multipart.py b/tests/test_multipart.py index fd7c4b16283..8518a082715 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -23,6 +23,11 @@ BOUNDARY = b'--:' +def pytest_generate_tests(metafunc): # pragma: no cover + if "newline" in metafunc.fixturenames: + metafunc.parametrize("newline", [b'\r\n', b'\n'], ids=str) + + @pytest.fixture def buf(): return bytearray() @@ -118,46 +123,53 @@ async def test_release_when_stream_at_eof(self) -> None: class TestPartReader: - async def test_next(self) -> None: + async def test_next(self, newline) -> None: + data = b'Hello, world!%s--:' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello, world!\r\n--:')) + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.next() assert b'Hello, world!' == result assert obj.at_eof() - async def test_next_next(self) -> None: + async def test_next_next(self, newline) -> None: + data = b'Hello, world!%s--:' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello, world!\r\n--:')) + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.next() assert b'Hello, world!' == result assert obj.at_eof() result = await obj.next() assert result is None - async def test_read(self) -> None: + async def test_read(self, newline) -> None: + data = b'Hello, world!%s--:' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello, world!\r\n--:')) + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.read() assert b'Hello, world!' == result assert obj.at_eof() async def test_read_chunk_at_eof(self) -> None: - obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'--:')) + obj = aiohttp.BodyPartReader(BOUNDARY, {}, Stream(b'--:')) obj._at_eof = True result = await obj.read_chunk() assert b'' == result - async def test_read_chunk_without_content_length(self) -> None: + async def test_read_chunk_without_content_length(self, newline) -> None: + data = b'Hello, world!%s--:' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello, world!\r\n--:')) + BOUNDARY, {}, Stream(data), _newline=newline + ) c1 = await obj.read_chunk(8) c2 = await obj.read_chunk(8) c3 = await obj.read_chunk(8) assert c1 + c2 == b'Hello, world!' assert c3 == b'' - async def test_read_incomplete_chunk(self) -> None: + async def test_read_incomplete_chunk(self, newline) -> None: loop = asyncio.get_event_loop() stream = Stream(b'') @@ -169,11 +181,12 @@ def prepare(data): with mock.patch.object(stream, 'read', side_effect=[ prepare(b'Hello, '), prepare(b'World'), - prepare(b'!\r\n--:'), + prepare(b'!%s--:' % newline), prepare(b'') ]): obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + BOUNDARY, {}, stream, _newline=newline + ) c1 = await obj.read_chunk(8) assert c1 == b'Hello, ' c2 = await obj.read_chunk(8) @@ -181,25 +194,29 @@ def prepare(data): c3 = await obj.read_chunk(8) assert c3 == b'!' - async def test_read_all_at_once(self) -> None: - stream = Stream(b'Hello, World!\r\n--:--\r\n') - obj = aiohttp.BodyPartReader(BOUNDARY, {}, stream) + async def test_read_all_at_once(self, newline) -> None: + data = b'Hello, World!%s--:--%s' % (newline, newline) + obj = aiohttp.BodyPartReader( + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.read_chunk() assert b'Hello, World!' == result result = await obj.read_chunk() assert b'' == result assert obj.at_eof() - async def test_read_incomplete_body_chunked(self) -> None: - stream = Stream(b'Hello, World!\r\n-') - obj = aiohttp.BodyPartReader(BOUNDARY, {}, stream) + async def test_read_incomplete_body_chunked(self, newline) -> None: + data = b'Hello, World!%s--' % newline + obj = aiohttp.BodyPartReader( + BOUNDARY, {}, Stream(data), _newline=newline + ) result = b'' with pytest.raises(AssertionError): for _ in range(4): result += await obj.read_chunk(7) - assert b'Hello, World!\r\n-' == result + assert data == result - async def test_read_boundary_with_incomplete_chunk(self) -> None: + async def test_read_boundary_with_incomplete_chunk(self, newline) -> None: loop = asyncio.get_event_loop() stream = Stream(b'') @@ -210,12 +227,13 @@ def prepare(data): with mock.patch.object(stream, 'read', side_effect=[ prepare(b'Hello, World'), - prepare(b'!\r\n'), + prepare(b'!%s' % newline), prepare(b'--:'), prepare(b'') ]): obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + BOUNDARY, {}, stream, _newline=newline + ) c1 = await obj.read_chunk(12) assert c1 == b'Hello, World' c2 = await obj.read_chunk(8) @@ -223,21 +241,28 @@ def prepare(data): c3 = await obj.read_chunk(8) assert c3 == b'' - async def test_multi_read_chunk(self) -> None: - stream = Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--') - obj = aiohttp.BodyPartReader(BOUNDARY, {}, stream) + async def test_multi_read_chunk(self, newline) -> None: + data = b'Hello,%s--:%s%sworld!%s--:--' % ((newline,) * 4) + obj = aiohttp.BodyPartReader( + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.read_chunk(8) assert b'Hello,' == result result = await obj.read_chunk(8) assert b'' == result assert obj.at_eof() - async def test_read_chunk_properly_counts_read_bytes(self) -> None: + async def test_read_chunk_properly_counts_read_bytes( + self, newline + ) -> None: expected = b'.' * 10 + tail = b'%s--:--' % newline size = len(expected) obj = aiohttp.BodyPartReader( BOUNDARY, {'CONTENT-LENGTH': size}, - StreamWithShortenRead(expected + b'\r\n--:--')) + StreamWithShortenRead(expected + tail), + _newline=newline, + ) result = bytearray() while True: chunk = await obj.read_chunk() @@ -248,86 +273,111 @@ async def test_read_chunk_properly_counts_read_bytes(self) -> None: assert b'.' * size == result assert obj.at_eof() - async def test_read_does_not_read_boundary(self) -> None: - stream = Stream(b'Hello, world!\r\n--:') - obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + async def test_read_does_not_read_boundary(self, newline) -> None: + data = b'Hello, world!%s--:' % newline + stream = Stream(data) + obj = aiohttp.BodyPartReader(BOUNDARY, {}, stream, _newline=newline) result = await obj.read() assert b'Hello, world!' == result assert b'--:' == (await stream.read()) - async def test_multiread(self) -> None: + async def test_multiread(self, newline) -> None: + data = b'Hello,%s--:%s%sworld!%s--:--' % ((newline,) * 4) obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--')) + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.read() assert b'Hello,' == result result = await obj.read() assert b'' == result assert obj.at_eof() - async def test_read_multiline(self) -> None: + async def test_read_multiline(self, newline) -> None: + data = b'Hello\n,\r\nworld!%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello\n,\r\nworld!\r\n--:--')) + BOUNDARY, {}, Stream(data), _newline=newline + ) result = await obj.read() assert b'Hello\n,\r\nworld!' == result result = await obj.read() assert b'' == result assert obj.at_eof() - async def test_read_respects_content_length(self) -> None: + async def test_read_respects_content_length(self, newline) -> None: + data = b'.' * 100500 + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {'CONTENT-LENGTH': 100500}, - Stream(b'.' * 100500 + b'\r\n--:--')) + BOUNDARY, + {'CONTENT-LENGTH': 100500}, + Stream(data + tail), + _newline=newline, + ) result = await obj.read() - assert b'.' * 100500 == result + assert data == result assert obj.at_eof() - async def test_read_with_content_encoding_gzip(self) -> None: + async def test_read_with_content_encoding_gzip(self, newline) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_ENCODING: 'gzip'}, Stream(b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x0b\xc9\xccMU' b'(\xc9W\x08J\xcdI\xacP\x04\x00$\xfb\x9eV\x0e\x00\x00\x00' - b'\r\n--:--')) + b'%s--:--' % newline), + _newline=newline, + ) result = await obj.read(decode=True) assert b'Time to Relax!' == result - async def test_read_with_content_encoding_deflate(self) -> None: + async def test_read_with_content_encoding_deflate(self, newline) -> None: + data = b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00' + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_ENCODING: 'deflate'}, - Stream(b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00\r\n--:--')) + Stream(data + tail), + _newline=newline, + ) result = await obj.read(decode=True) assert b'Time to Relax!' == result - async def test_read_with_content_encoding_identity(self) -> None: + async def test_read_with_content_encoding_identity(self, newline) -> None: thing = (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x0b\xc9\xccMU' - b'(\xc9W\x08J\xcdI\xacP\x04\x00$\xfb\x9eV\x0e\x00\x00\x00' - b'\r\n') + b'(\xc9W\x08J\xcdI\xacP\x04\x00$\xfb\x9eV\x0e\x00\x00\x00') obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_ENCODING: 'identity'}, - Stream(thing + b'--:--')) + Stream(thing + b'%s--:--' % newline), + _newline=newline, + ) result = await obj.read(decode=True) - assert thing[:-2] == result + assert thing == result - async def test_read_with_content_encoding_unknown(self) -> None: + async def test_read_with_content_encoding_unknown(self, newline) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_ENCODING: 'snappy'}, - Stream(b'\x0e4Time to Relax!\r\n--:--')) + Stream(b'\x0e4Time to Relax!%s--:--' % newline), + _newline=newline, + ) with pytest.raises(RuntimeError): await obj.read(decode=True) - async def test_read_with_content_transfer_encoding_base64(self) -> None: + async def test_read_with_content_transfer_encoding_base64( + self, newline + ) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TRANSFER_ENCODING: 'base64'}, - Stream(b'VGltZSB0byBSZWxheCE=\r\n--:--')) + Stream(b'VGltZSB0byBSZWxheCE=%s--:--' % newline), + _newline=newline, + ) result = await obj.read(decode=True) assert b'Time to Relax!' == result async def test_read_with_content_transfer_encoding_quoted_printable( - self) -> None: + self, newline + ) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TRANSFER_ENCODING: 'quoted-printable'}, Stream(b'=D0=9F=D1=80=D0=B8=D0=B2=D0=B5=D1=82,' - b' =D0=BC=D0=B8=D1=80!\r\n--:--')) + b' =D0=BC=D0=B8=D1=80!%s--:--' % newline), + _newline=newline, + ) result = await obj.read(decode=True) expected = (b'\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82,' b' \xd0\xbc\xd0\xb8\xd1\x80!') @@ -335,54 +385,86 @@ async def test_read_with_content_transfer_encoding_quoted_printable( @pytest.mark.parametrize('encoding', ('binary', '8bit', '7bit')) async def test_read_with_content_transfer_encoding_binary( - self, encoding) -> None: + self, encoding, newline + ) -> None: data = b'\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82,' \ b' \xd0\xbc\xd0\xb8\xd1\x80!' obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TRANSFER_ENCODING: encoding}, - Stream(data + b'\r\n--:--')) + Stream(data + b'%s--:--' % newline), + _newline=newline, + ) result = await obj.read(decode=True) assert data == result - async def test_read_with_content_transfer_encoding_unknown(self) -> None: + async def test_read_with_content_transfer_encoding_unknown( + self, newline + ) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TRANSFER_ENCODING: 'unknown'}, - Stream(b'\x0e4Time to Relax!\r\n--:--')) + Stream(b'\x0e4Time to Relax!%s--:--' % newline), + _newline=newline, + ) with pytest.raises(RuntimeError): await obj.read(decode=True) - async def test_read_text(self) -> None: + async def test_read_text(self, newline) -> None: obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello, world!\r\n--:--')) + BOUNDARY, + {}, + Stream(b'Hello, world!%s--:--' % newline), + _newline=newline, + ) result = await obj.text() assert 'Hello, world!' == result - async def test_read_text_default_encoding(self) -> None: + async def test_read_text_default_encoding(self, newline) -> None: + data = 'Привет, Мир!' + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, - Stream('Привет, Мир!\r\n--:--'.encode('utf-8'))) + BOUNDARY, + {}, + Stream(data.encode('utf-8') + tail), + _newline=newline, + ) result = await obj.text() - assert 'Привет, Мир!' == result + assert data == result - async def test_read_text_encoding(self) -> None: + async def test_read_text_encoding(self, newline) -> None: + data = 'Привет, Мир!' + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, - Stream('Привет, Мир!\r\n--:--'.encode('cp1251'))) + BOUNDARY, + {}, + Stream(data.encode('cp1251') + tail), + _newline=newline, + ) result = await obj.text(encoding='cp1251') - assert 'Привет, Мир!' == result + assert data == result - async def test_read_text_guess_encoding(self) -> None: + async def test_read_text_guess_encoding(self, newline) -> None: + data = 'Привет, Мир!' + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {CONTENT_TYPE: 'text/plain;charset=cp1251'}, - Stream('Привет, Мир!\r\n--:--'.encode('cp1251'))) + BOUNDARY, + {CONTENT_TYPE: 'text/plain;charset=cp1251'}, + Stream(data.encode('cp1251') + tail), + _newline=newline, + ) result = await obj.text() - assert 'Привет, Мир!' == result + assert data == result - async def test_read_text_compressed(self) -> None: + async def test_read_text_compressed(self, newline) -> None: + data = ( + b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00' + b'%s--:--' % newline + ) obj = aiohttp.BodyPartReader( - BOUNDARY, {CONTENT_ENCODING: 'deflate', - CONTENT_TYPE: 'text/plain'}, - Stream(b'\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00\r\n--:--')) + BOUNDARY, + {CONTENT_ENCODING: 'deflate', CONTENT_TYPE: 'text/plain'}, + Stream(data), + _newline=newline, + ) result = await obj.text() assert 'Time to Relax!' == result @@ -393,32 +475,45 @@ async def test_read_text_while_closed(self) -> None: result = await obj.text() assert '' == result - async def test_read_json(self) -> None: + async def test_read_json(self, newline) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/json'}, - Stream(b'{"test": "passed"}\r\n--:--')) + Stream(b'{"test": "passed"}%s--:--' % newline), + _newline=newline, + ) result = await obj.json() assert {'test': 'passed'} == result - async def test_read_json_encoding(self) -> None: + async def test_read_json_encoding(self, newline) -> None: + data = '{"тест": "пассед"}'.encode('cp1251') + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/json'}, - Stream('{"тест": "пассед"}\r\n--:--'.encode('cp1251'))) + Stream(data + tail), + _newline=newline, + ) result = await obj.json(encoding='cp1251') assert {'тест': 'пассед'} == result - async def test_read_json_guess_encoding(self) -> None: + async def test_read_json_guess_encoding(self, newline) -> None: + data = '{"тест": "пассед"}'.encode('cp1251') + tail = b'%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/json; charset=cp1251'}, - Stream('{"тест": "пассед"}\r\n--:--'.encode('cp1251'))) + Stream(data + tail), + _newline=newline, + ) result = await obj.json() assert {'тест': 'пассед'} == result - async def test_read_json_compressed(self) -> None: + async def test_read_json_compressed(self, newline) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_ENCODING: 'deflate', CONTENT_TYPE: 'application/json'}, - Stream(b'\xabV*I-.Q\xb2RP*H,.NMQ\xaa\x05\x00\r\n--:--')) + Stream(b'\xabV*I-.Q\xb2RP*H,.NMQ\xaa\x05\x00' + b'%s--:--' % newline), + _newline=newline, + ) result = await obj.json() assert {'test': 'passed'} == result @@ -430,25 +525,34 @@ async def test_read_json_while_closed(self) -> None: result = await obj.json() assert result is None - async def test_read_form(self) -> None: + async def test_read_form(self, newline) -> None: + data = b'foo=bar&foo=baz&boo=%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/x-www-form-urlencoded'}, - Stream(b'foo=bar&foo=baz&boo=\r\n--:--')) + Stream(data), + _newline=newline, + ) result = await obj.form() assert [('foo', 'bar'), ('foo', 'baz'), ('boo', '')] == result - async def test_read_form_encoding(self) -> None: + async def test_read_form_encoding(self, newline) -> None: + data = b'foo=bar&foo=baz&boo=%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/x-www-form-urlencoded'}, - Stream('foo=bar&foo=baz&boo=\r\n--:--'.encode('cp1251'))) + Stream(data), + _newline=newline, + ) result = await obj.form(encoding='cp1251') assert [('foo', 'bar'), ('foo', 'baz'), ('boo', '')] == result - async def test_read_form_guess_encoding(self) -> None: + async def test_read_form_guess_encoding(self, newline) -> None: + data = b'foo=bar&foo=baz&boo=%s--:--' % newline obj = aiohttp.BodyPartReader( BOUNDARY, {CONTENT_TYPE: 'application/x-www-form-urlencoded; charset=utf-8'}, - Stream('foo=bar&foo=baz&boo=\r\n--:--'.encode('utf-8'))) + Stream(data), + _newline=newline, + ) result = await obj.form() assert [('foo', 'bar'), ('foo', 'baz'), ('boo', '')] == result @@ -456,14 +560,21 @@ async def test_read_form_while_closed(self) -> None: stream = Stream(b'') obj = aiohttp.BodyPartReader( BOUNDARY, - {CONTENT_TYPE: 'application/x-www-form-urlencoded'}, stream) + {CONTENT_TYPE: 'application/x-www-form-urlencoded'}, + stream, + ) obj._at_eof = True result = await obj.form() assert not result - async def test_readline(self) -> None: + async def test_readline(self, newline) -> None: + data = b'Hello\n,\r\nworld!%s--:--' % newline obj = aiohttp.BodyPartReader( - BOUNDARY, {}, Stream(b'Hello\n,\r\nworld!\r\n--:--')) + BOUNDARY, + {}, + Stream(data), + _newline=newline, + ) result = await obj.readline() assert b'Hello\n' == result result = await obj.readline() @@ -474,29 +585,43 @@ async def test_readline(self) -> None: assert b'' == result assert obj.at_eof() - async def test_release(self) -> None: - stream = Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--') + async def test_release(self, newline) -> None: + data = b'Hello,%s--:\r\n\r\nworld!%s--:--' % (newline, newline) + stream = Stream(data) obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + BOUNDARY, + {}, + stream, + _newline=newline, + ) + remained = b'--:\r\n\r\nworld!%s--:--' % newline await obj.release() assert obj.at_eof() - assert b'--:\r\n\r\nworld!\r\n--:--' == stream.content.read() + assert remained == stream.content.read() - async def test_release_respects_content_length(self) -> None: + async def test_release_respects_content_length(self, newline) -> None: obj = aiohttp.BodyPartReader( BOUNDARY, {'CONTENT-LENGTH': 100500}, - Stream(b'.' * 100500 + b'\r\n--:--')) + Stream(b'.' * 100500 + b'%s--:--' % newline), + _newline=newline, + ) result = await obj.release() assert result is None assert obj.at_eof() - async def test_release_release(self) -> None: - stream = Stream(b'Hello,\r\n--:\r\n\r\nworld!\r\n--:--') + async def test_release_release(self, newline) -> None: + data = b'Hello,%s--:\r\n\r\nworld!%s--:--' % (newline, newline) + remained = b'--:\r\n\r\nworld!%s--:--' % newline + stream = Stream(data) obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + BOUNDARY, + {}, + stream, + _newline=newline, + ) await obj.release() await obj.release() - assert b'--:\r\n\r\nworld!\r\n--:--' == stream.content.read() + assert remained == stream.content.read() async def test_filename(self) -> None: part = aiohttp.BodyPartReader( @@ -505,23 +630,22 @@ async def test_filename(self) -> None: None) assert 'foo.html' == part.filename - async def test_reading_long_part(self) -> None: + async def test_reading_long_part(self, newline) -> None: size = 2 * stream_reader_default_limit protocol = mock.Mock(_reading_paused=False) stream = StreamReader(protocol) - stream.feed_data(b'0' * size + b'\r\n--:--') + stream.feed_data(b'0' * size + b'%s--:--' % newline) stream.feed_eof() - obj = aiohttp.BodyPartReader( - BOUNDARY, {}, stream) + obj = aiohttp.BodyPartReader(BOUNDARY, {}, stream, _newline=newline) data = await obj.read() assert len(data) == size class TestMultipartReader: - def test_from_response(self) -> None: + def test_from_response(self, newline) -> None: resp = Response({CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\nhello\r\n--:--')) + Stream(b'--:%s\r\nhello%s--:--' % (newline, newline))) res = aiohttp.MultipartReader.from_response(resp) assert isinstance(res, MultipartResponseWrapper) @@ -535,129 +659,154 @@ def test_bad_boundary(self) -> None: with pytest.raises(ValueError): aiohttp.MultipartReader.from_response(resp) - def test_dispatch(self) -> None: + def test_dispatch(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\necho\r\n--:--')) + Stream(b'--:%s\r\necho%s--:--' % (newline, newline))) res = reader._get_part_reader({CONTENT_TYPE: 'text/plain'}) assert isinstance(res, reader.part_reader_cls) - def test_dispatch_bodypart(self) -> None: + def test_dispatch_bodypart(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\necho\r\n--:--')) + Stream(b'--:%s\r\necho%s--:--' % (newline, newline))) res = reader._get_part_reader({CONTENT_TYPE: 'text/plain'}) assert isinstance(res, reader.part_reader_cls) - def test_dispatch_multipart(self) -> None: + def test_dispatch_multipart(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'----:--\r\n' - b'\r\n' - b'test\r\n' - b'----:--\r\n' - b'\r\n' - b'passed\r\n' - b'----:----\r\n' - b'--:--')) + Stream( + newline.join([ + b'----:--', + b'', + b'test', + b'----:--', + b'', + b'passed', + b'----:----' + b'--:--', + ]) + ) + ) res = reader._get_part_reader( {CONTENT_TYPE: 'multipart/related;boundary=--:--'}) assert isinstance(res, reader.__class__) - def test_dispatch_custom_multipart_reader(self) -> None: + def test_dispatch_custom_multipart_reader(self, newline) -> None: class CustomReader(aiohttp.MultipartReader): pass reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'----:--\r\n' - b'\r\n' - b'test\r\n' - b'----:--\r\n' - b'\r\n' - b'passed\r\n' - b'----:----\r\n' - b'--:--')) + Stream( + newline.join([ + b'----:--', + b'', + b'test', + b'----:--', + b'', + b'passed', + b'----:----', + b'--:--', + ]) + ) + ) reader.multipart_reader_cls = CustomReader res = reader._get_part_reader( {CONTENT_TYPE: 'multipart/related;boundary=--:--'}) assert isinstance(res, CustomReader) - async def test_emit_next(self) -> None: + async def test_emit_next(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\necho\r\n--:--')) + Stream(b'--:%s\r\necho%s--:--' % (newline, newline))) res = await reader.next() assert isinstance(res, reader.part_reader_cls) - async def test_invalid_boundary(self) -> None: + async def test_invalid_boundary(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'---:\r\n\r\necho\r\n---:--')) + Stream(b'---:%s\r\necho%s---:--' % (newline, newline))) with pytest.raises(ValueError): await reader.next() - async def test_release(self) -> None: + async def test_release(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/mixed;boundary=":"'}, - Stream(b'--:\r\n' - b'Content-Type: multipart/related;boundary=--:--\r\n' - b'\r\n' - b'----:--\r\n' - b'\r\n' - b'test\r\n' - b'----:--\r\n' - b'\r\n' - b'passed\r\n' - b'----:----\r\n' - b'\r\n' - b'--:--')) + Stream( + newline.join([ + b'--:', + b'Content-Type: multipart/related;boundary=--:--', + b'', + b'----:--', + b'', + b'test', + b'----:--', + b'', + b'passed', + b'----:----', + b'', + b'--:--', + ]) + ) + ) await reader.release() assert reader.at_eof() - async def test_release_release(self) -> None: + async def test_release_release(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\necho\r\n--:--')) + Stream(b'--:%s\r\necho%s--:--' % (newline, newline))) await reader.release() assert reader.at_eof() await reader.release() assert reader.at_eof() - async def test_release_next(self) -> None: + async def test_release_next(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n\r\necho\r\n--:--')) + Stream(b'--:%s\r\necho%s--:--' % (newline, newline))) await reader.release() assert reader.at_eof() res = await reader.next() assert res is None - async def test_second_next_releases_previous_object(self) -> None: + async def test_second_next_releases_previous_object(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n' - b'\r\n' - b'test\r\n' - b'--:\r\n' - b'\r\n' - b'passed\r\n' - b'--:--')) + Stream( + newline.join([ + b'--:', + b'', + b'test', + b'--:', + b'', + b'passed', + b'--:--', + ]) + ) + ) first = await reader.next() assert isinstance(first, aiohttp.BodyPartReader) second = await reader.next() assert first.at_eof() assert not second.at_eof() - async def test_release_without_read_the_last_object(self) -> None: + async def test_release_without_read_the_last_object(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n' - b'\r\n' - b'test\r\n' - b'--:\r\n' - b'\r\n' - b'passed\r\n' - b'--:--')) + Stream( + newline.join([ + b'--:', + b'', + b'test', + b'--:', + b'', + b'passed', + b'--:--', + ]) + ) + ) first = await reader.next() second = await reader.next() third = await reader.next() @@ -666,16 +815,23 @@ async def test_release_without_read_the_last_object(self) -> None: assert second.at_eof() assert third is None - async def test_read_chunk_by_length_doesnt_breaks_reader(self) -> None: + async def test_read_chunk_by_length_doesnt_breaks_reader( + self, newline + ) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n' - b'Content-Length: 4\r\n\r\n' - b'test' - b'\r\n--:\r\n' - b'Content-Length: 6\r\n\r\n' - b'passed' - b'\r\n--:--')) + Stream(newline.join([ + b'--:', + b'Content-Length: 4', + b'', + b'test', + b'--:', + b'Content-Length: 6', + b'', + b'passed', + b'--:--', + ])) + ) body_parts = [] while True: read_part = b'' @@ -687,16 +843,21 @@ async def test_read_chunk_by_length_doesnt_breaks_reader(self) -> None: body_parts.append(read_part) assert body_parts == [b'test', b'passed'] - async def test_read_chunk_from_stream_doesnt_breaks_reader(self) -> None: + async def test_read_chunk_from_stream_doesnt_breaks_reader( + self, newline + ) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'--:\r\n' - b'\r\n' - b'chunk' - b'\r\n--:\r\n' - b'\r\n' - b'two_chunks' - b'\r\n--:--')) + Stream(newline.join([ + b'--:', + b'', + b'chunk', + b'--:', + b'', + b'two_chunks', + b'--:--', + ])) + ) body_parts = [] while True: read_part = b'' @@ -710,24 +871,56 @@ async def test_read_chunk_from_stream_doesnt_breaks_reader(self) -> None: body_parts.append(read_part) assert body_parts == [b'chunk', b'two_chunks'] - async def test_reading_skips_prelude(self) -> None: + async def test_reading_skips_prelude(self, newline) -> None: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, - Stream(b'Multi-part data is not supported.\r\n' - b'\r\n' - b'--:\r\n' - b'\r\n' - b'test\r\n' - b'--:\r\n' - b'\r\n' - b'passed\r\n' - b'--:--')) + Stream(newline.join([ + b'Multi-part data is not supported.', + b'', + b'--:', + b'', + b'test', + b'--:', + b'', + b'passed', + b'--:--' + ])) + ) first = await reader.next() assert isinstance(first, aiohttp.BodyPartReader) second = await reader.next() assert first.at_eof() assert not second.at_eof() + async def test_read_mixed_newlines(self) -> None: + reader = aiohttp.MultipartReader( + {CONTENT_TYPE: 'multipart/mixed;boundary=":"'}, + Stream( + b''.join([ + b'--:\n', + b'Content-Type: multipart/related;boundary=--:--\n', + b'\n', + b'----:--\r\n', + b'\r\n', + b'test\r\n', + b'----:--\r\n', + b'\r\n', + b'passed\r\n', + b'----:----\r\n', + b'\n', + b'--:--', + ]) + ) + ) + while True: + part = await reader.next() + if part is None: + break + while True: + subpart = await part.next() + if subpart is None: + break + async def test_writer(writer) -> None: assert writer.size == 0