From 4531af83835cff7681d9bc85a28dee9df47c3f99 Mon Sep 17 00:00:00 2001 From: James Braza Date: Mon, 11 Dec 2023 09:23:10 -0500 Subject: [PATCH 1/7] Added test case exposing iter_text adding an empty string --- tests/test_decoders.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_decoders.py b/tests/test_decoders.py index 61c9a4acca..170a93453c 100644 --- a/tests/test_decoders.py +++ b/tests/test_decoders.py @@ -219,6 +219,17 @@ def test_text_decoder_empty_cases(): assert response.text == "" +@pytest.mark.parametrize( + ["data", "expected"], + [((b"Hello,", b" world!"), ["Hello,", " world!"])], +) +def test_streaming_text_decoder( + data: typing.Iterable[bytes], expected: typing.List[str] +) -> None: + response = httpx.Response(200, content=iter(data)) + assert list(response.iter_text()) == expected + + def test_line_decoder_nl(): response = httpx.Response(200, content=[b""]) assert list(response.iter_lines()) == [] From e9e4b13a71b4bd31d8c8e210cfe73979eb10d040 Mon Sep 17 00:00:00 2001 From: James Braza Date: Mon, 11 Dec 2023 09:23:24 -0500 Subject: [PATCH 2/7] Added missing else to TextChunker.decode for empty input --- httpx/_decoders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httpx/_decoders.py b/httpx/_decoders.py index 57c649297a..89415c63da 100644 --- a/httpx/_decoders.py +++ b/httpx/_decoders.py @@ -212,7 +212,7 @@ def __init__(self, chunk_size: typing.Optional[int] = None) -> None: def decode(self, content: str) -> typing.List[str]: if self._chunk_size is None: - return [content] + return [content] if content else [] self._buffer.write(content) if self._buffer.tell() >= self._chunk_size: From 05ed234ec8e47eee6d6076769b7dba8f35d867a7 Mon Sep 17 00:00:00 2001 From: James Braza Date: Mon, 11 Dec 2023 10:00:28 -0500 Subject: [PATCH 3/7] Using itertools.chain to not degrade coverage on empty chunker.decode in iter_text --- httpx/_models.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/httpx/_models.py b/httpx/_models.py index dac177c4f6..9ef02a91d5 100644 --- a/httpx/_models.py +++ b/httpx/_models.py @@ -1,5 +1,6 @@ import datetime import email.message +import itertools import json as jsonlib import typing import urllib.request @@ -851,10 +852,9 @@ def iter_text( text_content = decoder.decode(byte_content) for chunk in chunker.decode(text_content): yield chunk - text_content = decoder.flush() - for chunk in chunker.decode(text_content): - yield chunk - for chunk in chunker.flush(): + for chunk in itertools.chain( + chunker.decode(decoder.flush()), chunker.flush() + ): yield chunk def iter_lines(self) -> typing.Iterator[str]: @@ -955,10 +955,9 @@ async def aiter_text( text_content = decoder.decode(byte_content) for chunk in chunker.decode(text_content): yield chunk - text_content = decoder.flush() - for chunk in chunker.decode(text_content): - yield chunk - for chunk in chunker.flush(): + for chunk in itertools.chain( + chunker.decode(decoder.flush()), chunker.flush() + ): yield chunk async def aiter_lines(self) -> typing.AsyncIterator[str]: From 8dd9b54fb8a9cc49f574371a2dda4f36a252feee Mon Sep 17 00:00:00 2001 From: James Braza Date: Mon, 11 Dec 2023 10:07:07 -0500 Subject: [PATCH 4/7] Added test of LineDecoder.decode edge cases --- tests/test_decoders.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_decoders.py b/tests/test_decoders.py index 170a93453c..08a572a3c4 100644 --- a/tests/test_decoders.py +++ b/tests/test_decoders.py @@ -273,6 +273,11 @@ def test_line_decoder_crnl(): assert list(response.iter_lines()) == ["12345", "foo bar baz"] +@pytest.mark.parametrize(["text", "expected"], [("", [])]) +def test_line_decoding_edge_cases(text: str, expected: typing.List[str]) -> None: + assert httpx._decoders.LineDecoder().decode(text) == expected + + def test_invalid_content_encoding_header(): headers = [(b"Content-Encoding", b"invalid-header")] body = b"test 123" From 6d0980629f213a413f8863511f81808d0b80170e Mon Sep 17 00:00:00 2001 From: James Braza Date: Mon, 11 Dec 2023 11:14:31 -0500 Subject: [PATCH 5/7] Revert "Added test of LineDecoder.decode edge cases" This reverts commit e19a41801fcb9ec11d6bad4664a5bf20480bdc31. --- tests/test_decoders.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_decoders.py b/tests/test_decoders.py index 08a572a3c4..170a93453c 100644 --- a/tests/test_decoders.py +++ b/tests/test_decoders.py @@ -273,11 +273,6 @@ def test_line_decoder_crnl(): assert list(response.iter_lines()) == ["12345", "foo bar baz"] -@pytest.mark.parametrize(["text", "expected"], [("", [])]) -def test_line_decoding_edge_cases(text: str, expected: typing.List[str]) -> None: - assert httpx._decoders.LineDecoder().decode(text) == expected - - def test_invalid_content_encoding_header(): headers = [(b"Content-Encoding", b"invalid-header")] body = b"test 123" From 4d888ebcca62903333e95f92464c7c842ed814eb Mon Sep 17 00:00:00 2001 From: James Braza Date: Mon, 11 Dec 2023 11:14:48 -0500 Subject: [PATCH 6/7] Added no cover pragma to empty string edge case, with explanatory comment --- httpx/_decoders.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/httpx/_decoders.py b/httpx/_decoders.py index 89415c63da..3f507c8e04 100644 --- a/httpx/_decoders.py +++ b/httpx/_decoders.py @@ -280,7 +280,9 @@ def decode(self, text: str) -> typing.List[str]: text = text[:-1] if not text: - return [] + # NOTE: the edge case input of empty text doesn't occur in practice, + # because other httpx internals filter out this value + return [] # pragma: no cover trailing_newline = text[-1] in NEWLINE_CHARS lines = text.splitlines() From 2781f4db7ad4ce4b6d300e775325b49d4654fc17 Mon Sep 17 00:00:00 2001 From: James Braza Date: Mon, 11 Dec 2023 17:06:38 -0500 Subject: [PATCH 7/7] Removed itertools.chain in favor of pragma no cover comment --- httpx/_models.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/httpx/_models.py b/httpx/_models.py index 9ef02a91d5..b8617cdab5 100644 --- a/httpx/_models.py +++ b/httpx/_models.py @@ -1,6 +1,5 @@ import datetime import email.message -import itertools import json as jsonlib import typing import urllib.request @@ -852,9 +851,10 @@ def iter_text( text_content = decoder.decode(byte_content) for chunk in chunker.decode(text_content): yield chunk - for chunk in itertools.chain( - chunker.decode(decoder.flush()), chunker.flush() - ): + text_content = decoder.flush() + for chunk in chunker.decode(text_content): + yield chunk # pragma: no cover + for chunk in chunker.flush(): yield chunk def iter_lines(self) -> typing.Iterator[str]: @@ -955,9 +955,10 @@ async def aiter_text( text_content = decoder.decode(byte_content) for chunk in chunker.decode(text_content): yield chunk - for chunk in itertools.chain( - chunker.decode(decoder.flush()), chunker.flush() - ): + text_content = decoder.flush() + for chunk in chunker.decode(text_content): + yield chunk # pragma: no cover + for chunk in chunker.flush(): yield chunk async def aiter_lines(self) -> typing.AsyncIterator[str]: