From b0f5b4cf36a2c1cd6965888be0fc7481a1ff8433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marta=20G=C3=B3mez=20Mac=C3=ADas?= Date: Sat, 27 May 2023 18:17:16 +0200 Subject: [PATCH 1/4] fix(tokenizer): Additional NL token when using CRLF --- Lib/test/test_tokenize.py | 8 ++++++++ Parser/tokenizer.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index abb68859be944c..293592b3fd13db 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -84,6 +84,14 @@ def test_basic(self): NEWLINE '\\n' (4, 26) (4, 27) DEDENT '' (5, 0) (5, 0) """) + + self.check_tokenize("foo='bar'\r\n", """\ + NAME 'foo' (1, 0) (1, 3) + OP '=' (1, 3) (1, 4) + STRING "'bar'" (1, 4) (1, 9) + NEWLINE '\\n' (1, 9) (1, 10) + """) + indent_error_file = b"""\ def k(x): x += 2 diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index a7651b1bd86430..a84c2492b6b17a 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -800,7 +800,7 @@ translate_newlines(const char *s, int exec_input, struct tok_state *tok) { } /* If this is exec input, add a newline to the end of the string if there isn't one already. */ - if (exec_input && c != '\n') { + if (exec_input && c != '\n' && c != '\0') { *current = '\n'; current++; } From b5510abc0b54afe05dd176b1c594edbaf88779b8 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 16:23:17 +0000 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst new file mode 100644 index 00000000000000..c8d6956e2f4568 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst @@ -0,0 +1 @@ +Fix including an additional NL token when parsing files having CRLF lines. From c58df91876f063292d3caaa33fb2bd76b88fc2ca Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 27 May 2023 17:25:34 +0100 Subject: [PATCH 3/4] Update Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst --- .../2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst index c8d6956e2f4568..b2ec7b1b60d243 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst @@ -1 +1 @@ -Fix including an additional NL token when parsing files having CRLF lines. +Fix including an additional NL token when parsing files having CRLF lines. Patch by Marta Gómez. From 2f7fe532bed24bd755588f1a6e05b542b4732d56 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 27 May 2023 17:26:08 +0100 Subject: [PATCH 4/4] Update Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst --- .../2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst index b2ec7b1b60d243..d41a2169ccb3de 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-27-16-23-16.gh-issue-105017.KQrsC0.rst @@ -1 +1 @@ -Fix including an additional NL token when parsing files having CRLF lines. Patch by Marta Gómez. +Do not include an additional final ``NL`` token when parsing files having CRLF lines. Patch by Marta Gómez.