From f0294ad384b10bd42f0e91ab008c9f41a4f9d0d5 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 5 Jun 2023 17:34:20 +0200 Subject: [PATCH] gh-105324: Fix tokenize module main function for stdin --- Lib/tokenize.py | 3 +-- .../2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 380dc2ab468b57..a07a8bf45891ac 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -490,8 +490,7 @@ def error(message, filename=None, location=None): else: filename = "" tokens = _generate_tokens_from_c_tokenizer( - (x.encode('utf-8') for x in iter(sys.stdin.readline, "") - ), "utf-8", extra_tokens=True) + sys.stdin.readline, extra_tokens=True) # Output the tokenization diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst new file mode 100644 index 00000000000000..17275aed338d0d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-05-17-35-50.gh-issue-105324.BqhiJJ.rst @@ -0,0 +1,2 @@ +Fix the main function of the :mod:`tokenize` module when reading from +``sys.stdin``. Patch by Pablo Galindo