From f4e5643df64d0c2a009ed224560044b3409a47c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 13 Sep 2024 03:07:23 +0200 Subject: [PATCH 01/32] gh-124027: Support Del, PgUp, and PgDn on TERM=vt100 (#124028) pyrepl: Support Del, PgUp, and PgDn on TERM=vt100 From Fedora's /etc/inputrc: "\e[5~": history-search-backward "\e[6~": history-search-forward "\e[3~": delete-char Fixes https://github.com/python/cpython/issues/124027 --- Lib/_pyrepl/historical_reader.py | 2 ++ Lib/_pyrepl/reader.py | 1 + .../2024-09-13-02-25-06.gh-issue-124027.to_9DY.rst | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-13-02-25-06.gh-issue-124027.to_9DY.rst diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index f6e14bdffc3352..5d416f336ad5d2 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -266,7 +266,9 @@ def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: (r"\M-r", "restore-history"), (r"\M-.", "yank-arg"), (r"\", "history-search-forward"), + (r"\x1b[6~", "history-search-forward"), (r"\", "history-search-backward"), + (r"\x1b[5~", "history-search-backward"), ) def select_item(self, i: int) -> None: diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 54bd1ea0222a60..4b0700d069c621 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -150,6 +150,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: (r"\", "right"), (r"\C-\", "forward-word"), (r"\", "delete"), + (r"\x1b[3~", "delete"), (r"\", "backspace"), (r"\M-\", "backward-kill-word"), (r"\", "end-of-line"), # was 'end' diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-13-02-25-06.gh-issue-124027.to_9DY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-13-02-25-06.gh-issue-124027.to_9DY.rst new file mode 100644 index 00000000000000..1834ba0ba08bfb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-13-02-25-06.gh-issue-124027.to_9DY.rst @@ -0,0 +1,2 @@ +Support ````, ````, and ```` keys in the Python +REPL when ``$TERM`` is set to ``vt100``. From a47cd21890fee845d582b41ecf7d029801899ea5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 12 Sep 2024 22:29:08 -0400 Subject: [PATCH 02/32] gh-123976: Refresh docs around custom providers. (#123977) * gh-123976: Refresh docs around custom providers. * Remove excess whitespace. --- Doc/library/importlib.metadata.rst | 50 ++++++++++--------- ...-09-11-16-52-08.gh-issue-123976.jhOfNR.rst | 1 + 2 files changed, 28 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2024-09-11-16-52-08.gh-issue-123976.jhOfNR.rst diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 9c0879f5ca850f..66ba621084c898 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -19,7 +19,7 @@ such as its entry points or its top-level names (`Import Package `_\s, modules, if any). Built in part on Python's import system, this library intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with +API`_ and `metadata API`_ of ``pkg_resources``. Along with :mod:`importlib.resources`, this package can eliminate the need to use the older and less efficient ``pkg_resources`` package. @@ -46,7 +46,7 @@ and metadata defined by the `Core metadata specifications `_ you've installed -using ``pip``. We start by creating a virtual environment and installing +using ``pip``. We start by creating a virtual environment and installing something into it: .. code-block:: shell-session @@ -87,7 +87,7 @@ You can get the version string for ``wheel`` by running the following: '0.32.3' You can also get a collection of entry points selectable by properties of the EntryPoint (typically 'group' or 'name'), such as -``console_scripts``, ``distutils.commands`` and others. Each group contains a +``console_scripts``, ``distutils.commands`` and others. Each group contains a collection of :ref:`EntryPoint ` objects. You can get the :ref:`metadata for a distribution `:: @@ -114,7 +114,7 @@ Entry points The ``entry_points()`` function returns a collection of entry points. Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value. There are also ``.module``, +a ``.load()`` method to resolve the value. There are also ``.module``, ``.attr``, and ``.extras`` attributes for getting the components of the ``.value`` attribute. @@ -167,7 +167,7 @@ Inspect the resolved entry point:: The ``group`` and ``name`` are arbitrary values defined by the package author and usually a client will wish to resolve all entry points for a particular -group. Read `the setuptools docs +group. Read `the setuptools docs `_ for more information on entry points, their definition, and usage. @@ -240,12 +240,12 @@ number, as a string:: Distribution files ------------------ -You can also get the full set of files contained within a distribution. The +You can also get the full set of files contained within a distribution. The ``files()`` function takes a `Distribution Package `_ name and returns all of the -files installed by this distribution. Each file object returned is a +files installed by this distribution. Each file object returned is a ``PackagePath``, a :class:`pathlib.PurePath` derived object with additional ``dist``, -``size``, and ``hash`` properties as indicated by the metadata. For example:: +``size``, and ``hash`` properties as indicated by the metadata. For example:: >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP >>> util # doctest: +SKIP @@ -321,9 +321,9 @@ Distributions ============= While the above API is the most common and convenient usage, you can get all -of that information from the ``Distribution`` class. A ``Distribution`` is an +of that information from the ``Distribution`` class. A ``Distribution`` is an abstract object that represents the metadata for -a Python `Distribution Package `_. You can +a Python `Distribution Package `_. You can get the ``Distribution`` instance:: >>> from importlib.metadata import distribution # doctest: +SKIP @@ -366,21 +366,26 @@ This metadata finder search defaults to ``sys.path``, but varies slightly in how - ``importlib.metadata`` will incidentally honor :py:class:`pathlib.Path` objects on ``sys.path`` even though such values will be ignored for imports. -Extending the search algorithm -============================== +Implementing Custom Providers +============================= + +``importlib.metadata`` address two API surfaces, one for *consumers* +and another for *providers*. Most users are consumers, consuming +metadata provided by the packages. There are other use-cases, however, +where users wish to expose metadata through some other mechanism, +such as alongside a custom importer. Such a use case calls for a +*custom provider*. Because `Distribution Package `_ metadata is not available through :data:`sys.path` searches, or package loaders directly, the metadata for a distribution is found through import -system :ref:`finders `. To find a distribution package's metadata, +system :ref:`finders `. To find a distribution package's metadata, ``importlib.metadata`` queries the list of :term:`meta path finders ` on :data:`sys.meta_path`. -By default ``importlib.metadata`` installs a finder for distribution packages -found on the file system. -This finder doesn't actually find any *distributions*, -but it can find their metadata. +The implementation has hooks integrated into the ``PathFinder``, +serving metadata for distribution packages found on the file system. The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the interface expected of finders by Python's import system. @@ -391,16 +396,16 @@ interface expected of finders by Python's import system. method:: @abc.abstractmethod - def find_distributions(context=DistributionFinder.Context()): + def find_distributions(context=DistributionFinder.Context()) -> Iterable[Distribution]: """Return an iterable of all Distribution instances capable of loading the metadata for packages for the indicated ``context``. """ The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` properties indicating the path to search and name to match and may -supply other relevant context. +supply other relevant context sought by the consumer. -What this means in practice is that to support finding distribution package +In practice, to support finding distribution package metadata in locations other than the file system, subclass ``Distribution`` and implement the abstract methods. Then from a custom finder, return instances of this derived ``Distribution`` in the @@ -409,8 +414,7 @@ a custom finder, return instances of this derived ``Distribution`` in the Example ------- -Consider for example a custom finder that loads Python -modules from a database:: +Imagine a custom finder that loads Python modules from a database:: class DatabaseImporter(importlib.abc.MetaPathFinder): def __init__(self, db): diff --git a/Misc/NEWS.d/next/Documentation/2024-09-11-16-52-08.gh-issue-123976.jhOfNR.rst b/Misc/NEWS.d/next/Documentation/2024-09-11-16-52-08.gh-issue-123976.jhOfNR.rst new file mode 100644 index 00000000000000..7f8e5801ae6597 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-09-11-16-52-08.gh-issue-123976.jhOfNR.rst @@ -0,0 +1 @@ +Refresh docs around custom providers. From 1f9d163850c43ba85193ef853986c5e96b168c8c Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 13 Sep 2024 05:23:54 +0100 Subject: [PATCH 03/32] gh-116622: Android test script improvements (#124012) * Set Android test script stdout to line-buffered * Print warning logcat messages on stderr * Add a -vv option to display high-volume messages which are rarely useful * Documentation and comment improvements --- Android/README.md | 26 ++++++++++++++------------ Android/android.py | 25 ++++++++++++++++++++----- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/Android/README.md b/Android/README.md index bae9150ef057ac..f9e3f290ef46f5 100644 --- a/Android/README.md +++ b/Android/README.md @@ -12,8 +12,12 @@ approachable user experience: ## Prerequisites -Export the `ANDROID_HOME` environment variable to point at your Android SDK. If -you don't already have the SDK, here's how to install it: +First, make sure you have all the usual tools and libraries needed to build +Python for your development machine. + +Second, you'll need an Android SDK. If you already have the SDK installed, +export the `ANDROID_HOME` environment variable to point at its location. +Otherwise, here's how to install it: * Download the "Command line tools" from . * Create a directory `android-sdk/cmdline-tools`, and unzip the command line @@ -37,11 +41,6 @@ development tools, which currently means Linux or macOS. This involves doing a cross-build where you use a "build" Python (for your development machine) to help produce a "host" Python for Android. -First, make sure you have all the usual tools and libraries needed to build -Python for your development machine. The only Android tool you need to install -is the command line tools package above: the build script will download the -rest. - The easiest way to do a build is to use the `android.py` script. You can either have it perform the entire build process from start to finish in one step, or you can do it in discrete steps that mirror running `configure` and `make` for @@ -80,12 +79,15 @@ call. For example, if you want a pydebug build that also caches the results from ## Testing -The tests can be run on Linux, macOS, or Windows, although on Windows you'll -have to build the `cross-build/HOST` subdirectory on one of the other platforms -and copy it over. +The test suite can be run on Linux, macOS, or Windows: + +* On Linux, the emulator needs access to the KVM virtualization interface, and + a DISPLAY environment variable pointing at an X server. +* On Windows, you won't be able to do the build on the same machine, so you'll + have to copy the `cross-build/HOST` directory from somewhere else. -The test suite can usually be run on a device with 2 GB of RAM, though for some -configurations or test orders you may need to increase this. As of Android +The test suite can usually be run on a device with 2 GB of RAM, but this is +borderline, so you may need to increase it to 4 GB. As of Android Studio Koala, 2 GB is the default for all emulators, although the user interface may indicate otherwise. The effective setting is `hw.ramSize` in ~/.android/avd/*.avd/hardware-qemu.ini, whereas Android Studio displays the diff --git a/Android/android.py b/Android/android.py index bfa7832a4a83da..8696d9eaeca19c 100755 --- a/Android/android.py +++ b/Android/android.py @@ -259,8 +259,8 @@ def setup_testbed(): f"{temp_dir}/{outer_jar}", "gradle-wrapper.jar"]) -# run_testbed will build the app automatically, but it hides the Gradle output -# by default, so it's useful to have this as a separate command for the buildbot. +# run_testbed will build the app automatically, but it's useful to have this as +# a separate command to allow running the app outside of this script. def build_testbed(context): setup_sdk() setup_testbed() @@ -376,6 +376,8 @@ async def find_pid(serial): shown_error = False while True: try: + # `pidof` requires API level 24 or higher. The level 23 emulator + # includes it, but it doesn't work (it returns all processes). pid = (await async_check_output( adb, "-s", serial, "shell", "pidof", "-s", APP_ID )).strip() @@ -407,6 +409,7 @@ async def logcat_task(context, initial_devices): serial = await wait_for(find_device(context, initial_devices), startup_timeout) pid = await wait_for(find_pid(serial), startup_timeout) + # `--pid` requires API level 24 or higher. args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] hidden_output = [] async with async_process( @@ -421,11 +424,15 @@ async def logcat_task(context, initial_devices): # such messages, but other components might. level, message = None, line + # Exclude high-volume messages which are rarely useful. + if context.verbose < 2 and "from python test_syslog" in message: + continue + # Put high-level messages on stderr so they're highlighted in the # buildbot logs. This will include Python's own stderr. stream = ( sys.stderr - if level in ["E", "F"] # ERROR and FATAL (aka ASSERT) + if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka ASSERT) else sys.stdout ) @@ -573,8 +580,9 @@ def parse_args(): test = subcommands.add_parser( "test", help="Run the test suite") test.add_argument( - "-v", "--verbose", action="store_true", - help="Show Gradle output, and non-Python logcat messages") + "-v", "--verbose", action="count", default=0, + help="Show Gradle output, and non-Python logcat messages. " + "Use twice to include high-volume messages which are rarely useful.") device_group = test.add_mutually_exclusive_group(required=True) device_group.add_argument( "--connected", metavar="SERIAL", help="Run on a connected device. " @@ -591,6 +599,13 @@ def parse_args(): def main(): install_signal_handler() + + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + context = parse_args() dispatch = {"configure-build": configure_build_python, "make-build": make_build_python, From 584cdf8d4140406b3676515332a26c856c02618b Mon Sep 17 00:00:00 2001 From: Yngve Mardal Moe Date: Fri, 13 Sep 2024 06:36:17 +0200 Subject: [PATCH 04/32] gh-123614: Add save function to turtle.py (#123617) --- Doc/library/turtle.rst | 19 ++++++ Lib/test/test_turtle.py | 66 +++++++++++++++++++ Lib/turtle.py | 36 +++++++++- ...-09-02-20-39-10.gh-issue-123614.26TMHp.rst | 2 + 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-39-10.gh-issue-123614.26TMHp.rst diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index afda3685d606bb..da801d4dc1f5b3 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -427,6 +427,7 @@ Input methods Methods specific to Screen | :func:`bye` | :func:`exitonclick` + | :func:`save` | :func:`setup` | :func:`title` @@ -2269,6 +2270,24 @@ Methods specific to Screen, not inherited from TurtleScreen client script. +.. function:: save(filename, overwrite=False) + + Save the current turtle drawing (and turtles) as a PostScript file. + + :param filename: the path of the saved PostScript file + :param overwrite: if ``False`` and there already exists a file with the given + filename, then the function will raise a + ``FileExistsError``. If it is ``True``, the file will be + overwritten. + + .. doctest:: + :skipif: _tkinter is None + + >>> screen.save("my_drawing.ps") + >>> screen.save("my_drawing.ps", overwrite=True) + + .. versionadded:: 3.14 + .. function:: setup(width=_CFG["width"], height=_CFG["height"], startx=_CFG["leftright"], starty=_CFG["topbottom"]) Set the size and position of the main window. Default values of arguments diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 14121a590a5026..c75a002a89b4c4 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -1,5 +1,9 @@ +import os import pickle +import re import unittest +import unittest.mock +import tempfile from test import support from test.support import import_helper from test.support import os_helper @@ -130,6 +134,7 @@ def assertVectorsAlmostEqual(self, vec1, vec2): self.assertAlmostEqual( i, j, msg='values at index {} do not match'.format(idx)) + class Multiplier: def __mul__(self, other): @@ -461,6 +466,67 @@ def test_teleport(self): self.assertTrue(tpen.isdown()) +class TestTurtleScreen(unittest.TestCase): + def test_save_raises_if_wrong_extension(self) -> None: + screen = unittest.mock.Mock() + + msg = "Unknown file extension: '.png', must be one of {'.ps', '.eps'}" + with ( + tempfile.TemporaryDirectory() as tmpdir, + self.assertRaisesRegex(ValueError, re.escape(msg)) + ): + turtle.TurtleScreen.save(screen, os.path.join(tmpdir, "file.png")) + + def test_save_raises_if_parent_not_found(self) -> None: + screen = unittest.mock.Mock() + + with tempfile.TemporaryDirectory() as tmpdir: + parent = os.path.join(tmpdir, "unknown_parent") + msg = f"The directory '{parent}' does not exist. Cannot save to it" + + with self.assertRaisesRegex(FileNotFoundError, re.escape(msg)): + turtle.TurtleScreen.save(screen, os.path.join(parent, "a.ps")) + + def test_save_raises_if_file_found(self) -> None: + screen = unittest.mock.Mock() + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + with open(file_path, "w") as f: + f.write("some text") + + msg = ( + f"The file '{file_path}' already exists. To overwrite it use" + " the 'overwrite=True' argument of the save function." + ) + with self.assertRaisesRegex(FileExistsError, re.escape(msg)): + turtle.TurtleScreen.save(screen, file_path) + + def test_save_overwrites_if_specified(self) -> None: + screen = unittest.mock.Mock() + screen.cv.postscript.return_value = "postscript" + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + with open(file_path, "w") as f: + f.write("some text") + + turtle.TurtleScreen.save(screen, file_path, overwrite=True) + with open(file_path) as f: + assert f.read() == "postscript" + + def test_save(self) -> None: + screen = unittest.mock.Mock() + screen.cv.postscript.return_value = "postscript" + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, "some_file.ps") + + turtle.TurtleScreen.save(screen, file_path) + with open(file_path) as f: + assert f.read() == "postscript" + + class TestModuleLevel(unittest.TestCase): def test_all_signatures(self): import inspect diff --git a/Lib/turtle.py b/Lib/turtle.py index 99850ae5efe348..8a5801f2efe625 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -106,6 +106,7 @@ import sys from os.path import isfile, split, join +from pathlib import Path from copy import deepcopy from tkinter import simpledialog @@ -115,7 +116,7 @@ 'clearscreen', 'colormode', 'delay', 'exitonclick', 'getcanvas', 'getshapes', 'listen', 'mainloop', 'mode', 'numinput', 'onkey', 'onkeypress', 'onkeyrelease', 'onscreenclick', 'ontimer', - 'register_shape', 'resetscreen', 'screensize', 'setup', + 'register_shape', 'resetscreen', 'screensize', 'save', 'setup', 'setworldcoordinates', 'textinput', 'title', 'tracer', 'turtles', 'update', 'window_height', 'window_width'] _tg_turtle_functions = ['back', 'backward', 'begin_fill', 'begin_poly', 'bk', @@ -1492,6 +1493,39 @@ def screensize(self, canvwidth=None, canvheight=None, bg=None): """ return self._resize(canvwidth, canvheight, bg) + def save(self, filename, *, overwrite=False): + """Save the drawing as a PostScript file + + Arguments: + filename -- a string, the path of the created file. + Must end with '.ps' or '.eps'. + + Optional arguments: + overwrite -- boolean, if true, then existing files will be overwritten + + Example (for a TurtleScreen instance named screen): + >>> screen.save('my_drawing.eps') + """ + filename = Path(filename) + if not filename.parent.exists(): + raise FileNotFoundError( + f"The directory '{filename.parent}' does not exist." + " Cannot save to it." + ) + if not overwrite and filename.exists(): + raise FileExistsError( + f"The file '{filename}' already exists. To overwrite it use" + " the 'overwrite=True' argument of the save function." + ) + if (ext := filename.suffix) not in {".ps", ".eps"}: + raise ValueError( + f"Unknown file extension: '{ext}'," + " must be one of {'.ps', '.eps'}" + ) + + postscript = self.cv.postscript() + filename.write_text(postscript) + onscreenclick = onclick resetscreen = reset clearscreen = clear diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-39-10.gh-issue-123614.26TMHp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-39-10.gh-issue-123614.26TMHp.rst new file mode 100644 index 00000000000000..64a5eac9f7840a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-02-20-39-10.gh-issue-123614.26TMHp.rst @@ -0,0 +1,2 @@ +Add :func:`turtle.save` to easily save Turtle drawings as PostScript files. +Patch by Marie Roald and Yngve Mardal Moe. From f5548834256414c6a721e9ebfa511e043e73ef03 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 13 Sep 2024 05:58:11 +0100 Subject: [PATCH 05/32] gh-116622: Mock the passage of time in Android logcat rate limit tests (#124015) Mock the passage of time in Android logcat rate limit tests Co-authored-by: Russell Keith-Magee --- Lib/_android_support.py | 5 ++++- Lib/test/test_android.py | 32 ++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 066e54708fb75c..353b34fa36aca4 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -165,7 +165,10 @@ def write(self, prio, tag, message): now = time() self._bucket_level += ( (now - self._prev_write_time) * MAX_BYTES_PER_SECOND) - self._bucket_level = min(self._bucket_level, BUCKET_SIZE) + + # If the bucket level is still below zero, the clock must have gone + # backwards, so reset it to zero and continue. + self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE)) self._prev_write_time = now self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 09fa21cea877c2..2ef9f10fdcc1cc 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -10,7 +10,7 @@ from contextlib import ExitStack, contextmanager from threading import Thread from test.support import LOOPBACK_TIMEOUT -from time import sleep, time +from time import time from unittest.mock import patch @@ -42,7 +42,8 @@ def logcat_thread(): for line in self.logcat_process.stdout: self.logcat_queue.put(line.rstrip("\n")) self.logcat_process.stdout.close() - Thread(target=logcat_thread).start() + self.logcat_thread = Thread(target=logcat_thread) + self.logcat_thread.start() from ctypes import CDLL, c_char_p, c_int android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") @@ -78,6 +79,7 @@ def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): def tearDown(self): self.logcat_process.terminate() self.logcat_process.wait(LOOPBACK_TIMEOUT) + self.logcat_thread.join(LOOPBACK_TIMEOUT) @contextmanager def unbuffered(self, stream): @@ -369,6 +371,8 @@ def write(b, lines=None, *, write_len=None): ): stream.write(obj) + +class TestAndroidRateLimit(unittest.TestCase): def test_rate_limit(self): # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 PER_MESSAGE_OVERHEAD = 28 @@ -387,6 +391,19 @@ def test_rate_limit(self): 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0)) ) + "\n" + # To avoid depending on the performance of the test device, we mock the + # passage of time. + mock_now = time() + + def mock_time(): + # Avoid division by zero by simulating a small delay. + mock_sleep(0.0001) + return mock_now + + def mock_sleep(duration): + nonlocal mock_now + mock_now += duration + # See _android_support.py. The default values of these parameters work # well across a wide range of devices, but we'll use smaller values to # ensure a quick and reliable test that doesn't flood the log too much. @@ -395,21 +412,24 @@ def test_rate_limit(self): with ( patch("_android_support.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), patch("_android_support.BUCKET_SIZE", BUCKET_KB * 1024), + patch("_android_support.sleep", mock_sleep), + patch("_android_support.time", mock_time), ): # Make sure the token bucket is full. - sleep(BUCKET_KB / MAX_KB_PER_SECOND) + stream.write("Initial message to reset _prev_write_time") + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) line_num = 0 # Write BUCKET_KB messages, and return the rate at which they were # accepted in KB per second. def write_bucketful(): nonlocal line_num - start = time() + start = mock_time() max_line_num = line_num + BUCKET_KB while line_num < max_line_num: stream.write(message.format(line_num)) line_num += 1 - return BUCKET_KB / (time() - start) + return BUCKET_KB / (mock_time() - start) # The first bucketful should be written with minimal delay. The # factor of 2 here is not arbitrary: it verifies that the system can @@ -427,5 +447,5 @@ def write_bucketful(): ) # Once the token bucket refills, we should go back to full speed. - sleep(BUCKET_KB / MAX_KB_PER_SECOND) + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) From e5b0185e43c972ce98decd1493cd0b0c3a6b166b Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 12 Sep 2024 23:14:52 -0700 Subject: [PATCH 06/32] GH-101599: Update docs to remove redundant option in argparse tutorial (#124025) --- Doc/howto/argparse.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index ae5bab90bf8131..30d9ac700376e6 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -444,7 +444,7 @@ And the output: options: -h, --help show this help message and exit - -v {0,1,2}, --verbosity {0,1,2} + -v, --verbosity {0,1,2} increase output verbosity Note that the change also reflects both in the error message as well as the From 403f3ddedcab14f6c16ea78a93bb4acf49d06a07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 13 Sep 2024 12:33:13 +0200 Subject: [PATCH 07/32] gh-123961: Remove global variable `ModDict` in `_cursesmodule.c` (#123962) --- Modules/_cursesmodule.c | 79 ++++++++++++--------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 3 - 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 1d36b40d5236b8..be31cb4f54ddcc 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -3255,8 +3255,6 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) Py_RETURN_NONE; } -static PyObject *ModDict; - /*[clinic input] _curses.initscr @@ -3285,19 +3283,23 @@ _curses_initscr_impl(PyObject *module) initialised = initialised_setupterm = TRUE; -/* This was moved from initcurses() because it core dumped on SGI, - where they're not defined until you've called initscr() */ -#define SetDictInt(NAME, VALUE) \ - do { \ - PyObject *value = PyLong_FromLong((long)(VALUE)); \ - if (value == NULL) { \ - return NULL; \ - } \ - int rc = PyDict_SetItemString(ModDict, (NAME), value); \ - Py_DECREF(value); \ - if (rc < 0) { \ - return NULL; \ - } \ + PyObject *module_dict = PyModule_GetDict(module); // borrowed + if (module_dict == NULL) { + return NULL; + } + /* This was moved from initcurses() because it core dumped on SGI, + where they're not defined until you've called initscr() */ +#define SetDictInt(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return NULL; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return NULL; \ + } \ } while (0) /* Here are some graphic symbols you can use */ @@ -3976,11 +3978,11 @@ _curses_qiflush_impl(PyObject *module, int flag) Py_RETURN_NONE; } -/* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES - * and _curses.COLS */ #if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM) +/* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES + * and _curses.COLS. Returns 1 on success and 0 on failure. */ static int -update_lines_cols(void) +update_lines_cols(PyObject *private_module) { PyObject *exposed_module = NULL, *o = NULL; @@ -3992,6 +3994,10 @@ update_lines_cols(void) if (exposed_module_dict == NULL) { goto error; } + PyObject *private_module_dict = PyModule_GetDict(private_module); // borrowed + if (private_module_dict == NULL) { + goto error; + } o = PyLong_FromLong(LINES); if (o == NULL) { @@ -4000,7 +4006,7 @@ update_lines_cols(void) if (PyDict_SetItemString(exposed_module_dict, "LINES", o) < 0) { goto error; } - if (PyDict_SetItemString(ModDict, "LINES", o) < 0) { + if (PyDict_SetItemString(private_module_dict, "LINES", o) < 0) { goto error; } Py_DECREF(o); @@ -4012,7 +4018,7 @@ update_lines_cols(void) if (PyDict_SetItemString(exposed_module_dict, "COLS", o) < 0) { goto error; } - if (PyDict_SetItemString(ModDict, "COLS", o) < 0) { + if (PyDict_SetItemString(private_module_dict, "COLS", o) < 0) { goto error; } Py_DECREF(o); @@ -4034,7 +4040,7 @@ static PyObject * _curses_update_lines_cols_impl(PyObject *module) /*[clinic end generated code: output=423f2b1e63ed0f75 input=5f065ab7a28a5d90]*/ { - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { return NULL; } Py_RETURN_NONE; @@ -4121,7 +4127,7 @@ _curses_resizeterm_impl(PyObject *module, int nlines, int ncols) result = PyCursesCheckERR(resizeterm(nlines, ncols), "resizeterm"); if (!result) return NULL; - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { Py_DECREF(result); return NULL; } @@ -4160,7 +4166,7 @@ _curses_resize_term_impl(PyObject *module, int nlines, int ncols) result = PyCursesCheckERR(resize_term(nlines, ncols), "resize_term"); if (!result) return NULL; - if (!update_lines_cols()) { + if (!update_lines_cols(module)) { Py_DECREF(result); return NULL; } @@ -4232,17 +4238,21 @@ _curses_start_color_impl(PyObject *module) initialisedcolors = TRUE; -#define DICT_ADD_INT_VALUE(NAME, VALUE) \ - do { \ - PyObject *value = PyLong_FromLong((long)(VALUE)); \ - if (value == NULL) { \ - return NULL; \ - } \ - int rc = PyDict_SetItemString(ModDict, (NAME), value); \ - Py_DECREF(value); \ - if (rc < 0) { \ - return NULL; \ - } \ + PyObject *module_dict = PyModule_GetDict(module); // borrowed + if (module_dict == NULL) { + return NULL; + } +#define DICT_ADD_INT_VALUE(NAME, VALUE) \ + do { \ + PyObject *value = PyLong_FromLong((long)(VALUE)); \ + if (value == NULL) { \ + return NULL; \ + } \ + int rc = PyDict_SetItemString(module_dict, (NAME), value); \ + Py_DECREF(value); \ + if (rc < 0) { \ + return NULL; \ + } \ } while (0) DICT_ADD_INT_VALUE("COLORS", COLORS); @@ -4779,7 +4789,6 @@ cursesmodule_exec(PyObject *module) if (module_dict == NULL) { return -1; } - ModDict = module_dict; /* For PyCurses_InitScr to use later */ void **PyCurses_API = PyMem_Calloc(PyCurses_API_pointers, sizeof(void *)); if (PyCurses_API == NULL) { diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index cb9750a69a632b..71f2fded77fe07 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -391,9 +391,6 @@ Modules/xxmodule.c - ErrorObject - ##----------------------- ## other -## initialized once -Modules/_cursesmodule.c - ModDict - - ## state Modules/_datetimemodule.c - _datetime_global_state - Modules/_tkinter.c - tcl_lock - From d7e83398c188a0acd19a496ee2eeeeab52d64a11 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 13 Sep 2024 14:13:52 +0300 Subject: [PATCH 08/32] gh-108303: Remove the non-test `Lib/test/reperf.py` (GH-114356) --- Lib/test/_test_embed_structseq.py | 9 ++++++++- Lib/test/reperf.py | 23 ----------------------- Lib/test/test_embed.py | 11 +++++++---- 3 files changed, 15 insertions(+), 28 deletions(-) delete mode 100644 Lib/test/reperf.py diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 868f9f83e8be77..154662efce9412 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -48,7 +48,14 @@ def test_sys_funcs(self): try: - unittest.main() + unittest.main( + module=( + '__main__' + if __name__ == '__main__' + # Avoiding a circular import: + else sys.modules['test._test_embed_structseq'] + ) + ) except SystemExit as exc: if exc.args[0] != 0: raise diff --git a/Lib/test/reperf.py b/Lib/test/reperf.py deleted file mode 100644 index e93bacdb5843c6..00000000000000 --- a/Lib/test/reperf.py +++ /dev/null @@ -1,23 +0,0 @@ -import re -import time - -def main(): - s = "\13hello\14 \13world\14 " * 1000 - p = re.compile(r"([\13\14])") - timefunc(10, p.sub, "", s) - timefunc(10, p.split, s) - timefunc(10, p.findall, s) - -def timefunc(n, func, *args, **kw): - t0 = time.perf_counter() - try: - for i in range(n): - result = func(*args, **kw) - return result - finally: - t1 = time.perf_counter() - if n > 1: - print(n, "times", end=' ') - print(func.__name__, "%.3f" % (t1-t0), "CPU seconds") - -main() diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 6790326a2afa47..7c5cb855a397ab 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1971,7 +1971,11 @@ def test_no_memleak(self): @unittest.skipUnless(support.Py_DEBUG, '-X presite requires a Python debug build') def test_presite(self): - cmd = [sys.executable, "-I", "-X", "presite=test.reperf", "-c", "print('cmd')"] + cmd = [ + sys.executable, + "-I", "-X", "presite=test._test_embed_structseq", + "-c", "print('unique-python-message')", + ] proc = subprocess.run( cmd, stdout=subprocess.PIPE, @@ -1980,9 +1984,8 @@ def test_presite(self): ) self.assertEqual(proc.returncode, 0) out = proc.stdout.strip() - self.assertIn("10 times sub", out) - self.assertIn("CPU seconds", out) - self.assertIn("cmd", out) + self.assertIn("Tests passed", out) + self.assertIn("unique-python-message", out) class StdPrinterTests(EmbeddingTestsMixin, unittest.TestCase): From 432bf31327c6b9647acb8bdb0eac2d392fd9f60a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 13 Sep 2024 13:18:49 +0200 Subject: [PATCH 09/32] gh-123909: PyType_From*: Disallow metaclasses with custom tp_new (GH-123947) --- Doc/c-api/type.rst | 24 +++++++++--- Lib/test/test_capi/test_misc.py | 12 ++---- ...-09-10-16-54-27.gh-issue-123909.CTGxDR.rst | 3 ++ Objects/typeobject.c | 38 +++++-------------- 4 files changed, 34 insertions(+), 43 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-09-10-16-54-27.gh-issue-123909.CTGxDR.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 0cae5c09505ebe..fa04d6c0ad5da5 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -345,8 +345,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -362,8 +366,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) @@ -378,8 +386,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. raw:: html diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index d50217b695967e..18392c4fce5dc1 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -722,22 +722,16 @@ def test_heaptype_with_custom_metaclass_custom_new(self): with self.assertRaisesRegex(TypeError, msg): t = _testcapi.pytype_fromspec_meta(metaclass) - def test_heaptype_with_custom_metaclass_deprecation(self): + def test_heaptype_base_with_custom_metaclass(self): metaclass = _testcapi.HeapCTypeMetaclassCustomNew - # gh-103968: a metaclass with custom tp_new is deprecated, but still - # allowed for functions that existed in 3.11 - # (PyType_FromSpecWithBases is used here). class Base(metaclass=metaclass): pass # Class creation from C - with warnings_helper.check_warnings( - ('.* _testcapi.Subclass .* custom tp_new.*in Python 3.14.*', DeprecationWarning), - ): + msg = "Metaclasses with custom tp_new are not supported." + with self.assertRaisesRegex(TypeError, msg): sub = _testcapi.make_type_with_base(Base) - self.assertTrue(issubclass(sub, Base)) - self.assertIsInstance(sub, metaclass) def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): for weakref_cls in (_testcapi.HeapCTypeWithWeakref, diff --git a/Misc/NEWS.d/next/C_API/2024-09-10-16-54-27.gh-issue-123909.CTGxDR.rst b/Misc/NEWS.d/next/C_API/2024-09-10-16-54-27.gh-issue-123909.CTGxDR.rst new file mode 100644 index 00000000000000..b7a4913abbcb89 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-09-10-16-54-27.gh-issue-123909.CTGxDR.rst @@ -0,0 +1,3 @@ +:c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases` and +:c:func:`PyType_FromModuleAndSpec` will now fail if the metaclass of the new +type has custom :c:member:`~PyTypeObject.tp_new`. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a6483f74b7947d..28edd801284b81 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4668,10 +4668,10 @@ special_offset_from_member( return -1; } -static PyObject * -_PyType_FromMetaclass_impl( +PyObject * +PyType_FromMetaclass( PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in, int _allow_tp_new) + PyType_Spec *spec, PyObject *bases_in) { /* Invariant: A non-NULL value in one of these means this function holds * a strong reference or owns allocated memory. @@ -4848,21 +4848,10 @@ _PyType_FromMetaclass_impl( goto finally; } if (metaclass->tp_new && metaclass->tp_new != PyType_Type.tp_new) { - if (_allow_tp_new) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "Type %s uses PyType_Spec with a metaclass that has custom " - "tp_new. This is deprecated and will no longer be allowed in " - "Python 3.14.", spec->name) < 0) { - goto finally; - } - } - else { - PyErr_SetString( - PyExc_TypeError, - "Metaclasses with custom tp_new are not supported."); - goto finally; - } + PyErr_SetString( + PyExc_TypeError, + "Metaclasses with custom tp_new are not supported."); + goto finally; } /* Calculate best base, and check that all bases are type objects */ @@ -5109,29 +5098,22 @@ _PyType_FromMetaclass_impl( return (PyObject*)res; } -PyObject * -PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in) -{ - return _PyType_FromMetaclass_impl(metaclass, module, spec, bases_in, 0); -} - PyObject * PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { - return _PyType_FromMetaclass_impl(NULL, module, spec, bases, 1); + return PyType_FromMetaclass(NULL, module, spec, bases); } PyObject * PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { - return _PyType_FromMetaclass_impl(NULL, NULL, spec, bases, 1); + return PyType_FromMetaclass(NULL, NULL, spec, bases); } PyObject * PyType_FromSpec(PyType_Spec *spec) { - return _PyType_FromMetaclass_impl(NULL, NULL, spec, NULL, 1); + return PyType_FromMetaclass(NULL, NULL, spec, NULL); } PyObject * From e49d1b44d3ef2542c0ae165e14a7e5ffbc32b2d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 13 Sep 2024 13:26:19 +0200 Subject: [PATCH 10/32] gh-124044: protect macros expansions in `_cursesmodules.c` using `do { ... } while (0)` (#124045) --- Modules/_cursesmodule.c | 47 ++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index be31cb4f54ddcc..b1fd839d7a5f82 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -182,22 +182,31 @@ static char *screen_encoding = NULL; /* Utility Macros */ #define PyCursesSetupTermCalled \ - if (initialised_setupterm != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call (at least) setupterm() first"); \ - return 0; } + do { \ + if (initialised_setupterm != TRUE) { \ + PyErr_SetString(PyCursesError, \ + "must call (at least) setupterm() first"); \ + return 0; \ + } \ + } while (0) -#define PyCursesInitialised \ - if (initialised != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call initscr() first"); \ - return 0; } +#define PyCursesInitialised \ + do { \ + if (initialised != TRUE) { \ + PyErr_SetString(PyCursesError, \ + "must call initscr() first"); \ + return 0; \ + } \ + } while (0) #define PyCursesInitialisedColor \ - if (initialisedcolors != TRUE) { \ - PyErr_SetString(PyCursesError, \ - "must call start_color() first"); \ - return 0; } + do { \ + if (initialisedcolors != TRUE) { \ + PyErr_SetString(PyCursesError, \ + "must call start_color() first"); \ + return 0; \ + } \ + } while (0) /* Utility Functions */ @@ -2633,12 +2642,12 @@ PyTypeObject PyCursesWindow_Type = { #define NoArgNoReturnFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ return PyCursesCheckERR(X(), # X); } #define NoArgOrFlagNoReturnFunctionBody(X, flag) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ if (flag) \ return PyCursesCheckERR(X(), # X); \ else \ @@ -2647,23 +2656,23 @@ PyTypeObject PyCursesWindow_Type = { #define NoArgReturnIntFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ return PyLong_FromLong((long) X()); } #define NoArgReturnStringFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ return PyBytes_FromString(X()); } #define NoArgTrueFalseFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ return PyBool_FromLong(X()); } #define NoArgNoReturnVoidFunctionBody(X) \ { \ - PyCursesInitialised \ + PyCursesInitialised; \ X(); \ Py_RETURN_NONE; } From acb3f875fba0552d55989f1f2686c334fecaecca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:43:06 +0200 Subject: [PATCH 11/32] gh-123961: Add `curses` prefix to global variables in `_cursesmodule.c` (#124047) Use the `const char*` type instead of a `const *` for the encoding name. --- Modules/_cursesmodule.c | 28 ++++++++++----------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 8 +++--- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index b1fd839d7a5f82..c9ee5687c2b5d9 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -170,20 +170,20 @@ class _curses.window "PyCursesWindowObject *" "&PyCursesWindow_Type" static PyObject *PyCursesError; /* Tells whether setupterm() has been called to initialise terminfo. */ -static int initialised_setupterm = FALSE; +static int curses_setupterm_called = FALSE; /* Tells whether initscr() has been called to initialise curses. */ -static int initialised = FALSE; +static int curses_initscr_called = FALSE; /* Tells whether start_color() has been called to initialise color usage. */ -static int initialisedcolors = FALSE; +static int curses_start_color_called = FALSE; -static char *screen_encoding = NULL; +static const char *curses_screen_encoding = NULL; /* Utility Macros */ #define PyCursesSetupTermCalled \ do { \ - if (initialised_setupterm != TRUE) { \ + if (curses_setupterm_called != TRUE) { \ PyErr_SetString(PyCursesError, \ "must call (at least) setupterm() first"); \ return 0; \ @@ -192,7 +192,7 @@ static char *screen_encoding = NULL; #define PyCursesInitialised \ do { \ - if (initialised != TRUE) { \ + if (curses_initscr_called != TRUE) { \ PyErr_SetString(PyCursesError, \ "must call initscr() first"); \ return 0; \ @@ -201,7 +201,7 @@ static char *screen_encoding = NULL; #define PyCursesInitialisedColor \ do { \ - if (initialisedcolors != TRUE) { \ + if (curses_start_color_called != TRUE) { \ PyErr_SetString(PyCursesError, \ "must call start_color() first"); \ return 0; \ @@ -267,7 +267,7 @@ PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) if (win) encoding = win->encoding; else - encoding = screen_encoding; + encoding = curses_screen_encoding; bytes = PyUnicode_AsEncodedString(obj, encoding, NULL); if (bytes == NULL) return 0; @@ -3278,7 +3278,7 @@ _curses_initscr_impl(PyObject *module) { WINDOW *win; - if (initialised) { + if (curses_initscr_called) { wrefresh(stdscr); return (PyObject *)PyCursesWindow_New(stdscr, NULL); } @@ -3290,7 +3290,7 @@ _curses_initscr_impl(PyObject *module) return NULL; } - initialised = initialised_setupterm = TRUE; + curses_initscr_called = curses_setupterm_called = TRUE; PyObject *module_dict = PyModule_GetDict(module); // borrowed if (module_dict == NULL) { @@ -3386,7 +3386,7 @@ _curses_initscr_impl(PyObject *module) if (winobj == NULL) { return NULL; } - screen_encoding = winobj->encoding; + curses_screen_encoding = winobj->encoding; return (PyObject *)winobj; } @@ -3428,7 +3428,7 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) } } - if (!initialised_setupterm && setupterm((char *)term, fd, &err) == ERR) { + if (!curses_setupterm_called && setupterm((char *)term, fd, &err) == ERR) { const char* s = "setupterm: unknown error"; if (err == 0) { @@ -3441,7 +3441,7 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) return NULL; } - initialised_setupterm = TRUE; + curses_setupterm_called = TRUE; Py_RETURN_NONE; } @@ -4245,7 +4245,7 @@ _curses_start_color_impl(PyObject *module) return NULL; } - initialisedcolors = TRUE; + curses_start_color_called = TRUE; PyObject *module_dict = PyModule_GetDict(module); // borrowed if (module_dict == NULL) { diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 71f2fded77fe07..e1c07f88b963bc 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -409,10 +409,10 @@ Modules/_tkinter.c - trbInCmd - Include/datetime.h - PyDateTimeAPI - Modules/_ctypes/cfield.c _ctypes_get_fielddesc initialized - Modules/_ctypes/malloc_closure.c - _pagesize - -Modules/_cursesmodule.c - initialised - -Modules/_cursesmodule.c - initialised_setupterm - -Modules/_cursesmodule.c - initialisedcolors - -Modules/_cursesmodule.c - screen_encoding - +Modules/_cursesmodule.c - curses_initscr_called - +Modules/_cursesmodule.c - curses_setupterm_called - +Modules/_cursesmodule.c - curses_start_color_called - +Modules/_cursesmodule.c - curses_screen_encoding - Modules/_elementtree.c - expat_capi - Modules/readline.c - libedit_append_replace_history_offset - Modules/readline.c - using_libedit_emulation - From b46c65ed2b78214cb8914779ac4e8d343ac4775e Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Fri, 13 Sep 2024 15:49:13 +0300 Subject: [PATCH 12/32] gh-123811: Test that round(Decimal) can return signed zero (GH-124007) --- Lib/test/test_decimal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 46755107de0102..12479e32d0f5db 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -2071,7 +2071,9 @@ def test_tonum_methods(self): #to quantize, which is already extensively tested test_triples = [ ('123.456', -4, '0E+4'), + ('-123.456', -4, '-0E+4'), ('123.456', -3, '0E+3'), + ('-123.456', -3, '-0E+3'), ('123.456', -2, '1E+2'), ('123.456', -1, '1.2E+2'), ('123.456', 0, '123'), From cfe6074d1fa81cf0684fbf8a623616441a1966e7 Mon Sep 17 00:00:00 2001 From: Nate Ohlson Date: Fri, 13 Sep 2024 08:40:04 -0500 Subject: [PATCH 13/32] gh-112301: Enable warning emitting options and ignore warnings in CI (#123020) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- .github/workflows/reusable-macos.yml | 18 +- .github/workflows/reusable-ubuntu.yml | 18 +- ...-08-14-19-43-57.gh-issue-112301.IQUcOy.rst | 1 + Tools/build/.warningignore_macos | 227 +++++++++++++++ Tools/build/.warningignore_ubuntu | 260 ++++++++++++++++++ Tools/build/check_warnings.py | 146 ++++------ configure | 195 +++++++++++++ configure.ac | 5 + 8 files changed, 779 insertions(+), 91 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index d77723ef27c2dc..49613a8ddaebf5 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -35,7 +35,7 @@ jobs: path: config.cache key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk make - name: Configure CPython run: | GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ @@ -48,10 +48,22 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt + if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + run: gmake -j8 + - name: Build CPython for compiler warning check + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: >- + python3 Tools/build/check_warnings.py + --compiler-output-file-path=compiler_output_macos.txt + --warning-ignore-file-path=Tools/build/.warningignore_macos + --compiler-output-type=clang + --fail-on-regression + --fail-on-improvement + --path-prefix="./" - name: Tests run: make test diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index b197db814b2743..f2c83798027d10 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -67,20 +67,32 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: >- ../cpython-ro-srcdir/configure - CFLAGS="-fdiagnostics-format=json" --config-cache --with-pydebug --enable-slower-safety --with-openssl=$OPENSSL_DIR ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Build CPython out-of-tree + if: ${{ inputs.free-threading }} working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt + run: make -j4 + - name: Build CPython out-of-tree (for compiler warning check) + if: ${{ !inputs.free-threading}} + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: set -o pipefail; make -j4 2>&1 | tee compiler_output_ubuntu.txt - name: Display build info working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json + if: ${{ !inputs.free-threading }} + run: >- + python Tools/build/check_warnings.py + --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt + --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu + --compiler-output-type=gcc + --fail-on-regression + --fail-on-improvement + --path-prefix="../cpython-ro-srcdir/" - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw diff --git a/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst b/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst new file mode 100644 index 00000000000000..9750cf203eef86 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst @@ -0,0 +1 @@ +Enable compiler options that warn of potential security vulnerabilities. diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos index 67f50119db7310..fffb828eb4a8ab 100644 --- a/Tools/build/.warningignore_macos +++ b/Tools/build/.warningignore_macos @@ -3,3 +3,230 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) +Include/internal/mimalloc/mimalloc/internal.h 4 +Include/internal/pycore_backoff.h 1 +Include/internal/pycore_dict.h 2 +Include/internal/pycore_gc.h 1 +Include/internal/pycore_long.h 2 +Include/internal/pycore_object.h 4 +Modules/_asynciomodule.c 3 +Modules/_bisectmodule.c 2 +Modules/_bz2module.c 5 +Modules/_collectionsmodule.c 2 +Modules/_csv.c 3 +Modules/_ctypes/_ctypes.c 37 +Modules/_ctypes/_ctypes_test_generated.c.h 141 +Modules/_ctypes/callbacks.c 6 +Modules/_ctypes/callproc.c 15 +Modules/_ctypes/cfield.c 59 +Modules/_ctypes/malloc_closure.c 3 +Modules/_ctypes/stgdict.c 17 +Modules/_cursesmodule.c 24 +Modules/_datetimemodule.c 28 +Modules/_dbmmodule.c 8 +Modules/_decimal/_decimal.c 15 +Modules/_elementtree.c 42 +Modules/_functoolsmodule.c 6 +Modules/_gdbmmodule.c 5 +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 +Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 24 +Modules/_hashopenssl.c 16 +Modules/_interpchannelsmodule.c 1 +Modules/_interpqueuesmodule.c 1 +Modules/_io/_iomodule.c 1 +Modules/_io/bufferedio.c 4 +Modules/_io/bytesio.c 11 +Modules/_io/fileio.c 9 +Modules/_io/stringio.c 8 +Modules/_io/textio.c 11 +Modules/_json.c 19 +Modules/_localemodule.c 3 +Modules/_lzmamodule.c 10 +Modules/_multiprocessing/semaphore.c 2 +Modules/_operator.c 5 +Modules/_pickle.c 71 +Modules/_posixsubprocess.c 8 +Modules/_queuemodule.c 4 +Modules/_randommodule.c 3 +Modules/_scproxy.c 3 +Modules/_sqlite/connection.c 4 +Modules/_sqlite/cursor.c 3 +Modules/_sqlite/module.c 2 +Modules/_sre/sre.c 18 +Modules/_sre/sre_lib.h 62 +Modules/_ssl.c 29 +Modules/_struct.c 1 +Modules/_testbuffer.c 22 +Modules/_testcapi/heaptype.c 1 +Modules/_testcapi/long.c 2 +Modules/_testcapi/mem.c 2 +Modules/_testcapi/monitoring.c 3 +Modules/_testcapi/pyatomic.c 1 +Modules/_testcapi/unicode.c 2 +Modules/_testcapi/vectorcall.c 3 +Modules/_testcapi/watchers.c 3 +Modules/_testcapimodule.c 3 +Modules/_testclinic.c 14 +Modules/_testexternalinspection.c 8 +Modules/_testinternalcapi.c 8 +Modules/_testinternalcapi/pytime.c 8 +Modules/_testinternalcapi/test_critical_sections.c 1 +Modules/_testinternalcapi/test_lock.c 2 +Modules/_testlimitedcapi/heaptype_relative.c 4 +Modules/_testlimitedcapi/object.c 2 +Modules/_testlimitedcapi/unicode.c 2 +Modules/_threadmodule.c 2 +Modules/_tkinter.c 6 +Modules/_xxtestfuzz/_xxtestfuzz.c 1 +Modules/_xxtestfuzz/fuzzer.c 11 +Modules/_zoneinfo.c 14 +Modules/arraymodule.c 32 +Modules/atexitmodule.c 1 +Modules/binascii.c 206 +Modules/blake2module.c 6 +Modules/cjkcodecs/_codecs_cn.c 1 +Modules/cjkcodecs/_codecs_iso2022.c 2 +Modules/cjkcodecs/_codecs_jp.c 14 +Modules/cjkcodecs/_codecs_kr.c 3 +Modules/cjkcodecs/cjkcodecs.h 1 +Modules/cjkcodecs/multibytecodec.c 2 +Modules/clinic/_testclinic.c.h 1 +Modules/clinic/arraymodule.c.h 1 +Modules/clinic/unicodedata.c.h 10 +Modules/cmathmodule.c 1 +Modules/expat/siphash.h 8 +Modules/expat/xmlparse.c 45 +Modules/expat/xmltok.c 17 +Modules/expat/xmltok_impl.c 34 +Modules/faulthandler.c 3 +Modules/fcntlmodule.c 1 +Modules/getpath.c 7 +Modules/grpmodule.c 4 +Modules/itertoolsmodule.c 7 +Modules/main.c 2 +Modules/mathmodule.c 15 +Modules/mmapmodule.c 20 +Modules/posixmodule.c 67 +Modules/pwdmodule.c 4 +Modules/pyexpat.c 20 +Modules/readline.c 1 +Modules/resource.c 3 +Modules/rotatingtree.c 1 +Modules/selectmodule.c 6 +Modules/sha3module.c 4 +Modules/signalmodule.c 1 +Modules/socketmodule.c 44 +Modules/syslogmodule.c 3 +Modules/timemodule.c 4 +Modules/unicodedata.c 28 +Modules/unicodedata_db.h 1 +Modules/xxsubtype.c 2 +Modules/zlibmodule.c 16 +Objects/abstract.c 2 +Objects/bytearrayobject.c 34 +Objects/bytes_methods.c 9 +Objects/bytesobject.c 35 +Objects/call.c 13 +Objects/classobject.c 4 +Objects/codeobject.c 15 +Objects/descrobject.c 2 +Objects/dictobject.c 28 +Objects/fileobject.c 3 +Objects/floatobject.c 30 +Objects/frameobject.c 19 +Objects/funcobject.c 1 +Objects/genobject.c 5 +Objects/listobject.c 43 +Objects/longobject.c 46 +Objects/memoryobject.c 6 +Objects/methodobject.c 1 +Objects/mimalloc/alloc.c 6 +Objects/mimalloc/arena.c 6 +Objects/mimalloc/heap.c 1 +Objects/mimalloc/init.c 2 +Objects/mimalloc/options.c 1 +Objects/mimalloc/os.c 4 +Objects/mimalloc/page-queue.c 2 +Objects/mimalloc/page.c 1 +Objects/mimalloc/prim/osx/../unix/prim.c 2 +Objects/mimalloc/random.c 1 +Objects/mimalloc/segment.c 11 +Objects/mimalloc/stats.c 1 +Objects/moduleobject.c 2 +Objects/object.c 1 +Objects/obmalloc.c 6 +Objects/odictobject.c 3 +Objects/rangeobject.c 10 +Objects/setobject.c 13 +Objects/sliceobject.c 4 +Objects/stringlib/codecs.h 26 +Objects/stringlib/eq.h 1 +Objects/stringlib/fastsearch.h 14 +Objects/stringlib/join.h 1 +Objects/stringlib/replace.h 4 +Objects/stringlib/repr.h 21 +Objects/stringlib/transmogrify.h 5 +Objects/structseq.c 14 +Objects/tupleobject.c 10 +Objects/typeobject.c 17 +Objects/unicodectype.c 7 +Objects/unicodeobject.c 113 +Parser/action_helpers.c 4 +Parser/lexer/buffer.c 1 +Parser/lexer/lexer.c 12 +Parser/parser.c 116 +Parser/pegen.c 7 +Parser/string_parser.c 7 +Parser/tokenizer/file_tokenizer.c 8 +Parser/tokenizer/helpers.c 7 +Parser/tokenizer/readline_tokenizer.c 3 +Programs/_freeze_module.c 1 +Python/Python-ast.c 15 +Python/asdl.c 3 +Python/assemble.c 7 +Python/ast_opt.c 7 +Python/bltinmodule.c 9 +Python/bootstrap_hash.c 4 +Python/ceval.c 8 +Python/ceval_gil.c 2 +Python/codecs.c 32 +Python/codegen.c 6 +Python/compile.c 2 +Python/context.c 1 +Python/crossinterp.c 2 +Python/crossinterp_data_lookup.h 1 +Python/dtoa.c 34 +Python/errors.c 1 +Python/fileutils.c 7 +Python/flowgraph.c 8 +Python/formatter_unicode.c 7 +Python/frame.c 4 +Python/gc.c 8 +Python/generated_cases.c.h 35 +Python/getargs.c 11 +Python/import.c 5 +Python/initconfig.c 11 +Python/instrumentation.c 31 +Python/intrinsics.c 1 +Python/legacy_tracing.c 3 +Python/lock.c 4 +Python/marshal.c 11 +Python/modsupport.c 3 +Python/mystrtoul.c 4 +Python/pathconfig.c 1 +Python/preconfig.c 2 +Python/pyarena.c 1 +Python/pyhash.c 2 +Python/pylifecycle.c 7 +Python/pystate.c 6 +Python/pystrhex.c 19 +Python/pystrtod.c 3 +Python/qsbr.c 2 +Python/specialize.c 10 +Python/suggestions.c 12 +Python/symtable.c 18 +Python/sysmodule.c 2 +Python/thread_pthread.h 1 +Python/traceback.c 6 +Python/tracemalloc.c 6 diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 469c727abfb11c..e98305e81808d6 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,3 +3,263 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) +/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/evp.h 2 +/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/ssl.h 4 +/usr/include/tcl8.6/tclTomMathDecls.h 1 +Include/cpython/bytearrayobject.h 1 +Include/cpython/bytesobject.h 3 +Include/cpython/dictobject.h 2 +Include/cpython/listobject.h 1 +Include/cpython/pyctype.h 2 +Include/cpython/tupleobject.h 1 +Include/cpython/unicodeobject.h 7 +Include/internal/mimalloc/mimalloc/internal.h 4 +Include/internal/mimalloc/mimalloc/types.h 2 +Include/internal/pycore_asdl.h 1 +Include/internal/pycore_backoff.h 3 +Include/internal/pycore_blocks_output_buffer.h 1 +Include/internal/pycore_dict.h 2 +Include/internal/pycore_gc.h 1 +Include/internal/pycore_gc.h 1 +Include/internal/pycore_interp.h 1 +Include/internal/pycore_list.h 1 +Include/internal/pycore_long.h 3 +Include/internal/pycore_object.h 4 +Include/internal/pycore_obmalloc.h 1 +Include/internal/pycore_pymath.h 1 +Include/internal/pycore_runtime_init.h 1 +Include/longobject.h 1 +Include/object.h 4 +Include/opcode_ids.h 1 +Include/pymacro.h 4 +Include/pymath.h 1 +Include/pymem.h 2 +Include/pyport.h 2 +Modules/_asynciomodule.c 3 +Modules/_bisectmodule.c 4 +Modules/_bz2module.c 5 +Modules/_collectionsmodule.c 2 +Modules/_csv.c 2 +Modules/_ctypes/_ctypes.c 53 +Modules/_ctypes/_ctypes_test.c 7 +Modules/_ctypes/_ctypes_test_generated.c.h 2 +Modules/_ctypes/callbacks.c 3 +Modules/_ctypes/callproc.c 13 +Modules/_ctypes/cfield.c 33 +Modules/_ctypes/stgdict.c 17 +Modules/_cursesmodule.c 27 +Modules/_datetimemodule.c 38 +Modules/_datetimemodule.c 38 +Modules/_dbmmodule.c 7 +Modules/_decimal/_decimal.c 19 +Modules/_elementtree.c 37 +Modules/_functoolsmodule.c 6 +Modules/_gdbmmodule.c 4 +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 +Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 +Modules/_hashopenssl.c 13 +Modules/_io/_iomodule.c 1 +Modules/_io/bufferedio.c 15 +Modules/_io/bytesio.c 14 +Modules/_io/fileio.c 9 +Modules/_io/stringio.c 8 +Modules/_io/textio.c 17 +Modules/_json.c 19 +Modules/_localemodule.c 2 +Modules/_lsprof.c 5 +Modules/_lzmamodule.c 6 +Modules/_multiprocessing/posixshmem.c 1 +Modules/_multiprocessing/semaphore.c 1 +Modules/_operator.c 5 +Modules/_pickle.c 73 +Modules/_posixsubprocess.c 11 +Modules/_queuemodule.c 4 +Modules/_randommodule.c 3 +Modules/_sqlite/connection.c 5 +Modules/_sqlite/cursor.c 3 +Modules/_sqlite/module.c 2 +Modules/_sre/sre.c 14 +Modules/_sre/sre_lib.h 25 +Modules/_ssl.c 26 +Modules/_struct.c 3 +Modules/_testbuffer.c 27 +Modules/_testcapi/bytes.c 1 +Modules/_testcapi/heaptype.c 1 +Modules/_testcapi/long.c 2 +Modules/_testcapi/mem.c 2 +Modules/_testcapi/monitoring.c 3 +Modules/_testcapi/pyatomic.c 4 +Modules/_testcapi/pyatomic.c 4 +Modules/_testcapi/unicode.c 1 +Modules/_testcapi/vectorcall.c 3 +Modules/_testcapi/watchers.c 3 +Modules/_testcapimodule.c 1 +Modules/_testclinic.c 14 +Modules/_testclinic.c 14 +Modules/_testexternalinspection.c 7 +Modules/_testinternalcapi.c 10 +Modules/_testinternalcapi/test_critical_sections.c 1 +Modules/_testinternalcapi/test_lock.c 4 +Modules/_testlimitedcapi/heaptype_relative.c 3 +Modules/_testlimitedcapi/object.c 2 +Modules/_testlimitedcapi/unicode.c 1 +Modules/_testmultiphase.c 1 +Modules/_tkinter.c 8 +Modules/_xxtestfuzz/_xxtestfuzz.c 1 +Modules/_xxtestfuzz/fuzzer.c 13 +Modules/_zoneinfo.c 17 +Modules/arraymodule.c 48 +Modules/binascii.c 208 +Modules/blake2module.c 8 +Modules/cjkcodecs/_codecs_iso2022.c 1 +Modules/cjkcodecs/_codecs_jp.c 17 +Modules/cjkcodecs/_codecs_kr.c 7 +Modules/cjkcodecs/alg_jisx0201.h 2 +Modules/cjkcodecs/cjkcodecs.h 1 +Modules/cjkcodecs/multibytecodec.c 12 +Modules/expat/pyexpatns.h 3 +Modules/expat/siphash.h 1 +Modules/expat/xmlparse.c 43 +Modules/expat/xmltok.c 15 +Modules/expat/xmltok.c 15 +Modules/expat/xmltok_impl.c 8 +Modules/faulthandler.c 5 +Modules/fcntlmodule.c 6 +Modules/getpath.c 7 +Modules/grpmodule.c 4 +Modules/itertoolsmodule.c 4 +Modules/main.c 2 +Modules/mathmodule.c 14 +Modules/mmapmodule.c 22 +Modules/mmapmodule.c 22 +Modules/posixmodule.c 79 +Modules/pwdmodule.c 4 +Modules/pyexpat.c 10 +Modules/readline.c 1 +Modules/resource.c 4 +Modules/rotatingtree.c 2 +Modules/selectmodule.c 1 +Modules/sha3module.c 4 +Modules/signalmodule.c 3 +Modules/socketmodule.c 75 +Modules/syslogmodule.c 3 +Modules/termios.c 1 +Modules/timemodule.c 10 +Modules/unicodedata.c 24 +Modules/unicodedata_db.h 1 +Modules/zlibmodule.c 24 +Objects/abstract.c 6 +Objects/bytearrayobject.c 42 +Objects/bytes_methods.c 4 +Objects/bytesobject.c 45 +Objects/call.c 12 +Objects/classobject.c 4 +Objects/codeobject.c 19 +Objects/descrobject.c 2 +Objects/dictobject.c 31 +Objects/fileobject.c 3 +Objects/floatobject.c 10 +Objects/frameobject.c 16 +Objects/funcobject.c 1 +Objects/genobject.c 3 +Objects/listobject.c 38 +Objects/longobject.c 47 +Objects/memoryobject.c 12 +Objects/methodobject.c 1 +Objects/mimalloc/alloc.c 6 +Objects/mimalloc/arena.c 6 +Objects/mimalloc/heap.c 2 +Objects/mimalloc/init.c 2 +Objects/mimalloc/options.c 4 +Objects/mimalloc/os.c 4 +Objects/mimalloc/page-queue.c 2 +Objects/mimalloc/page.c 2 +Objects/mimalloc/prim/unix/prim.c 6 +Objects/mimalloc/random.c 1 +Objects/mimalloc/segment.c 11 +Objects/mimalloc/stats.c 5 +Objects/moduleobject.c 4 +Objects/object.c 1 +Objects/obmalloc.c 6 +Objects/odictobject.c 6 +Objects/rangeobject.c 10 +Objects/setobject.c 13 +Objects/sliceobject.c 2 +Objects/stringlib/codecs.h 12 +Objects/stringlib/eq.h 1 +Objects/stringlib/fastsearch.h 8 +Objects/stringlib/join.h 3 +Objects/stringlib/replace.h 4 +Objects/stringlib/repr.h 21 +Objects/stringlib/transmogrify.h 26 +Objects/structseq.c 10 +Objects/tupleobject.c 8 +Objects/typeobject.c 38 +Objects/unicodectype.c 7 +Objects/unicodeobject.c 135 +Parser/action_helpers.c 3 +Parser/lexer/buffer.c 1 +Parser/lexer/lexer.c 14 +Parser/parser.c 116 +Parser/pegen.c 8 +Parser/string_parser.c 7 +Parser/tokenizer/file_tokenizer.c 9 +Parser/tokenizer/helpers.c 7 +Parser/tokenizer/readline_tokenizer.c 4 +Python/assemble.c 11 +Python/ast_opt.c 5 +Python/bltinmodule.c 8 +Python/bootstrap_hash.c 7 +Python/ceval.c 8 +Python/ceval_gil.c 2 +Python/codecs.c 28 +Python/codegen.c 6 +Python/compile.c 2 +Python/context.c 1 +Python/crossinterp.c 2 +Python/crossinterp_data_lookup.h 1 +Python/dtoa.c 30 +Python/errors.c 1 +Python/fileutils.c 11 +Python/flowgraph.c 7 +Python/formatter_unicode.c 6 +Python/frame.c 3 +Python/gc.c 9 +Python/gc.c 9 +Python/generated_cases.c.h 27 +Python/generated_cases.c.h 27 +Python/getargs.c 7 +Python/hashtable.c 1 +Python/import.c 6 +Python/import.c 7 +Python/initconfig.c 11 +Python/instrumentation.c 43 +Python/intrinsics.c 1 +Python/legacy_tracing.c 3 +Python/lock.c 4 +Python/marshal.c 16 +Python/modsupport.c 3 +Python/mystrtoul.c 4 +Python/pathconfig.c 1 +Python/perf_jit_trampoline.c 32 +Python/perf_trampoline.c 12 +Python/preconfig.c 2 +Python/pyarena.c 1 +Python/pyhash.c 4 +Python/pylifecycle.c 3 +Python/pystate.c 4 +Python/pystrhex.c 15 +Python/pystrtod.c 12 +Python/pytime.c 2 +Python/qsbr.c 2 +Python/specialize.c 9 +Python/suggestions.c 12 +Python/symtable.c 15 +Python/sysmodule.c 2 +Python/thread.c 1 +Python/thread_pthread.h 6 +Python/traceback.c 6 +Python/tracemalloc.c 6 diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 1ed83447b6b668..a9d0c1eeddbc1f 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -1,38 +1,49 @@ """ -Parses compiler output with -fdiagnostics-format=json and checks that warnings +Parses compiler output from Clang or GCC and checks that warnings exist only in files that are expected to have warnings. """ import argparse from collections import defaultdict -import json import re import sys from pathlib import Path from typing import NamedTuple + class FileWarnings(NamedTuple): name: str count: int -def extract_warnings_from_compiler_output_clang( +def extract_warnings_from_compiler_output( compiler_output: str, + compiler_output_type: str, + path_prefix: str = "", ) -> list[dict]: """ - Extracts warnings from the compiler output when using clang + Extracts warnings from the compiler output based on compiler + output type. Removes path prefix from file paths if provided. + Compatible with GCC and Clang compiler output. """ - # Regex to find warnings in the compiler output - clang_warning_regex = re.compile( - r"(?P.*):(?P\d+):(?P\d+): warning: " - r"(?P.*) (?P