From 5a991da32961ef5780996d58b8816d5f2085f540 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 17 Dec 2022 00:14:27 +0000 Subject: [PATCH 01/12] gh-78707: deprecate passing >1 argument to `PurePath.[is_]relative_to()` (GH-94469) This brings `relative_to()` and `is_relative_to()` more in line with other pathlib methods like `rename()` and `symlink_to()`. Resolves #78707. --- Doc/library/pathlib.rst | 14 ++++++++--- Lib/pathlib.py | 24 ++++++++++++------- Lib/test/test_pathlib.py | 8 ++++--- ...2-07-01-00-01-22.gh-issue-78707.fHGSuM.rst | 3 +++ 4 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 6537637f33c70e..47687400c14e3a 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -490,7 +490,7 @@ Pure paths provide the following methods and properties: True -.. method:: PurePath.is_relative_to(*other) +.. method:: PurePath.is_relative_to(other) Return whether or not this path is relative to the *other* path. @@ -502,6 +502,10 @@ Pure paths provide the following methods and properties: .. versionadded:: 3.9 + .. deprecated-removed:: 3.12 3.14 + + Passing additional arguments is deprecated; if supplied, they are joined + with *other*. .. method:: PurePath.is_reserved() @@ -564,7 +568,7 @@ Pure paths provide the following methods and properties: True -.. method:: PurePath.relative_to(*other, walk_up=False) +.. method:: PurePath.relative_to(other, walk_up=False) Compute a version of this path relative to the path represented by *other*. If it's impossible, :exc:`ValueError` is raised:: @@ -581,7 +585,7 @@ Pure paths provide the following methods and properties: raise ValueError(error_message.format(str(self), str(formatted))) ValueError: '/etc/passwd' is not in the subpath of '/usr' OR one path is relative and the other is absolute. -When *walk_up* is False (the default), the path must start with *other*. + When *walk_up* is False (the default), the path must start with *other*. When the argument is True, ``..`` entries may be added to form the relative path. In all other cases, such as the paths referencing different drives, :exc:`ValueError` is raised.:: @@ -605,6 +609,10 @@ When *walk_up* is False (the default), the path must start with *other*. .. versionadded:: 3.12 The *walk_up* argument (old behavior is the same as ``walk_up=False``). + .. deprecated-removed:: 3.12 3.14 + + Passing additional positional arguments is deprecated; if supplied, + they are joined with *other*. .. method:: PurePath.with_name(name) diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f31eb3010368d5..7890fdade42056 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -624,7 +624,7 @@ def with_suffix(self, suffix): return self._from_parsed_parts(self._drv, self._root, self._parts[:-1] + [name]) - def relative_to(self, *other, walk_up=False): + def relative_to(self, other, /, *_deprecated, walk_up=False): """Return the relative path to another path identified by the passed arguments. If the operation is not possible (because this is not related to the other path), raise ValueError. @@ -632,10 +632,14 @@ def relative_to(self, *other, walk_up=False): The *walk_up* parameter controls whether `..` may be used to resolve the path. """ - if not other: - raise TypeError("need at least one argument") + if _deprecated: + msg = ("support for supplying more than one positional argument " + "to pathlib.PurePath.relative_to() is deprecated and " + "scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath.relative_to(*args)", msg, + remove=(3, 14)) path_cls = type(self) - other = path_cls(*other) + other = path_cls(other, *_deprecated) for step, path in enumerate([other] + list(other.parents)): if self.is_relative_to(path): break @@ -646,12 +650,16 @@ def relative_to(self, *other, walk_up=False): parts = ('..',) * step + self.parts[len(path.parts):] return path_cls(*parts) - def is_relative_to(self, *other): + def is_relative_to(self, other, /, *_deprecated): """Return True if the path is relative to another path or False. """ - if not other: - raise TypeError("need at least one argument") - other = type(self)(*other) + if _deprecated: + msg = ("support for supplying more than one argument to " + "pathlib.PurePath.is_relative_to() is deprecated and " + "scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath.is_relative_to(*args)", + msg, remove=(3, 14)) + other = type(self)(other, *_deprecated) return other == self or other in self.parents @property diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 1d01d3cbd91d14..fa6ea0ac63d8da 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -654,8 +654,9 @@ def test_relative_to_common(self): self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) # With several args. - self.assertEqual(p.relative_to('a', 'b'), P()) - self.assertEqual(p.relative_to('a', 'b', walk_up=True), P()) + with self.assertWarns(DeprecationWarning): + p.relative_to('a', 'b') + p.relative_to('a', 'b', walk_up=True) # Unrelated paths. self.assertRaises(ValueError, p.relative_to, P('c')) self.assertRaises(ValueError, p.relative_to, P('a/b/c')) @@ -706,7 +707,8 @@ def test_is_relative_to_common(self): self.assertTrue(p.is_relative_to(P('a/b'))) self.assertTrue(p.is_relative_to('a/b')) # With several args. - self.assertTrue(p.is_relative_to('a', 'b')) + with self.assertWarns(DeprecationWarning): + p.is_relative_to('a', 'b') # Unrelated paths. self.assertFalse(p.is_relative_to(P('c'))) self.assertFalse(p.is_relative_to(P('a/b/c'))) diff --git a/Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst b/Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst new file mode 100644 index 00000000000000..c490a3c1bd7ecc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-07-01-00-01-22.gh-issue-78707.fHGSuM.rst @@ -0,0 +1,3 @@ +Deprecate passing more than one positional argument to +:meth:`pathlib.PurePath.relative_to` and +:meth:`~pathlib.PurePath.is_relative_to`. From 9cdd2fa63b7549d00830bccf19a34e9d69d0b15e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 16 Dec 2022 20:45:55 -0800 Subject: [PATCH 02/12] GH-98831: Add DECREF_INPUTS(), expanding to DECREF() each stack input (#100205) The presence of this macro indicates that a particular instruction may be considered for conversion to a register-based format (see https://github.com/faster-cpython/ideas/issues/485). An invariant (currently unchecked) is that `DEOPT_IF()` may only occur *before* `DECREF_INPUTS()`, and `ERROR_IF()` may only occur *after* it. One reason not to check this is that there are a few places where we insert *two* `DECREF_INPUTS()` calls, in different branches of the code. The invariant checking would have to be able to do some flow control analysis to understand this. Note that many instructions, especially specialized ones, can't be converted to use this macro straightforwardly. This is because the generator currently only generates plain `Py_DECREF(variable)` statements, and cannot generate things like `_Py_DECREF_SPECIALIZED()` let alone deal with `_PyList_AppendTakeRef()`. --- Python/bytecodes.c | 31 +++++++++++-------------- Tools/cases_generator/generate_cases.py | 6 ++++- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8e95b737030252..b29e16e080e961 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -161,7 +161,7 @@ dummy_func( super(LOAD_CONST__LOAD_FAST) = LOAD_CONST + LOAD_FAST; inst(POP_TOP, (value --)) { - Py_DECREF(value); + DECREF_INPUTS(); } inst(PUSH_NULL, (-- res)) { @@ -172,19 +172,19 @@ dummy_func( inst(UNARY_POSITIVE, (value -- res)) { res = PyNumber_Positive(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } inst(UNARY_NEGATIVE, (value -- res)) { res = PyNumber_Negative(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } inst(UNARY_NOT, (value -- res)) { int err = PyObject_IsTrue(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(err < 0, error); if (err == 0) { res = Py_True; @@ -197,7 +197,7 @@ dummy_func( inst(UNARY_INVERT, (value -- res)) { res = PyNumber_Invert(value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -351,8 +351,7 @@ dummy_func( STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -438,8 +437,7 @@ dummy_func( ERROR_IF(true, error); } Py_INCREF(res); // Do this before DECREF'ing dict, sub - Py_DECREF(dict); - Py_DECREF(sub); + DECREF_INPUTS(); } inst(BINARY_SUBSCR_GETITEM, (unused/1, type_version/2, func_version/1, container, sub -- unused)) { @@ -500,9 +498,7 @@ dummy_func( DECREMENT_ADAPTIVE_COUNTER(cache->counter); /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(err, error); } @@ -538,8 +534,7 @@ dummy_func( inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ int err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); + DECREF_INPUTS(); ERROR_IF(err, error); } @@ -550,11 +545,11 @@ dummy_func( if (hook == NULL) { _PyErr_SetString(tstate, PyExc_RuntimeError, "lost sys.displayhook"); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(true, error); } res = PyObject_CallOneArg(hook, value); - Py_DECREF(value); + DECREF_INPUTS(); ERROR_IF(res == NULL, error); Py_DECREF(res); } @@ -625,12 +620,12 @@ dummy_func( "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); + DECREF_INPUTS(); ERROR_IF(true, error); } iter = (*getter)(obj); - Py_DECREF(obj); + DECREF_INPUTS(); ERROR_IF(iter == NULL, error); if (Py_TYPE(iter)->tp_as_async == NULL || diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 2dfc76f2560eae..85a7c6098e0b95 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -209,7 +209,7 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None cache_offset += ceffect.size assert cache_offset == self.cache_offset + cache_adjust - # Write the body, substituting a goto for ERROR_IF() + # Write the body, substituting a goto for ERROR_IF() and other stuff assert dedent <= 0 extra = " " * -dedent for line in self.block_text: @@ -232,6 +232,10 @@ def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None ) else: out.write_raw(f"{extra}{space}if ({cond}) goto {label};\n") + elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*$", line): + space = m.group(1) + for ieff in self.input_effects: + out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n") else: out.write_raw(extra + line) From efbb1eb9f54cad4f7bf5df03eed3a6aba02d99f4 Mon Sep 17 00:00:00 2001 From: colorfulappl Date: Sat, 17 Dec 2022 14:37:44 +0800 Subject: [PATCH 03/12] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (#99890) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Co-authored-by: Erlend E. Aasland --- Lib/test/test_capi/test_getargs.py | 4 ++++ ...2-11-30-16-39-22.gh-issue-99240.67nAX-.rst | 2 ++ Modules/_testcapi/getargs.c | 19 +++++++++++++++++++ Python/getargs.c | 10 +++++----- 4 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 1d5c7fbaa1510a..3792d1a6515b44 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1085,6 +1085,10 @@ def test_Z_hash(self): with self.assertWarns(DeprecationWarning): self.assertIsNone(getargs_Z_hash(None)) + def test_gh_99240_clear_args(self): + from _testcapi import gh_99240_clear_args + self.assertRaises(TypeError, gh_99240_clear_args, 'a', '\0b') + class Object_TestCase(unittest.TestCase): def test_S(self): diff --git a/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst b/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst new file mode 100644 index 00000000000000..9a1bb3cf5a73f0 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-30-16-39-22.gh-issue-99240.67nAX-.rst @@ -0,0 +1,2 @@ +In argument parsing, after deallocating newly allocated memory, reset its +pointer to NULL. diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index 25a8e5fe8b605b..aa201319950de7 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -854,6 +854,24 @@ getargs_s_hash_int2(PyObject *self, PyObject *args, PyObject *kwargs) Py_RETURN_NONE; } +static PyObject * +gh_99240_clear_args(PyObject *self, PyObject *args) +{ + char *a = NULL; + char *b = NULL; + + if (!PyArg_ParseTuple(args, "eses", "idna", &a, "idna", &b)) { + if (a || b) { + PyErr_Clear(); + PyErr_SetString(PyExc_AssertionError, "Arguments are not cleared."); + } + return NULL; + } + PyMem_Free(a); + PyMem_Free(b); + Py_RETURN_NONE; +} + static PyMethodDef test_methods[] = { {"get_args", get_args, METH_VARARGS}, {"get_kwargs", _PyCFunction_CAST(get_kwargs), METH_VARARGS|METH_KEYWORDS}, @@ -906,6 +924,7 @@ static PyMethodDef test_methods[] = { {"test_empty_argparse", test_empty_argparse, METH_NOARGS}, {"test_k_code", test_k_code, METH_NOARGS}, {"test_s_code", test_s_code, METH_NOARGS}, + {"gh_99240_clear_args", gh_99240_clear_args, METH_VARARGS}, {NULL}, }; diff --git a/Python/getargs.c b/Python/getargs.c index 0167dd753d88f7..66dd90877fe6ff 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -202,9 +202,9 @@ _PyArg_VaParse_SizeT(PyObject *args, const char *format, va_list va) static int cleanup_ptr(PyObject *self, void *ptr) { - if (ptr) { - PyMem_Free(ptr); - } + void **pptr = (void **)ptr; + PyMem_Free(*pptr); + *pptr = NULL; return 0; } @@ -1116,7 +1116,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, PyErr_NoMemory(); RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr( "(cleanup problem)", @@ -1162,7 +1162,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, PyErr_NoMemory(); RETURN_ERR_OCCURRED; } - if (addcleanup(*buffer, freelist, cleanup_ptr)) { + if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr("(cleanup problem)", arg, msgbuf, bufsize); From a6f82f1fc68cb24e2d88d35fde4cfb663213a744 Mon Sep 17 00:00:00 2001 From: colorfulappl Date: Sat, 17 Dec 2022 14:40:51 +0800 Subject: [PATCH 04/12] gh-99240: Reset pointer to NULL when the pointed memory is freed in argument parsing (#99890) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Co-authored-by: Erlend E. Aasland From 8edcb30c3f8bdd8099a093146fedbd9b63a3f667 Mon Sep 17 00:00:00 2001 From: ram vikram singh Date: Sat, 17 Dec 2022 12:42:39 +0530 Subject: [PATCH 05/12] gh-97909: Fix markup for `PyMethodDef` members (#100089) --- Doc/c-api/structures.rst | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 827d624fc99edb..be6e7b5faf9f79 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -228,29 +228,30 @@ Implementing functions and methods Structure used to describe a method of an extension type. This structure has four fields: - +------------------+---------------+-------------------------------+ - | Field | C Type | Meaning | - +==================+===============+===============================+ - | :attr:`ml_name` | const char \* | name of the method | - +------------------+---------------+-------------------------------+ - | :attr:`ml_meth` | PyCFunction | pointer to the C | - | | | implementation | - +------------------+---------------+-------------------------------+ - | :attr:`ml_flags` | int | flag bits indicating how the | - | | | call should be constructed | - +------------------+---------------+-------------------------------+ - | :attr:`ml_doc` | const char \* | points to the contents of the | - | | | docstring | - +------------------+---------------+-------------------------------+ - -The :attr:`ml_meth` is a C function pointer. The functions may be of different + .. c:member:: const char* ml_name + + name of the method + + .. c:member:: PyCFunction ml_meth + + pointer to the C implementation + + .. c:member:: int ml_flags + + flags bits indicating how the call should be constructed + + .. c:member:: const char* ml_doc + + points to the contents of the docstring + +The :c:member:`ml_meth` is a C function pointer. The functions may be of different types, but they always return :c:expr:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as :c:expr:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. -The :attr:`ml_flags` field is a bitfield which can include the following flags. +The :c:member:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding convention. From 0264f634f720fbf12afaf1715a53cd1495fbd85b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sat, 17 Dec 2022 09:42:18 +0200 Subject: [PATCH 06/12] Docs: Use `PY_VERSION_HEX` for version comparison (#100179) --- Doc/c-api/apiabiversion.rst | 2 ++ Doc/whatsnew/3.11.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 85b6e2f373877f..62d542966622ce 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -58,6 +58,8 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is hexversion ``0x030a00f0``. + Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. + This version is also available via the symbol :data:`Py_Version`. .. c:var:: const unsigned long Py_Version diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 7931988ed0b04d..6f283f1d800202 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2319,7 +2319,7 @@ Porting to Python 3.11 can define the following macros and use them throughout the code (credit: these were copied from the ``mypy`` codebase):: - #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8 + #if PY_VERSION_HEX >= 0x03080000 # define CPy_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_BEGIN(op, dealloc) # define CPy_TRASHCAN_END(op) Py_TRASHCAN_END #else From 2b38a9aa747785f6d540e95e6846d6510de6b306 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 17 Dec 2022 09:53:36 +0100 Subject: [PATCH 07/12] gh-93649: Split tracemalloc tests from _testcapimodule.c (#99551) --- Lib/test/test_capi/test_mem.py | 169 ++++++++++++++++++++++++++++++++ Lib/test/test_capi/test_misc.py | 154 ----------------------------- Modules/_testcapi/mem.c | 81 +++++++++++++++ Modules/_testcapimodule.c | 75 -------------- 4 files changed, 250 insertions(+), 229 deletions(-) create mode 100644 Lib/test/test_capi/test_mem.py diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py new file mode 100644 index 00000000000000..a9ff410cb93ab8 --- /dev/null +++ b/Lib/test/test_capi/test_mem.py @@ -0,0 +1,169 @@ +import re +import textwrap +import unittest + + +from test import support +from test.support import import_helper, requires_subprocess +from test.support.script_helper import assert_python_failure, assert_python_ok + + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + +@requires_subprocess() +class PyMemDebugTests(unittest.TestCase): + PYTHONMALLOC = 'debug' + # '0x04c06e0' or '04C06E0' + PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' + + def check(self, code): + with support.SuppressCrashReport(): + out = assert_python_failure( + '-c', code, + PYTHONMALLOC=self.PYTHONMALLOC, + # FreeBSD: instruct jemalloc to not fill freed() memory + # with junk byte 0x5a, see JEMALLOC(3) + MALLOC_CONF="junk:false", + ) + stderr = out.err + return stderr.decode('ascii', 'replace') + + def test_buffer_overflow(self): + out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" + r" 16 bytes originally requested\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n" + r" at tail\+0: 0x78 \*\*\* OUCH\n" + r" at tail\+1: 0xfd\n" + r" at tail\+2: 0xfd\n" + r" .*\n" + r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" + r" Data at p: cd cd cd .*\n" + r"\n" + r"Enable tracemalloc to get the memory block allocation traceback\n" + r"\n" + r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte") + regex = regex.format(ptr=self.PTR_REGEX) + regex = re.compile(regex, flags=re.DOTALL) + self.assertRegex(out, regex) + + def test_api_misuse(self): + out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" + r" 16 bytes originally requested\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" + r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" + r" Data at p: cd cd cd .*\n" + r"\n" + r"Enable tracemalloc to get the memory block allocation traceback\n" + r"\n" + r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n") + regex = regex.format(ptr=self.PTR_REGEX) + self.assertRegex(out, regex) + + def check_malloc_without_gil(self, code): + out = self.check(code) + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without holding the GIL') + self.assertIn(expected, out) + + def test_pymem_malloc_without_gil(self): + # Debug hooks must raise an error if PyMem_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def test_pyobject_malloc_without_gil(self): + # Debug hooks must raise an error if PyObject_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def check_pyobject_is_freed(self, func_name): + code = textwrap.dedent(f''' + import gc, os, sys, _testcapi + # Disable the GC to avoid crash on GC collection + gc.disable() + try: + _testcapi.{func_name}() + # Exit immediately to avoid a crash while deallocating + # the invalid object + os._exit(0) + except _testcapi.error: + os._exit(1) + ''') + assert_python_ok( + '-c', code, + PYTHONMALLOC=self.PYTHONMALLOC, + MALLOC_CONF="junk:false", + ) + + def test_pyobject_null_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_null_is_freed') + + def test_pyobject_uninitialized_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed') + + def test_pyobject_forbidden_bytes_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed') + + def test_pyobject_freed_is_freed(self): + self.check_pyobject_is_freed('check_pyobject_freed_is_freed') + + def test_set_nomemory(self): + code = """if 1: + import _testcapi + + class C(): pass + + # The first loop tests both functions and that remove_mem_hooks() + # can be called twice in a row. The second loop checks a call to + # set_nomemory() after a call to remove_mem_hooks(). The third + # loop checks the start and stop arguments of set_nomemory(). + for outer_cnt in range(1, 4): + start = 10 * outer_cnt + for j in range(100): + if j == 0: + if outer_cnt != 3: + _testcapi.set_nomemory(start) + else: + _testcapi.set_nomemory(start, start + 1) + try: + C() + except MemoryError as e: + if outer_cnt != 3: + _testcapi.remove_mem_hooks() + print('MemoryError', outer_cnt, j) + _testcapi.remove_mem_hooks() + break + """ + rc, out, err = assert_python_ok('-c', code) + lines = out.splitlines() + for i, line in enumerate(lines, 1): + self.assertIn(b'MemoryError', out) + *_, count = line.split(b' ') + count = int(count) + self.assertLessEqual(count, i*5) + self.assertGreaterEqual(count, i*5-2) + + +class PyMemMallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'malloc_debug' + + +@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') +class PyMemPymallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'pymalloc_debug' + + +@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') +class PyMemDefaultTests(PyMemDebugTests): + # test default allocator of Python compiled in debug mode + PYTHONMALLOC = '' + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 06a51aa3cc219a..dace37c362e569 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -323,42 +323,6 @@ def test_getitem_with_error(self): def test_buildvalue_N(self): _testcapi.test_buildvalue_N() - def test_set_nomemory(self): - code = """if 1: - import _testcapi - - class C(): pass - - # The first loop tests both functions and that remove_mem_hooks() - # can be called twice in a row. The second loop checks a call to - # set_nomemory() after a call to remove_mem_hooks(). The third - # loop checks the start and stop arguments of set_nomemory(). - for outer_cnt in range(1, 4): - start = 10 * outer_cnt - for j in range(100): - if j == 0: - if outer_cnt != 3: - _testcapi.set_nomemory(start) - else: - _testcapi.set_nomemory(start, start + 1) - try: - C() - except MemoryError as e: - if outer_cnt != 3: - _testcapi.remove_mem_hooks() - print('MemoryError', outer_cnt, j) - _testcapi.remove_mem_hooks() - break - """ - rc, out, err = assert_python_ok('-c', code) - lines = out.splitlines() - for i, line in enumerate(lines, 1): - self.assertIn(b'MemoryError', out) - *_, count = line.split(b' ') - count = int(count) - self.assertLessEqual(count, i*5) - self.assertGreaterEqual(count, i*5-2) - def test_mapping_keys_values_items(self): class Mapping1(dict): def keys(self): @@ -1470,124 +1434,6 @@ class Test_testinternalcapi(unittest.TestCase): if name.startswith('test_')) -@support.requires_subprocess() -class PyMemDebugTests(unittest.TestCase): - PYTHONMALLOC = 'debug' - # '0x04c06e0' or '04C06E0' - PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' - - def check(self, code): - with support.SuppressCrashReport(): - out = assert_python_failure( - '-c', code, - PYTHONMALLOC=self.PYTHONMALLOC, - # FreeBSD: instruct jemalloc to not fill freed() memory - # with junk byte 0x5a, see JEMALLOC(3) - MALLOC_CONF="junk:false", - ) - stderr = out.err - return stderr.decode('ascii', 'replace') - - def test_buffer_overflow(self): - out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') - regex = (r"Debug memory block at address p={ptr}: API 'm'\n" - r" 16 bytes originally requested\n" - r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" - r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n" - r" at tail\+0: 0x78 \*\*\* OUCH\n" - r" at tail\+1: 0xfd\n" - r" at tail\+2: 0xfd\n" - r" .*\n" - r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" - r" Data at p: cd cd cd .*\n" - r"\n" - r"Enable tracemalloc to get the memory block allocation traceback\n" - r"\n" - r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte") - regex = regex.format(ptr=self.PTR_REGEX) - regex = re.compile(regex, flags=re.DOTALL) - self.assertRegex(out, regex) - - def test_api_misuse(self): - out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') - regex = (r"Debug memory block at address p={ptr}: API 'm'\n" - r" 16 bytes originally requested\n" - r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" - r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" - r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" - r" Data at p: cd cd cd .*\n" - r"\n" - r"Enable tracemalloc to get the memory block allocation traceback\n" - r"\n" - r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n") - regex = regex.format(ptr=self.PTR_REGEX) - self.assertRegex(out, regex) - - def check_malloc_without_gil(self, code): - out = self.check(code) - expected = ('Fatal Python error: _PyMem_DebugMalloc: ' - 'Python memory allocator called without holding the GIL') - self.assertIn(expected, out) - - def test_pymem_malloc_without_gil(self): - # Debug hooks must raise an error if PyMem_Malloc() is called - # without holding the GIL - code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' - self.check_malloc_without_gil(code) - - def test_pyobject_malloc_without_gil(self): - # Debug hooks must raise an error if PyObject_Malloc() is called - # without holding the GIL - code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' - self.check_malloc_without_gil(code) - - def check_pyobject_is_freed(self, func_name): - code = textwrap.dedent(f''' - import gc, os, sys, _testcapi - # Disable the GC to avoid crash on GC collection - gc.disable() - try: - _testcapi.{func_name}() - # Exit immediately to avoid a crash while deallocating - # the invalid object - os._exit(0) - except _testcapi.error: - os._exit(1) - ''') - assert_python_ok( - '-c', code, - PYTHONMALLOC=self.PYTHONMALLOC, - MALLOC_CONF="junk:false", - ) - - def test_pyobject_null_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_null_is_freed') - - def test_pyobject_uninitialized_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed') - - def test_pyobject_forbidden_bytes_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed') - - def test_pyobject_freed_is_freed(self): - self.check_pyobject_is_freed('check_pyobject_freed_is_freed') - - -class PyMemMallocDebugTests(PyMemDebugTests): - PYTHONMALLOC = 'malloc_debug' - - -@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') -class PyMemPymallocDebugTests(PyMemDebugTests): - PYTHONMALLOC = 'pymalloc_debug' - - -@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') -class PyMemDefaultTests(PyMemDebugTests): - # test default allocator of Python compiled in debug mode - PYTHONMALLOC = '' - - @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") class Test_ModuleStateAccess(unittest.TestCase): """Test access to module start (PEP 573)""" diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index ef723bf0658533..ae3f7a4372dcd8 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -596,6 +596,82 @@ check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) #endif } +// Tracemalloc tests +static PyObject * +tracemalloc_track(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + Py_ssize_t size; + int release_gil = 0; + + if (!PyArg_ParseTuple(args, "IOn|i", + &domain, &ptr_obj, &size, &release_gil)) + { + return NULL; + } + void *ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) { + return NULL; + } + + int res; + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); + Py_END_ALLOW_THREADS + } + else { + res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); + } + if (res < 0) { + PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +tracemalloc_untrack(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + return NULL; + } + void *ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) { + return NULL; + } + + int res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); + if (res < 0) { + PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +tracemalloc_get_traceback(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) { + return NULL; + } + void *ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) { + return NULL; + } + + return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr); +} + static PyMethodDef test_methods[] = { {"check_pyobject_forbidden_bytes_is_freed", check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, @@ -617,6 +693,11 @@ static PyMethodDef test_methods[] = { {"test_pymem_setrawallocators", test_pymem_setrawallocators, METH_NOARGS}, {"test_pyobject_new", test_pyobject_new, METH_NOARGS}, {"test_pyobject_setallocators", test_pyobject_setallocators, METH_NOARGS}, + + // Tracemalloc tests + {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, + {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, + {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 83eef73a875d9d..35c895d9ceb2a1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2065,78 +2065,6 @@ getitem_with_error(PyObject *self, PyObject *args) return PyObject_GetItem(map, key); } -static PyObject * -tracemalloc_track(PyObject *self, PyObject *args) -{ - unsigned int domain; - PyObject *ptr_obj; - void *ptr; - Py_ssize_t size; - int release_gil = 0; - int res; - - if (!PyArg_ParseTuple(args, "IOn|i", &domain, &ptr_obj, &size, &release_gil)) - return NULL; - ptr = PyLong_AsVoidPtr(ptr_obj); - if (PyErr_Occurred()) - return NULL; - - if (release_gil) { - Py_BEGIN_ALLOW_THREADS - res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); - Py_END_ALLOW_THREADS - } - else { - res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size); - } - - if (res < 0) { - PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error"); - return NULL; - } - - Py_RETURN_NONE; -} - -static PyObject * -tracemalloc_untrack(PyObject *self, PyObject *args) -{ - unsigned int domain; - PyObject *ptr_obj; - void *ptr; - int res; - - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) - return NULL; - ptr = PyLong_AsVoidPtr(ptr_obj); - if (PyErr_Occurred()) - return NULL; - - res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr); - if (res < 0) { - PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error"); - return NULL; - } - - Py_RETURN_NONE; -} - -static PyObject * -tracemalloc_get_traceback(PyObject *self, PyObject *args) -{ - unsigned int domain; - PyObject *ptr_obj; - void *ptr; - - if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) - return NULL; - ptr = PyLong_AsVoidPtr(ptr_obj); - if (PyErr_Occurred()) - return NULL; - - return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr); -} - static PyObject * dict_get_version(PyObject *self, PyObject *args) { @@ -3301,9 +3229,6 @@ static PyMethodDef TestMethods[] = { {"return_result_with_error", return_result_with_error, METH_NOARGS}, {"getitem_with_error", getitem_with_error, METH_VARARGS}, {"Py_CompileString", pycompilestring, METH_O}, - {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, - {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, - {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {"dict_get_version", dict_get_version, METH_VARARGS}, {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, {"stack_pointer", stack_pointer, METH_NOARGS}, From 0fe61d0838218b18bcff4bf434beba3e8dbcc42b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 17 Dec 2022 14:32:48 +0200 Subject: [PATCH 08/12] gh-100272: Fix JSON serialization of OrderedDict (GH-100273) It now preserves the order of keys. --- Lib/test/test_json/test_default.py | 11 +++++++++++ .../2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst | 1 + Modules/_json.c | 7 +++---- 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst diff --git a/Lib/test/test_json/test_default.py b/Lib/test/test_json/test_default.py index 9b8325e9c3816b..3ce16684a08272 100644 --- a/Lib/test/test_json/test_default.py +++ b/Lib/test/test_json/test_default.py @@ -1,3 +1,4 @@ +import collections from test.test_json import PyTest, CTest @@ -7,6 +8,16 @@ def test_default(self): self.dumps(type, default=repr), self.dumps(repr(type))) + def test_ordereddict(self): + od = collections.OrderedDict(a=1, b=2, c=3, d=4) + od.move_to_end('b') + self.assertEqual( + self.dumps(od), + '{"a": 1, "c": 3, "d": 4, "b": 2}') + self.assertEqual( + self.dumps(od, sort_keys=True), + '{"a": 1, "b": 2, "c": 3, "d": 4}') + class TestPyDefault(TestDefault, PyTest): pass class TestCDefault(TestDefault, CTest): pass diff --git a/Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst b/Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst new file mode 100644 index 00000000000000..2fb08926179211 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-12-15-18-28-13.gh-issue-100272.D1O9Ey.rst @@ -0,0 +1 @@ +Fix JSON serialization of OrderedDict. It now preserves the order of keys. diff --git a/Modules/_json.c b/Modules/_json.c index 429b4ee0fa8d8d..6879ad3d0722b6 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1570,10 +1570,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, */ } - if (s->sort_keys) { - - items = PyDict_Items(dct); - if (items == NULL || PyList_Sort(items) < 0) + if (s->sort_keys || !PyDict_CheckExact(dct)) { + items = PyMapping_Items(dct); + if (items == NULL || (s->sort_keys && PyList_Sort(items) < 0)) goto bail; for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); i++) { From 1cf3d78c92eb07dc09d15cc2e773b0b1b9436825 Mon Sep 17 00:00:00 2001 From: Jeremy Paige Date: Sat, 17 Dec 2022 23:11:21 -0800 Subject: [PATCH 09/12] Correct CVE-2020-10735 documentation (#100306) --- Doc/library/stdtypes.rst | 6 +++--- Python/clinic/sysmodule.c.h | 4 ++-- Python/sysmodule.c | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c785336944f50a..5a6b7d83edf75a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5480,7 +5480,7 @@ to mitigate denial of service attacks. This limit *only* applies to decimal or other non-power-of-two number bases. Hexadecimal, octal, and binary conversions are unlimited. The limit can be configured. -The :class:`int` type in CPython is an abitrary length number stored in binary +The :class:`int` type in CPython is an arbitrary length number stored in binary form (commonly known as a "bignum"). There exists no algorithm that can convert a string to a binary integer or a binary integer to a string in linear time, *unless* the base is a power of 2. Even the best known algorithms for base 10 @@ -5544,7 +5544,7 @@ and :class:`str` or :class:`bytes`: * ``int(string)`` with default base 10. * ``int(string, base)`` for all bases that are not a power of 2. * ``str(integer)``. -* ``repr(integer)`` +* ``repr(integer)``. * any other string conversion to base 10, for example ``f"{integer}"``, ``"{}".format(integer)``, or ``b"%d" % integer``. @@ -5572,7 +5572,7 @@ command line flag to configure the limit: :envvar:`PYTHONINTMAXSTRDIGITS` or :option:`-X int_max_str_digits <-X>`. If both the env var and the ``-X`` option are set, the ``-X`` option takes precedence. A value of *-1* indicates that both were unset, thus a value of - :data:`sys.int_info.default_max_str_digits` was used during initilization. + :data:`sys.int_info.default_max_str_digits` was used during initialization. From code, you can inspect the current limit and set a new one using these :mod:`sys` APIs: diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 5678d0ac2a608b..03eeda8126ebbb 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -749,7 +749,7 @@ PyDoc_STRVAR(sys_get_int_max_str_digits__doc__, "get_int_max_str_digits($module, /)\n" "--\n" "\n" -"Set the maximum string digits limit for non-binary int<->str conversions."); +"Return the maximum string digits limit for non-binary int<->str conversions."); #define SYS_GET_INT_MAX_STR_DIGITS_METHODDEF \ {"get_int_max_str_digits", (PyCFunction)sys_get_int_max_str_digits, METH_NOARGS, sys_get_int_max_str_digits__doc__}, @@ -1318,4 +1318,4 @@ sys_is_stack_trampoline_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=79228e569529129c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b32b444538dfd354 input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 91f5c487c98fe3..3f0baf98890b44 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1699,12 +1699,12 @@ sys_mdebug_impl(PyObject *module, int flag) /*[clinic input] sys.get_int_max_str_digits -Set the maximum string digits limit for non-binary int<->str conversions. +Return the maximum string digits limit for non-binary int<->str conversions. [clinic start generated code]*/ static PyObject * sys_get_int_max_str_digits_impl(PyObject *module) -/*[clinic end generated code: output=0042f5e8ae0e8631 input=8dab13e2023e60d5]*/ +/*[clinic end generated code: output=0042f5e8ae0e8631 input=61bf9f99bc8b112d]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); return PyLong_FromLong(interp->long_state.max_str_digits); From 532aa4e4e019812d0388920768ede7c04232ebe1 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Sun, 18 Dec 2022 20:13:24 +0100 Subject: [PATCH 10/12] gh-94912: Added marker for non-standard coroutine function detection (#99247) This introduces a new decorator `@inspect.markcoroutinefunction`, which, applied to a sync function, makes it appear async to `inspect.iscoroutinefunction()`. --- Doc/library/inspect.rst | 25 ++++++++++- Doc/whatsnew/3.12.rst | 6 +++ Lib/inspect.py | 26 ++++++++++- Lib/test/test_inspect.py | 45 +++++++++++++++++++ ...2-11-17-10-02-18.gh-issue-94912.G2aa-E.rst | 2 + 5 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 6705577551dcc5..58b84a35a890e3 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -343,8 +343,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: iscoroutinefunction(object) - Return ``True`` if the object is a :term:`coroutine function` - (a function defined with an :keyword:`async def` syntax). + Return ``True`` if the object is a :term:`coroutine function` (a function + defined with an :keyword:`async def` syntax), a :func:`functools.partial` + wrapping a :term:`coroutine function`, or a sync function marked with + :func:`markcoroutinefunction`. .. versionadded:: 3.5 @@ -352,6 +354,25 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a :term:`coroutine function`. + .. versionchanged:: 3.12 + Sync functions marked with :func:`markcoroutinefunction` now return + ``True``. + + +.. function:: markcoroutinefunction(func) + + Decorator to mark a callable as a :term:`coroutine function` if it would not + otherwise be detected by :func:`iscoroutinefunction`. + + This may be of use for sync functions that return a :term:`coroutine`, if + the function is passed to an API that requires :func:`iscoroutinefunction`. + + When possible, using an :keyword:`async def` function is preferred. Also + acceptable is calling the function and testing the return with + :func:`iscoroutine`. + + .. versionadded:: 3.12 + .. function:: iscoroutine(object) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 73dc462f0b3303..0cc4471364b671 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -225,6 +225,12 @@ asyncio a custom event loop factory. (Contributed by Kumar Aditya in :gh:`99388`.) +inspect +------- + +* Add :func:`inspect.markcoroutinefunction` to mark sync functions that return + a :term:`coroutine` for use with :func:`iscoroutinefunction`. + (Contributed Carlton Gibson in :gh:`99247`.) pathlib ------- diff --git a/Lib/inspect.py b/Lib/inspect.py index e92c355220fd8d..052f0bfe64b607 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -125,6 +125,7 @@ "ismodule", "isroutine", "istraceback", + "markcoroutinefunction", "signature", "stack", "trace", @@ -391,12 +392,33 @@ def isgeneratorfunction(obj): See help(isfunction) for a list of attributes.""" return _has_code_flag(obj, CO_GENERATOR) +# A marker for markcoroutinefunction and iscoroutinefunction. +_is_coroutine_marker = object() + +def _has_coroutine_mark(f): + while ismethod(f): + f = f.__func__ + f = functools._unwrap_partial(f) + if not (isfunction(f) or _signature_is_functionlike(f)): + return False + return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_marker + +def markcoroutinefunction(func): + """ + Decorator to ensure callable is recognised as a coroutine function. + """ + if hasattr(func, '__func__'): + func = func.__func__ + func._is_coroutine_marker = _is_coroutine_marker + return func + def iscoroutinefunction(obj): """Return true if the object is a coroutine function. - Coroutine functions are defined with "async def" syntax. + Coroutine functions are normally defined with "async def" syntax, but may + be marked via markcoroutinefunction. """ - return _has_code_flag(obj, CO_COROUTINE) + return _has_code_flag(obj, CO_COROUTINE) or _has_coroutine_mark(obj) def isasyncgenfunction(obj): """Return true if the object is an asynchronous generator function. diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 0f8217ba3b7550..78e6e9e34e5019 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -202,6 +202,51 @@ def test_iscoroutine(self): gen_coroutine_function_example)))) self.assertTrue(inspect.isgenerator(gen_coro)) + async def _fn3(): + pass + + @inspect.markcoroutinefunction + def fn3(): + return _fn3() + + self.assertTrue(inspect.iscoroutinefunction(fn3)) + self.assertTrue( + inspect.iscoroutinefunction( + inspect.markcoroutinefunction(lambda: _fn3()) + ) + ) + + class Cl: + async def __call__(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(Cl)) + # instances with async def __call__ are NOT recognised. + self.assertFalse(inspect.iscoroutinefunction(Cl())) + + class Cl2: + @inspect.markcoroutinefunction + def __call__(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(Cl2)) + # instances with marked __call__ are NOT recognised. + self.assertFalse(inspect.iscoroutinefunction(Cl2())) + + class Cl3: + @inspect.markcoroutinefunction + @classmethod + def do_something_classy(cls): + pass + + @inspect.markcoroutinefunction + @staticmethod + def do_something_static(): + pass + + self.assertTrue(inspect.iscoroutinefunction(Cl3.do_something_classy)) + self.assertTrue(inspect.iscoroutinefunction(Cl3.do_something_static)) + self.assertFalse( inspect.iscoroutinefunction(unittest.mock.Mock())) self.assertTrue( diff --git a/Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst b/Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst new file mode 100644 index 00000000000000..ee00f9d8d03f2c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-17-10-02-18.gh-issue-94912.G2aa-E.rst @@ -0,0 +1,2 @@ +Add :func:`inspect.markcoroutinefunction` decorator which manually marks +a function as a coroutine for the benefit of :func:`iscoroutinefunction`. From 702a5bc4637c82dc011e98b84f0cede98eb08dda Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 19 Dec 2022 10:40:11 +0200 Subject: [PATCH 11/12] Docs: Don't upload CI artifacts (#100330) --- .github/workflows/doc.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 44a1f206df1eb9..465da12fa1be80 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -50,18 +50,8 @@ jobs: run: make -C Doc/ venv - name: 'Check documentation' run: make -C Doc/ check - - name: 'Upload NEWS' - uses: actions/upload-artifact@v3 - with: - name: NEWS - path: Doc/build/NEWS - name: 'Build HTML documentation' run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html - - name: 'Upload docs' - uses: actions/upload-artifact@v3 - with: - name: doc-html - path: Doc/build/html # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: From 797edb28c3dd02a5727f0374e937e906a389ab77 Mon Sep 17 00:00:00 2001 From: Jon Burdo Date: Mon, 19 Dec 2022 13:59:01 -0500 Subject: [PATCH 12/12] gh-89727: Fix os.walk RecursionError on deep trees (#99803) Use a stack to implement os.walk iteratively instead of recursively to avoid hitting recursion limits on deeply nested trees. --- Lib/os.py | 160 +++++++++--------- Lib/test/support/__init__.py | 16 +- Lib/test/test_os.py | 43 +++++ ...2-11-29-20-44-54.gh-issue-89727.UJZjkk.rst | 3 + 4 files changed, 139 insertions(+), 83 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst diff --git a/Lib/os.py b/Lib/os.py index fd1e774fdcbcfa..73a5442ee8b83f 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -340,89 +340,95 @@ def walk(top, topdown=True, onerror=None, followlinks=False): """ sys.audit("os.walk", top, topdown, onerror, followlinks) - return _walk(fspath(top), topdown, onerror, followlinks) - -def _walk(top, topdown, onerror, followlinks): - dirs = [] - nondirs = [] - walk_dirs = [] - - # We may not have read permission for top, in which case we can't - # get a list of the files the directory contains. os.walk - # always suppressed the exception then, rather than blow up for a - # minor reason when (say) a thousand readable directories are still - # left to visit. That logic is copied here. - try: - # Note that scandir is global in this module due - # to earlier import-*. - scandir_it = scandir(top) - except OSError as error: - if onerror is not None: - onerror(error) - return - with scandir_it: - while True: - try: + stack = [(False, fspath(top))] + islink, join = path.islink, path.join + while stack: + must_yield, top = stack.pop() + if must_yield: + yield top + continue + + dirs = [] + nondirs = [] + walk_dirs = [] + + # We may not have read permission for top, in which case we can't + # get a list of the files the directory contains. + # We suppress the exception here, rather than blow up for a + # minor reason when (say) a thousand readable directories are still + # left to visit. + try: + scandir_it = scandir(top) + except OSError as error: + if onerror is not None: + onerror(error) + continue + + cont = False + with scandir_it: + while True: try: - entry = next(scandir_it) - except StopIteration: + try: + entry = next(scandir_it) + except StopIteration: + break + except OSError as error: + if onerror is not None: + onerror(error) + cont = True break - except OSError as error: - if onerror is not None: - onerror(error) - return - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider that the entry is not - # a directory, same behaviour than os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) + try: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False - if not topdown and is_dir: - # Bottom-up: recurse into sub-directory, but exclude symlinks to - # directories if followlinks is False - if followlinks: - walk_into = True + if is_dir: + dirs.append(entry.name) else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider that the - # entry is not a symbolic link, same behaviour than - # os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - - # Yield before recursion if going top down - if topdown: - yield top, dirs, nondirs - - # Recurse into sub-directories - islink, join = path.islink, path.join - for dirname in dirs: - new_path = join(top, dirname) - # Issue #23605: os.path.islink() is used instead of caching - # entry.is_symlink() result during the loop on os.scandir() because - # the caller can replace the directory entry during the "yield" - # above. - if followlinks or not islink(new_path): - yield from _walk(new_path, topdown, onerror, followlinks) - else: - # Recurse into sub-directories - for new_path in walk_dirs: - yield from _walk(new_path, topdown, onerror, followlinks) - # Yield after recursion if going bottom up - yield top, dirs, nondirs + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) + if cont: + continue + + if topdown: + # Yield before sub-directory traversal if going top down + yield top, dirs, nondirs + # Traverse into sub-directories + for dirname in reversed(dirs): + new_path = join(top, dirname) + # bpo-23605: os.path.islink() is used instead of caching + # entry.is_symlink() result during the loop on os.scandir() because + # the caller can replace the directory entry during the "yield" + # above. + if followlinks or not islink(new_path): + stack.append((False, new_path)) + else: + # Yield after sub-directory traversal if going bottom up + stack.append((True, (top, dirs, nondirs))) + # Traverse into sub-directories + for new_path in reversed(walk_dirs): + stack.append((False, new_path)) __all__.append("walk") diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a631bfc80cfaf0..b7186057990ac1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2178,19 +2178,23 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) @contextlib.contextmanager +def set_recursion_limit(limit): + """Temporarily change the recursion limit.""" + original_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(limit) + yield + finally: + sys.setrecursionlimit(original_limit) + def infinite_recursion(max_depth=75): """Set a lower limit for tests that interact with infinite recursions (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some debug windows builds, due to not enough functions being inlined the stack size might not handle the default recursion limit (1000). See bpo-11105 for details.""" + return set_recursion_limit(max_depth) - original_depth = sys.getrecursionlimit() - try: - sys.setrecursionlimit(max_depth) - yield - finally: - sys.setrecursionlimit(original_depth) def ignore_deprecations_from(module: str, *, like: str) -> object: token = object() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e0577916428a08..e6e25b507de051 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -33,6 +33,7 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper +from test.support import set_recursion_limit from test.support import warnings_helper from platform import win32_is_iot @@ -1471,6 +1472,46 @@ def test_walk_many_open_files(self): self.assertEqual(next(it), expected) p = os.path.join(p, 'd') + def test_walk_above_recursion_limit(self): + depth = 50 + os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) + with set_recursion_limit(depth - 5): + all = list(self.walk(self.walk_path)) + + sub2_path = self.sub2_tree[0] + for root, dirs, files in all: + if root == sub2_path: + dirs.sort() + files.sort() + + d_entries = [] + d_path = self.walk_path + for _ in range(depth): + d_path = os.path.join(d_path, "d") + d_entries.append((d_path, ["d"], [])) + d_entries[-1][1].clear() + + # Sub-sequences where the order is known + sections = { + "SUB1": [ + (self.sub1_path, ["SUB11"], ["tmp2"]), + (self.sub11_path, [], []), + ], + "SUB2": [self.sub2_tree], + "d": d_entries, + } + + # The ordering of sub-dirs is arbitrary but determines the order in + # which sub-sequences appear + dirs = all[0][1] + expected = [(self.walk_path, dirs, ["tmp1"])] + for d in dirs: + expected.extend(sections[d]) + + self.assertEqual(len(all), depth + 4) + self.assertEqual(sorted(dirs), ["SUB1", "SUB2", "d"]) + self.assertEqual(all, expected) + @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class FwalkTests(WalkTests): @@ -1545,6 +1586,8 @@ def test_fd_leak(self): # fwalk() keeps file descriptors open test_walk_many_open_files = None + # fwalk() still uses recursion + test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): diff --git a/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst b/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst new file mode 100644 index 00000000000000..8a5fdb64b87f82 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-29-20-44-54.gh-issue-89727.UJZjkk.rst @@ -0,0 +1,3 @@ +Fix issue with :func:`os.walk` where a :exc:`RecursionError` would occur on +deep directory structures by adjusting the implementation of +:func:`os.walk` to be iterative instead of recursive.