From 9e011e7c77dad7d0bbb944c44891531606caeb21 Mon Sep 17 00:00:00 2001 From: Joshua Herman Date: Sat, 29 Apr 2023 20:26:24 -0500 Subject: [PATCH 01/43] gh-82054: allow test runner to split test_asyncio to execute in parallel by sharding. (#103927) This runs test_asyncio sub-tests in parallel using sharding from Cinder. This suite is typically the longest-pole in runs because it is a test package with a lot of further sub-tests otherwise run serially. By breaking out the sub-tests as independent modules we can run a lot more in parallel. After porting we can see the direct impact on a multicore system. Without this change: Running make test is 5 min 26 seconds With this change: Running make test takes 3 min 39 seconds That'll vary based on system and parallelism. On a `-j 4` run similar to what CI and buildbot systems often do, it reduced the overall test suite completion latency by 10%. The drawbacks are that this implementation is hacky and due to the sorting of the tests it obscures when the asyncio tests occur and involves changing CPython test infrastructure but, the wall time saved it is worth it, especially in low-core count CI runs as it pulls a long tail. The win for productivity and reserved CI resource usage is significant. Future tests that deserve to be refactored into split up suites to benefit from are test_concurrent_futures and the way the _test_multiprocessing suite gets run for all start methods. As exposed by passing the -o flag to python -m test to get a list of the 10 longest running tests. --------- Co-authored-by: Carl Meyer Co-authored-by: Gregory P. Smith [Google, LLC] --- Lib/test/libregrtest/runtest.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index e9bb72a7d77ee1..61595277ed6d5a 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -143,6 +143,14 @@ def __str__(self) -> str: # set of tests that we don't want to be executed when using regrtest NOTTESTS = set() +#If these test directories are encountered recurse into them and treat each +# test_ .py or dir as a separate test module. This can increase parallelism. +# Beware this can't generally be done for any directory with sub-tests as the +# __init__.py may do things which alter what tests are to be run. + +SPLITTESTDIRS = { + "test_asyncio", +} # Storage of uncollectable objects FOUND_GARBAGE = [] @@ -158,7 +166,7 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_dirs=SPLITTESTDIRS, base_mod=""): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) names = os.listdir(testdir) @@ -166,8 +174,13 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): others = set(stdtests) | nottests for name in names: mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) + if mod[:5] == "test_" and mod not in others: + if mod in split_test_dirs: + subdir = os.path.join(testdir, mod) + mod = f"{base_mod or 'test'}.{mod}" + tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod)) + elif ext in (".py", ""): + tests.append(f"{base_mod}.{mod}" if base_mod else mod) return stdtests + sorted(tests) From 4b10ecc29f6ae69e599a5475a62d8e96a8711f90 Mon Sep 17 00:00:00 2001 From: Amethyst Reese Date: Sat, 29 Apr 2023 20:21:20 -0700 Subject: [PATCH 02/43] Update name in acknowledgements and add mailmap (#103696) I changed my name last year, and would like to update my name in the acknowledgements and git history accordingly. git-mailmap reference: https://git-scm.com/docs/gitmailmap Co-authored-by: Jelle Zijlstra --- .mailmap | 3 +++ Misc/ACKS | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000000000..013c839ed6b7a4 --- /dev/null +++ b/.mailmap @@ -0,0 +1,3 @@ +# This file sets the canonical name for contributors to the repository. +# Documentation: https://git-scm.com/docs/gitmailmap +Amethyst Reese diff --git a/Misc/ACKS b/Misc/ACKS index 65be5cfc3c7945..42ec059a7c4ec2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1488,7 +1488,7 @@ John Redford Kalyan Reddy Terry J. Reedy Gareth Rees -John Reese +Amethyst Reese Steve Reeves Lennart Regebro John Regehr From accb417c338630ac6e836a5c811a89d54a3cd1d3 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 30 Apr 2023 08:02:03 +0300 Subject: [PATCH 03/43] Replace Netlify with Read the Docs build previews (#103843) Co-authored-by: Oleg Iarygin Co-authored-by: C.A.M. Gerlach --- .github/workflows/documentation-links.yml | 27 +++++++++++++++++++++++ .readthedocs.yml | 18 +++++++++++++++ Doc/conf.py | 11 ++++----- Doc/tools/templates/layout.html | 5 ----- netlify.toml | 11 --------- 5 files changed, 51 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/documentation-links.yml create mode 100644 .readthedocs.yml delete mode 100644 netlify.toml diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml new file mode 100644 index 00000000000000..43a7afec73884e --- /dev/null +++ b/.github/workflows/documentation-links.yml @@ -0,0 +1,27 @@ +name: Read the Docs PR preview +# Automatically edits a pull request's descriptions with a link +# to the documentation's preview on Read the Docs. + +on: + pull_request_target: + types: + - opened + paths: + - 'Doc/**' + - '.github/workflows/doc.yml' + +permissions: + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/actions/preview@v1 + with: + project-slug: "cpython-previews" + single-version: "true" diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000000000..898a9ae89dbb92 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,18 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +# Project page: https://readthedocs.org/projects/cpython-previews/ + +version: 2 + +sphinx: + configuration: Doc/conf.py + +build: + os: ubuntu-22.04 + tools: + python: "3" + + commands: + - make -C Doc venv html + - mkdir _readthedocs + - mv Doc/build/html _readthedocs/html diff --git a/Doc/conf.py b/Doc/conf.py index 42c23bf77c7034..cef2a0e2837f6a 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -114,12 +114,13 @@ # Short title used e.g. for HTML tags. html_short_title = '%s Documentation' % release -# Deployment preview information, from Netlify -# (See netlify.toml and https://docs.netlify.com/configure-builds/environment-variables/#git-metadata) +# Deployment preview information +# (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) +repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL") html_context = { - "is_deployment_preview": os.getenv("IS_DEPLOYMENT_PREVIEW"), - "repository_url": os.getenv("REPOSITORY_URL"), - "pr_id": os.getenv("REVIEW_ID") + "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external", + "repository_url": repository_url.removesuffix(".git") if repository_url else None, + "pr_id": os.getenv("READTHEDOCS_VERSION") } # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 460161cd320223..b91f8138553e62 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -11,11 +11,6 @@ {%- if is_deployment_preview %} <div id="deployment-preview-warning" style="padding: .5em; text-align: center; background-color: #fff2ba; color: #6a580e;"> - <div style="float: right; margin-top: -10px; margin-left: 10px;"> - <a href="https://www.netlify.com"> - <img src="https://www.netlify.com/img/global/badges/netlify-color-accent.svg" alt="Deploys by Netlify" /> - </a> - </div> {% trans %}This is a deploy preview created from a <a href="{{ repository_url }}/pull/{{ pr_id }}">pull request</a>. For authoritative documentation, see the {% endtrans %} <a href="https://docs.python.org/3/{{ pagename }}{{ file_suffix }}">{% trans %} the current stable release{% endtrans %}</a>. diff --git a/netlify.toml b/netlify.toml deleted file mode 100644 index f5790fc5fec74f..00000000000000 --- a/netlify.toml +++ /dev/null @@ -1,11 +0,0 @@ -[build] - base = "Doc/" - command = "make html" - publish = "build/html" - # Do not trigger netlify builds if docs were not changed. - # Changed files should be in sync with `.github/workflows/doc.yml` - ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF . ../netlify.toml" - -[build.environment] - PYTHON_VERSION = "3.8" - IS_DEPLOYMENT_PREVIEW = "true" From ed95e8cbd4cbc813666c7ce7760257cc0f169d03 Mon Sep 17 00:00:00 2001 From: Ken Jin <kenjin@python.org> Date: Sun, 30 Apr 2023 21:08:26 +0800 Subject: [PATCH 04/43] gh-98003: Inline call frames for CALL_FUNCTION_EX (GH-98004) --- Include/internal/pycore_call.h | 10 ++ ...2-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 3 + Objects/call.c | 20 ++-- Python/bytecodes.c | 19 ++++ Python/ceval.c | 46 ++++++++ Python/generated_cases.c.h | 105 +++++++++++------- 6 files changed, 148 insertions(+), 55 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 55378e3dfebf24..5d9342b562b002 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -116,6 +116,16 @@ _PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); } +PyObject *const * +_PyStack_UnpackDict(PyThreadState *tstate, + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject **p_kwnames); + +void +_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, + PyObject *kwnames); + +void _PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames); #ifdef __cplusplus } diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst new file mode 100644 index 00000000000000..f9e71bc1344bb3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -0,0 +1,3 @@ +Complex function calls are now faster and consume no C stack +space. + diff --git a/Objects/call.c b/Objects/call.c index bd027e41f8a9a5..cf6e357a990441 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -8,16 +8,6 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() -static PyObject *const * -_PyStack_UnpackDict(PyThreadState *tstate, - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject **p_kwnames); - -static void -_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, - PyObject *kwnames); - - static PyObject * null_error(PyThreadState *tstate) { @@ -965,7 +955,7 @@ _PyStack_AsDict(PyObject *const *values, PyObject *kwnames) The newly allocated argument vector supports PY_VECTORCALL_ARGUMENTS_OFFSET. When done, you must call _PyStack_UnpackDict_Free(stack, nargs, kwnames) */ -static PyObject *const * +PyObject *const * _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames) @@ -1034,7 +1024,7 @@ _PyStack_UnpackDict(PyThreadState *tstate, return stack; } -static void +void _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames) { @@ -1042,6 +1032,12 @@ _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, for (Py_ssize_t i = 0; i < n; i++) { Py_DECREF(stack[i]); } + _PyStack_UnpackDict_FreeNoDecRef(stack, kwnames); +} + +void +_PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames) +{ PyMem_Free((PyObject **)stack - 1); Py_DECREF(kwnames); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9de0d92e382d3d..e83894e8902872 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3103,6 +3103,25 @@ dummy_func( } } else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); + } result = PyObject_Call(func, callargs, kwargs); } DECREF_INPUTS(); diff --git a/Python/ceval.c b/Python/ceval.c index 5d5221b2e40990..958689debc87f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -212,6 +212,9 @@ static _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, PyObject* const* args, size_t argcount, PyObject *kwnames); +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); static void _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); @@ -1501,6 +1504,49 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, return NULL; } +/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. + Steals references to func, callargs and kwargs. +*/ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) +{ + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs; + if (has_dict) { + newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); + if (newargs == NULL) { + Py_DECREF(func); + goto error; + } + } + else { + newargs = &PyTuple_GET_ITEM(callargs, 0); + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + /* No need to decref func here because the reference has been stolen by + _PyEvalFramePushAndInit. + */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return new_frame; +error: + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return NULL; +} + PyObject * _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 864a4f7bcaff0f..069a7ced0a4c25 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4296,16 +4296,35 @@ } } else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); + } result = PyObject_Call(func, callargs, kwargs); } - #line 4302 "Python/generated_cases.c.h" + #line 4321 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3109 "Python/bytecodes.c" + #line 3128 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4309 "Python/generated_cases.c.h" + #line 4328 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4320,7 +4339,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3119 "Python/bytecodes.c" + #line 3138 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4349,14 +4368,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4353 "Python/generated_cases.c.h" + #line 4372 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3150 "Python/bytecodes.c" + #line 3169 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4377,7 +4396,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4381 "Python/generated_cases.c.h" + #line 4400 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4385,15 +4404,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3173 "Python/bytecodes.c" + #line 3192 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4391 "Python/generated_cases.c.h" + #line 4410 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3175 "Python/bytecodes.c" + #line 3194 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4397 "Python/generated_cases.c.h" + #line 4416 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4404,7 +4423,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3179 "Python/bytecodes.c" + #line 3198 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4439,7 +4458,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4443 "Python/generated_cases.c.h" + #line 4462 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4448,10 +4467,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3216 "Python/bytecodes.c" + #line 3235 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4455 "Python/generated_cases.c.h" + #line 4474 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4463,7 +4482,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3221 "Python/bytecodes.c" + #line 3240 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4478,12 +4497,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4482 "Python/generated_cases.c.h" + #line 4501 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3236 "Python/bytecodes.c" + #line 3255 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4487 "Python/generated_cases.c.h" + #line 4506 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4493,16 +4512,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3241 "Python/bytecodes.c" + #line 3260 "Python/bytecodes.c" assert(oparg >= 2); - #line 4499 "Python/generated_cases.c.h" + #line 4518 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3245 "Python/bytecodes.c" + #line 3264 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4522,11 +4541,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4526 "Python/generated_cases.c.h" + #line 4545 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3267 "Python/bytecodes.c" + #line 3286 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4538,26 +4557,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4542 "Python/generated_cases.c.h" + #line 4561 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3281 "Python/bytecodes.c" + #line 3300 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4548 "Python/generated_cases.c.h" + #line 4567 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3285 "Python/bytecodes.c" + #line 3304 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4555 "Python/generated_cases.c.h" + #line 4574 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3290 "Python/bytecodes.c" + #line 3309 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4566,12 +4585,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4570 "Python/generated_cases.c.h" + #line 4589 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3301 "Python/bytecodes.c" + #line 3320 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4580,12 +4599,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4584 "Python/generated_cases.c.h" + #line 4603 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3312 "Python/bytecodes.c" + #line 3331 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4598,12 +4617,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4602 "Python/generated_cases.c.h" + #line 4621 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3327 "Python/bytecodes.c" + #line 3346 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4616,30 +4635,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4620 "Python/generated_cases.c.h" + #line 4639 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3342 "Python/bytecodes.c" + #line 3361 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4631 "Python/generated_cases.c.h" + #line 4650 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3350 "Python/bytecodes.c" + #line 3369 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4638 "Python/generated_cases.c.h" + #line 4657 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3355 "Python/bytecodes.c" + #line 3374 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4645 "Python/generated_cases.c.h" + #line 4664 "Python/generated_cases.c.h" } From f186557dc3e77495ba65b9b7e492ab6ddb7cecc3 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak <pieter.eendebak@gmail.com> Date: Sun, 30 Apr 2023 17:36:19 +0200 Subject: [PATCH 05/43] gh-103977: compile re expressions in platform.py only if required (#103981) Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com> --- Lib/platform.py | 75 ++++++++++--------- ...-04-28-19-08-50.gh-issue-103977.msF70A.rst | 1 + 2 files changed, 40 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst diff --git a/Lib/platform.py b/Lib/platform.py index 790ef860bf106e..7bb222088d5061 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -136,11 +136,11 @@ 'pl': 200, 'p': 200, } -_component_re = re.compile(r'([0-9]+|[._+-])') def _comparable_version(version): + component_re = re.compile(r'([0-9]+|[._+-])') result = [] - for v in _component_re.split(version): + for v in component_re.split(version): if v not in '._+-': try: v = int(v, 10) @@ -152,11 +152,6 @@ def _comparable_version(version): ### Platform specific APIs -_libc_search = re.compile(b'(__libc_init)' - b'|' - b'(GLIBC_([0-9.]+))' - b'|' - br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) def libc_ver(executable=None, lib='', version='', chunksize=16384): @@ -190,6 +185,12 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): # sys.executable is not set. return lib, version + libc_search = re.compile(b'(__libc_init)' + b'|' + b'(GLIBC_([0-9.]+))' + b'|' + br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) + V = _comparable_version # We use os.path.realpath() # here to work around problems with Cygwin not being @@ -200,7 +201,7 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): pos = 0 while pos < len(binary): if b'libc' in binary or b'GLIBC' in binary: - m = _libc_search.search(binary, pos) + m = libc_search.search(binary, pos) else: m = None if not m or m.end() == len(binary): @@ -247,9 +248,6 @@ def _norm_version(version, build=''): version = '.'.join(strings[:3]) return version -_ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' - r'.*' - r'\[.* ([\d.]+)\])') # Examples of VER command output: # @@ -295,9 +293,13 @@ def _syscmd_ver(system='', release='', version='', else: return system, release, version + ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' + r'.*' + r'\[.* ([\d.]+)\])') + # Parse the output info = info.strip() - m = _ver_output.match(info) + m = ver_output.match(info) if m is not None: system, release, version = m.groups() # Strip trailing dots from version and release @@ -1033,18 +1035,6 @@ def processor(): ### Various APIs for extracting information from sys.version -_sys_version_parser = re.compile( - r'([\w.+]+)\s*' # "version<space>" - r'\(#?([^,]+)' # "(#buildno" - r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)<space>" - r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" - -_pypy_sys_version_parser = re.compile( - r'([\w.+]+)\s*' - r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' - r'\[PyPy [^\]]+\]?') - _sys_version_cache = {} def _sys_version(sys_version=None): @@ -1076,10 +1066,17 @@ def _sys_version(sys_version=None): if result is not None: return result + sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version<space>" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)<space>" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" + if sys.platform.startswith('java'): # Jython name = 'Jython' - match = _sys_version_parser.match(sys_version) + match = sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % @@ -1091,8 +1088,13 @@ def _sys_version(sys_version=None): elif "PyPy" in sys_version: # PyPy + pypy_sys_version_parser = re.compile( + r'([\w.+]+)\s*' + r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' + r'\[PyPy [^\]]+\]?') + name = "PyPy" - match = _pypy_sys_version_parser.match(sys_version) + match = pypy_sys_version_parser.match(sys_version) if match is None: raise ValueError("failed to parse PyPy sys.version: %s" % repr(sys_version)) @@ -1101,7 +1103,7 @@ def _sys_version(sys_version=None): else: # CPython - match = _sys_version_parser.match(sys_version) + match = sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % @@ -1290,13 +1292,6 @@ def platform(aliased=False, terse=False): ### freedesktop.org os-release standard # https://www.freedesktop.org/software/systemd/man/os-release.html -# NAME=value with optional quotes (' or "). The regular expression is less -# strict than shell lexer, but that's ok. -_os_release_line = re.compile( - "^(?P<name>[a-zA-Z0-9_]+)=(?P<quote>[\"\']?)(?P<value>.*)(?P=quote)$" -) -# unescape five special characters mentioned in the standard -_os_release_unescape = re.compile(r"\\([\\\$\"\'`])") # /etc takes precedence over /usr/lib _os_release_candidates = ("/etc/os-release", "/usr/lib/os-release") _os_release_cache = None @@ -1311,10 +1306,18 @@ def _parse_os_release(lines): "PRETTY_NAME": "Linux", } + # NAME=value with optional quotes (' or "). The regular expression is less + # strict than shell lexer, but that's ok. + os_release_line = re.compile( + "^(?P<name>[a-zA-Z0-9_]+)=(?P<quote>[\"\']?)(?P<value>.*)(?P=quote)$" + ) + # unescape five special characters mentioned in the standard + os_release_unescape = re.compile(r"\\([\\\$\"\'`])") + for line in lines: - mo = _os_release_line.match(line) + mo = os_release_line.match(line) if mo is not None: - info[mo.group('name')] = _os_release_unescape.sub( + info[mo.group('name')] = os_release_unescape.sub( r"\1", mo.group('value') ) diff --git a/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst b/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst new file mode 100644 index 00000000000000..ff4005774a95d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst @@ -0,0 +1 @@ +Improve import time of :mod:`platform` module. From 7d3931e94a76491111a6e391e111cb066236cff4 Mon Sep 17 00:00:00 2001 From: Alex Waygood <Alex.Waygood@Gmail.com> Date: Sun, 30 Apr 2023 16:51:46 +0100 Subject: [PATCH 06/43] gh-104012: Ensure test_calendar.CalendarTestCase.test_deprecation_warning consistently passes (#104014) --- Lib/test/test_calendar.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 03388e8c55d5a8..24e472b5fee828 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -493,11 +493,11 @@ def test_format(self): class CalendarTestCase(unittest.TestCase): def test_deprecation_warning(self): - with warnings.catch_warnings(record=True) as w: + with self.assertWarnsRegex( + DeprecationWarning, + "The 'January' attribute is deprecated, use 'JANUARY' instead" + ): calendar.January - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, DeprecationWarning) - self.assertIn("The 'January' attribute is deprecated, use 'JANUARY' instead", str(w[0].message)) def test_isleap(self): # Make sure that the return is right for a few years, and From 654d44b3a4d3ee4d92b690668aa5189acf4f9d8f Mon Sep 17 00:00:00 2001 From: Nikita Sobolev <mail@sobolevn.me> Date: Sun, 30 Apr 2023 20:16:55 +0300 Subject: [PATCH 07/43] gh-104015: Fix direct invocation of `test_dataclasses` (#104017) Previously, `python -m test test_dataclasses` passed, but `./python.exe Lib/test/test_dataclasses.py` failed --- Lib/test/test_dataclasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 7dd81a8855f1be..7b48b26f9e7743 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -3671,7 +3671,7 @@ def test_text_annotations(self): ByMakeDataClass = make_dataclass('ByMakeDataClass', [('x', int)]) ManualModuleMakeDataClass = make_dataclass('ManualModuleMakeDataClass', [('x', int)], - module='test.test_dataclasses') + module=__name__) WrongNameMakeDataclass = make_dataclass('Wrong', [('x', int)]) WrongModuleMakeDataclass = make_dataclass('WrongModuleMakeDataclass', [('x', int)], From 74a2b79c6265c92ef381b5ff0dc63903bf0178ac Mon Sep 17 00:00:00 2001 From: Liam Gersten <gerstenliam@gmail.com> Date: Sun, 30 Apr 2023 16:17:36 -0400 Subject: [PATCH 08/43] gh-88773: Added teleport method to Turtle library (#103974) Add a `teleport` method to `turtle` module turtle instances that acts a lot like `goto`, _but_ ensures the pen is up while warping to the new position to and can control shape filling behavior as part of the jump. Based on an educator user feature request. --------- Co-authored-by: Terry Jan Reedy <tjreedy@udel.edu> Co-authored-by: Hugo van Kemenade <hugovk@users.noreply.github.com> Co-authored-by: Gregory P. Smith <greg@krypto.org> --- Doc/library/turtle.rst | 53 +++++++++++---- Lib/test/test_turtle.py | 20 ++++++ Lib/turtle.py | 66 ++++++++++++++++++- ...3-04-28-18-04-23.gh-issue-88773.xXCNJw.rst | 1 + 4 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 05392d04e52263..10138f4f406f85 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -107,6 +107,7 @@ Turtle motion | :func:`right` | :func:`rt` | :func:`left` | :func:`lt` | :func:`goto` | :func:`setpos` | :func:`setposition` + | :func:`teleport` | :func:`setx` | :func:`sety` | :func:`setheading` | :func:`seth` @@ -372,6 +373,44 @@ Turtle motion (0.00,0.00) +.. function:: teleport(x, y=None, *, fill_gap=False) + + :param x: a number or ``None`` + :param y: a number or ``None`` + :param fill_gap: a boolean + + Move turtle to an absolute position. Unlike goto(x, y), a line will not + be drawn. The turtle's orientation does not change. If currently + filling, the polygon(s) teleported from will be filled after leaving, + and filling will begin again after teleporting. This can be disabled + with fill_gap=True, which makes the imaginary line traveled during + teleporting act as a fill barrier like in goto(x, y). + + .. doctest:: + :skipif: _tkinter is None + :hide: + + >>> turtle.goto(0, 0) + + .. doctest:: + :skipif: _tkinter is None + + >>> tp = turtle.pos() + >>> tp + (0.00,0.00) + >>> turtle.teleport(60) + >>> turtle.pos() + (60.00,0.00) + >>> turtle.teleport(y=10) + >>> turtle.pos() + (60.00,10.00) + >>> turtle.teleport(20, 30) + >>> turtle.pos() + (20.00,30.00) + + .. versionadded: 3.12 + + .. function:: setx(x) :param x: a number (integer or float) @@ -537,8 +576,7 @@ Turtle motion :skipif: _tkinter is None >>> turtle.color("blue") - >>> turtle.stamp() - 11 + >>> stamp_id = turtle.stamp() >>> turtle.fd(50) @@ -575,15 +613,8 @@ Turtle motion .. doctest:: >>> for i in range(8): - ... turtle.stamp(); turtle.fd(30) - 13 - 14 - 15 - 16 - 17 - 18 - 19 - 20 + ... unused_stamp_id = turtle.stamp() + ... turtle.fd(30) >>> turtle.clearstamps(2) >>> turtle.clearstamps(-2) >>> turtle.clearstamps() diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 95af84e3779824..3f9f129a3dd200 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -267,6 +267,14 @@ def test_goto(self): self.assertAlmostEqual(self.nav.xcor(), 100) self.assertAlmostEqual(self.nav.ycor(), -100) + def test_teleport(self): + self.nav.teleport(20, -30, fill_gap=True) + self.assertAlmostEqual(self.nav.xcor(), 20) + self.assertAlmostEqual(self.nav.ycor(), -30) + self.nav.teleport(-20, 30, fill_gap=False) + self.assertAlmostEqual(self.nav.xcor(), -20) + self.assertAlmostEqual(self.nav.ycor(), 30) + def test_pos(self): self.assertEqual(self.nav.pos(), self.nav._position) self.nav.goto(100, -100) @@ -440,6 +448,18 @@ def test_showturtle_hideturtle_and_isvisible(self): tpen.showturtle() self.assertTrue(tpen.isvisible()) + def test_teleport(self): + + tpen = turtle.TPen() + + for fill_gap_value in [True, False]: + tpen.penup() + tpen.teleport(100, 100, fill_gap=fill_gap_value) + self.assertFalse(tpen.isdown()) + tpen.pendown() + tpen.teleport(-100, -100, fill_gap=fill_gap_value) + self.assertTrue(tpen.isdown()) + if __name__ == '__main__': unittest.main() diff --git a/Lib/turtle.py b/Lib/turtle.py index 1b369327bc8eff..2de406e0f517af 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -135,7 +135,7 @@ 'pu', 'radians', 'right', 'reset', 'resizemode', 'rt', 'seth', 'setheading', 'setpos', 'setposition', 'settiltangle', 'setundobuffer', 'setx', 'sety', 'shape', 'shapesize', 'shapetransform', 'shearfactor', 'showturtle', - 'speed', 'st', 'stamp', 'tilt', 'tiltangle', 'towards', + 'speed', 'st', 'stamp', 'teleport', 'tilt', 'tiltangle', 'towards', 'turtlesize', 'undo', 'undobufferentries', 'up', 'width', 'write', 'xcor', 'ycor'] _tg_utilities = ['write_docstringdict', 'done'] @@ -1614,6 +1614,13 @@ def _goto(self, end): """move turtle to position end.""" self._position = end + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: + """To be overwritten by child class RawTurtle. + Includes no TPen references.""" + new_x = x if x is not None else self._position[0] + new_y = y if y is not None else self._position[1] + self._position = Vec2D(new_x, new_y) + def forward(self, distance): """Move the turtle forward by the specified distance. @@ -2293,6 +2300,15 @@ def fillcolor(self, *args): else: return self._color(self._fillcolor) + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: + """To be overwritten by child class RawTurtle. + Includes no TNavigator references. + """ + pendown = self.isdown() + if pendown: + self.pen(pendown=False) + self.pen(pendown=pendown) + def showturtle(self): """Makes the turtle visible. @@ -2710,6 +2726,54 @@ def _cc(self, args): if not ((0 <= r <= 255) and (0 <= g <= 255) and (0 <= b <= 255)): raise TurtleGraphicsError("bad color sequence: %s" % str(args)) return "#%02x%02x%02x" % (r, g, b) + + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: + """Instantly move turtle to an absolute position. + + Arguments: + x -- a number or None + y -- a number None + fill_gap -- a boolean This argument must be specified by name. + + call: teleport(x, y) # two coordinates + --or: teleport(x) # teleport to x position, keeping y as is + --or: teleport(y=y) # teleport to y position, keeping x as is + --or: teleport(x, y, fill_gap=True) + # teleport but fill the gap in between + + Move turtle to an absolute position. Unlike goto(x, y), a line will not + be drawn. The turtle's orientation does not change. If currently + filling, the polygon(s) teleported from will be filled after leaving, + and filling will begin again after teleporting. This can be disabled + with fill_gap=True, which makes the imaginary line traveled during + teleporting act as a fill barrier like in goto(x, y). + + Example (for a Turtle instance named turtle): + >>> tp = turtle.pos() + >>> tp + (0.00,0.00) + >>> turtle.teleport(60) + >>> turtle.pos() + (60.00,0.00) + >>> turtle.teleport(y=10) + >>> turtle.pos() + (60.00,10.00) + >>> turtle.teleport(20, 30) + >>> turtle.pos() + (20.00,30.00) + """ + pendown = self.isdown() + was_filling = self.filling() + if pendown: + self.pen(pendown=False) + if was_filling and not fill_gap: + self.end_fill() + new_x = x if x is not None else self._position[0] + new_y = y if y is not None else self._position[1] + self._position = Vec2D(new_x, new_y) + self.pen(pendown=pendown) + if was_filling and not fill_gap: + self.begin_fill() def clone(self): """Create and return a clone of the turtle. diff --git a/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst b/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst new file mode 100644 index 00000000000000..f14c9533f3af87 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst @@ -0,0 +1 @@ +Added :func:`turtle.teleport` to the :mod:`turtle` module to move a turtle to a new point without tracing a line, visible or invisible. Patch by Liam Gersten. From 69bc86cb1aed49db27afc0095e0f4bcd8f1f3983 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 30 Apr 2023 17:16:38 -0700 Subject: [PATCH 09/43] Improve int test coverage (#104024) Following discussion in https://discuss.python.org/t/bug-in-int-42/26360/5 This tests some of the things documented in https://github.com/python/cpython/pull/100436 Co-authored-by: Gregory P. Smith <greg@krypto.org> --- Lib/test/test_int.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 334fea0774be51..5545ee39d8e942 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -155,6 +155,8 @@ def test_basic(self): self.assertEqual(int(' 0O123 ', 0), 83) self.assertEqual(int(' 0X123 ', 0), 291) self.assertEqual(int(' 0B100 ', 0), 4) + with self.assertRaises(ValueError): + int('010', 0) # without base still base 10 self.assertEqual(int('0123'), 123) @@ -221,6 +223,24 @@ def test_basic(self): self.assertEqual(int('2br45qc', 35), 4294967297) self.assertEqual(int('1z141z5', 36), 4294967297) + def test_invalid_signs(self): + with self.assertRaises(ValueError): + int('+') + with self.assertRaises(ValueError): + int('-') + with self.assertRaises(ValueError): + int('- 1') + with self.assertRaises(ValueError): + int('+ 1') + with self.assertRaises(ValueError): + int(' + 1 ') + + def test_unicode(self): + self.assertEqual(int("१२३४५६७८९०1234567890"), 12345678901234567890) + self.assertEqual(int('١٢٣٤٥٦٧٨٩٠'), 1234567890) + self.assertEqual(int("१२३४५६७८९०1234567890", 0), 12345678901234567890) + self.assertEqual(int('١٢٣٤٥٦٧٨٩٠', 0), 1234567890) + def test_underscores(self): for lit in VALID_UNDERSCORE_LITERALS: if any(ch in lit for ch in '.eEjJ'): From 4b27972f5fe816d3616f97f8643d8ad922473ab5 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy <tjreedy@udel.edu> Date: Sun, 30 Apr 2023 21:36:27 -0400 Subject: [PATCH 10/43] gh-88496: Fix IDLE test hang on macOS (#104025) Replace widget.update() with widget.update_idletasks in two places. --- Lib/idlelib/colorizer.py | 2 +- Lib/idlelib/outwin.py | 2 +- .../next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index e9f19c145c8673..b4df353012b788 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -310,7 +310,7 @@ def recolorize_main(self): # crumb telling the next invocation to resume here # in case update tells us to leave. self.tag_add("TODO", next) - self.update() + self.update_idletasks() if self.stop_colorizing: if DEBUG: print("colorizing stopped") return diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index ac67c904ab9797..610031e26f1dff 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -112,7 +112,7 @@ def write(self, s, tags=(), mark="insert"): assert isinstance(s, str) self.text.insert(mark, s, tags) self.text.see(mark) - self.text.update() + self.text.update_idletasks() return len(s) def writelines(self, lines): diff --git a/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst b/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst new file mode 100644 index 00000000000000..4f390d189d23b5 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst @@ -0,0 +1 @@ +Fix IDLE test hang on macOS. From 93107aa2a49a9354ffb10b3cd263dc3e99ebdeff Mon Sep 17 00:00:00 2001 From: Ben Faulhaber <111227622+faulhaberben@users.noreply.github.com> Date: Mon, 1 May 2023 07:47:34 +0200 Subject: [PATCH 11/43] Adjust expression from `==` to `!=` in alignment with the meaning of the paragraph. (GH-104021) --- Doc/library/venv.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 52bf99e5bb0f67..9e5672545dea35 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -55,7 +55,7 @@ point to the directories of the virtual environment, whereas :data:`sys.base_prefix` and :data:`sys.base_exec_prefix` point to those of the base Python used to create the environment. It is sufficient to check -``sys.prefix == sys.base_prefix`` to determine if the current interpreter is +``sys.prefix != sys.base_prefix`` to determine if the current interpreter is running from a virtual environment. A virtual environment may be "activated" using a script in its binary directory From 487f55d5801a9ae7d79d37e259e8c377c9acd39b Mon Sep 17 00:00:00 2001 From: Carey Metcalfe <carey@cmetcalfe.ca> Date: Mon, 1 May 2023 01:32:04 -0600 Subject: [PATCH 12/43] gh-103895: Improve how invalid `Exception.__notes__` are displayed (#103897) --- Lib/test/test_traceback.py | 12 ++++++++++-- Lib/traceback.py | 8 ++++++-- .../2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst | 3 +++ Python/pythonrun.c | 5 ++++- 4 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 5e2b353782994e..19a2be88d2c1bc 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1539,11 +1539,11 @@ def __repr__(self): e.__notes__ = BadThing() notes_repr = 'bad repr' - self.assertEqual(self.get_report(e), vanilla + notes_repr) + self.assertEqual(self.get_report(e), vanilla + notes_repr + '\n') e.__notes__ = Unprintable() err_msg = '<__notes__ repr() failed>' - self.assertEqual(self.get_report(e), vanilla + err_msg) + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') # non-string item in the __notes__ sequence e.__notes__ = [BadThing(), 'Final Note'] @@ -1555,6 +1555,14 @@ def __repr__(self): err_msg = '<note str() failed>' self.assertEqual(self.get_report(e), vanilla + err_msg + '\nFinal Note\n') + e.__notes__ = "please do not explode me" + err_msg = "'please do not explode me'" + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + + e.__notes__ = b"please do not show me as numbers" + err_msg = "b'please do not show me as numbers'" + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + def test_exception_with_note_with_multiple_notes(self): e = ValueError(42) vanilla = self.get_report(e) diff --git a/Lib/traceback.py b/Lib/traceback.py index 9e720ac9948fce..ba4a9ffd001b53 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -852,12 +852,16 @@ def format_exception_only(self): yield _format_final_exc_line(stype, self._str) else: yield from self._format_syntax_error(stype) - if isinstance(self.__notes__, collections.abc.Sequence): + + if ( + isinstance(self.__notes__, collections.abc.Sequence) + and not isinstance(self.__notes__, (str, bytes)) + ): for note in self.__notes__: note = _safe_string(note, 'note') yield from [l + '\n' for l in note.split('\n')] elif self.__notes__ is not None: - yield _safe_string(self.__notes__, '__notes__', func=repr) + yield "{}\n".format(_safe_string(self.__notes__, '__notes__', func=repr)) def _format_syntax_error(self, stype): """Format SyntaxError exceptions (internal helper).""" diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst new file mode 100644 index 00000000000000..6fed304c9132b3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst @@ -0,0 +1,3 @@ +Improve handling of edge cases in showing ``Exception.__notes__``. Ensures +that the messages always end with a newline and that string/bytes are not +exploded over multiple lines. Patch by Carey Metcalfe. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b16d3f53f89fb9..05e7b4370869af 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1107,7 +1107,7 @@ print_exception_notes(struct exception_print_context *ctx, PyObject *value) if (notes == NULL) { return -1; } - if (!PySequence_Check(notes)) { + if (!PySequence_Check(notes) || PyUnicode_Check(notes) || PyBytes_Check(notes)) { int res = 0; if (write_indented_margin(ctx, f) < 0) { res = -1; @@ -1122,6 +1122,9 @@ print_exception_notes(struct exception_print_context *ctx, PyObject *value) Py_DECREF(s); } Py_DECREF(notes); + if (PyFile_WriteString("\n", f) < 0) { + res = -1; + } return res; } Py_ssize_t num_notes = PySequence_Length(notes); From 59c27fa5cb95e2d608747a50fc675bbe2fc96beb Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Mon, 1 May 2023 18:10:35 +0800 Subject: [PATCH 13/43] gh-102213: Optimize the performance of `__getattr__` (GH-103761) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kirill <80244920+Eclips4@users.noreply.github.com> Co-authored-by: Łukasz Langa <lukasz@langa.pl> Co-authored-by: Xiang Wang <34048878+wangxiang-hz@users.noreply.github.com> --- Lib/test/test_descr.py | 15 +++++++++++++- ...-05-01-08-08-05.gh-issue-102213.nfH-4C.rst | 1 + Objects/typeobject.c | 20 ++++++++++++------- 3 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index f17bb1813b9d87..ad3eefba365856 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5004,7 +5004,7 @@ class Child(Parent): self.assertEqual(Parent.__subclasses__(), []) def test_attr_raise_through_property(self): - # add test case for gh-103272 + # test case for gh-103272 class A: def __getattr__(self, name): raise ValueError("FOO") @@ -5016,6 +5016,19 @@ def foo(self): with self.assertRaisesRegex(ValueError, "FOO"): A().foo + # test case for gh-103551 + class B: + @property + def __getattr__(self, name): + raise ValueError("FOO") + + @property + def foo(self): + raise NotImplementedError("BAR") + + with self.assertRaisesRegex(NotImplementedError, "BAR"): + B().foo + class DictProxyTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst new file mode 100644 index 00000000000000..997bef226e713f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst @@ -0,0 +1 @@ +Fix performance loss when accessing an object's attributes with ``__getattr__`` defined. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 38b99315457a58..e807cc90faa16a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8306,17 +8306,23 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) if (getattribute == NULL || (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == - (void *)PyObject_GenericGetAttr)) - res = PyObject_GenericGetAttr(self, name); - else { + (void *)PyObject_GenericGetAttr)) { + res = _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); + /* if res == NULL with no exception set, then it must be an + AttributeError suppressed by us. */ + if (res == NULL && !PyErr_Occurred()) { + res = call_attribute(self, getattr, name); + } + } else { Py_INCREF(getattribute); res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); + if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + res = call_attribute(self, getattr, name); + } } - if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - res = call_attribute(self, getattr, name); - } + Py_DECREF(getattr); return res; } From 4181d078fc945313568eb39965cb9190881606b5 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora <80244920+Eclips4@users.noreply.github.com> Date: Mon, 1 May 2023 16:42:59 +0300 Subject: [PATCH 14/43] gh-104036: Fix direct invocation of test_typing (#104037) Previously, `python -m test test_typing` worked, but `python Lib/test/test_typing.py` did not. --- Lib/test/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f36bb958c88ef9..7c6a521c3c48f8 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -117,7 +117,7 @@ def test_repr(self): class Sub(Any): pass self.assertEqual( repr(Sub), - "<class 'test.test_typing.AnyTests.test_repr.<locals>.Sub'>", + f"<class '{__name__}.AnyTests.test_repr.<locals>.Sub'>", ) def test_errors(self): From e1476942525ae847875dab55541bef4a8a99dd3d Mon Sep 17 00:00:00 2001 From: Dong-hee Na <donghee.na@python.org> Date: Mon, 1 May 2023 23:03:24 +0900 Subject: [PATCH 15/43] gh-104028: Reduce object creation while calling callback function from gc (gh-104030) --- .../2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst | 2 ++ Modules/gcmodule.c | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst new file mode 100644 index 00000000000000..9c35ea88499dce --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst @@ -0,0 +1,2 @@ +Reduce object creation while calling callback function from gc. +Patch by Dong-hee Na. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 966c1e615502ef..3fd5f4cd70e832 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1388,10 +1388,19 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase, return; } } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_WriteUnraisable(NULL); + return; + } + + PyObject *stack[] = {phase_obj, info}; for (Py_ssize_t i=0; i<PyList_GET_SIZE(gcstate->callbacks); i++) { PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_CallFunction(cb, "sO", phase, info); + r = PyObject_Vectorcall(cb, stack, 2, NULL); if (r == NULL) { PyErr_WriteUnraisable(cb); } @@ -1400,6 +1409,7 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase, } Py_DECREF(cb); } + Py_DECREF(phase_obj); Py_XDECREF(info); assert(!_PyErr_Occurred(tstate)); } From 99aab610622fc4b4c4fe56b77c0760cf77066a53 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev <mail@sobolevn.me> Date: Mon, 1 May 2023 18:19:06 +0300 Subject: [PATCH 16/43] gh-104035: Do not ignore user-defined `__{get,set}state__` in slotted frozen dataclasses (#104041) --- Lib/dataclasses.py | 6 +- Lib/test/test_dataclasses.py | 68 +++++++++++++++++++ ...-05-01-16-43-28.gh-issue-104035.MrJBw8.rst | 2 + 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index a73cdc22a5f4b3..b0b8a773b7594f 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1227,8 +1227,10 @@ def _add_slots(cls, is_frozen, weakref_slot): if is_frozen: # Need this for pickling frozen classes with slots. - cls.__getstate__ = _dataclass_getstate - cls.__setstate__ = _dataclass_setstate + if '__getstate__' not in cls_dict: + cls.__getstate__ = _dataclass_getstate + if '__setstate__' not in cls_dict: + cls.__setstate__ = _dataclass_setstate return cls diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 7b48b26f9e7743..6669f1c57e2e78 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -3184,6 +3184,74 @@ def test_frozen_pickle(self): self.assertIsNot(obj, p) self.assertEqual(obj, p) + @dataclass(frozen=True, slots=True) + class FrozenSlotsGetStateClass: + foo: str + bar: int + + getstate_called: bool = field(default=False, compare=False) + + def __getstate__(self): + object.__setattr__(self, 'getstate_called', True) + return [self.foo, self.bar] + + @dataclass(frozen=True, slots=True) + class FrozenSlotsSetStateClass: + foo: str + bar: int + + setstate_called: bool = field(default=False, compare=False) + + def __setstate__(self, state): + object.__setattr__(self, 'setstate_called', True) + object.__setattr__(self, 'foo', state[0]) + object.__setattr__(self, 'bar', state[1]) + + @dataclass(frozen=True, slots=True) + class FrozenSlotsAllStateClass: + foo: str + bar: int + + getstate_called: bool = field(default=False, compare=False) + setstate_called: bool = field(default=False, compare=False) + + def __getstate__(self): + object.__setattr__(self, 'getstate_called', True) + return [self.foo, self.bar] + + def __setstate__(self, state): + object.__setattr__(self, 'setstate_called', True) + object.__setattr__(self, 'foo', state[0]) + object.__setattr__(self, 'bar', state[1]) + + def test_frozen_slots_pickle_custom_state(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + obj = self.FrozenSlotsGetStateClass('a', 1) + dumped = pickle.dumps(obj, protocol=proto) + + self.assertTrue(obj.getstate_called) + self.assertEqual(obj, pickle.loads(dumped)) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + obj = self.FrozenSlotsSetStateClass('a', 1) + obj2 = pickle.loads(pickle.dumps(obj, protocol=proto)) + + self.assertTrue(obj2.setstate_called) + self.assertEqual(obj, obj2) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + obj = self.FrozenSlotsAllStateClass('a', 1) + dumped = pickle.dumps(obj, protocol=proto) + + self.assertTrue(obj.getstate_called) + + obj2 = pickle.loads(dumped) + self.assertTrue(obj2.setstate_called) + self.assertEqual(obj, obj2) + def test_slots_with_default_no_init(self): # Originally reported in bpo-44649. @dataclass(slots=True) diff --git a/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst b/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst new file mode 100644 index 00000000000000..8c8e3d6ba5fbc1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst @@ -0,0 +1,2 @@ +Do not ignore user-defined ``__getstate__`` and ``__setstate__`` methods for +slotted frozen dataclasses. From d5a97074d24cd14cb2a35a2b1ad3074863cde264 Mon Sep 17 00:00:00 2001 From: chgnrdv <52372310+chgnrdv@users.noreply.github.com> Date: Mon, 1 May 2023 18:26:43 +0300 Subject: [PATCH 17/43] gh-103824: fix use-after-free error in Parser/tokenizer.c (#103993) --- Lib/test/test_tokenize.py | 15 ++++++++++++++- Parser/tokenizer.c | 4 ++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 283a7c23609e67..911b53e5816588 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -11,7 +11,7 @@ from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) from test.support import os_helper -from test.support.script_helper import run_test_script, make_script +from test.support.script_helper import run_test_script, make_script, run_python_until_end import os import token @@ -1470,6 +1470,19 @@ def test_comment_at_the_end_of_the_source_without_newline(self): self.assertEqual(tok_name[tokens[i + 1].exact_type], tok_name[expected_tokens[i]]) self.assertEqual(tok_name[tokens[-1].exact_type], tok_name[token.ENDMARKER]) + def test_invalid_character_in_fstring_middle(self): + # See gh-103824 + script = b'''F""" + \xe5"""''' + + with os_helper.temp_dir() as temp_dir: + filename = os.path.join(temp_dir, "script.py") + with open(filename, 'wb') as file: + file.write(script) + rs, _ = run_python_until_end(filename) + self.assertIn(b"SyntaxError", rs.err) + + class UntokenizeTest(TestCase): def test_bad_input_order(self): diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 8de0572a1fc459..8fb9be7bfd0182 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -2552,6 +2552,10 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct while (end_quote_size != current_tok->f_string_quote_size) { int c = tok_nextc(tok); if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) { + if (tok->decoding_erred) { + return MAKE_TOKEN(ERRORTOKEN); + } + assert(tok->multi_line_start != NULL); // shift the tok_state's location into // the start of string, and report the error From 2a884ceb36e799c900129d4b5b6248262004efb5 Mon Sep 17 00:00:00 2001 From: chgnrdv <52372310+chgnrdv@users.noreply.github.com> Date: Mon, 1 May 2023 18:45:50 +0300 Subject: [PATCH 18/43] Fix typo in "expected" word in few source files (#104034) --- Include/unicodeobject.h | 2 +- Modules/_ctypes/callproc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 74474f5bb8f976..5839c747a29275 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -626,7 +626,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String( /* --- ASCII Codecs ------------------------------------------------------- - Only 7-bit ASCII data is excepted. All other codes generate errors. + Only 7-bit ASCII data is expected. All other codes generate errors. */ diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 93bc784df5386f..f10cf58216acf7 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1823,7 +1823,7 @@ resize(PyObject *self, PyObject *args) dict = PyObject_stgdict((PyObject *)obj); if (dict == NULL) { PyErr_SetString(PyExc_TypeError, - "excepted ctypes instance"); + "expected ctypes instance"); return NULL; } if (size < dict->size) { From a679c3d58d10aafd9ac9355fdd16151607e37d65 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 1 May 2023 18:19:47 +0100 Subject: [PATCH 19/43] gh-102799: replace sys.exc_info by sys.exception in inspect and traceback modules (#104032) --- Lib/inspect.py | 4 +++- Lib/traceback.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index 6d1d7b766cb3bb..92c2675cfd7d32 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1766,7 +1766,9 @@ def stack(context=1): def trace(context=1): """Return a list of records for the stack below the current exception.""" - return getinnerframes(sys.exc_info()[2], context) + exc = sys.exception() + tb = None if exc is None else exc.__traceback__ + return getinnerframes(tb, context) # ------------------------------------------------ static version of getattr diff --git a/Lib/traceback.py b/Lib/traceback.py index ba4a9ffd001b53..419f6e81b5e1be 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -179,12 +179,12 @@ def _safe_string(value, what, func=str): # -- def print_exc(limit=None, file=None, chain=True): - """Shorthand for 'print_exception(*sys.exc_info(), limit, file, chain)'.""" - print_exception(*sys.exc_info(), limit=limit, file=file, chain=chain) + """Shorthand for 'print_exception(sys.exception(), limit, file, chain)'.""" + print_exception(sys.exception(), limit=limit, file=file, chain=chain) def format_exc(limit=None, chain=True): """Like print_exc() but return a string.""" - return "".join(format_exception(*sys.exc_info(), limit=limit, chain=chain)) + return "".join(format_exception(sys.exception(), limit=limit, chain=chain)) def print_last(limit=None, file=None, chain=True): """This is a shorthand for 'print_exception(sys.last_exc, limit, file, chain)'.""" From 2d526cd32fe8b286aae38956648e508070729f8f Mon Sep 17 00:00:00 2001 From: Franek Magiera <framagie@gmail.com> Date: Mon, 1 May 2023 19:58:50 +0200 Subject: [PATCH 20/43] GH-103629: Update Unpack's repr in compliance with PEP 692 (#104048) --- Lib/test/test_typing.py | 59 ++++++++++--------- Lib/typing.py | 13 +++- ...-05-01-19-10-05.gh-issue-103629.81bpZz.rst | 1 + 3 files changed, 45 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 7c6a521c3c48f8..f162e587810ac0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -880,6 +880,11 @@ def test_cannot_be_called(self): with self.assertRaises(TypeError): Unpack() + def test_usage_with_kwargs(self): + Movie = TypedDict('Movie', {'name': str, 'year': int}) + def foo(**kwargs: Unpack[Movie]): ... + self.assertEqual(repr(foo.__annotations__['kwargs']), + f"typing.Unpack[{__name__}.Movie]") class TypeVarTupleTests(BaseTestCase): @@ -1050,14 +1055,14 @@ class G2(Generic[Unpack[Ts]]): pass self.assertEqual(repr(Ts), 'Ts') - self.assertEqual(repr((*Ts,)[0]), '*Ts') - self.assertEqual(repr(Unpack[Ts]), '*Ts') + self.assertEqual(repr((*Ts,)[0]), 'typing.Unpack[Ts]') + self.assertEqual(repr(Unpack[Ts]), 'typing.Unpack[Ts]') - self.assertEqual(repr(tuple[*Ts]), 'tuple[*Ts]') - self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[*Ts]') + self.assertEqual(repr(tuple[*Ts]), 'tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[typing.Unpack[Ts]]') - self.assertEqual(repr(*tuple[*Ts]), '*tuple[*Ts]') - self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), '*typing.Tuple[*Ts]') + self.assertEqual(repr(*tuple[*Ts]), '*tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), 'typing.Unpack[typing.Tuple[typing.Unpack[Ts]]]') def test_variadic_class_repr_is_correct(self): Ts = TypeVarTuple('Ts') @@ -1074,86 +1079,86 @@ class B(Generic[Unpack[Ts]]): pass self.assertEndsWith(repr(A[*tuple[int, ...]]), 'A[*tuple[int, ...]]') self.assertEndsWith(repr(B[Unpack[Tuple[int, ...]]]), - 'B[*typing.Tuple[int, ...]]') + 'B[typing.Unpack[typing.Tuple[int, ...]]]') self.assertEndsWith(repr(A[float, *tuple[int, ...]]), 'A[float, *tuple[int, ...]]') self.assertEndsWith(repr(A[float, Unpack[Tuple[int, ...]]]), - 'A[float, *typing.Tuple[int, ...]]') + 'A[float, typing.Unpack[typing.Tuple[int, ...]]]') self.assertEndsWith(repr(A[*tuple[int, ...], str]), 'A[*tuple[int, ...], str]') self.assertEndsWith(repr(B[Unpack[Tuple[int, ...]], str]), - 'B[*typing.Tuple[int, ...], str]') + 'B[typing.Unpack[typing.Tuple[int, ...]], str]') self.assertEndsWith(repr(A[float, *tuple[int, ...], str]), 'A[float, *tuple[int, ...], str]') self.assertEndsWith(repr(B[float, Unpack[Tuple[int, ...]], str]), - 'B[float, *typing.Tuple[int, ...], str]') + 'B[float, typing.Unpack[typing.Tuple[int, ...]], str]') def test_variadic_class_alias_repr_is_correct(self): Ts = TypeVarTuple('Ts') class A(Generic[Unpack[Ts]]): pass B = A[*Ts] - self.assertEndsWith(repr(B), 'A[*Ts]') + self.assertEndsWith(repr(B), 'A[typing.Unpack[Ts]]') self.assertEndsWith(repr(B[()]), 'A[()]') self.assertEndsWith(repr(B[float]), 'A[float]') self.assertEndsWith(repr(B[float, str]), 'A[float, str]') C = A[Unpack[Ts]] - self.assertEndsWith(repr(C), 'A[*Ts]') + self.assertEndsWith(repr(C), 'A[typing.Unpack[Ts]]') self.assertEndsWith(repr(C[()]), 'A[()]') self.assertEndsWith(repr(C[float]), 'A[float]') self.assertEndsWith(repr(C[float, str]), 'A[float, str]') D = A[*Ts, int] - self.assertEndsWith(repr(D), 'A[*Ts, int]') + self.assertEndsWith(repr(D), 'A[typing.Unpack[Ts], int]') self.assertEndsWith(repr(D[()]), 'A[int]') self.assertEndsWith(repr(D[float]), 'A[float, int]') self.assertEndsWith(repr(D[float, str]), 'A[float, str, int]') E = A[Unpack[Ts], int] - self.assertEndsWith(repr(E), 'A[*Ts, int]') + self.assertEndsWith(repr(E), 'A[typing.Unpack[Ts], int]') self.assertEndsWith(repr(E[()]), 'A[int]') self.assertEndsWith(repr(E[float]), 'A[float, int]') self.assertEndsWith(repr(E[float, str]), 'A[float, str, int]') F = A[int, *Ts] - self.assertEndsWith(repr(F), 'A[int, *Ts]') + self.assertEndsWith(repr(F), 'A[int, typing.Unpack[Ts]]') self.assertEndsWith(repr(F[()]), 'A[int]') self.assertEndsWith(repr(F[float]), 'A[int, float]') self.assertEndsWith(repr(F[float, str]), 'A[int, float, str]') G = A[int, Unpack[Ts]] - self.assertEndsWith(repr(G), 'A[int, *Ts]') + self.assertEndsWith(repr(G), 'A[int, typing.Unpack[Ts]]') self.assertEndsWith(repr(G[()]), 'A[int]') self.assertEndsWith(repr(G[float]), 'A[int, float]') self.assertEndsWith(repr(G[float, str]), 'A[int, float, str]') H = A[int, *Ts, str] - self.assertEndsWith(repr(H), 'A[int, *Ts, str]') + self.assertEndsWith(repr(H), 'A[int, typing.Unpack[Ts], str]') self.assertEndsWith(repr(H[()]), 'A[int, str]') self.assertEndsWith(repr(H[float]), 'A[int, float, str]') self.assertEndsWith(repr(H[float, str]), 'A[int, float, str, str]') I = A[int, Unpack[Ts], str] - self.assertEndsWith(repr(I), 'A[int, *Ts, str]') + self.assertEndsWith(repr(I), 'A[int, typing.Unpack[Ts], str]') self.assertEndsWith(repr(I[()]), 'A[int, str]') self.assertEndsWith(repr(I[float]), 'A[int, float, str]') self.assertEndsWith(repr(I[float, str]), 'A[int, float, str, str]') J = A[*Ts, *tuple[str, ...]] - self.assertEndsWith(repr(J), 'A[*Ts, *tuple[str, ...]]') + self.assertEndsWith(repr(J), 'A[typing.Unpack[Ts], *tuple[str, ...]]') self.assertEndsWith(repr(J[()]), 'A[*tuple[str, ...]]') self.assertEndsWith(repr(J[float]), 'A[float, *tuple[str, ...]]') self.assertEndsWith(repr(J[float, str]), 'A[float, str, *tuple[str, ...]]') K = A[Unpack[Ts], Unpack[Tuple[str, ...]]] - self.assertEndsWith(repr(K), 'A[*Ts, *typing.Tuple[str, ...]]') - self.assertEndsWith(repr(K[()]), 'A[*typing.Tuple[str, ...]]') - self.assertEndsWith(repr(K[float]), 'A[float, *typing.Tuple[str, ...]]') - self.assertEndsWith(repr(K[float, str]), 'A[float, str, *typing.Tuple[str, ...]]') + self.assertEndsWith(repr(K), 'A[typing.Unpack[Ts], typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[()]), 'A[typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[float]), 'A[float, typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[float, str]), 'A[float, str, typing.Unpack[typing.Tuple[str, ...]]]') def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): @@ -1171,9 +1176,9 @@ class C(type(Unpack[Ts])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Unpack'): class C(Unpack): pass - with self.assertRaisesRegex(TypeError, r'Cannot subclass \*Ts'): + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): class C(*Ts): pass - with self.assertRaisesRegex(TypeError, r'Cannot subclass \*Ts'): + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): class C(Unpack[Ts]): pass def test_variadic_class_args_are_correct(self): @@ -4108,13 +4113,13 @@ class TsP(Generic[*Ts, P]): MyCallable[[int], bool]: "MyCallable[[int], bool]", MyCallable[[int, str], bool]: "MyCallable[[int, str], bool]", MyCallable[[int, list[int]], bool]: "MyCallable[[int, list[int]], bool]", - MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[*Ts, ~P], ~T]", + MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[typing.Unpack[Ts], ~P], ~T]", DoubleSpec[P2, P, T]: "DoubleSpec[~P2, ~P, ~T]", DoubleSpec[[int], [str], bool]: "DoubleSpec[[int], [str], bool]", DoubleSpec[[int, int], [str, str], bool]: "DoubleSpec[[int, int], [str, str], bool]", - TsP[*Ts, P]: "TsP[*Ts, ~P]", + TsP[*Ts, P]: "TsP[typing.Unpack[Ts], ~P]", TsP[int, str, list[int], []]: "TsP[int, str, list[int], []]", TsP[int, [str, list[int]]]: "TsP[int, [str, list[int]]]", diff --git a/Lib/typing.py b/Lib/typing.py index 354bc80eb3abfa..1a1c989dbaf37d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1753,6 +1753,17 @@ class Bar(Generic[Unpack[Ts]]): ... Foo[*tuple[int, str]] class Bar(Generic[*Ts]): ... + The operator can also be used along with a `TypedDict` to annotate + `**kwargs` in a function signature. For instance: + + class Movie(TypedDict): + name: str + year: int + + # This function expects two keyword arguments - *name* of type `str` and + # *year* of type `int`. + def foo(**kwargs: Unpack[Movie]): ... + Note that there is only some runtime checking of this operator. Not everything the runtime allows may be accepted by static type checkers. @@ -1767,7 +1778,7 @@ class _UnpackGenericAlias(_GenericAlias, _root=True): def __repr__(self): # `Unpack` only takes one argument, so __args__ should contain only # a single item. - return '*' + repr(self.__args__[0]) + return f'typing.Unpack[{_type_repr(self.__args__[0])}]' def __getitem__(self, args): if self.__typing_is_unpacked_typevartuple__: diff --git a/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst b/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst new file mode 100644 index 00000000000000..7971ab66359c3d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst @@ -0,0 +1 @@ +Update the ``repr`` of :class:`typing.Unpack` according to :pep:`692`. From 5078eedc5b18f0d208af6e30f60b33419132d1b6 Mon Sep 17 00:00:00 2001 From: jx124 <64946984+jx124@users.noreply.github.com> Date: Tue, 2 May 2023 03:15:47 +0800 Subject: [PATCH 21/43] gh-104016: Fixed off by 1 error in f string tokenizer (#104047) Co-authored-by: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Co-authored-by: Ken Jin <kenjin@python.org> Co-authored-by: Pablo Galindo <pablogsal@gmail.com> --- Lib/test/test_fstring.py | 16 ++++++++++++++++ Parser/tokenizer.c | 7 +++++-- Parser/tokenizer.h | 7 ++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 5e94c99ae65af1..5c5176dc54a6d9 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -565,7 +565,23 @@ def test_fstring_nested_too_deeply(self): self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", ['f"{1+2:{1+2:{1+1:{1}}}}"']) + + def create_nested_fstring(n): + if n == 0: + return "1+1" + prev = create_nested_fstring(n-1) + return f'f"{{{prev}}}"' + self.assertAllRaise(SyntaxError, + "too many nested f-strings", + [create_nested_fstring(160)]) + + def test_syntax_error_in_nested_fstring(self): + # See gh-104016 for more information on this crash + self.assertAllRaise(SyntaxError, + "invalid syntax", + ['f"{1 1:' + ('{f"1:' * 199)]) + def test_double_braces(self): self.assertEqual(f'{{', '{') self.assertEqual(f'a{{', 'a{') diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 8fb9be7bfd0182..d2f9fee110ebf5 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -43,12 +43,12 @@ #ifdef Py_DEBUG static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) { assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index < MAXLEVEL); + assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL); return &(tok->tok_mode_stack[tok->tok_mode_stack_index]); } static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index < MAXLEVEL); + assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]); } #else @@ -2235,6 +2235,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t p_start = tok->start; p_end = tok->cur; + if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) { + return MAKE_TOKEN(syntaxerror(tok, "too many nested f-strings")); + } tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok); the_current_tok->kind = TOK_FSTRING_MODE; the_current_tok->f_string_quote = quote; diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 8b4213c4ce3b5a..5e2171885ac75b 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -10,8 +10,9 @@ extern "C" { #include "pycore_token.h" /* For token types */ -#define MAXINDENT 100 /* Max indentation level */ -#define MAXLEVEL 200 /* Max parentheses level */ +#define MAXINDENT 100 /* Max indentation level */ +#define MAXLEVEL 200 /* Max parentheses level */ +#define MAXFSTRINGLEVEL 150 /* Max f-string nesting level */ enum decoding_state { STATE_INIT, @@ -123,7 +124,7 @@ struct tok_state { enum interactive_underflow_t interactive_underflow; int report_warnings; // TODO: Factor this into its own thing - tokenizer_mode tok_mode_stack[MAXLEVEL]; + tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL]; int tok_mode_stack_index; int tok_report_warnings; #ifdef Py_DEBUG From 3ed8c882902a6982fd67e898a5b8a2d619fb5ddf Mon Sep 17 00:00:00 2001 From: John Belmonte <john@neggie.net> Date: Tue, 2 May 2023 04:47:14 +0900 Subject: [PATCH 22/43] gh-104018: disallow "z" format specifier in %-format of byte strings (GH-104033) PEP-0682 specified that %-formatting would not support the "z" specifier, but it was unintentionally allowed for bytes. This PR makes use of the "z" flag an error for %-formatting in a bytestring. Issue: #104018 --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/test/test_format.py | 2 ++ .../2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst | 1 + Objects/bytesobject.c | 1 - 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 69b0d5f1c5a515..6fa49dbc0b730c 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -619,6 +619,8 @@ def test_specifier_z_error(self): error_msg = re.escape("unsupported format character 'z'") with self.assertRaisesRegex(ValueError, error_msg): "%z.1f" % 0 # not allowed in old style string interpolation + with self.assertRaisesRegex(ValueError, error_msg): + b"%z.1f" % 0 if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst new file mode 100644 index 00000000000000..f3cadaee0e32d9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst @@ -0,0 +1 @@ +Disallow the "z" format specifier in %-format of bytes objects. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 27b2ad4f2cb38f..e7e85cc19cda75 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -705,7 +705,6 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, case ' ': flags |= F_BLANK; continue; case '#': flags |= F_ALT; continue; case '0': flags |= F_ZERO; continue; - case 'z': flags |= F_NO_NEG_0; continue; } break; } From e665563f8301d0db5cb0847d75fc879f074aa100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?An=C5=BEe=20Pe=C4=8Dar?= <anze@pecar.me> Date: Tue, 2 May 2023 00:49:54 +0500 Subject: [PATCH 23/43] gh-100458: Clarify Enum.__format__() change of mixed-in types in the whatsnew/3.11.rst (GH-100387) Co-authored-by: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Co-authored-by: Ethan Furman <ethan@stoneleaf.us> --- Doc/whatsnew/3.11.rst | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 10fcfb6a0b5639..687719a260a61c 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -666,19 +666,11 @@ enum for :meth:`~object.__str__` and :meth:`~object.__format__` (used by :func:`str`, :func:`format` and :term:`f-string`\s). -* Changed :class:`~enum.IntEnum`, :class:`~enum.IntFlag` and :class:`~enum.StrEnum` - to now inherit from :class:`~enum.ReprEnum`, - so their :func:`str` output now matches :func:`format` - (both ``str(AnIntEnum.ONE)`` and ``format(AnIntEnum.ONE)`` return ``'1'``, - whereas before ``str(AnIntEnum.ONE)`` returned ``'AnIntEnum.ONE'``. - -* Changed :meth:`Enum.__format__() <enum.Enum.__format__>` - (the default for :func:`format`, :meth:`str.format` and :term:`f-string`\s) - of enums with mixed-in types (e.g. :class:`int`, :class:`str`) - to also include the class name in the output, not just the member's key. - This matches the existing behavior of :meth:`enum.Enum.__str__`, - returning e.g. ``'AnEnum.MEMBER'`` for an enum ``AnEnum(str, Enum)`` - instead of just ``'MEMBER'``. +* Changed :meth:`Enum.__format__() <enum.Enum.__format__>` (the default for + :func:`format`, :meth:`str.format` and :term:`f-string`\s) to always produce + the same result as :meth:`Enum.__str__()`: for enums inheriting from + :class:`~enum.ReprEnum` it will be the member's value; for all other enums + it will be the enum and member name (e.g. ``Color.RED``). * Added a new *boundary* class parameter to :class:`~enum.Flag` enums and the :class:`~enum.FlagBoundary` enum with its options, From d448fcb0323bf00cb4ff4a1e65e8424a73b5f0d4 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora <80244920+Eclips4@users.noreply.github.com> Date: Mon, 1 May 2023 23:17:47 +0300 Subject: [PATCH 24/43] gh-104057: Fix direct invocation of test_module (GH-104059) --- Lib/test/test_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 70e4efea69359a..c7eb92290e1b6d 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -236,7 +236,7 @@ def test_module_repr_with_full_loader(self): # Yes, a class not an instance. m.__loader__ = FullLoader self.assertEqual( - repr(m), "<module 'foo' (<class 'test.test_module.FullLoader'>)>") + repr(m), f"<module 'foo' (<class '{__name__}.FullLoader'>)>") def test_module_repr_with_bare_loader_and_filename(self): m = ModuleType('foo') From 59bc36aacddd5a3acd32c80c0dfd0726135a7817 Mon Sep 17 00:00:00 2001 From: Eric Snow <ericsnowcurrently@gmail.com> Date: Mon, 1 May 2023 15:08:34 -0600 Subject: [PATCH 25/43] gh-84436: Immortalize in _PyStructSequence_InitBuiltinWithFlags() (gh-104054) This also does some cleanup. --- Include/internal/pycore_structseq.h | 4 +- Objects/floatobject.c | 4 +- Objects/longobject.c | 4 +- Objects/structseq.c | 102 ++++++++++++++++------------ Objects/typeobject.c | 1 + Python/errors.c | 4 +- Python/sysmodule.c | 10 +-- Python/thread.c | 4 +- 8 files changed, 74 insertions(+), 59 deletions(-) diff --git a/Include/internal/pycore_structseq.h b/Include/internal/pycore_structseq.h index d10a921c55ff8b..bd1e85c6883f01 100644 --- a/Include/internal/pycore_structseq.h +++ b/Include/internal/pycore_structseq.h @@ -15,7 +15,7 @@ PyAPI_FUNC(PyTypeObject *) _PyStructSequence_NewType( PyStructSequence_Desc *desc, unsigned long tp_flags); -PyAPI_FUNC(int) _PyStructSequence_InitBuiltinWithFlags( +extern int _PyStructSequence_InitBuiltinWithFlags( PyTypeObject *type, PyStructSequence_Desc *desc, unsigned long tp_flags); @@ -27,7 +27,7 @@ _PyStructSequence_InitBuiltin(PyTypeObject *type, return _PyStructSequence_InitBuiltinWithFlags(type, desc, 0); } -extern void _PyStructSequence_FiniType(PyTypeObject *type); +extern void _PyStructSequence_FiniBuiltin(PyTypeObject *type); #ifdef __cplusplus } diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 9c2315781bed36..a694ddcd019ee8 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -12,7 +12,7 @@ #include "pycore_object.h" // _PyObject_Init() #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_structseq.h" // _PyStructSequence_FiniType() +#include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include <ctype.h> #include <float.h> @@ -2029,7 +2029,7 @@ void _PyFloat_FiniType(PyInterpreterState *interp) { if (_Py_IsMainInterpreter(interp)) { - _PyStructSequence_FiniType(&FloatInfoType); + _PyStructSequence_FiniBuiltin(&FloatInfoType); } } diff --git a/Objects/longobject.c b/Objects/longobject.c index f84809b8a8986a..de043488d7a173 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -9,7 +9,7 @@ #include "pycore_object.h" // _PyObject_Init() #include "pycore_pystate.h" // _Py_IsMainInterpreter() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS -#include "pycore_structseq.h" // _PyStructSequence_FiniType() +#include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include <ctype.h> #include <float.h> @@ -6367,5 +6367,5 @@ _PyLong_FiniTypes(PyInterpreterState *interp) return; } - _PyStructSequence_FiniType(&Int_InfoType); + _PyStructSequence_FiniBuiltin(&Int_InfoType); } diff --git a/Objects/structseq.c b/Objects/structseq.c index 88a71bc52958f5..d8f55dc1eae5ed 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -433,12 +433,10 @@ initialize_structseq_dict(PyStructSequence_Desc *desc, PyObject* dict, static PyMemberDef * initialize_members(PyStructSequence_Desc *desc, - Py_ssize_t *pn_members, Py_ssize_t *pn_unnamed_members) + Py_ssize_t n_members, Py_ssize_t n_unnamed_members) { PyMemberDef *members; - Py_ssize_t n_members, n_unnamed_members; - n_members = count_members(desc, &n_unnamed_members); members = PyMem_NEW(PyMemberDef, n_members - n_unnamed_members + 1); if (members == NULL) { PyErr_NoMemory(); @@ -463,8 +461,6 @@ initialize_members(PyStructSequence_Desc *desc, } members[k].name = NULL; - *pn_members = n_members; - *pn_unnamed_members = n_unnamed_members; return members; } @@ -510,39 +506,58 @@ _PyStructSequence_InitBuiltinWithFlags(PyTypeObject *type, PyStructSequence_Desc *desc, unsigned long tp_flags) { - if (type->tp_flags & Py_TPFLAGS_READY) { - if (_PyStaticType_InitBuiltin(type) < 0) { - goto failed_init_builtin; + Py_ssize_t n_unnamed_members; + Py_ssize_t n_members = count_members(desc, &n_unnamed_members); + PyMemberDef *members = NULL; + + int initialized = 1; + if ((type->tp_flags & Py_TPFLAGS_READY) == 0) { + assert(type->tp_name == NULL); + assert(type->tp_members == NULL); + assert(type->tp_base == NULL); + + members = initialize_members(desc, n_members, n_unnamed_members); + if (members == NULL) { + goto error; } - return 0; - } + initialize_static_fields(type, desc, members, tp_flags); - PyMemberDef *members; - Py_ssize_t n_members, n_unnamed_members; - - members = initialize_members(desc, &n_members, &n_unnamed_members); - if (members == NULL) { - return -1; + _Py_SetImmortal(type); + initialized = 0; + } +#ifndef NDEBUG + else { + // Ensure that the type was initialized. + assert(type->tp_name != NULL); + assert(type->tp_members != NULL); + assert(type->tp_base == &PyTuple_Type); + assert((type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + assert(_Py_IsImmortal(type)); } - initialize_static_fields(type, desc, members, tp_flags); +#endif - Py_INCREF(type); // XXX It should be immortal. if (_PyStaticType_InitBuiltin(type) < 0) { - PyMem_Free(members); - goto failed_init_builtin; + PyErr_Format(PyExc_RuntimeError, + "Can't initialize builtin type %s", + desc->name); + goto error; + } + // This should be dropped if tp_dict is made per-interpreter. + if (initialized) { + return 0; } if (initialize_structseq_dict( desc, type->tp_dict, n_members, n_unnamed_members) < 0) { - PyMem_Free(members); - return -1; + goto error; } + return 0; -failed_init_builtin: - PyErr_Format(PyExc_RuntimeError, - "Can't initialize builtin type %s", - desc->name); +error: + if (members != NULL) { + PyMem_Free(members); + } return -1; } @@ -566,7 +581,8 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) return -1; } - members = initialize_members(desc, &n_members, &n_unnamed_members); + n_members = count_members(desc, &n_unnamed_members); + members = initialize_members(desc, n_members, n_unnamed_members); if (members == NULL) { return -1; } @@ -585,35 +601,32 @@ PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) } +/* This is exposed in the internal API, not the public API. + It is only called on builtin static types, which are all + initialized via _PyStructSequence_InitBuiltinWithFlags(). */ + void -_PyStructSequence_FiniType(PyTypeObject *type) +_PyStructSequence_FiniBuiltin(PyTypeObject *type) { // Ensure that the type is initialized assert(type->tp_name != NULL); assert(type->tp_base == &PyTuple_Type); + assert((type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + assert(_Py_IsImmortal(type)); // Cannot delete a type if it still has subclasses if (_PyType_HasSubclasses(type)) { + // XXX Shouldn't this be an error? return; } - // Undo PyStructSequence_NewType() - type->tp_name = NULL; - PyMem_Free(type->tp_members); - _PyStaticType_Dealloc(type); - assert(Py_REFCNT(type) == 1); - // Undo Py_INCREF(type) of _PyStructSequence_InitType(). - // Don't use Py_DECREF(): static type must not be deallocated - Py_SET_REFCNT(type, 0); -#ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); -#endif - // Make sure that _PyStructSequence_InitType() will initialize - // the type again - assert(Py_REFCNT(type) == 0); - assert(type->tp_name == NULL); + // Undo _PyStructSequence_InitBuiltinWithFlags(). + type->tp_name = NULL; + PyMem_Free(type->tp_members); + type->tp_members = NULL; + type->tp_base = NULL; } @@ -627,7 +640,8 @@ _PyStructSequence_NewType(PyStructSequence_Desc *desc, unsigned long tp_flags) Py_ssize_t n_members, n_unnamed_members; /* Initialize MemberDefs */ - members = initialize_members(desc, &n_members, &n_unnamed_members); + n_members = count_members(desc, &n_unnamed_members); + members = initialize_members(desc, n_members, n_unnamed_members); if (members == NULL) { return NULL; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index e807cc90faa16a..060d14e254ab2d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7031,6 +7031,7 @@ PyType_Ready(PyTypeObject *type) int _PyStaticType_InitBuiltin(PyTypeObject *self) { + assert(_Py_IsImmortal((PyObject *)self)); assert(!(self->tp_flags & Py_TPFLAGS_HEAPTYPE)); if (self->tp_flags & Py_TPFLAGS_READY) { diff --git a/Python/errors.c b/Python/errors.c index 7fc267385c569b..ce72049b92de27 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -6,7 +6,7 @@ #include "pycore_initconfig.h" // _PyStatus_ERR() #include "pycore_pyerrors.h" // _PyErr_Format() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_structseq.h" // _PyStructSequence_FiniType() +#include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_FromFrame() @@ -1357,7 +1357,7 @@ _PyErr_FiniTypes(PyInterpreterState *interp) return; } - _PyStructSequence_FiniType(&UnraisableHookArgsType); + _PyStructSequence_FiniBuiltin(&UnraisableHookArgsType); } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d673e40af5e1de..81dabe6102f18d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3492,13 +3492,13 @@ void _PySys_Fini(PyInterpreterState *interp) { if (_Py_IsMainInterpreter(interp)) { - _PyStructSequence_FiniType(&VersionInfoType); - _PyStructSequence_FiniType(&FlagsType); + _PyStructSequence_FiniBuiltin(&VersionInfoType); + _PyStructSequence_FiniBuiltin(&FlagsType); #if defined(MS_WINDOWS) - _PyStructSequence_FiniType(&WindowsVersionType); + _PyStructSequence_FiniBuiltin(&WindowsVersionType); #endif - _PyStructSequence_FiniType(&Hash_InfoType); - _PyStructSequence_FiniType(&AsyncGenHooksType); + _PyStructSequence_FiniBuiltin(&Hash_InfoType); + _PyStructSequence_FiniBuiltin(&AsyncGenHooksType); #ifdef __EMSCRIPTEN__ Py_CLEAR(EmscriptenInfoType); #endif diff --git a/Python/thread.c b/Python/thread.c index 7fdedb0b9b7e26..f90cd34a073540 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -7,7 +7,7 @@ #include "Python.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_structseq.h" // _PyStructSequence_FiniType() +#include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include "pycore_pythread.h" #ifndef DONT_HAVE_STDIO_H @@ -195,5 +195,5 @@ _PyThread_FiniType(PyInterpreterState *interp) return; } - _PyStructSequence_FiniType(&ThreadInfoType); + _PyStructSequence_FiniBuiltin(&ThreadInfoType); } From a474e04388c2ef6aca75c26cb70a1b6200235feb Mon Sep 17 00:00:00 2001 From: Itamar Ostricher <itamarost@gmail.com> Date: Mon, 1 May 2023 14:10:13 -0700 Subject: [PATCH 26/43] gh-97696: asyncio eager tasks factory (#102853) Co-authored-by: Jacob Bower <jbower@meta.com> Co-authored-by: Carol Willing <carolcode@willingconsulting.com> --- Doc/library/asyncio-task.rst | 36 ++ Doc/whatsnew/3.12.rst | 5 + .../pycore_global_objects_fini_generated.h | 2 + Include/internal/pycore_global_strings.h | 2 + .../internal/pycore_runtime_init_generated.h | 2 + .../internal/pycore_unicodeobject_generated.h | 6 + Lib/asyncio/base_tasks.py | 10 +- Lib/asyncio/tasks.py | 122 +++++-- .../test_asyncio/test_eager_task_factory.py | 344 ++++++++++++++++++ ...3-03-15-12-18-07.gh-issue-97696.DtnpIC.rst | 6 + Modules/_asynciomodule.c | 254 ++++++++++++- Modules/clinic/_asynciomodule.c.h | 203 ++++++++++- 12 files changed, 945 insertions(+), 47 deletions(-) create mode 100644 Lib/test/test_asyncio/test_eager_task_factory.py create mode 100644 Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index ba0f909c405a34..f8727b98066990 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -527,6 +527,42 @@ Running Tasks Concurrently and there is no running event loop. +Eager Task Factory +================== + +.. function:: eager_task_factory(loop, coro, *, name=None, context=None) + + A task factory for eager task execution. + + When using this factory (via :meth:`loop.set_task_factory(asyncio.eager_task_factory) <loop.set_task_factory>`), + coroutines begin execution synchronously during :class:`Task` construction. + Tasks are only scheduled on the event loop if they block. + This can be a performance improvement as the overhead of loop scheduling + is avoided for coroutines that complete synchronously. + + A common example where this is beneficial is coroutines which employ + caching or memoization to avoid actual I/O when possible. + + .. note:: + + Immediate execution of the coroutine is a semantic change. + If the coroutine returns or raises, the task is never scheduled + to the event loop. If the coroutine execution blocks, the task is + scheduled to the event loop. This change may introduce behavior + changes to existing applications. For example, + the application's task execution order is likely to change. + + .. versionadded:: 3.12 + +.. function:: create_eager_task_factory(custom_task_constructor) + + Create an eager task factory, similar to :func:`eager_task_factory`, + using the provided *custom_task_constructor* when creating a new task instead + of the default :class:`Task`. + + .. versionadded:: 3.12 + + Shielding From Cancellation =========================== diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index f4ee30b0d4d9eb..a3fce7ccacf7c1 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -613,6 +613,11 @@ Optimizations * Speed up :class:`asyncio.Task` creation by deferring expensive string formatting. (Contributed by Itamar O in :gh:`103793`.) +* Added :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory` + functions to allow opting an event loop in to eager task execution, + speeding up some use-cases by up to 50%. + (Contributed by Jacob Bower & Itamar O in :gh:`102853`) + CPython bytecode changes ======================== diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 4fa15d74b3ad64..5e8a8d78412786 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -882,6 +882,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst_dir_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(duration)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(e)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encode)); @@ -972,6 +973,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(instructions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intern)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intersection)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_running)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isatty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isinstance)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isoformat)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index e19d8ff1b50468..28e82203d8f639 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -370,6 +370,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(dst_dir_fd) STRUCT_FOR_ID(duration) STRUCT_FOR_ID(e) + STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) STRUCT_FOR_ID(element_factory) STRUCT_FOR_ID(encode) @@ -460,6 +461,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(instructions) STRUCT_FOR_ID(intern) STRUCT_FOR_ID(intersection) + STRUCT_FOR_ID(is_running) STRUCT_FOR_ID(isatty) STRUCT_FOR_ID(isinstance) STRUCT_FOR_ID(isoformat) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 42c4874d9466bf..dd4471160a4a4c 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -876,6 +876,7 @@ extern "C" { INIT_ID(dst_dir_fd), \ INIT_ID(duration), \ INIT_ID(e), \ + INIT_ID(eager_start), \ INIT_ID(effective_ids), \ INIT_ID(element_factory), \ INIT_ID(encode), \ @@ -966,6 +967,7 @@ extern "C" { INIT_ID(instructions), \ INIT_ID(intern), \ INIT_ID(intersection), \ + INIT_ID(is_running), \ INIT_ID(isatty), \ INIT_ID(isinstance), \ INIT_ID(isoformat), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 6d9cd24d9f3a13..1a8338b341fd68 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -963,6 +963,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(e); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(eager_start); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(effective_ids); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1233,6 +1236,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(intersection); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(is_running); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(isatty); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/asyncio/base_tasks.py b/Lib/asyncio/base_tasks.py index 26298e638cbf0d..c907b683413732 100644 --- a/Lib/asyncio/base_tasks.py +++ b/Lib/asyncio/base_tasks.py @@ -15,11 +15,13 @@ def _task_repr_info(task): info.insert(1, 'name=%r' % task.get_name()) - coro = coroutines._format_coroutine(task._coro) - info.insert(2, f'coro=<{coro}>') - if task._fut_waiter is not None: - info.insert(3, f'wait_for={task._fut_waiter!r}') + info.insert(2, f'wait_for={task._fut_waiter!r}') + + if task._coro: + coro = coroutines._format_coroutine(task._coro) + info.insert(2, f'coro=<{coro}>') + return info diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index c90d32c97add78..aa5269ade19a7f 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -6,6 +6,7 @@ 'wait', 'wait_for', 'as_completed', 'sleep', 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', 'current_task', 'all_tasks', + 'create_eager_task_factory', 'eager_task_factory', '_register_task', '_unregister_task', '_enter_task', '_leave_task', ) @@ -43,22 +44,26 @@ def all_tasks(loop=None): """Return a set of all tasks for the loop.""" if loop is None: loop = events.get_running_loop() - # Looping over a WeakSet (_all_tasks) isn't safe as it can be updated from another - # thread while we do so. Therefore we cast it to list prior to filtering. The list - # cast itself requires iteration, so we repeat it several times ignoring - # RuntimeErrors (which are not very likely to occur). See issues 34970 and 36607 for - # details. + # capturing the set of eager tasks first, so if an eager task "graduates" + # to a regular task in another thread, we don't risk missing it. + eager_tasks = list(_eager_tasks) + # Looping over the WeakSet isn't safe as it can be updated from another + # thread, therefore we cast it to list prior to filtering. The list cast + # itself requires iteration, so we repeat it several times ignoring + # RuntimeErrors (which are not very likely to occur). + # See issues 34970 and 36607 for details. + scheduled_tasks = None i = 0 while True: try: - tasks = list(_all_tasks) + scheduled_tasks = list(_scheduled_tasks) except RuntimeError: i += 1 if i >= 1000: raise else: break - return {t for t in tasks + return {t for t in itertools.chain(scheduled_tasks, eager_tasks) if futures._get_loop(t) is loop and not t.done()} @@ -93,7 +98,8 @@ class Task(futures._PyFuture): # Inherit Python Task implementation # status is still pending _log_destroy_pending = True - def __init__(self, coro, *, loop=None, name=None, context=None): + def __init__(self, coro, *, loop=None, name=None, context=None, + eager_start=False): super().__init__(loop=loop) if self._source_traceback: del self._source_traceback[-1] @@ -117,8 +123,11 @@ def __init__(self, coro, *, loop=None, name=None, context=None): else: self._context = context - self._loop.call_soon(self.__step, context=self._context) - _register_task(self) + if eager_start and self._loop.is_running(): + self.__eager_start() + else: + self._loop.call_soon(self.__step, context=self._context) + _register_task(self) def __del__(self): if self._state == futures._PENDING and self._log_destroy_pending: @@ -250,6 +259,25 @@ def uncancel(self): self._num_cancels_requested -= 1 return self._num_cancels_requested + def __eager_start(self): + prev_task = _swap_current_task(self._loop, self) + try: + _register_eager_task(self) + try: + self._context.run(self.__step_run_and_handle_result, None) + finally: + _unregister_eager_task(self) + finally: + try: + curtask = _swap_current_task(self._loop, prev_task) + assert curtask is self + finally: + if self.done(): + self._coro = None + self = None # Needed to break cycles when an exception occurs. + else: + _register_task(self) + def __step(self, exc=None): if self.done(): raise exceptions.InvalidStateError( @@ -258,11 +286,17 @@ def __step(self, exc=None): if not isinstance(exc, exceptions.CancelledError): exc = self._make_cancelled_error() self._must_cancel = False - coro = self._coro self._fut_waiter = None _enter_task(self._loop, self) - # Call either coro.throw(exc) or coro.send(None). + try: + self.__step_run_and_handle_result(exc) + finally: + _leave_task(self._loop, self) + self = None # Needed to break cycles when an exception occurs. + + def __step_run_and_handle_result(self, exc): + coro = self._coro try: if exc is None: # We use the `send` method directly, because coroutines @@ -334,7 +368,6 @@ def __step(self, exc=None): self._loop.call_soon( self.__step, new_exc, context=self._context) finally: - _leave_task(self._loop, self) self = None # Needed to break cycles when an exception occurs. def __wakeup(self, future): @@ -897,8 +930,27 @@ def callback(): return future -# WeakSet containing all alive tasks. -_all_tasks = weakref.WeakSet() +def create_eager_task_factory(custom_task_constructor): + + if "eager_start" not in inspect.signature(custom_task_constructor).parameters: + raise TypeError( + "Provided constructor does not support eager task execution") + + def factory(loop, coro, *, name=None, context=None): + return custom_task_constructor( + coro, loop=loop, name=name, context=context, eager_start=True) + + + return factory + +eager_task_factory = create_eager_task_factory(Task) + + +# Collectively these two sets hold references to the complete set of active +# tasks. Eagerly executed tasks use a faster regular set as an optimization +# but may graduate to a WeakSet if the task blocks on IO. +_scheduled_tasks = weakref.WeakSet() +_eager_tasks = set() # Dictionary containing tasks that are currently active in # all running event loops. {EventLoop: Task} @@ -906,8 +958,13 @@ def callback(): def _register_task(task): - """Register a new task in asyncio as executed by loop.""" - _all_tasks.add(task) + """Register an asyncio Task scheduled to run on an event loop.""" + _scheduled_tasks.add(task) + + +def _register_eager_task(task): + """Register an asyncio Task about to be eagerly executed.""" + _eager_tasks.add(task) def _enter_task(loop, task): @@ -926,28 +983,49 @@ def _leave_task(loop, task): del _current_tasks[loop] +def _swap_current_task(loop, task): + prev_task = _current_tasks.get(loop) + if task is None: + del _current_tasks[loop] + else: + _current_tasks[loop] = task + return prev_task + + def _unregister_task(task): - """Unregister a task.""" - _all_tasks.discard(task) + """Unregister a completed, scheduled Task.""" + _scheduled_tasks.discard(task) + + +def _unregister_eager_task(task): + """Unregister a task which finished its first eager step.""" + _eager_tasks.discard(task) _py_current_task = current_task _py_register_task = _register_task +_py_register_eager_task = _register_eager_task _py_unregister_task = _unregister_task +_py_unregister_eager_task = _unregister_eager_task _py_enter_task = _enter_task _py_leave_task = _leave_task +_py_swap_current_task = _swap_current_task try: - from _asyncio import (_register_task, _unregister_task, - _enter_task, _leave_task, - _all_tasks, _current_tasks, + from _asyncio import (_register_task, _register_eager_task, + _unregister_task, _unregister_eager_task, + _enter_task, _leave_task, _swap_current_task, + _scheduled_tasks, _eager_tasks, _current_tasks, current_task) except ImportError: pass else: _c_current_task = current_task _c_register_task = _register_task + _c_register_eager_task = _register_eager_task _c_unregister_task = _unregister_task + _c_unregister_eager_task = _unregister_eager_task _c_enter_task = _enter_task _c_leave_task = _leave_task + _c_swap_current_task = _swap_current_task diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py new file mode 100644 index 00000000000000..fe690934292a86 --- /dev/null +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -0,0 +1,344 @@ +"""Tests for base_events.py""" + +import asyncio +import contextvars +import gc +import time +import unittest + +from types import GenericAlias +from unittest import mock +from asyncio import base_events +from asyncio import tasks +from test.test_asyncio import utils as test_utils +from test.test_asyncio.test_tasks import get_innermost_context +from test import support + +MOCK_ANY = mock.ANY + + +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + +class EagerTaskFactoryLoopTests: + + Task = None + + def run_coro(self, coro): + """ + Helper method to run the `coro` coroutine in the test event loop. + It helps with making sure the event loop is running before starting + to execute `coro`. This is important for testing the eager step + functionality, since an eager step is taken only if the event loop + is already running. + """ + + async def coro_runner(): + self.assertTrue(asyncio.get_event_loop().is_running()) + return await coro + + return self.loop.run_until_complete(coro) + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.eager_task_factory = asyncio.create_eager_task_factory(self.Task) + self.loop.set_task_factory(self.eager_task_factory) + self.set_event_loop(self.loop) + + def test_eager_task_factory_set(self): + self.assertIsNotNone(self.eager_task_factory) + self.assertIs(self.loop.get_task_factory(), self.eager_task_factory) + + async def noop(): pass + + async def run(): + t = self.loop.create_task(noop()) + self.assertIsInstance(t, self.Task) + await t + + self.run_coro(run()) + + def test_await_future_during_eager_step(self): + + async def set_result(fut, val): + fut.set_result(val) + + async def run(): + fut = self.loop.create_future() + t = self.loop.create_task(set_result(fut, 'my message')) + # assert the eager step completed the task + self.assertTrue(t.done()) + return await fut + + self.assertEqual(self.run_coro(run()), 'my message') + + def test_eager_completion(self): + + async def coro(): + return 'hello' + + async def run(): + t = self.loop.create_task(coro()) + # assert the eager step completed the task + self.assertTrue(t.done()) + return await t + + self.assertEqual(self.run_coro(run()), 'hello') + + def test_block_after_eager_step(self): + + async def coro(): + await asyncio.sleep(0.1) + return 'finished after blocking' + + async def run(): + t = self.loop.create_task(coro()) + self.assertFalse(t.done()) + result = await t + self.assertTrue(t.done()) + return result + + self.assertEqual(self.run_coro(run()), 'finished after blocking') + + def test_cancellation_after_eager_completion(self): + + async def coro(): + return 'finished without blocking' + + async def run(): + t = self.loop.create_task(coro()) + t.cancel() + result = await t + # finished task can't be cancelled + self.assertFalse(t.cancelled()) + return result + + self.assertEqual(self.run_coro(run()), 'finished without blocking') + + def test_cancellation_after_eager_step_blocks(self): + + async def coro(): + await asyncio.sleep(0.1) + return 'finished after blocking' + + async def run(): + t = self.loop.create_task(coro()) + t.cancel('cancellation message') + self.assertGreater(t.cancelling(), 0) + result = await t + + with self.assertRaises(asyncio.CancelledError) as cm: + self.run_coro(run()) + + self.assertEqual('cancellation message', cm.exception.args[0]) + + def test_current_task(self): + captured_current_task = None + + async def coro(): + nonlocal captured_current_task + captured_current_task = asyncio.current_task() + # verify the task before and after blocking is identical + await asyncio.sleep(0.1) + self.assertIs(asyncio.current_task(), captured_current_task) + + async def run(): + t = self.loop.create_task(coro()) + self.assertIs(captured_current_task, t) + await t + + self.run_coro(run()) + captured_current_task = None + + def test_all_tasks_with_eager_completion(self): + captured_all_tasks = None + + async def coro(): + nonlocal captured_all_tasks + captured_all_tasks = asyncio.all_tasks() + + async def run(): + t = self.loop.create_task(coro()) + self.assertIn(t, captured_all_tasks) + self.assertNotIn(t, asyncio.all_tasks()) + + self.run_coro(run()) + + def test_all_tasks_with_blocking(self): + captured_eager_all_tasks = None + + async def coro(fut1, fut2): + nonlocal captured_eager_all_tasks + captured_eager_all_tasks = asyncio.all_tasks() + await fut1 + fut2.set_result(None) + + async def run(): + fut1 = self.loop.create_future() + fut2 = self.loop.create_future() + t = self.loop.create_task(coro(fut1, fut2)) + self.assertIn(t, captured_eager_all_tasks) + self.assertIn(t, asyncio.all_tasks()) + fut1.set_result(None) + await fut2 + self.assertNotIn(t, asyncio.all_tasks()) + + self.run_coro(run()) + + def test_context_vars(self): + cv = contextvars.ContextVar('cv', default=0) + + coro_first_step_ran = False + coro_second_step_ran = False + + async def coro(): + nonlocal coro_first_step_ran + nonlocal coro_second_step_ran + self.assertEqual(cv.get(), 1) + cv.set(2) + self.assertEqual(cv.get(), 2) + coro_first_step_ran = True + await asyncio.sleep(0.1) + self.assertEqual(cv.get(), 2) + cv.set(3) + self.assertEqual(cv.get(), 3) + coro_second_step_ran = True + + async def run(): + cv.set(1) + t = self.loop.create_task(coro()) + self.assertTrue(coro_first_step_ran) + self.assertFalse(coro_second_step_ran) + self.assertEqual(cv.get(), 1) + await t + self.assertTrue(coro_second_step_ran) + self.assertEqual(cv.get(), 1) + + self.run_coro(run()) + + +class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): + Task = tasks._PyTask + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + + +class AsyncTaskCounter: + def __init__(self, loop, *, task_class, eager): + self.suspense_count = 0 + self.task_count = 0 + + def CountingTask(*args, eager_start=False, **kwargs): + if not eager_start: + self.task_count += 1 + kwargs["eager_start"] = eager_start + return task_class(*args, **kwargs) + + if eager: + factory = asyncio.create_eager_task_factory(CountingTask) + else: + def factory(loop, coro, **kwargs): + return CountingTask(coro, loop=loop, **kwargs) + loop.set_task_factory(factory) + + def get(self): + return self.task_count + + +async def awaitable_chain(depth): + if depth == 0: + return 0 + return 1 + await awaitable_chain(depth - 1) + + +async def recursive_taskgroups(width, depth): + if depth == 0: + return + + async with asyncio.TaskGroup() as tg: + futures = [ + tg.create_task(recursive_taskgroups(width, depth - 1)) + for _ in range(width) + ] + + +async def recursive_gather(width, depth): + if depth == 0: + return + + await asyncio.gather( + *[recursive_gather(width, depth - 1) for _ in range(width)] + ) + + +class BaseTaskCountingTests: + + Task = None + eager = None + expected_task_count = None + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.counter = AsyncTaskCounter(self.loop, task_class=self.Task, eager=self.eager) + self.set_event_loop(self.loop) + + def test_awaitables_chain(self): + observed_depth = self.loop.run_until_complete(awaitable_chain(100)) + self.assertEqual(observed_depth, 100) + self.assertEqual(self.counter.get(), 0 if self.eager else 1) + + def test_recursive_taskgroups(self): + num_tasks = self.loop.run_until_complete(recursive_taskgroups(5, 4)) + self.assertEqual(self.counter.get(), self.expected_task_count) + + def test_recursive_gather(self): + self.loop.run_until_complete(recursive_gather(5, 4)) + self.assertEqual(self.counter.get(), self.expected_task_count) + + +class BaseNonEagerTaskFactoryTests(BaseTaskCountingTests): + eager = False + expected_task_count = 781 # 1 + 5 + 5^2 + 5^3 + 5^4 + + +class BaseEagerTaskFactoryTests(BaseTaskCountingTests): + eager = True + expected_task_count = 0 + + +class NonEagerTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = asyncio.Task + + +class EagerTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = asyncio.Task + + +class NonEagerPyTaskTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = tasks._PyTask + + +class EagerPyTaskTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = tasks._PyTask + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class NonEagerCTaskTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class EagerCTaskTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst b/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst new file mode 100644 index 00000000000000..0b3854d74eb991 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst @@ -0,0 +1,6 @@ +Implemented an eager task factory in asyncio. +When used as a task factory on an event loop, it performs eager execution of +coroutines. Coroutines that are able to complete synchronously (e.g. return or +raise without blocking) are returned immediately as a finished task, and the +task is never scheduled to the event loop. If the coroutine blocks, the +(pending) task is scheduled and returned. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 82dbc087322aa9..8b1a29b6d33e8b 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -8,6 +8,7 @@ #include "pycore_runtime_init.h" // _Py_ID() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef +#include "cpython/context.h" #include <stddef.h> // offsetof() @@ -31,8 +32,11 @@ typedef struct { all running event loops. {EventLoop: Task} */ PyObject *current_tasks; - /* WeakSet containing all alive tasks. */ - PyObject *all_tasks; + /* WeakSet containing all tasks scheduled to run on event loops. */ + PyObject *scheduled_tasks; + + /* Set containing all eagerly executing tasks. */ + PyObject *eager_tasks; /* An isinstance type cache for the 'is_coroutine()' function. */ PyObject *iscoroutine_typecache; @@ -156,6 +160,9 @@ class _asyncio.Future "FutureObj *" "&Future_Type" /* Get FutureIter from Future */ static PyObject * future_new_iter(PyObject *); +static PyObject * +task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *result); + static int _is_coroutine(asyncio_state *state, PyObject *coro) @@ -1830,6 +1837,7 @@ class _asyncio.Task "TaskObj *" "&Task_Type" static int task_call_step_soon(asyncio_state *state, TaskObj *, PyObject *); static PyObject * task_wakeup(TaskObj *, PyObject *); static PyObject * task_step(asyncio_state *, TaskObj *, PyObject *); +static int task_eager_start(asyncio_state *state, TaskObj *task); /* ----- Task._step wrapper */ @@ -1940,7 +1948,7 @@ static PyMethodDef TaskWakeupDef = { static int register_task(asyncio_state *state, PyObject *task) { - PyObject *res = PyObject_CallMethodOneArg(state->all_tasks, + PyObject *res = PyObject_CallMethodOneArg(state->scheduled_tasks, &_Py_ID(add), task); if (res == NULL) { return -1; @@ -1949,11 +1957,16 @@ register_task(asyncio_state *state, PyObject *task) return 0; } +static int +register_eager_task(asyncio_state *state, PyObject *task) +{ + return PySet_Add(state->eager_tasks, task); +} static int unregister_task(asyncio_state *state, PyObject *task) { - PyObject *res = PyObject_CallMethodOneArg(state->all_tasks, + PyObject *res = PyObject_CallMethodOneArg(state->scheduled_tasks, &_Py_ID(discard), task); if (res == NULL) { return -1; @@ -1962,6 +1975,11 @@ unregister_task(asyncio_state *state, PyObject *task) return 0; } +static int +unregister_eager_task(asyncio_state *state, PyObject *task) +{ + return PySet_Discard(state->eager_tasks, task); +} static int enter_task(asyncio_state *state, PyObject *loop, PyObject *task) @@ -2015,6 +2033,54 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) return _PyDict_DelItem_KnownHash(state->current_tasks, loop, hash); } +static PyObject * +swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) +{ + PyObject *prev_task; + Py_hash_t hash; + hash = PyObject_Hash(loop); + if (hash == -1) { + return NULL; + } + + prev_task = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); + if (prev_task == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + prev_task = Py_None; + } + + if (task == Py_None) { + if (_PyDict_DelItem_KnownHash(state->current_tasks, loop, hash) == -1) { + return NULL; + } + } else { + if (_PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash) == -1) { + return NULL; + } + } + + Py_INCREF(prev_task); + + return prev_task; +} + +static int +is_loop_running(PyObject *loop) +{ + PyObject *func = PyObject_GetAttr(loop, &_Py_ID(is_running)); + if (func == NULL) { + PyErr_Format(PyExc_TypeError, "Loop missing is_running()"); + return -1; + } + PyObject *res = PyObject_CallNoArgs(func); + int retval = Py_IsTrue(res); + Py_DECREF(func); + Py_DECREF(res); + return !!retval; +} + /* ----- Task */ /*[clinic input] @@ -2025,15 +2091,16 @@ _asyncio.Task.__init__ loop: object = None name: object = None context: object = None + eager_start: bool = False A coroutine wrapped in a Future. [clinic start generated code]*/ static int _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, - PyObject *name, PyObject *context) -/*[clinic end generated code: output=49ac96fe33d0e5c7 input=924522490c8ce825]*/ - + PyObject *name, PyObject *context, + int eager_start) +/*[clinic end generated code: output=7aced2d27836f1a1 input=18e3f113a51b829d]*/ { if (future_init((FutureObj*)self, loop)) { return -1; @@ -2083,6 +2150,19 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, return -1; } + if (eager_start) { + int loop_running = is_loop_running(self->task_loop); + if (loop_running == -1) { + return -1; + } + if (loop_running) { + if (task_eager_start(state, self)) { + return -1; + } + return 0; + } + } + if (task_call_step_soon(state, self, NULL)) { return -1; } @@ -2831,6 +2911,20 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) Py_RETURN_NONE; } + PyObject *ret = task_step_handle_result_impl(state, task, result); + return ret; + +fail: + return NULL; +} + + +static PyObject * +task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *result) +{ + int res; + PyObject *o; + if (result == (PyObject*)task) { /* We have a task that wants to await on itself */ goto self_await; @@ -3062,6 +3156,65 @@ task_step(asyncio_state *state, TaskObj *task, PyObject *exc) } } +static int +task_eager_start(asyncio_state *state, TaskObj *task) +{ + assert(task != NULL); + PyObject *prevtask = swap_current_task(state, task->task_loop, (PyObject *)task); + if (prevtask == NULL) { + return -1; + } + + if (register_eager_task(state, (PyObject *)task) == -1) { + Py_DECREF(prevtask); + return -1; + } + + if (PyContext_Enter(task->task_context) == -1) { + Py_DECREF(prevtask); + return -1; + } + + int retval = 0; + + PyObject *stepres = task_step_impl(state, task, NULL); + if (stepres == NULL) { + PyObject *exc = PyErr_GetRaisedException(); + _PyErr_ChainExceptions1(exc); + retval = -1; + } else { + Py_DECREF(stepres); + } + + PyObject *curtask = swap_current_task(state, task->task_loop, prevtask); + Py_DECREF(prevtask); + if (curtask == NULL) { + retval = -1; + } else { + assert(curtask == (PyObject *)task); + Py_DECREF(curtask); + } + + if (unregister_eager_task(state, (PyObject *)task) == -1) { + retval = -1; + } + + if (PyContext_Exit(task->task_context) == -1) { + retval = -1; + } + + if (task->task_state == STATE_PENDING) { + if (register_task(state, (PyObject *)task) == -1) { + retval = -1; + } + } else { + // This seems to really help performance on pyperformance benchmarks + Py_CLEAR(task->task_coro); + } + + return retval; +} + static PyObject * task_wakeup(TaskObj *task, PyObject *o) { @@ -3225,6 +3378,27 @@ _asyncio__register_task_impl(PyObject *module, PyObject *task) Py_RETURN_NONE; } +/*[clinic input] +_asyncio._register_eager_task + + task: object + +Register a new task in asyncio as executed by loop. + +Returns None. +[clinic start generated code]*/ + +static PyObject * +_asyncio__register_eager_task_impl(PyObject *module, PyObject *task) +/*[clinic end generated code: output=dfe1d45367c73f1a input=237f684683398c51]*/ +{ + asyncio_state *state = get_asyncio_state(module); + if (register_eager_task(state, task) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] _asyncio._unregister_task @@ -3247,6 +3421,27 @@ _asyncio__unregister_task_impl(PyObject *module, PyObject *task) Py_RETURN_NONE; } +/*[clinic input] +_asyncio._unregister_eager_task + + task: object + +Unregister a task. + +Returns None. +[clinic start generated code]*/ + +static PyObject * +_asyncio__unregister_eager_task_impl(PyObject *module, PyObject *task) +/*[clinic end generated code: output=a426922bd07f23d1 input=9d07401ef14ee048]*/ +{ + asyncio_state *state = get_asyncio_state(module); + if (unregister_eager_task(state, task) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] _asyncio._enter_task @@ -3298,6 +3493,27 @@ _asyncio__leave_task_impl(PyObject *module, PyObject *loop, PyObject *task) } +/*[clinic input] +_asyncio._swap_current_task + + loop: object + task: object + +Temporarily swap in the supplied task and return the original one (or None). + +This is intended for use during eager coroutine execution. + +[clinic start generated code]*/ + +static PyObject * +_asyncio__swap_current_task_impl(PyObject *module, PyObject *loop, + PyObject *task) +/*[clinic end generated code: output=9f88de958df74c7e input=c9c72208d3d38b6c]*/ +{ + return swap_current_task(get_asyncio_state(module), loop, task); +} + + /*[clinic input] _asyncio.current_task @@ -3379,7 +3595,8 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->asyncio_InvalidStateError); Py_VISIT(state->asyncio_CancelledError); - Py_VISIT(state->all_tasks); + Py_VISIT(state->scheduled_tasks); + Py_VISIT(state->eager_tasks); Py_VISIT(state->current_tasks); Py_VISIT(state->iscoroutine_typecache); @@ -3416,7 +3633,8 @@ module_clear(PyObject *mod) Py_CLEAR(state->asyncio_InvalidStateError); Py_CLEAR(state->asyncio_CancelledError); - Py_CLEAR(state->all_tasks); + Py_CLEAR(state->scheduled_tasks); + Py_CLEAR(state->eager_tasks); Py_CLEAR(state->current_tasks); Py_CLEAR(state->iscoroutine_typecache); @@ -3496,9 +3714,14 @@ module_init(asyncio_state *state) PyObject *weak_set; WITH_MOD("weakref") GET_MOD_ATTR(weak_set, "WeakSet"); - state->all_tasks = PyObject_CallNoArgs(weak_set); + state->scheduled_tasks = PyObject_CallNoArgs(weak_set); Py_CLEAR(weak_set); - if (state->all_tasks == NULL) { + if (state->scheduled_tasks == NULL) { + goto fail; + } + + state->eager_tasks = PySet_New(NULL); + if (state->eager_tasks == NULL) { goto fail; } @@ -3522,9 +3745,12 @@ static PyMethodDef asyncio_methods[] = { _ASYNCIO__GET_RUNNING_LOOP_METHODDEF _ASYNCIO__SET_RUNNING_LOOP_METHODDEF _ASYNCIO__REGISTER_TASK_METHODDEF + _ASYNCIO__REGISTER_EAGER_TASK_METHODDEF _ASYNCIO__UNREGISTER_TASK_METHODDEF + _ASYNCIO__UNREGISTER_EAGER_TASK_METHODDEF _ASYNCIO__ENTER_TASK_METHODDEF _ASYNCIO__LEAVE_TASK_METHODDEF + _ASYNCIO__SWAP_CURRENT_TASK_METHODDEF {NULL, NULL} }; @@ -3561,7 +3787,11 @@ module_exec(PyObject *mod) return -1; } - if (PyModule_AddObjectRef(mod, "_all_tasks", state->all_tasks) < 0) { + if (PyModule_AddObjectRef(mod, "_scheduled_tasks", state->scheduled_tasks) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(mod, "_eager_tasks", state->eager_tasks) < 0) { return -1; } diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 43c5d771798634..6a780a80cd0bc4 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -482,14 +482,15 @@ _asyncio_Future__make_cancelled_error(FutureObj *self, PyObject *Py_UNUSED(ignor } PyDoc_STRVAR(_asyncio_Task___init____doc__, -"Task(coro, *, loop=None, name=None, context=None)\n" +"Task(coro, *, loop=None, name=None, context=None, eager_start=False)\n" "--\n" "\n" "A coroutine wrapped in a Future."); static int _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, - PyObject *name, PyObject *context); + PyObject *name, PyObject *context, + int eager_start); static int _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) @@ -497,14 +498,14 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) int return_value = -1; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 4 + #define NUM_KEYWORDS 5 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(coro), &_Py_ID(loop), &_Py_ID(name), &_Py_ID(context), }, + .ob_item = { &_Py_ID(coro), &_Py_ID(loop), &_Py_ID(name), &_Py_ID(context), &_Py_ID(eager_start), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -513,14 +514,14 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"coro", "loop", "name", "context", NULL}; + static const char * const _keywords[] = {"coro", "loop", "name", "context", "eager_start", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "Task", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[5]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; @@ -528,6 +529,7 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *loop = Py_None; PyObject *name = Py_None; PyObject *context = Py_None; + int eager_start = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { @@ -549,9 +551,18 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - context = fastargs[3]; + if (fastargs[3]) { + context = fastargs[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + eager_start = PyObject_IsTrue(fastargs[4]); + if (eager_start < 0) { + goto exit; + } skip_optional_kwonly: - return_value = _asyncio_Task___init___impl((TaskObj *)self, coro, loop, name, context); + return_value = _asyncio_Task___init___impl((TaskObj *)self, coro, loop, name, context, eager_start); exit: return return_value; @@ -1064,6 +1075,63 @@ _asyncio__register_task(PyObject *module, PyObject *const *args, Py_ssize_t narg return return_value; } +PyDoc_STRVAR(_asyncio__register_eager_task__doc__, +"_register_eager_task($module, /, task)\n" +"--\n" +"\n" +"Register a new task in asyncio as executed by loop.\n" +"\n" +"Returns None."); + +#define _ASYNCIO__REGISTER_EAGER_TASK_METHODDEF \ + {"_register_eager_task", _PyCFunction_CAST(_asyncio__register_eager_task), METH_FASTCALL|METH_KEYWORDS, _asyncio__register_eager_task__doc__}, + +static PyObject * +_asyncio__register_eager_task_impl(PyObject *module, PyObject *task); + +static PyObject * +_asyncio__register_eager_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(task), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"task", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_register_eager_task", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *task; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + task = args[0]; + return_value = _asyncio__register_eager_task_impl(module, task); + +exit: + return return_value; +} + PyDoc_STRVAR(_asyncio__unregister_task__doc__, "_unregister_task($module, /, task)\n" "--\n" @@ -1121,6 +1189,63 @@ _asyncio__unregister_task(PyObject *module, PyObject *const *args, Py_ssize_t na return return_value; } +PyDoc_STRVAR(_asyncio__unregister_eager_task__doc__, +"_unregister_eager_task($module, /, task)\n" +"--\n" +"\n" +"Unregister a task.\n" +"\n" +"Returns None."); + +#define _ASYNCIO__UNREGISTER_EAGER_TASK_METHODDEF \ + {"_unregister_eager_task", _PyCFunction_CAST(_asyncio__unregister_eager_task), METH_FASTCALL|METH_KEYWORDS, _asyncio__unregister_eager_task__doc__}, + +static PyObject * +_asyncio__unregister_eager_task_impl(PyObject *module, PyObject *task); + +static PyObject * +_asyncio__unregister_eager_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(task), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"task", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_unregister_eager_task", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *task; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + task = args[0]; + return_value = _asyncio__unregister_eager_task_impl(module, task); + +exit: + return return_value; +} + PyDoc_STRVAR(_asyncio__enter_task__doc__, "_enter_task($module, /, loop, task)\n" "--\n" @@ -1243,6 +1368,66 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } +PyDoc_STRVAR(_asyncio__swap_current_task__doc__, +"_swap_current_task($module, /, loop, task)\n" +"--\n" +"\n" +"Temporarily swap in the supplied task and return the original one (or None).\n" +"\n" +"This is intended for use during eager coroutine execution."); + +#define _ASYNCIO__SWAP_CURRENT_TASK_METHODDEF \ + {"_swap_current_task", _PyCFunction_CAST(_asyncio__swap_current_task), METH_FASTCALL|METH_KEYWORDS, _asyncio__swap_current_task__doc__}, + +static PyObject * +_asyncio__swap_current_task_impl(PyObject *module, PyObject *loop, + PyObject *task); + +static PyObject * +_asyncio__swap_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(loop), &_Py_ID(task), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"loop", "task", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_swap_current_task", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *loop; + PyObject *task; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + loop = args[0]; + task = args[1]; + return_value = _asyncio__swap_current_task_impl(module, loop, task); + +exit: + return return_value; +} + PyDoc_STRVAR(_asyncio_current_task__doc__, "current_task($module, /, loop=None)\n" "--\n" @@ -1302,4 +1487,4 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=00f494214f2fd008 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b0e283177b07639 input=a9049054013a1b77]*/ From 80b714835d6f5e1cb8fbc486f9575b5eee9f007e Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 1 May 2023 22:29:30 +0100 Subject: [PATCH 27/43] gh-87092: Expose assembler to unit tests (#103988) --- Include/internal/pycore_compile.h | 4 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/support/bytecode_helper.py | 32 +++-- Lib/test/test_compiler_assemble.py | 71 ++++++++++ Modules/_testinternalcapi.c | 65 +++++++++- Modules/clinic/_testinternalcapi.c.h | 64 ++++++++- Python/compile.c | 121 +++++++++++++----- Python/flowgraph.c | 14 ++ 11 files changed, 329 insertions(+), 48 deletions(-) create mode 100644 Lib/test/test_compiler_assemble.py diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 1a032f652dddaf..4bd4ef57238f98 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -103,6 +103,10 @@ PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, PyObject *consts); +PyAPI_FUNC(PyCodeObject*) +_PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, + PyObject *instructions); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 5e8a8d78412786..f0740b68dd1114 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1029,6 +1029,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(memlimit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(message)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metaclass)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 28e82203d8f639..234d5e2a09892c 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -517,6 +517,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(memlimit) STRUCT_FOR_ID(message) STRUCT_FOR_ID(metaclass) + STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index dd4471160a4a4c..16f2147aa8e93b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1023,6 +1023,7 @@ extern "C" { INIT_ID(memlimit), \ INIT_ID(message), \ INIT_ID(metaclass), \ + INIT_ID(metadata), \ INIT_ID(method), \ INIT_ID(mod), \ INIT_ID(mode), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 1a8338b341fd68..cd41b731537f0e 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1404,6 +1404,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(metaclass); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(metadata); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(method); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 1d9b889c920986..357ec44dbc218d 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,7 +3,7 @@ import unittest import dis import io -from _testinternalcapi import compiler_codegen, optimize_cfg +from _testinternalcapi import compiler_codegen, optimize_cfg, assemble_code_object _UNSPECIFIED = object() @@ -108,6 +108,18 @@ def normalize_insts(self, insts): res.append((opcode, arg, *loc)) return res + def complete_insts_info(self, insts): + # fill in omitted fields in location, and oparg 0 for ops with no arg. + res = [] + for item in insts: + assert isinstance(item, tuple) + inst = list(item) + opcode = dis.opmap[inst[0]] + oparg = inst[1] + loc = inst[2:] + [-1] * (6 - len(inst)) + res.append((opcode, oparg, *loc)) + return res + class CodegenTestCase(CompilationStepTestCase): @@ -118,20 +130,14 @@ def generate_code(self, ast): class CfgOptimizationTestCase(CompilationStepTestCase): - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] - for item in insts: - assert isinstance(item, tuple) - inst = list(reversed(item)) - opcode = dis.opmap[inst.pop()] - oparg = inst.pop() - loc = inst + [-1] * (4 - len(inst)) - res.append((opcode, oparg, *loc)) - return res - def get_optimized(self, insts, consts): insts = self.normalize_insts(insts) insts = self.complete_insts_info(insts) insts = optimize_cfg(insts, consts) return insts, consts + +class AssemblerTestCase(CompilationStepTestCase): + + def get_code_object(self, filename, insts, metadata): + co = assemble_code_object(filename, insts, metadata) + return co diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py new file mode 100644 index 00000000000000..96c1691e24a9c8 --- /dev/null +++ b/Lib/test/test_compiler_assemble.py @@ -0,0 +1,71 @@ + +import ast +import types + +from test.support.bytecode_helper import AssemblerTestCase + + +# Tests for the code-object creation stage of the compiler. + +class IsolatedAssembleTests(AssemblerTestCase): + + def complete_metadata(self, metadata, filename="myfile.py"): + if metadata is None: + metadata = {} + for key in ['name', 'qualname']: + metadata.setdefault(key, key) + for key in ['consts']: + metadata.setdefault(key, []) + for key in ['names', 'varnames', 'cellvars', 'freevars']: + metadata.setdefault(key, {}) + for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']: + metadata.setdefault(key, 0) + metadata.setdefault('firstlineno', 1) + metadata.setdefault('filename', filename) + return metadata + + def assemble_test(self, insts, metadata, expected): + metadata = self.complete_metadata(metadata) + insts = self.complete_insts_info(insts) + + co = self.get_code_object(metadata['filename'], insts, metadata) + self.assertIsInstance(co, types.CodeType) + + expected_metadata = {} + for key, value in metadata.items(): + if isinstance(value, list): + expected_metadata[key] = tuple(value) + elif isinstance(value, dict): + expected_metadata[key] = tuple(value.keys()) + else: + expected_metadata[key] = value + + for key, value in expected_metadata.items(): + self.assertEqual(getattr(co, "co_" + key), value) + + f = types.FunctionType(co, {}) + for args, res in expected.items(): + self.assertEqual(f(*args), res) + + def test_simple_expr(self): + metadata = { + 'filename' : 'avg.py', + 'name' : 'avg', + 'qualname' : 'stats.avg', + 'consts' : [2], + 'argcount' : 2, + 'varnames' : {'x' : 0, 'y' : 1}, + } + + # code for "return (x+y)/2" + insts = [ + ('RESUME', 0), + ('LOAD_FAST', 0, 1), # 'x' + ('LOAD_FAST', 1, 1), # 'y' + ('BINARY_OP', 0, 1), # '+' + ('LOAD_CONST', 0, 1), # 2 + ('BINARY_OP', 11, 1), # '/' + ('RETURN_VALUE', 1), + ] + expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} + self.assemble_test(insts, metadata, expected) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 632fac2de0c419..1e38f1aa63499b 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -14,7 +14,7 @@ #include "Python.h" #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg +#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble #include "pycore_fileutils.h" // _Py_normpath #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head @@ -625,6 +625,68 @@ _testinternalcapi_optimize_cfg_impl(PyObject *module, PyObject *instructions, return _PyCompile_OptimizeCfg(instructions, consts); } +static int +get_nonnegative_int_from_dict(PyObject *dict, const char *key) { + PyObject *obj = PyDict_GetItemString(dict, key); + if (obj == NULL) { + return -1; + } + return PyLong_AsLong(obj); +} + +/*[clinic input] + +_testinternalcapi.assemble_code_object -> object + + filename: object + instructions: object + metadata: object + +Create a code object for the given instructions. +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_assemble_code_object_impl(PyObject *module, + PyObject *filename, + PyObject *instructions, + PyObject *metadata) +/*[clinic end generated code: output=38003dc16a930f48 input=e713ad77f08fb3a8]*/ + +{ + assert(PyDict_Check(metadata)); + _PyCompile_CodeUnitMetadata umd; + + umd.u_name = PyDict_GetItemString(metadata, "name"); + umd.u_qualname = PyDict_GetItemString(metadata, "qualname"); + + assert(PyUnicode_Check(umd.u_name)); + assert(PyUnicode_Check(umd.u_qualname)); + + umd.u_consts = PyDict_GetItemString(metadata, "consts"); + umd.u_names = PyDict_GetItemString(metadata, "names"); + umd.u_varnames = PyDict_GetItemString(metadata, "varnames"); + umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars"); + umd.u_freevars = PyDict_GetItemString(metadata, "freevars"); + + assert(PyList_Check(umd.u_consts)); + assert(PyDict_Check(umd.u_names)); + assert(PyDict_Check(umd.u_varnames)); + assert(PyDict_Check(umd.u_cellvars)); + assert(PyDict_Check(umd.u_freevars)); + + umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount"); + umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount"); + umd.u_kwonlyargcount = get_nonnegative_int_from_dict(metadata, "kwonlyargcount"); + umd.u_firstlineno = get_nonnegative_int_from_dict(metadata, "firstlineno"); + + assert(umd.u_argcount >= 0); + assert(umd.u_posonlyargcount >= 0); + assert(umd.u_kwonlyargcount >= 0); + assert(umd.u_firstlineno >= 0); + + return (PyObject*)_PyCompile_Assemble(&umd, filename, instructions); +} + static PyObject * get_interp_settings(PyObject *self, PyObject *args) @@ -705,6 +767,7 @@ static PyMethodDef module_functions[] = { {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF + _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF {"get_interp_settings", get_interp_settings, METH_VARARGS, NULL}, {"clear_extension", clear_extension, METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index e8d5681b194916..89573222572594 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -128,4 +128,66 @@ _testinternalcapi_optimize_cfg(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=efe95836482fd542 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_testinternalcapi_assemble_code_object__doc__, +"assemble_code_object($module, /, filename, instructions, metadata)\n" +"--\n" +"\n" +"Create a code object for the given instructions."); + +#define _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF \ + {"assemble_code_object", _PyCFunction_CAST(_testinternalcapi_assemble_code_object), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_assemble_code_object__doc__}, + +static PyObject * +_testinternalcapi_assemble_code_object_impl(PyObject *module, + PyObject *filename, + PyObject *instructions, + PyObject *metadata); + +static PyObject * +_testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(filename), &_Py_ID(instructions), &_Py_ID(metadata), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"filename", "instructions", "metadata", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "assemble_code_object", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *filename; + PyObject *instructions; + PyObject *metadata; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + filename = args[0]; + instructions = args[1]; + metadata = args[2]; + return_value = _testinternalcapi_assemble_code_object_impl(module, filename, instructions, metadata); + +exit: + return return_value; +} +/*[clinic end generated code: output=d5e08c9d67f9721f input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index e8789def867308..cbe5403aafbc87 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -308,7 +308,6 @@ instr_sequence_fini(instr_sequence *seq) { seq->s_instrs = NULL; } - static int instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { memset(g, 0, sizeof(cfg_builder)); @@ -6754,11 +6753,11 @@ _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) static int * -build_cellfixedoffsets(struct compiler_unit *u) +build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) { - int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(u->u_metadata.u_freevars); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); int noffsets = ncellvars + nfreevars; int *fixed = PyMem_New(int, noffsets); @@ -6772,8 +6771,8 @@ build_cellfixedoffsets(struct compiler_unit *u) PyObject *varname, *cellindex; Py_ssize_t pos = 0; - while (PyDict_Next(u->u_metadata.u_cellvars, &pos, &varname, &cellindex)) { - PyObject *varindex = PyDict_GetItem(u->u_metadata.u_varnames, varname); + while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { + PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname); if (varindex != NULL) { assert(PyLong_AS_LONG(cellindex) < INT_MAX); assert(PyLong_AS_LONG(varindex) < INT_MAX); @@ -6787,17 +6786,17 @@ build_cellfixedoffsets(struct compiler_unit *u) } static int -insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, +insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixed, int nfreevars, int code_flags) { - assert(u->u_metadata.u_firstlineno > 0); + assert(umd->u_firstlineno > 0); /* Add the generator prefix instructions. */ if (code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { cfg_instr make_gen = { .i_opcode = RETURN_GENERATOR, .i_oparg = 0, - .i_loc = LOCATION(u->u_metadata.u_firstlineno, u->u_metadata.u_firstlineno, -1, -1), + .i_loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1), .i_target = NULL, }; RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 0, &make_gen)); @@ -6811,12 +6810,12 @@ insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, } /* Set up cells for any variable that escapes, to be put in a closure. */ - const int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); + const int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); if (ncellvars) { - // u->u_metadata.u_cellvars has the cells out of order so we sort them + // umd->u_cellvars has the cells out of order so we sort them // before adding the MAKE_CELL instructions. Note that we // adjust for arg cells, which come first. - const int nvars = ncellvars + (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); + const int nvars = ncellvars + (int)PyDict_GET_SIZE(umd->u_varnames); int *sorted = PyMem_RawCalloc(nvars, sizeof(int)); if (sorted == NULL) { PyErr_NoMemory(); @@ -6860,11 +6859,11 @@ insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, } static int -fix_cell_offsets(struct compiler_unit *u, basicblock *entryblock, int *fixedmap) +fix_cell_offsets(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixedmap) { - int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(u->u_metadata.u_freevars); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); int noffsets = ncellvars + nfreevars; // First deal with duplicates (arg cells). @@ -6906,30 +6905,30 @@ fix_cell_offsets(struct compiler_unit *u, basicblock *entryblock, int *fixedmap) static int -prepare_localsplus(struct compiler_unit* u, cfg_builder *g, int code_flags) +prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_flags) { - assert(PyDict_GET_SIZE(u->u_metadata.u_varnames) < INT_MAX); - assert(PyDict_GET_SIZE(u->u_metadata.u_cellvars) < INT_MAX); - assert(PyDict_GET_SIZE(u->u_metadata.u_freevars) < INT_MAX); - int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(u->u_metadata.u_freevars); + assert(PyDict_GET_SIZE(umd->u_varnames) < INT_MAX); + assert(PyDict_GET_SIZE(umd->u_cellvars) < INT_MAX); + assert(PyDict_GET_SIZE(umd->u_freevars) < INT_MAX); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); assert(INT_MAX - nlocals - ncellvars > 0); assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); int nlocalsplus = nlocals + ncellvars + nfreevars; - int* cellfixedoffsets = build_cellfixedoffsets(u); + int* cellfixedoffsets = build_cellfixedoffsets(umd); if (cellfixedoffsets == NULL) { return ERROR; } // This must be called before fix_cell_offsets(). - if (insert_prefix_instructions(u, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { + if (insert_prefix_instructions(umd, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { PyMem_Free(cellfixedoffsets); return ERROR; } - int numdropped = fix_cell_offsets(u, g->g_entryblock, cellfixedoffsets); + int numdropped = fix_cell_offsets(umd, g->g_entryblock, cellfixedoffsets); PyMem_Free(cellfixedoffsets); // At this point we're done with it. cellfixedoffsets = NULL; if (numdropped < 0) { @@ -6980,7 +6979,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, } /** Assembly **/ - int nlocalsplus = prepare_localsplus(u, &g, code_flags); + int nlocalsplus = prepare_localsplus(&u->u_metadata, &g, code_flags); if (nlocalsplus < 0) { goto error; } @@ -7157,11 +7156,6 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) goto error; } } - if (seq->s_used && !IS_TERMINATOR_OPCODE(seq->s_instrs[seq->s_used-1].i_opcode)) { - if (instr_sequence_addop(seq, RETURN_VALUE, 0, NO_LOCATION) < 0) { - goto error; - } - } PyMem_Free(is_target); return SUCCESS; error: @@ -7328,6 +7322,67 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) return res; } +int _PyCfg_JumpLabelsToTargets(basicblock *entryblock); + +PyCodeObject * +_PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, + PyObject *instructions) +{ + PyCodeObject *co = NULL; + instr_sequence optimized_instrs; + memset(&optimized_instrs, 0, sizeof(instr_sequence)); + + PyObject *const_cache = PyDict_New(); + if (const_cache == NULL) { + return NULL; + } + + cfg_builder g; + if (instructions_to_cfg(instructions, &g) < 0) { + goto error; + } + + if (_PyCfg_JumpLabelsToTargets(g.g_entryblock) < 0) { + goto error; + } + + int code_flags = 0; + int nlocalsplus = prepare_localsplus(umd, &g, code_flags); + if (nlocalsplus < 0) { + goto error; + } + + int maxdepth = _PyCfg_Stackdepth(g.g_entryblock, code_flags); + if (maxdepth < 0) { + goto error; + } + + _PyCfg_ConvertExceptionHandlersToNops(g.g_entryblock); + + /* Order of basic blocks must have been determined by now */ + + if (_PyCfg_ResolveJumps(&g) < 0) { + goto error; + } + + /* Can't modify the bytecode after computing jump offsets. */ + + if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) { + goto error; + } + + PyObject *consts = umd->u_consts; + co = _PyAssemble_MakeCodeObject(umd, const_cache, + consts, maxdepth, &optimized_instrs, + nlocalsplus, code_flags, filename); + +error: + Py_DECREF(const_cache); + _PyCfgBuilder_Fini(&g); + instr_sequence_fini(&optimized_instrs); + return co; +} + /* Retained for API compatibility. * Optimization is now done in _PyCfg_OptimizeCodeUnit */ diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 6f83a910cab392..f79afb4c66cd8b 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -223,6 +223,15 @@ dump_basicblock(const basicblock *b) } } } + +void +_PyCfgBuilder_DumpGraph(const basicblock *entryblock) +{ + for (const basicblock *b = entryblock; b != NULL; b = b->b_next) { + dump_basicblock(b); + } +} + #endif @@ -592,6 +601,11 @@ translate_jump_labels_to_targets(basicblock *entryblock) return SUCCESS; } +int +_PyCfg_JumpLabelsToTargets(basicblock *entryblock) +{ + return translate_jump_labels_to_targets(entryblock); +} static int mark_except_handlers(basicblock *entryblock) { From 605f8785db26c3acdde90cfd4ecebb208362a1b8 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora <80244920+Eclips4@users.noreply.github.com> Date: Tue, 2 May 2023 01:14:49 +0300 Subject: [PATCH 28/43] gh-104057: Fix direct invocation of test_super (#104064) --- Lib/test/test_super.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index ed773a3cff2a6d..698ab48f48eaa1 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -359,7 +359,7 @@ class C: def method(self): return super().msg - with patch("test.test_super.super", MySuper) as m: + with patch(f"{__name__}.super", MySuper) as m: self.assertEqual(C().method(), "super super") def test_shadowed_dynamic_two_arg(self): @@ -373,7 +373,7 @@ class C: def method(self): return super(1, 2).msg - with patch("test.test_super.super", MySuper) as m: + with patch(f"{__name__}.super", MySuper) as m: self.assertEqual(C().method(), "super super") self.assertEqual(call_args, [(1, 2)]) From b1ca34d4d5e463b8108eea20090f12292390f0cf Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado <Pablogsal@gmail.com> Date: Tue, 2 May 2023 01:55:41 +0200 Subject: [PATCH 29/43] gh-104016: Skip test for deeply neste f-strings on wasi (#104071) --- Lib/test/test_fstring.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 5c5176dc54a6d9..be71fde5aaba54 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -561,11 +561,12 @@ def test_mismatched_parens(self): ]) self.assertRaises(SyntaxError, eval, "f'{" + "("*500 + "}'") + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_fstring_nested_too_deeply(self): self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", ['f"{1+2:{1+2:{1+1:{1}}}}"']) - + def create_nested_fstring(n): if n == 0: return "1+1" @@ -575,13 +576,13 @@ def create_nested_fstring(n): self.assertAllRaise(SyntaxError, "too many nested f-strings", [create_nested_fstring(160)]) - + def test_syntax_error_in_nested_fstring(self): # See gh-104016 for more information on this crash self.assertAllRaise(SyntaxError, "invalid syntax", ['f"{1 1:' + ('{f"1:' * 199)]) - + def test_double_braces(self): self.assertEqual(f'{{', '{') self.assertEqual(f'a{{', 'a{') From fdd878650d325297cd801305bc2d1b0e903e42b4 Mon Sep 17 00:00:00 2001 From: Eric Snow <ericsnowcurrently@gmail.com> Date: Mon, 1 May 2023 19:36:00 -0600 Subject: [PATCH 30/43] gh-94673: Properly Initialize and Finalize Static Builtin Types for Each Interpreter (gh-104072) Until now, we haven't been initializing nor finalizing the per-interpreter state properly. --- Include/internal/pycore_object.h | 3 +- Include/internal/pycore_pylifecycle.h | 2 +- Include/internal/pycore_structseq.h | 10 ++- Include/internal/pycore_typeobject.h | 8 +- Modules/_io/_iomodule.c | 8 +- Objects/exceptions.c | 8 +- Objects/floatobject.c | 9 +- Objects/longobject.c | 11 +-- Objects/object.c | 8 +- Objects/structseq.c | 21 +++-- Objects/typeobject.c | 117 ++++++++++++++++---------- Objects/unicodeobject.c | 16 ++-- Objects/weakrefobject.c | 4 +- Python/errors.c | 11 +-- Python/pylifecycle.c | 6 +- Python/sysmodule.c | 28 +++--- Python/thread.c | 9 +- 17 files changed, 145 insertions(+), 134 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 2ca047846e0935..2ee0180c0554d2 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -272,8 +272,9 @@ _PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) { if (PyType_Check(op) && ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); static_builtin_state *state = _PyStaticType_GetState( - (PyTypeObject *)op); + interp, (PyTypeObject *)op); return _PyStaticType_GET_WEAKREFS_LISTPTR(state); } // Essentially _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(): diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index f96261a650dac7..7f8cc643ec0c96 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -39,7 +39,7 @@ extern PyStatus _PySys_Create( extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options); extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config); extern int _PySys_UpdateConfig(PyThreadState *tstate); -extern void _PySys_Fini(PyInterpreterState *interp); +extern void _PySys_FiniTypes(PyInterpreterState *interp); extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); diff --git a/Include/internal/pycore_structseq.h b/Include/internal/pycore_structseq.h index bd1e85c6883f01..6f5dfc12707cf8 100644 --- a/Include/internal/pycore_structseq.h +++ b/Include/internal/pycore_structseq.h @@ -16,18 +16,22 @@ PyAPI_FUNC(PyTypeObject *) _PyStructSequence_NewType( unsigned long tp_flags); extern int _PyStructSequence_InitBuiltinWithFlags( + PyInterpreterState *interp, PyTypeObject *type, PyStructSequence_Desc *desc, unsigned long tp_flags); static inline int -_PyStructSequence_InitBuiltin(PyTypeObject *type, +_PyStructSequence_InitBuiltin(PyInterpreterState *interp, + PyTypeObject *type, PyStructSequence_Desc *desc) { - return _PyStructSequence_InitBuiltinWithFlags(type, desc, 0); + return _PyStructSequence_InitBuiltinWithFlags(interp, type, desc, 0); } -extern void _PyStructSequence_FiniBuiltin(PyTypeObject *type); +extern void _PyStructSequence_FiniBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); #ifdef __cplusplus } diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 76253fd5fd864c..5bd04736c01d2d 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -104,10 +104,10 @@ _PyType_GetModuleState(PyTypeObject *type) } -extern int _PyStaticType_InitBuiltin(PyTypeObject *type); -extern static_builtin_state * _PyStaticType_GetState(PyTypeObject *); -extern void _PyStaticType_ClearWeakRefs(PyTypeObject *type); -extern void _PyStaticType_Dealloc(PyTypeObject *type); +extern int _PyStaticType_InitBuiltin(PyInterpreterState *, PyTypeObject *type); +extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTypeObject *); +extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); +extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); PyObject * _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index a3bfbc9ac5a1b1..8ec3a6081c98d9 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -680,7 +680,7 @@ _PyIO_InitTypes(PyInterpreterState *interp) for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; - if (_PyStaticType_InitBuiltin(type) < 0) { + if (_PyStaticType_InitBuiltin(interp, type) < 0) { return _PyStatus_ERR("Can't initialize builtin type"); } } @@ -691,15 +691,11 @@ _PyIO_InitTypes(PyInterpreterState *interp) void _PyIO_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - // Deallocate types in the reverse order to deallocate subclasses before // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types) - 1; i >= 0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(type); + _PyStaticType_Dealloc(interp, type); } } diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 6c9dfbd9b415cf..ba5ee291f08b0c 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3598,7 +3598,7 @@ _PyExc_InitTypes(PyInterpreterState *interp) { for (size_t i=0; i < Py_ARRAY_LENGTH(static_exceptions); i++) { PyTypeObject *exc = static_exceptions[i].exc; - if (_PyStaticType_InitBuiltin(exc) < 0) { + if (_PyStaticType_InitBuiltin(interp, exc) < 0) { return -1; } } @@ -3609,13 +3609,9 @@ _PyExc_InitTypes(PyInterpreterState *interp) static void _PyExc_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - for (Py_ssize_t i=Py_ARRAY_LENGTH(static_exceptions) - 1; i >= 0; i--) { PyTypeObject *exc = static_exceptions[i].exc; - _PyStaticType_Dealloc(exc); + _PyStaticType_Dealloc(interp, exc); } } diff --git a/Objects/floatobject.c b/Objects/floatobject.c index a694ddcd019ee8..d257857d9c619c 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1991,8 +1991,9 @@ PyStatus _PyFloat_InitTypes(PyInterpreterState *interp) { /* Init float info */ - if (_PyStructSequence_InitBuiltin(&FloatInfoType, - &floatinfo_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &FloatInfoType, + &floatinfo_desc) < 0) + { return _PyStatus_ERR("can't init float info type"); } @@ -2028,9 +2029,7 @@ _PyFloat_Fini(PyInterpreterState *interp) void _PyFloat_FiniType(PyInterpreterState *interp) { - if (_Py_IsMainInterpreter(interp)) { - _PyStructSequence_FiniBuiltin(&FloatInfoType); - } + _PyStructSequence_FiniBuiltin(interp, &FloatInfoType); } /* Print summary info about the state of the optimized allocator */ diff --git a/Objects/longobject.c b/Objects/longobject.c index de043488d7a173..853e934e2107ea 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -7,7 +7,6 @@ #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() -#include "pycore_pystate.h" // _Py_IsMainInterpreter() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() @@ -6352,7 +6351,9 @@ PyStatus _PyLong_InitTypes(PyInterpreterState *interp) { /* initialize int_info */ - if (_PyStructSequence_InitBuiltin(&Int_InfoType, &int_info_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &Int_InfoType, + &int_info_desc) < 0) + { return _PyStatus_ERR("can't init int info type"); } @@ -6363,9 +6364,5 @@ _PyLong_InitTypes(PyInterpreterState *interp) void _PyLong_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStructSequence_FiniBuiltin(&Int_InfoType); + _PyStructSequence_FiniBuiltin(interp, &Int_InfoType); } diff --git a/Objects/object.c b/Objects/object.c index 4ce10cf1192d3f..ee8690101d3cc0 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2105,7 +2105,7 @@ _PyTypes_InitTypes(PyInterpreterState *interp) // All other static types (unless initialized elsewhere) for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; - if (_PyStaticType_InitBuiltin(type) < 0) { + if (_PyStaticType_InitBuiltin(interp, type) < 0) { return _PyStatus_ERR("Can't initialize builtin type"); } if (type == &PyType_Type) { @@ -2128,15 +2128,11 @@ _PyTypes_InitTypes(PyInterpreterState *interp) void _PyTypes_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - // Deallocate types in the reverse order to deallocate subclasses before // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types)-1; i>=0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(type); + _PyStaticType_Dealloc(interp, type); } } diff --git a/Objects/structseq.c b/Objects/structseq.c index d8f55dc1eae5ed..ea476bf7a6a954 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -502,7 +502,8 @@ initialize_static_type(PyTypeObject *type, PyStructSequence_Desc *desc, } int -_PyStructSequence_InitBuiltinWithFlags(PyTypeObject *type, +_PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, + PyTypeObject *type, PyStructSequence_Desc *desc, unsigned long tp_flags) { @@ -536,7 +537,7 @@ _PyStructSequence_InitBuiltinWithFlags(PyTypeObject *type, } #endif - if (_PyStaticType_InitBuiltin(type) < 0) { + if (_PyStaticType_InitBuiltin(interp, type) < 0) { PyErr_Format(PyExc_RuntimeError, "Can't initialize builtin type %s", desc->name); @@ -606,7 +607,7 @@ PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) initialized via _PyStructSequence_InitBuiltinWithFlags(). */ void -_PyStructSequence_FiniBuiltin(PyTypeObject *type) +_PyStructSequence_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) { // Ensure that the type is initialized assert(type->tp_name != NULL); @@ -620,13 +621,15 @@ _PyStructSequence_FiniBuiltin(PyTypeObject *type) return; } - _PyStaticType_Dealloc(type); + _PyStaticType_Dealloc(interp, type); - // Undo _PyStructSequence_InitBuiltinWithFlags(). - type->tp_name = NULL; - PyMem_Free(type->tp_members); - type->tp_members = NULL; - type->tp_base = NULL; + if (_Py_IsMainInterpreter(interp)) { + // Undo _PyStructSequence_InitBuiltinWithFlags(). + type->tp_name = NULL; + PyMem_Free(type->tp_members); + type->tp_members = NULL; + type->tp_base = NULL; + } } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 060d14e254ab2d..2ed806fb01554f 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -69,13 +69,11 @@ static inline PyTypeObject * subclass_from_ref(PyObject *ref); /* helpers for for static builtin types */ -#ifndef NDEBUG static inline int static_builtin_index_is_set(PyTypeObject *self) { return self->tp_subclasses != NULL; } -#endif static inline size_t static_builtin_index_get(PyTypeObject *self) @@ -107,43 +105,46 @@ static_builtin_state_get(PyInterpreterState *interp, PyTypeObject *self) /* For static types we store some state in an array on each interpreter. */ static_builtin_state * -_PyStaticType_GetState(PyTypeObject *self) +_PyStaticType_GetState(PyInterpreterState *interp, PyTypeObject *self) { assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - PyInterpreterState *interp = _PyInterpreterState_GET(); return static_builtin_state_get(interp, self); } +/* Set the type's per-interpreter state. */ static void -static_builtin_state_init(PyTypeObject *self) +static_builtin_state_init(PyInterpreterState *interp, PyTypeObject *self) { - /* Set the type's per-interpreter state. */ - PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!static_builtin_index_is_set(self)) { + static_builtin_index_set(self, interp->types.num_builtins_initialized); + } + static_builtin_state *state = static_builtin_state_get(interp, self); /* It should only be called once for each builtin type. */ - assert(!static_builtin_index_is_set(self)); - - static_builtin_index_set(self, interp->types.num_builtins_initialized); - interp->types.num_builtins_initialized++; - - static_builtin_state *state = static_builtin_state_get(interp, self); + assert(state->type == NULL); state->type = self; + /* state->tp_subclasses is left NULL until init_subclasses() sets it. */ /* state->tp_weaklist is left NULL until insert_head() or insert_after() (in weakrefobject.c) sets it. */ + + interp->types.num_builtins_initialized++; } +/* Reset the type's per-interpreter state. + This basically undoes what static_builtin_state_init() did. */ static void -static_builtin_state_clear(PyTypeObject *self) +static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) { - /* Reset the type's per-interpreter state. - This basically undoes what static_builtin_state_init() did. */ - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, self); + + assert(state->type != NULL); state->type = NULL; assert(state->tp_weaklist == NULL); // It was already cleared out. - static_builtin_index_clear(self); + + if (_Py_IsMainInterpreter(interp)) { + static_builtin_index_clear(self); + } assert(interp->types.num_builtins_initialized > 0); interp->types.num_builtins_initialized--; @@ -4491,33 +4492,37 @@ clear_static_tp_subclasses(PyTypeObject *type) clear_subclasses(type); } +static void +clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) +{ + if (_Py_IsMainInterpreter(interp)) { + Py_CLEAR(type->tp_dict); + Py_CLEAR(type->tp_bases); + Py_CLEAR(type->tp_mro); + Py_CLEAR(type->tp_cache); + } + clear_static_tp_subclasses(type); +} + void -_PyStaticType_Dealloc(PyTypeObject *type) +_PyStaticType_Dealloc(PyInterpreterState *interp, PyTypeObject *type) { - assert(!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)); + assert(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); + assert(_Py_IsImmortal((PyObject *)type)); type_dealloc_common(type); - Py_CLEAR(type->tp_dict); - Py_CLEAR(type->tp_bases); - Py_CLEAR(type->tp_mro); - Py_CLEAR(type->tp_cache); - clear_static_tp_subclasses(type); + clear_static_type_objects(interp, type); - // PyObject_ClearWeakRefs() raises an exception if Py_REFCNT() != 0 - if (Py_REFCNT(type) == 0) { - PyObject_ClearWeakRefs((PyObject *)type); + if (_Py_IsMainInterpreter(interp)) { + type->tp_flags &= ~Py_TPFLAGS_READY; + type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; + type->tp_version_tag = 0; } - type->tp_flags &= ~Py_TPFLAGS_READY; - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; - - if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - _PyStaticType_ClearWeakRefs(type); - static_builtin_state_clear(type); - /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ - } + _PyStaticType_ClearWeakRefs(interp, type); + static_builtin_state_clear(interp, type); + /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ } @@ -4564,7 +4569,8 @@ static PyObject * lookup_subclasses(PyTypeObject *self) { if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - static_builtin_state *state = _PyStaticType_GetState(self); + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); assert(state != NULL); return state->tp_subclasses; } @@ -4574,8 +4580,9 @@ lookup_subclasses(PyTypeObject *self) int _PyType_HasSubclasses(PyTypeObject *self) { + PyInterpreterState *interp = _PyInterpreterState_GET(); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN && - _PyStaticType_GetState(self) == NULL) { + _PyStaticType_GetState(interp, self) == NULL) { return 0; } if (lookup_subclasses(self) == NULL) { @@ -6938,7 +6945,8 @@ type_ready_post_checks(PyTypeObject *type) else if (type->tp_dictoffset < (Py_ssize_t)sizeof(PyObject)) { if (type->tp_dictoffset + type->tp_basicsize <= 0) { PyErr_Format(PyExc_SystemError, - "type %s has a tp_dictoffset that is too small"); + "type %s has a tp_dictoffset that is too small", + type->tp_name); } } return 0; @@ -7029,17 +7037,32 @@ PyType_Ready(PyTypeObject *type) } int -_PyStaticType_InitBuiltin(PyTypeObject *self) +_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) { assert(_Py_IsImmortal((PyObject *)self)); assert(!(self->tp_flags & Py_TPFLAGS_HEAPTYPE)); + assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); + assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); +#ifndef NDEBUG + int ismain = _Py_IsMainInterpreter(interp); +#endif if (self->tp_flags & Py_TPFLAGS_READY) { + assert(!ismain); assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); + assert(self->tp_flags & Py_TPFLAGS_VALID_VERSION_TAG); + + static_builtin_state_init(interp, self); + + /* Per-interpreter tp_subclasses is done lazily. + Otherwise we would initialize it here. */ + assert(_PyType_CheckConsistency(self)); return 0; } + assert(ismain); + self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; @@ -7047,11 +7070,11 @@ _PyStaticType_InitBuiltin(PyTypeObject *self) self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; - static_builtin_state_init(self); + static_builtin_state_init(interp, self); int res = type_ready(self); if (res < 0) { - static_builtin_state_clear(self); + static_builtin_state_clear(interp, self); } return res; } @@ -7065,7 +7088,8 @@ init_subclasses(PyTypeObject *self) return NULL; } if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - static_builtin_state *state = _PyStaticType_GetState(self); + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); state->tp_subclasses = subclasses; return subclasses; } @@ -7080,7 +7104,8 @@ clear_subclasses(PyTypeObject *self) callers also test if tp_subclasses is NULL to check if a static type has no subclass. */ if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - static_builtin_state *state = _PyStaticType_GetState(self); + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); Py_CLEAR(state->tp_subclasses); return; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7537c12e92680c..6ae68cc20f7dee 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14573,13 +14573,13 @@ _PyUnicode_InitGlobalObjects(PyInterpreterState *interp) PyStatus _PyUnicode_InitTypes(PyInterpreterState *interp) { - if (_PyStaticType_InitBuiltin(&EncodingMapType) < 0) { + if (_PyStaticType_InitBuiltin(interp, &EncodingMapType) < 0) { goto error; } - if (_PyStaticType_InitBuiltin(&PyFieldNameIter_Type) < 0) { + if (_PyStaticType_InitBuiltin(interp, &PyFieldNameIter_Type) < 0) { goto error; } - if (_PyStaticType_InitBuiltin(&PyFormatterIter_Type) < 0) { + if (_PyStaticType_InitBuiltin(interp, &PyFormatterIter_Type) < 0) { goto error; } return _PyStatus_OK(); @@ -15158,13 +15158,9 @@ unicode_is_finalizing(void) void _PyUnicode_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStaticType_Dealloc(&EncodingMapType); - _PyStaticType_Dealloc(&PyFieldNameIter_Type); - _PyStaticType_Dealloc(&PyFormatterIter_Type); + _PyStaticType_Dealloc(interp, &EncodingMapType); + _PyStaticType_Dealloc(interp, &PyFieldNameIter_Type); + _PyStaticType_Dealloc(interp, &PyFormatterIter_Type); } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index c1afe63ecf66f6..aee79fc1410b29 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1017,9 +1017,9 @@ PyObject_ClearWeakRefs(PyObject *object) * or anything else. */ void -_PyStaticType_ClearWeakRefs(PyTypeObject *type) +_PyStaticType_ClearWeakRefs(PyInterpreterState *interp, PyTypeObject *type) { - static_builtin_state *state = _PyStaticType_GetState(type); + static_builtin_state *state = _PyStaticType_GetState(interp, type); PyObject **list = _PyStaticType_GET_WEAKREFS_LISTPTR(state); while (*list != NULL) { /* Note that clear_weakref() pops the first ref off the type's diff --git a/Python/errors.c b/Python/errors.c index ce72049b92de27..a8000ac94918db 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1342,8 +1342,9 @@ static PyStructSequence_Desc UnraisableHookArgs_desc = { PyStatus _PyErr_InitTypes(PyInterpreterState *interp) { - if (_PyStructSequence_InitBuiltin(&UnraisableHookArgsType, - &UnraisableHookArgs_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &UnraisableHookArgsType, + &UnraisableHookArgs_desc) < 0) + { return _PyStatus_ERR("failed to initialize UnraisableHookArgs type"); } return _PyStatus_OK(); @@ -1353,11 +1354,7 @@ _PyErr_InitTypes(PyInterpreterState *interp) void _PyErr_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStructSequence_FiniBuiltin(&UnraisableHookArgsType); + _PyStructSequence_FiniBuiltin(interp, &UnraisableHookArgsType); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ba248d208e425a..b8a115236900b9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1663,8 +1663,10 @@ flush_std_files(void) static void finalize_interp_types(PyInterpreterState *interp) { + _PyIO_FiniTypes(interp); + _PyUnicode_FiniTypes(interp); - _PySys_Fini(interp); + _PySys_FiniTypes(interp); _PyExc_Fini(interp); _PyAsyncGen_Fini(interp); _PyContext_Fini(interp); @@ -1706,8 +1708,6 @@ finalize_interp_clear(PyThreadState *tstate) /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); - _PyIO_FiniTypes(tstate->interp); - /* Clear all loghooks */ /* Both _PySys_Audit function and users still need PyObject, such as tuple. Call _PySys_ClearAuditHooks when PyObject available. */ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 81dabe6102f18d..781588b0df4ead 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3141,6 +3141,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) { PyObject *version_info; int res; + PyInterpreterState *interp = tstate->interp; /* stdin/stdout/stderr are set in pylifecycle.c */ @@ -3166,7 +3167,9 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("float_info", PyFloat_GetInfo()); SET_SYS("int_info", PyLong_GetInfo()); /* initialize hash_info */ - if (_PyStructSequence_InitBuiltin(&Hash_InfoType, &hash_info_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &Hash_InfoType, + &hash_info_desc) < 0) + { goto type_init_failed; } SET_SYS("hash_info", get_hash_info(tstate)); @@ -3190,7 +3193,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) #define ENSURE_INFO_TYPE(TYPE, DESC) \ do { \ if (_PyStructSequence_InitBuiltinWithFlags( \ - &TYPE, &DESC, Py_TPFLAGS_DISALLOW_INSTANTIATION) < 0) { \ + interp, &TYPE, &DESC, Py_TPFLAGS_DISALLOW_INSTANTIATION) < 0) { \ goto type_init_failed; \ } \ } while (0) @@ -3226,8 +3229,9 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("thread_info", PyThread_GetInfo()); /* initialize asyncgen_hooks */ - if (_PyStructSequence_InitBuiltin( - &AsyncGenHooksType, &asyncgen_hooks_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &AsyncGenHooksType, + &asyncgen_hooks_desc) < 0) + { goto type_init_failed; } @@ -3489,20 +3493,20 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) void -_PySys_Fini(PyInterpreterState *interp) +_PySys_FiniTypes(PyInterpreterState *interp) { - if (_Py_IsMainInterpreter(interp)) { - _PyStructSequence_FiniBuiltin(&VersionInfoType); - _PyStructSequence_FiniBuiltin(&FlagsType); + _PyStructSequence_FiniBuiltin(interp, &VersionInfoType); + _PyStructSequence_FiniBuiltin(interp, &FlagsType); #if defined(MS_WINDOWS) - _PyStructSequence_FiniBuiltin(&WindowsVersionType); + _PyStructSequence_FiniBuiltin(interp, &WindowsVersionType); #endif - _PyStructSequence_FiniBuiltin(&Hash_InfoType); - _PyStructSequence_FiniBuiltin(&AsyncGenHooksType); + _PyStructSequence_FiniBuiltin(interp, &Hash_InfoType); + _PyStructSequence_FiniBuiltin(interp, &AsyncGenHooksType); #ifdef __EMSCRIPTEN__ + if (_Py_IsMainInterpreter(interp)) { Py_CLEAR(EmscriptenInfoType); -#endif } +#endif } diff --git a/Python/thread.c b/Python/thread.c index f90cd34a073540..7fc53f9b61360b 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -137,7 +137,8 @@ PyThread_GetInfo(void) int len; #endif - if (_PyStructSequence_InitBuiltin(&ThreadInfoType, &threadinfo_desc) < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyStructSequence_InitBuiltin(interp, &ThreadInfoType, &threadinfo_desc) < 0) { return NULL; } @@ -191,9 +192,5 @@ PyThread_GetInfo(void) void _PyThread_FiniType(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStructSequence_FiniBuiltin(&ThreadInfoType); + _PyStructSequence_FiniBuiltin(interp, &ThreadInfoType); } From f73abf8e03fd370c86fbb2a249fe1e065f7d84b4 Mon Sep 17 00:00:00 2001 From: Eric Snow <ericsnowcurrently@gmail.com> Date: Mon, 1 May 2023 20:34:43 -0600 Subject: [PATCH 31/43] gh-94673: Hide Objects in PyTypeObject Behind Accessors (gh-104074) This makes it much cleaner to move more PyTypeObject fields to PyInterpreterState. --- Include/internal/pycore_object.h | 6 - Include/internal/pycore_typeobject.h | 14 + Modules/_abc.c | 7 +- Objects/structseq.c | 8 +- Objects/typeobject.c | 559 ++++++++++++++++----------- Python/context.c | 2 +- 6 files changed, 352 insertions(+), 244 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 2ee0180c0554d2..91853ad0525b55 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -331,10 +331,6 @@ extern int _Py_CheckSlotResult( const char *slot_name, int success); -// PyType_Ready() must be called if _PyType_IsReady() is false. -// See also the Py_TPFLAGS_READY flag. -#define _PyType_IsReady(type) ((type)->tp_dict != NULL) - // Test if a type supports weak references static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { return (type->tp_weaklistoffset != 0); @@ -392,8 +388,6 @@ _PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); -extern int _PyType_HasSubclasses(PyTypeObject *); -extern PyObject* _PyType_GetSubclasses(PyTypeObject *); // Access macro to the members which are floating "behind" the object static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 5bd04736c01d2d..f865e51aeba503 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -109,6 +109,20 @@ extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTyp extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); +PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); +extern PyObject * _PyType_GetBases(PyTypeObject *type); +extern PyObject * _PyType_GetMRO(PyTypeObject *type); +extern PyObject* _PyType_GetSubclasses(PyTypeObject *); +extern int _PyType_HasSubclasses(PyTypeObject *); + +// PyType_Ready() must be called if _PyType_IsReady() is false. +// See also the Py_TPFLAGS_READY flag. +static inline int +_PyType_IsReady(PyTypeObject *type) +{ + return _PyType_GetDict(type) != NULL; +} + PyObject * _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); PyObject * diff --git a/Modules/_abc.c b/Modules/_abc.c index 9d6654b4e58aad..997b618d557ab2 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -452,7 +452,8 @@ _abc__abc_init(PyObject *module, PyObject *self) * their special status w.r.t. pattern matching. */ if (PyType_Check(self)) { PyTypeObject *cls = (PyTypeObject *)self; - PyObject *flags = PyDict_GetItemWithError(cls->tp_dict, + PyObject *dict = _PyType_GetDict(cls); + PyObject *flags = PyDict_GetItemWithError(dict, &_Py_ID(__abc_tpflags__)); if (flags == NULL) { if (PyErr_Occurred()) { @@ -471,7 +472,7 @@ _abc__abc_init(PyObject *module, PyObject *self) } ((PyTypeObject *)self)->tp_flags |= (val & COLLECTION_FLAGS); } - if (PyDict_DelItem(cls->tp_dict, &_Py_ID(__abc_tpflags__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__abc_tpflags__)) < 0) { return NULL; } } @@ -742,7 +743,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, Py_DECREF(ok); /* 4. Check if it's a direct subclass. */ - PyObject *mro = ((PyTypeObject *)subclass)->tp_mro; + PyObject *mro = _PyType_GetMRO((PyTypeObject *)subclass); assert(PyTuple_Check(mro)); for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); diff --git a/Objects/structseq.c b/Objects/structseq.c index ea476bf7a6a954..f63660acb639c3 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -26,7 +26,7 @@ const char * const PyStructSequence_UnnamedField = "unnamed field"; static Py_ssize_t get_type_attr_as_size(PyTypeObject *tp, PyObject *name) { - PyObject *v = PyDict_GetItemWithError(tp->tp_dict, name); + PyObject *v = PyDict_GetItemWithError(_PyType_GetDict(tp), name); if (v == NULL && !PyErr_Occurred()) { PyErr_Format(PyExc_TypeError, "Missed attribute '%U' of type %s", @@ -493,7 +493,7 @@ initialize_static_type(PyTypeObject *type, PyStructSequence_Desc *desc, Py_INCREF(type); if (initialize_structseq_dict( - desc, type->tp_dict, n_members, n_unnamed_members) < 0) { + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { Py_DECREF(type); return -1; } @@ -549,7 +549,7 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, } if (initialize_structseq_dict( - desc, type->tp_dict, n_members, n_unnamed_members) < 0) { + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { goto error; } @@ -675,7 +675,7 @@ _PyStructSequence_NewType(PyStructSequence_Desc *desc, unsigned long tp_flags) } if (initialize_structseq_dict( - desc, type->tp_dict, n_members, n_unnamed_members) < 0) { + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { Py_DECREF(type); return NULL; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2ed806fb01554f..a9d3a69263fb40 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -64,7 +64,19 @@ lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound); static int slot_tp_setattro(PyObject *self, PyObject *name, PyObject *value); -static inline PyTypeObject * subclass_from_ref(PyObject *ref); + +static inline PyTypeObject * +type_from_ref(PyObject *ref) +{ + assert(PyWeakref_CheckRef(ref)); + PyObject *obj = PyWeakref_GET_OBJECT(ref); // borrowed ref + assert(obj != NULL); + if (obj == Py_None) { + return NULL; + } + assert(PyType_Check(obj)); + return _PyType_CAST(obj); +} /* helpers for for static builtin types */ @@ -155,6 +167,178 @@ static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) /* end static builtin helpers */ +/* accessors for objects stored on PyTypeObject */ + +static inline PyObject * +lookup_tp_dict(PyTypeObject *self) +{ + return self->tp_dict; +} + +PyObject * +_PyType_GetDict(PyTypeObject *self) +{ + return lookup_tp_dict(self); +} + +static inline void +set_tp_dict(PyTypeObject *self, PyObject *dict) +{ + self->tp_dict = dict; +} + +static inline void +clear_tp_dict(PyTypeObject *self) +{ + Py_CLEAR(self->tp_dict); +} + + +static inline PyObject * +lookup_tp_bases(PyTypeObject *self) +{ + return self->tp_bases; +} + +PyObject * +_PyType_GetBases(PyTypeObject *self) +{ + return lookup_tp_bases(self); +} + +static inline void +set_tp_bases(PyTypeObject *self, PyObject *bases) +{ + self->tp_bases = bases; +} + +static inline void +clear_tp_bases(PyTypeObject *self) +{ + Py_CLEAR(self->tp_bases); +} + + +static inline PyObject * +lookup_tp_mro(PyTypeObject *self) +{ + return self->tp_mro; +} + +PyObject * +_PyType_GetMRO(PyTypeObject *self) +{ + return lookup_tp_mro(self); +} + +static inline void +set_tp_mro(PyTypeObject *self, PyObject *mro) +{ + self->tp_mro = mro; +} + +static inline void +clear_tp_mro(PyTypeObject *self) +{ + Py_CLEAR(self->tp_mro); +} + + +static PyObject * +init_tp_subclasses(PyTypeObject *self) +{ + PyObject *subclasses = PyDict_New(); + if (subclasses == NULL) { + return NULL; + } + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + state->tp_subclasses = subclasses; + return subclasses; + } + self->tp_subclasses = (void *)subclasses; + return subclasses; +} + +static void +clear_tp_subclasses(PyTypeObject *self) +{ + /* Delete the dictionary to save memory. _PyStaticType_Dealloc() + callers also test if tp_subclasses is NULL to check if a static type + has no subclass. */ + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + Py_CLEAR(state->tp_subclasses); + return; + } + Py_CLEAR(self->tp_subclasses); +} + +static inline PyObject * +lookup_tp_subclasses(PyTypeObject *self) +{ + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + return state->tp_subclasses; + } + return (PyObject *)self->tp_subclasses; +} + +int +_PyType_HasSubclasses(PyTypeObject *self) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN + // XXX _PyStaticType_GetState() should never return NULL. + && _PyStaticType_GetState(interp, self) == NULL) + { + return 0; + } + if (lookup_tp_subclasses(self) == NULL) { + return 0; + } + return 1; +} + +PyObject* +_PyType_GetSubclasses(PyTypeObject *self) +{ + PyObject *list = PyList_New(0); + if (list == NULL) { + return NULL; + } + + PyObject *subclasses = lookup_tp_subclasses(self); // borrowed ref + if (subclasses == NULL) { + return list; + } + assert(PyDict_CheckExact(subclasses)); + // The loop cannot modify tp_subclasses, there is no need + // to hold a strong reference (use a borrowed reference). + + Py_ssize_t i = 0; + PyObject *ref; // borrowed ref + while (PyDict_Next(subclasses, &i, NULL, &ref)) { + PyTypeObject *subclass = type_from_ref(ref); // borrowed + if (subclass == NULL) { + continue; + } + + if (PyList_Append(list, _PyObject_CAST(subclass)) < 0) { + Py_DECREF(list); + return NULL; + } + } + return list; +} + +/* end accessors for objects stored on PyTypeObject */ + + /* * finds the beginning of the docstring's introspection signature. * if present, returns a pointer pointing to the first '('. @@ -225,7 +409,7 @@ _PyType_CheckConsistency(PyTypeObject *type) CHECK(PyType_Check(type)); CHECK(!(type->tp_flags & Py_TPFLAGS_READYING)); - CHECK(type->tp_dict != NULL); + CHECK(lookup_tp_dict(type) != NULL); if (type->tp_flags & Py_TPFLAGS_HAVE_GC) { // bpo-44263: tp_traverse is required if Py_TPFLAGS_HAVE_GC is set. @@ -235,7 +419,7 @@ _PyType_CheckConsistency(PyTypeObject *type) if (type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION) { CHECK(type->tp_new == NULL); - CHECK(PyDict_Contains(type->tp_dict, &_Py_ID(__new__)) == 0); + CHECK(PyDict_Contains(lookup_tp_dict(type), &_Py_ID(__new__)) == 0); } return 1; @@ -361,8 +545,6 @@ _PyTypes_Fini(PyInterpreterState *interp) } -static PyObject * lookup_subclasses(PyTypeObject *); - int PyType_AddWatcher(PyType_WatchCallback callback) { @@ -462,14 +644,14 @@ PyType_Modified(PyTypeObject *type) return; } - PyObject *subclasses = lookup_subclasses(type); + PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses != NULL) { assert(PyDict_CheckExact(subclasses)); Py_ssize_t i = 0; PyObject *ref; while (PyDict_Next(subclasses, &i, NULL, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == NULL) { continue; } @@ -598,7 +780,7 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) assert (type->tp_version_tag != 0); } - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); Py_ssize_t n = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < n; i++) { PyObject *b = PyTuple_GET_ITEM(bases, i); @@ -749,7 +931,8 @@ type_module(PyTypeObject *type, void *context) PyObject *mod; if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { - mod = PyDict_GetItemWithError(type->tp_dict, &_Py_ID(__module__)); + PyObject *dict = lookup_tp_dict(type); + mod = PyDict_GetItemWithError(dict, &_Py_ID(__module__)); if (mod == NULL) { if (!PyErr_Occurred()) { PyErr_Format(PyExc_AttributeError, "__module__"); @@ -781,7 +964,8 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyType_Modified(type); - return PyDict_SetItem(type->tp_dict, &_Py_ID(__module__), value); + PyObject *dict = lookup_tp_dict(type); + return PyDict_SetItem(dict, &_Py_ID(__module__), value); } static PyObject * @@ -790,9 +974,10 @@ type_abstractmethods(PyTypeObject *type, void *context) PyObject *mod = NULL; /* type itself has an __abstractmethods__ descriptor (this). Don't return that. */ - if (type != &PyType_Type) - mod = PyDict_GetItemWithError(type->tp_dict, - &_Py_ID(__abstractmethods__)); + if (type != &PyType_Type) { + PyObject *dict = lookup_tp_dict(type); + mod = PyDict_GetItemWithError(dict, &_Py_ID(__abstractmethods__)); + } if (!mod) { if (!PyErr_Occurred()) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); @@ -810,15 +995,16 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) special to update subclasses. */ int abstract, res; + PyObject *dict = lookup_tp_dict(type); if (value != NULL) { abstract = PyObject_IsTrue(value); if (abstract < 0) return -1; - res = PyDict_SetItem(type->tp_dict, &_Py_ID(__abstractmethods__), value); + res = PyDict_SetItem(dict, &_Py_ID(__abstractmethods__), value); } else { abstract = 0; - res = PyDict_DelItem(type->tp_dict, &_Py_ID(__abstractmethods__)); + res = PyDict_DelItem(dict, &_Py_ID(__abstractmethods__)); if (res && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); return -1; @@ -837,7 +1023,7 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) static PyObject * type_get_bases(PyTypeObject *type, void *context) { - return Py_NewRef(type->tp_bases); + return Py_NewRef(lookup_tp_bases(type)); } static PyTypeObject *best_base(PyObject *); @@ -865,7 +1051,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) /* error / reentrance */ return res; } - PyObject *new_mro = type->tp_mro; + PyObject *new_mro = lookup_tp_mro(type); PyObject *tuple; if (old_mro != NULL) { @@ -884,7 +1070,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) Py_XDECREF(tuple); if (res < 0) { - type->tp_mro = old_mro; + set_tp_mro(type, old_mro); Py_DECREF(new_mro); return -1; } @@ -963,7 +1149,8 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) below), which in turn may cause an inheritance cycle through tp_base chain. And this is definitely not what you want to ever happen. */ - (base->tp_mro != NULL && type_is_subtype_base_chain(base, type))) + (lookup_tp_mro(base) != NULL + && type_is_subtype_base_chain(base, type))) { PyErr_SetString(PyExc_TypeError, "a __bases__ item causes an inheritance cycle"); @@ -980,11 +1167,11 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) return -1; } - PyObject *old_bases = type->tp_bases; + PyObject *old_bases = lookup_tp_bases(type); assert(old_bases != NULL); PyTypeObject *old_base = type->tp_base; - type->tp_bases = Py_NewRef(new_bases); + set_tp_bases(type, Py_NewRef(new_bases)); type->tp_base = (PyTypeObject *)Py_NewRef(new_base); PyObject *temp = PyList_New(0); @@ -999,7 +1186,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) /* Take no action in case if type->tp_bases has been replaced through reentrance. */ int res; - if (type->tp_bases == new_bases) { + if (lookup_tp_bases(type) == new_bases) { /* any base that was in __bases__ but now isn't, we need to remove |type| from its tp_subclasses. conversely, any class now in __bases__ that wasn't @@ -1030,18 +1217,18 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) PyArg_UnpackTuple(PyList_GET_ITEM(temp, i), "", 2, 3, &cls, &new_mro, &old_mro); /* Do not rollback if cls has a newer version of MRO. */ - if (cls->tp_mro == new_mro) { - cls->tp_mro = Py_XNewRef(old_mro); + if (lookup_tp_mro(cls) == new_mro) { + set_tp_mro(cls, Py_XNewRef(old_mro)); Py_DECREF(new_mro); } } Py_DECREF(temp); bail: - if (type->tp_bases == new_bases) { + if (lookup_tp_bases(type) == new_bases) { assert(type->tp_base == new_base); - type->tp_bases = old_bases; + set_tp_bases(type, old_bases); type->tp_base = old_base; Py_DECREF(new_bases); @@ -1059,10 +1246,11 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) static PyObject * type_dict(PyTypeObject *type, void *context) { - if (type->tp_dict == NULL) { + PyObject *dict = lookup_tp_dict(type); + if (dict == NULL) { Py_RETURN_NONE; } - return PyDictProxy_New(type->tp_dict); + return PyDictProxy_New(dict); } static PyObject * @@ -1072,7 +1260,8 @@ type_get_doc(PyTypeObject *type, void *context) if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE) && type->tp_doc != NULL) { return _PyType_GetDocFromInternalDoc(type->tp_name, type->tp_doc); } - result = PyDict_GetItemWithError(type->tp_dict, &_Py_ID(__doc__)); + PyObject *dict = lookup_tp_dict(type); + result = PyDict_GetItemWithError(dict, &_Py_ID(__doc__)); if (result == NULL) { if (!PyErr_Occurred()) { result = Py_NewRef(Py_None); @@ -1100,7 +1289,8 @@ type_set_doc(PyTypeObject *type, PyObject *value, void *context) if (!check_set_special_type_attr(type, value, "__doc__")) return -1; PyType_Modified(type); - return PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), value); + PyObject *dict = lookup_tp_dict(type); + return PyDict_SetItem(dict, &_Py_ID(__doc__), value); } static PyObject * @@ -1113,9 +1303,9 @@ type_get_annotations(PyTypeObject *type, void *context) PyObject *annotations; /* there's no _PyDict_GetItemId without WithError, so let's LBYL. */ - if (PyDict_Contains(type->tp_dict, &_Py_ID(__annotations__))) { - annotations = PyDict_GetItemWithError( - type->tp_dict, &_Py_ID(__annotations__)); + PyObject *dict = lookup_tp_dict(type); + if (PyDict_Contains(dict, &_Py_ID(__annotations__))) { + annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); /* ** PyDict_GetItemWithError could still fail, ** for instance with a well-timed Ctrl-C or a MemoryError. @@ -1133,7 +1323,7 @@ type_get_annotations(PyTypeObject *type, void *context) annotations = PyDict_New(); if (annotations) { int result = PyDict_SetItem( - type->tp_dict, &_Py_ID(__annotations__), annotations); + dict, &_Py_ID(__annotations__), annotations); if (result) { Py_CLEAR(annotations); } else { @@ -1155,16 +1345,17 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) } int result; + PyObject *dict = lookup_tp_dict(type); if (value != NULL) { /* set */ - result = PyDict_SetItem(type->tp_dict, &_Py_ID(__annotations__), value); + result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); } else { /* delete */ - if (!PyDict_Contains(type->tp_dict, &_Py_ID(__annotations__))) { + if (!PyDict_Contains(dict, &_Py_ID(__annotations__))) { PyErr_Format(PyExc_AttributeError, "__annotations__"); return -1; } - result = PyDict_DelItem(type->tp_dict, &_Py_ID(__annotations__)); + result = PyDict_DelItem(dict, &_Py_ID(__annotations__)); } if (result == 0) { @@ -1751,7 +1942,7 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) { PyObject *mro; - mro = a->tp_mro; + mro = lookup_tp_mro(a); if (mro != NULL) { /* Deal with multiple inheritance without recursion by walking the MRO tuple */ @@ -2135,17 +2326,17 @@ mro_implementation(PyTypeObject *type) return NULL; } - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); Py_ssize_t n = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); - if (base->tp_mro == NULL) { + if (lookup_tp_mro(base) == NULL) { PyErr_Format(PyExc_TypeError, "Cannot extend an incomplete type '%.100s'", base->tp_name); return NULL; } - assert(PyTuple_Check(base->tp_mro)); + assert(PyTuple_Check(lookup_tp_mro(base))); } if (n == 1) { @@ -2153,7 +2344,8 @@ mro_implementation(PyTypeObject *type) * is trivial. */ PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, 0)); - Py_ssize_t k = PyTuple_GET_SIZE(base->tp_mro); + PyObject *base_mro = lookup_tp_mro(base); + Py_ssize_t k = PyTuple_GET_SIZE(base_mro); PyObject *result = PyTuple_New(k + 1); if (result == NULL) { return NULL; @@ -2162,7 +2354,7 @@ mro_implementation(PyTypeObject *type) ; PyTuple_SET_ITEM(result, 0, Py_NewRef(type)); for (Py_ssize_t i = 0; i < k; i++) { - PyObject *cls = PyTuple_GET_ITEM(base->tp_mro, i); + PyObject *cls = PyTuple_GET_ITEM(base_mro, i); PyTuple_SET_ITEM(result, i + 1, Py_NewRef(cls)); } return result; @@ -2189,7 +2381,7 @@ mro_implementation(PyTypeObject *type) for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); - to_merge[i] = base->tp_mro; + to_merge[i] = lookup_tp_mro(base); } to_merge[n] = bases; @@ -2344,9 +2536,9 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) /* Keep a reference to be able to do a reentrancy check below. Don't let old_mro be GC'ed and its address be reused for another object, like (suddenly!) a new tp_mro. */ - old_mro = Py_XNewRef(type->tp_mro); + old_mro = Py_XNewRef(lookup_tp_mro(type)); new_mro = mro_invoke(type); /* might cause reentrance */ - reent = (type->tp_mro != old_mro); + reent = (lookup_tp_mro(type) != old_mro); Py_XDECREF(old_mro); if (new_mro == NULL) { return -1; @@ -2357,12 +2549,12 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) return 0; } - type->tp_mro = new_mro; + set_tp_mro(type, new_mro); - type_mro_modified(type, type->tp_mro); + type_mro_modified(type, new_mro); /* corner case: the super class might have been hidden from the custom MRO */ - type_mro_modified(type, type->tp_bases); + type_mro_modified(type, lookup_tp_bases(type)); // XXX Expand this to Py_TPFLAGS_IMMUTABLETYPE? if (!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)) { @@ -2964,7 +3156,7 @@ type_new_alloc(type_new_ctx *ctx) type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; - type->tp_bases = Py_NewRef(ctx->bases); + set_tp_bases(type, Py_NewRef(ctx->bases)); type->tp_base = (PyTypeObject *)Py_NewRef(ctx->base); type->tp_dealloc = subtype_dealloc; @@ -3004,7 +3196,8 @@ type_new_set_name(const type_new_ctx *ctx, PyTypeObject *type) static int type_new_set_module(PyTypeObject *type) { - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__module__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__module__)); if (r < 0) { return -1; } @@ -3025,7 +3218,7 @@ type_new_set_module(PyTypeObject *type) return 0; } - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__module__), module) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__module__), module) < 0) { return -1; } return 0; @@ -3038,8 +3231,8 @@ static int type_new_set_ht_name(PyTypeObject *type) { PyHeapTypeObject *et = (PyHeapTypeObject *)type; - PyObject *qualname = PyDict_GetItemWithError( - type->tp_dict, &_Py_ID(__qualname__)); + PyObject *dict = lookup_tp_dict(type); + PyObject *qualname = PyDict_GetItemWithError(dict, &_Py_ID(__qualname__)); if (qualname != NULL) { if (!PyUnicode_Check(qualname)) { PyErr_Format(PyExc_TypeError, @@ -3048,7 +3241,7 @@ type_new_set_ht_name(PyTypeObject *type) return -1; } et->ht_qualname = Py_NewRef(qualname); - if (PyDict_DelItem(type->tp_dict, &_Py_ID(__qualname__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__qualname__)) < 0) { return -1; } } @@ -3068,7 +3261,8 @@ type_new_set_ht_name(PyTypeObject *type) static int type_new_set_doc(PyTypeObject *type) { - PyObject *doc = PyDict_GetItemWithError(type->tp_dict, &_Py_ID(__doc__)); + PyObject *dict = lookup_tp_dict(type); + PyObject *doc = PyDict_GetItemWithError(dict, &_Py_ID(__doc__)); if (doc == NULL) { if (PyErr_Occurred()) { return -1; @@ -3103,7 +3297,8 @@ type_new_set_doc(PyTypeObject *type) static int type_new_staticmethod(PyTypeObject *type, PyObject *attr) { - PyObject *func = PyDict_GetItemWithError(type->tp_dict, attr); + PyObject *dict = lookup_tp_dict(type); + PyObject *func = PyDict_GetItemWithError(dict, attr); if (func == NULL) { if (PyErr_Occurred()) { return -1; @@ -3118,7 +3313,7 @@ type_new_staticmethod(PyTypeObject *type, PyObject *attr) if (static_func == NULL) { return -1; } - if (PyDict_SetItem(type->tp_dict, attr, static_func) < 0) { + if (PyDict_SetItem(dict, attr, static_func) < 0) { Py_DECREF(static_func); return -1; } @@ -3130,7 +3325,8 @@ type_new_staticmethod(PyTypeObject *type, PyObject *attr) static int type_new_classmethod(PyTypeObject *type, PyObject *attr) { - PyObject *func = PyDict_GetItemWithError(type->tp_dict, attr); + PyObject *dict = lookup_tp_dict(type); + PyObject *func = PyDict_GetItemWithError(dict, attr); if (func == NULL) { if (PyErr_Occurred()) { return -1; @@ -3146,7 +3342,7 @@ type_new_classmethod(PyTypeObject *type, PyObject *attr) return -1; } - if (PyDict_SetItem(type->tp_dict, attr, method) < 0) { + if (PyDict_SetItem(dict, attr, method) < 0) { Py_DECREF(method); return -1; } @@ -3232,8 +3428,8 @@ type_new_set_slots(const type_new_ctx *ctx, PyTypeObject *type) static int type_new_set_classcell(PyTypeObject *type) { - PyObject *cell = PyDict_GetItemWithError( - type->tp_dict, &_Py_ID(__classcell__)); + PyObject *dict = lookup_tp_dict(type); + PyObject *cell = PyDict_GetItemWithError(dict, &_Py_ID(__classcell__)); if (cell == NULL) { if (PyErr_Occurred()) { return -1; @@ -3250,7 +3446,7 @@ type_new_set_classcell(PyTypeObject *type) } (void)PyCell_Set(cell, (PyObject *) type); - if (PyDict_DelItem(type->tp_dict, &_Py_ID(__classcell__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__classcell__)) < 0) { return -1; } return 0; @@ -3357,7 +3553,7 @@ type_new_init(type_new_ctx *ctx) goto error; } - type->tp_dict = dict; + set_tp_dict(type, dict); PyHeapTypeObject *et = (PyHeapTypeObject*)type; et->ht_slots = ctx->slots; @@ -3856,7 +4052,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, /* Set slots we have prepared */ type->tp_base = (PyTypeObject *)Py_NewRef(base); - type->tp_bases = bases; + set_tp_bases(type, bases); bases = NULL; // We give our reference to bases to the type type->tp_doc = tp_doc; @@ -3936,12 +4132,13 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, goto finally; } + PyObject *dict = lookup_tp_dict(type); if (type->tp_doc) { PyObject *__doc__ = PyUnicode_FromString(_PyType_DocWithoutSignature(type->tp_name, type->tp_doc)); if (!__doc__) { goto finally; } - r = PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), __doc__); + r = PyDict_SetItem(dict, &_Py_ID(__doc__), __doc__); Py_DECREF(__doc__); if (r < 0) { goto finally; @@ -3949,18 +4146,18 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, } if (weaklistoffset) { - if (PyDict_DelItem((PyObject *)type->tp_dict, &_Py_ID(__weaklistoffset__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__weaklistoffset__)) < 0) { goto finally; } } if (dictoffset) { - if (PyDict_DelItem((PyObject *)type->tp_dict, &_Py_ID(__dictoffset__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__dictoffset__)) < 0) { goto finally; } } /* Set type.__module__ */ - r = PyDict_Contains(type->tp_dict, &_Py_ID(__module__)); + r = PyDict_Contains(dict, &_Py_ID(__module__)); if (r < 0) { goto finally; } @@ -3972,7 +4169,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, if (modname == NULL) { goto finally; } - r = PyDict_SetItem(type->tp_dict, &_Py_ID(__module__), modname); + r = PyDict_SetItem(dict, &_Py_ID(__module__), modname); Py_DECREF(modname); if (r != 0) { goto finally; @@ -4094,7 +4291,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); - PyObject *mro = type->tp_mro; + PyObject *mro = lookup_tp_mro(type); // The type must be ready assert(mro != NULL); assert(PyTuple_Check(mro)); @@ -4143,14 +4340,14 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) } /* Look in tp_dict of types in MRO */ - PyObject *mro = type->tp_mro; + PyObject *mro = lookup_tp_mro(type); if (mro == NULL) { if ((type->tp_flags & Py_TPFLAGS_READYING) == 0) { if (PyType_Ready(type) < 0) { *error = -1; return NULL; } - mro = type->tp_mro; + mro = lookup_tp_mro(type); } if (mro == NULL) { *error = 1; @@ -4165,7 +4362,7 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 0; i < n; i++) { PyObject *base = PyTuple_GET_ITEM(mro, i); - PyObject *dict = _PyType_CAST(base)->tp_dict; + PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); assert(dict && PyDict_Check(dict)); res = _PyDict_GetItem_KnownHash(dict, name, hash); if (res != NULL) { @@ -4439,20 +4636,19 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys); static void type_dealloc_common(PyTypeObject *type) { - if (type->tp_bases != NULL) { + PyObject *bases = lookup_tp_bases(type); + if (bases != NULL) { PyObject *exc = PyErr_GetRaisedException(); - remove_all_subclasses(type, type->tp_bases); + remove_all_subclasses(type, bases); PyErr_SetRaisedException(exc); } } -static void clear_subclasses(PyTypeObject *self); - static void clear_static_tp_subclasses(PyTypeObject *type) { - PyObject *subclasses = lookup_subclasses(type); + PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses == NULL) { return; } @@ -4481,7 +4677,7 @@ clear_static_tp_subclasses(PyTypeObject *type) Py_ssize_t i = 0; PyObject *key, *ref; // borrowed ref while (PyDict_Next(subclasses, &i, &key, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == NULL) { continue; } @@ -4489,16 +4685,16 @@ clear_static_tp_subclasses(PyTypeObject *type) assert(!(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); } - clear_subclasses(type); + clear_tp_subclasses(type); } static void clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) { if (_Py_IsMainInterpreter(interp)) { - Py_CLEAR(type->tp_dict); - Py_CLEAR(type->tp_bases); - Py_CLEAR(type->tp_mro); + clear_tp_dict(type); + clear_tp_bases(type); + clear_tp_mro(type); Py_CLEAR(type->tp_cache); } clear_static_tp_subclasses(type); @@ -4545,7 +4741,7 @@ type_dealloc(PyTypeObject *type) Py_XDECREF(type->tp_bases); Py_XDECREF(type->tp_mro); Py_XDECREF(type->tp_cache); - clear_subclasses(type); + clear_tp_subclasses(type); /* A type's tp_doc is heap allocated, unlike the tp_doc slots * of most other objects. It's okay to cast it to char *. @@ -4565,65 +4761,6 @@ type_dealloc(PyTypeObject *type) } -static PyObject * -lookup_subclasses(PyTypeObject *self) -{ - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - assert(state != NULL); - return state->tp_subclasses; - } - return (PyObject *)self->tp_subclasses; -} - -int -_PyType_HasSubclasses(PyTypeObject *self) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN && - _PyStaticType_GetState(interp, self) == NULL) { - return 0; - } - if (lookup_subclasses(self) == NULL) { - return 0; - } - return 1; -} - -PyObject* -_PyType_GetSubclasses(PyTypeObject *self) -{ - PyObject *list = PyList_New(0); - if (list == NULL) { - return NULL; - } - - PyObject *subclasses = lookup_subclasses(self); // borrowed ref - if (subclasses == NULL) { - return list; - } - assert(PyDict_CheckExact(subclasses)); - // The loop cannot modify tp_subclasses, there is no need - // to hold a strong reference (use a borrowed reference). - - Py_ssize_t i = 0; - PyObject *ref; // borrowed ref - while (PyDict_Next(subclasses, &i, NULL, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed - if (subclass == NULL) { - continue; - } - - if (PyList_Append(list, _PyObject_CAST(subclass)) < 0) { - Py_DECREF(list); - return NULL; - } - } - return list; -} - - /*[clinic input] type.__subclasses__ @@ -4837,8 +4974,9 @@ type_clear(PyTypeObject *type) */ PyType_Modified(type); - if (type->tp_dict) { - PyDict_Clear(type->tp_dict); + PyObject *dict = lookup_tp_dict(type); + if (dict) { + PyDict_Clear(dict); } Py_CLEAR(((PyHeapTypeObject *)type)->ht_module); @@ -5387,7 +5525,8 @@ _PyType_GetSlotNames(PyTypeObject *cls) assert(PyType_Check(cls)); /* Get the slot names from the cache in the class if possible. */ - slotnames = PyDict_GetItemWithError(cls->tp_dict, &_Py_ID(__slotnames__)); + PyObject *dict = lookup_tp_dict(cls); + slotnames = PyDict_GetItemWithError(dict, &_Py_ID(__slotnames__)); if (slotnames != NULL) { if (slotnames != Py_None && !PyList_Check(slotnames)) { PyErr_Format(PyExc_TypeError, @@ -5887,8 +6026,8 @@ object___reduce_ex___impl(PyObject *self, int protocol) PyObject *reduce, *res; if (objreduce == NULL) { - objreduce = PyDict_GetItemWithError( - PyBaseObject_Type.tp_dict, &_Py_ID(__reduce__)); + PyObject *dict = lookup_tp_dict(&PyBaseObject_Type); + objreduce = PyDict_GetItemWithError(dict, &_Py_ID(__reduce__)); if (objreduce == NULL && PyErr_Occurred()) { return NULL; } @@ -6154,11 +6293,12 @@ type_add_method(PyTypeObject *type, PyMethodDef *meth) } int err; + PyObject *dict = lookup_tp_dict(type); if (!(meth->ml_flags & METH_COEXIST)) { - err = PyDict_SetDefault(type->tp_dict, name, descr) == NULL; + err = PyDict_SetDefault(dict, name, descr) == NULL; } else { - err = PyDict_SetItem(type->tp_dict, name, descr) < 0; + err = PyDict_SetItem(dict, name, descr) < 0; } if (!isdescr) { Py_DECREF(name); @@ -6197,7 +6337,7 @@ type_add_members(PyTypeObject *type) return 0; } - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); for (; memb->name != NULL; memb++) { PyObject *descr = PyDescr_NewMember(type, memb); if (descr == NULL) @@ -6221,7 +6361,7 @@ type_add_getset(PyTypeObject *type) return 0; } - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); for (; gsp->name != NULL; gsp++) { PyObject *descr = PyDescr_NewGetSet(type, gsp); if (descr == NULL) { @@ -6300,7 +6440,7 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) static int overrides_hash(PyTypeObject *type) { - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); assert(dict != NULL); int r = PyDict_Contains(dict, &_Py_ID(__eq__)); @@ -6579,7 +6719,7 @@ type_ready_set_bases(PyTypeObject *type) } /* Initialize tp_bases */ - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); if (bases == NULL) { PyTypeObject *base = type->tp_base; if (base == NULL) { @@ -6591,7 +6731,7 @@ type_ready_set_bases(PyTypeObject *type) if (bases == NULL) { return -1; } - type->tp_bases = bases; + set_tp_bases(type, bases); } return 0; } @@ -6600,7 +6740,7 @@ type_ready_set_bases(PyTypeObject *type) static int type_ready_set_dict(PyTypeObject *type) { - if (type->tp_dict != NULL) { + if (lookup_tp_dict(type) != NULL) { return 0; } @@ -6608,7 +6748,7 @@ type_ready_set_dict(PyTypeObject *type) if (dict == NULL) { return -1; } - type->tp_dict = dict; + set_tp_dict(type, dict); return 0; } @@ -6618,7 +6758,8 @@ type_ready_set_dict(PyTypeObject *type) static int type_dict_set_doc(PyTypeObject *type) { - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__doc__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__doc__)); if (r < 0) { return -1; } @@ -6634,14 +6775,14 @@ type_dict_set_doc(PyTypeObject *type) return -1; } - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), doc) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__doc__), doc) < 0) { Py_DECREF(doc); return -1; } Py_DECREF(doc); } else { - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), Py_None) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__doc__), Py_None) < 0) { return -1; } } @@ -6706,14 +6847,14 @@ type_ready_mro(PyTypeObject *type) if (mro_internal(type, NULL) < 0) { return -1; } - assert(type->tp_mro != NULL); - assert(PyTuple_Check(type->tp_mro)); + PyObject *mro = lookup_tp_mro(type); + assert(mro != NULL); + assert(PyTuple_Check(mro)); /* All bases of statically allocated type should be statically allocated, and static builtin types must have static builtin bases. */ if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { assert(type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); - PyObject *mro = type->tp_mro; Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(mro, i)); @@ -6774,8 +6915,8 @@ type_ready_inherit(PyTypeObject *type) } // Inherit slots - PyObject *mro = type->tp_mro; - Py_ssize_t n = PyTuple_GET_SIZE(type->tp_mro); + PyObject *mro = lookup_tp_mro(type); + Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 1; i < n; i++) { PyObject *b = PyTuple_GET_ITEM(mro, i); if (PyType_Check(b)) { @@ -6820,7 +6961,8 @@ type_ready_set_hash(PyTypeObject *type) return 0; } - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__hash__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__hash__)); if (r < 0) { return -1; } @@ -6828,7 +6970,7 @@ type_ready_set_hash(PyTypeObject *type) return 0; } - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__hash__), Py_None) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__hash__), Py_None) < 0) { return -1; } type->tp_hash = PyObject_HashNotImplemented; @@ -6840,7 +6982,7 @@ type_ready_set_hash(PyTypeObject *type) static int type_ready_add_subclasses(PyTypeObject *type) { - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); Py_ssize_t nbase = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < nbase; i++) { PyObject *b = PyTuple_GET_ITEM(bases, i); @@ -7080,38 +7222,6 @@ _PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) } -static PyObject * -init_subclasses(PyTypeObject *self) -{ - PyObject *subclasses = PyDict_New(); - if (subclasses == NULL) { - return NULL; - } - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - state->tp_subclasses = subclasses; - return subclasses; - } - self->tp_subclasses = (void *)subclasses; - return subclasses; -} - -static void -clear_subclasses(PyTypeObject *self) -{ - /* Delete the dictionary to save memory. _PyStaticType_Dealloc() - callers also test if tp_subclasses is NULL to check if a static type - has no subclass. */ - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = _PyStaticType_GetState(interp, self); - Py_CLEAR(state->tp_subclasses); - return; - } - Py_CLEAR(self->tp_subclasses); -} - static int add_subclass(PyTypeObject *base, PyTypeObject *type) { @@ -7128,9 +7238,9 @@ add_subclass(PyTypeObject *base, PyTypeObject *type) // Only get tp_subclasses after creating the key and value. // PyWeakref_NewRef() can trigger a garbage collection which can execute // arbitrary Python code and so modify base->tp_subclasses. - PyObject *subclasses = lookup_subclasses(base); + PyObject *subclasses = lookup_tp_subclasses(base); if (subclasses == NULL) { - subclasses = init_subclasses(base); + subclasses = init_tp_subclasses(base); if (subclasses == NULL) { Py_DECREF(key); Py_DECREF(ref); @@ -7161,19 +7271,6 @@ add_all_subclasses(PyTypeObject *type, PyObject *bases) return res; } -static inline PyTypeObject * -subclass_from_ref(PyObject *ref) -{ - assert(PyWeakref_CheckRef(ref)); - PyObject *obj = PyWeakref_GET_OBJECT(ref); // borrowed ref - assert(obj != NULL); - if (obj == Py_None) { - return NULL; - } - assert(PyType_Check(obj)); - return _PyType_CAST(obj); -} - static PyObject * get_subclasses_key(PyTypeObject *type, PyTypeObject *base) { @@ -7187,10 +7284,10 @@ get_subclasses_key(PyTypeObject *type, PyTypeObject *base) We fall back to manually traversing the values. */ Py_ssize_t i = 0; PyObject *ref; // borrowed ref - PyObject *subclasses = lookup_subclasses(base); + PyObject *subclasses = lookup_tp_subclasses(base); if (subclasses != NULL) { while (PyDict_Next(subclasses, &i, &key, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == type) { return Py_NewRef(key); } @@ -7203,7 +7300,7 @@ get_subclasses_key(PyTypeObject *type, PyTypeObject *base) static void remove_subclass(PyTypeObject *base, PyTypeObject *type) { - PyObject *subclasses = lookup_subclasses(base); // borrowed ref + PyObject *subclasses = lookup_tp_subclasses(base); // borrowed ref if (subclasses == NULL) { return; } @@ -7219,7 +7316,7 @@ remove_subclass(PyTypeObject *base, PyTypeObject *type) Py_XDECREF(key); if (PyDict_Size(subclasses) == 0) { - clear_subclasses(base); + clear_tp_subclasses(base); } } @@ -7517,7 +7614,7 @@ static int hackcheck(PyObject *self, setattrofunc func, const char *what) { PyTypeObject *type = Py_TYPE(self); - PyObject *mro = type->tp_mro; + PyObject *mro = lookup_tp_mro(type); if (!mro) { /* Probably ok not to check the call in this case. */ return 1; @@ -7805,7 +7902,8 @@ static struct PyMethodDef tp_new_methoddef[] = { static int add_tp_new_wrapper(PyTypeObject *type) { - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__new__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__new__)); if (r > 0) { return 0; } @@ -7817,7 +7915,7 @@ add_tp_new_wrapper(PyTypeObject *type) if (func == NULL) { return -1; } - r = PyDict_SetItem(type->tp_dict, &_Py_ID(__new__), func); + r = PyDict_SetItem(dict, &_Py_ID(__new__), func); Py_DECREF(func); return r; } @@ -9181,7 +9279,8 @@ update_all_slots(PyTypeObject* type) static int type_new_set_names(PyTypeObject *type) { - PyObject *names_to_set = PyDict_Copy(type->tp_dict); + PyObject *dict = lookup_tp_dict(type); + PyObject *names_to_set = PyDict_Copy(dict); if (names_to_set == NULL) { return -1; } @@ -9270,7 +9369,7 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, // It is safe to use a borrowed reference because update_subclasses() is // only used with update_slots_callback() which doesn't modify // tp_subclasses. - PyObject *subclasses = lookup_subclasses(type); // borrowed ref + PyObject *subclasses = lookup_tp_subclasses(type); // borrowed ref if (subclasses == NULL) { return 0; } @@ -9279,13 +9378,13 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, Py_ssize_t i = 0; PyObject *ref; while (PyDict_Next(subclasses, &i, NULL, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == NULL) { continue; } /* Avoid recursing down into unaffected classes */ - PyObject *dict = subclass->tp_dict; + PyObject *dict = lookup_tp_dict(subclass); if (dict != NULL && PyDict_Check(dict)) { int r = PyDict_Contains(dict, attr_name); if (r < 0) { @@ -9336,7 +9435,7 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, static int add_operators(PyTypeObject *type) { - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); pytype_slotdef *p; PyObject *descr; void **ptr; @@ -9432,7 +9531,7 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * PyObject *mro, *res; Py_ssize_t i, n; - mro = su_obj_type->tp_mro; + mro = lookup_tp_mro(su_obj_type); if (mro == NULL) return NULL; @@ -9453,7 +9552,7 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * Py_INCREF(mro); do { PyObject *obj = PyTuple_GET_ITEM(mro, i); - PyObject *dict = _PyType_CAST(obj)->tp_dict; + PyObject *dict = lookup_tp_dict(_PyType_CAST(obj)); assert(dict != NULL && PyDict_Check(dict)); res = PyDict_GetItemWithError(dict, name); diff --git a/Python/context.c b/Python/context.c index 5d385508405ede..1ffae9871be7b3 100644 --- a/Python/context.c +++ b/Python/context.c @@ -1309,7 +1309,7 @@ _PyContext_Init(PyInterpreterState *interp) PyObject *missing = get_token_missing(); if (PyDict_SetItemString( - PyContextToken_Type.tp_dict, "MISSING", missing)) + _PyType_GetDict(&PyContextToken_Type), "MISSING", missing)) { Py_DECREF(missing); return _PyStatus_ERR("can't init context types"); From 690df4c16ca4f0054d27a6148da9e6af809a2658 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy <tjreedy@udel.edu> Date: Mon, 1 May 2023 22:53:16 -0400 Subject: [PATCH 32/43] gh-88496: IDLE - fix another test on macOS (#104075) Needed for Catalina: test_sidebar add 'idletasks' and skip assert. --- Lib/idlelib/idle_test/test_sidebar.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index 5506fd2b0e22a5..fb52b3a0179553 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -57,7 +57,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.editwin.per.close() - cls.root.update() + cls.root.update_idletasks() cls.root.destroy() del cls.text, cls.text_frame, cls.editwin, cls.root @@ -695,7 +695,8 @@ def test_mousewheel(self): delta = -1 if sys.platform == 'darwin' else 120 sidebar.canvas.event_generate('<MouseWheel>', x=0, y=0, delta=delta) yield - self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) + if sys.platform != 'darwin': # .update_idletasks() does not work. + self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) # Scroll back down using the <Button-5> event. sidebar.canvas.event_generate('<Button-5>', x=0, y=0) From 9de0cf20fa0485e327e57cc0864c7476da85cfad Mon Sep 17 00:00:00 2001 From: Thomas Grainger <tagrain@gmail.com> Date: Tue, 2 May 2023 04:59:42 +0100 Subject: [PATCH 33/43] GH-103472: close response in HTTPConnection._tunnel (#103473) Avoid a potential `ResourceWarning` in `http.client.HTTPConnection` by closing the proxy / tunnel's CONNECT response explicitly. --------- Co-authored-by: Gregory P. Smith <greg@krypto.org> --- Lib/http/client.py | 33 ++++++++++--------- Lib/test/test_httplib.py | 23 +++++++++++++ ...-04-12-13-04-16.gh-issue-103472.C6bOHv.rst | 2 ++ 3 files changed, 43 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index 0f5cd35247ae82..74f7bcb68fb6bc 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -941,23 +941,26 @@ def _tunnel(self): del headers response = self.response_class(self.sock, method=self._method) - (version, code, message) = response._read_status() + try: + (version, code, message) = response._read_status() - if code != http.HTTPStatus.OK: - self.close() - raise OSError(f"Tunnel connection failed: {code} {message.strip()}") - while True: - line = response.fp.readline(_MAXLINE + 1) - if len(line) > _MAXLINE: - raise LineTooLong("header line") - if not line: - # for sites which EOF without sending a trailer - break - if line in (b'\r\n', b'\n', b''): - break + if code != http.HTTPStatus.OK: + self.close() + raise OSError(f"Tunnel connection failed: {code} {message.strip()}") + while True: + line = response.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("header line") + if not line: + # for sites which EOF without sending a trailer + break + if line in (b'\r\n', b'\n', b''): + break - if self.debuglevel > 0: - print('header:', line.decode()) + if self.debuglevel > 0: + print('header:', line.decode()) + finally: + response.close() def connect(self): """Connect to the host and port specified in __init__.""" diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index b4f4e2b14351a6..37f77fe0a320c7 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2390,6 +2390,29 @@ def test_tunnel_debuglog(self): lines = output.getvalue().splitlines() self.assertIn('header: {}'.format(expected_header), lines) + def test_tunnel_leak(self): + sock = None + + def _create_connection(address, timeout=None, source_address=None): + nonlocal sock + sock = FakeSocket( + 'HTTP/1.1 404 NOT FOUND\r\n\r\n', + host=address[0], + port=address[1], + ) + return sock + + self.conn._create_connection = _create_connection + self.conn.set_tunnel('destination.com') + exc = None + try: + self.conn.request('HEAD', '/', '') + except OSError as e: + # keeping a reference to exc keeps response alive in the traceback + exc = e + self.assertIsNotNone(exc) + self.assertTrue(sock.file_closed) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst b/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst new file mode 100644 index 00000000000000..01d84f024bd4a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst @@ -0,0 +1,2 @@ +Avoid a potential :exc:`ResourceWarning` in :class:`http.client.HTTPConnection` +by closing the proxy / tunnel's CONNECT response explicitly. From f0ad4567319ee4ae878d570ab7709ab63df9123e Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak <felisiak.mariusz@gmail.com> Date: Tue, 2 May 2023 06:30:43 +0200 Subject: [PATCH 34/43] gh-102997: Update macOS installer to SQLite 3.41.2. (GH-102998) --- Mac/BuildScript/build-installer.py | 6 +++--- .../macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 63fa21b2b33d17..2f5937489ac03d 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.40.1", - url="https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz", - checksum="42175b1a1d23529cb133bbd2b5900afd", + name="SQLite 3.41.2", + url="https://sqlite.org/2023/sqlite-autoconf-3410200.tar.gz", + checksum="862075fd1c38324878ef809eda39edfe", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' diff --git a/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst b/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst new file mode 100644 index 00000000000000..d0b390a896b743 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst @@ -0,0 +1 @@ +Update macOS installer to SQLite 3.41.2. From 82ba6ce303d04a7b21034e38d220e23ca9f1dc0a Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 May 2023 23:05:25 -0700 Subject: [PATCH 35/43] Improve assert_type phrasing (#104081) I'd like to make the fact that this does nothing at runtime really obvious, since I suspect this is unintuitive for users who are unfamiliar with static type checking. I thought of this because of https://discuss.python.org/t/add-arg-check-type-to-types/26384 wherein I'm skeptical that the user really did want `assert_type`. --- Doc/library/typing.rst | 7 ++++--- Lib/typing.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 409a95d528b5d3..c22fc0b28a50d0 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2484,15 +2484,16 @@ Functions and decorators Ask a static type checker to confirm that *val* has an inferred type of *typ*. - When the type checker encounters a call to ``assert_type()``, it + At runtime this does nothing: it returns the first argument unchanged with no + checks or side effects, no matter the actual type of the argument. + + When a static type checker encounters a call to ``assert_type()``, it emits an error if the value is not of the specified type:: def greet(name: str) -> None: assert_type(name, str) # OK, inferred type of `name` is `str` assert_type(name, int) # type checker error - At runtime this returns the first argument unchanged with no side effects. - This function is useful for ensuring the type checker's understanding of a script is in line with the developer's intentions:: diff --git a/Lib/typing.py b/Lib/typing.py index 1a1c989dbaf37d..0dacdd9031a776 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2319,15 +2319,16 @@ def cast(typ, val): def assert_type(val, typ, /): """Ask a static type checker to confirm that the value is of the given type. - When the type checker encounters a call to assert_type(), it + At runtime this does nothing: it returns the first argument unchanged with no + checks or side effects, no matter the actual type of the argument. + + When a static type checker encounters a call to assert_type(), it emits an error if the value is not of the specified type:: def greet(name: str) -> None: assert_type(name, str) # ok assert_type(name, int) # type checker error - At runtime this returns the first argument unchanged and otherwise - does nothing. """ return val From 68ed2a2a3f1e715dc10724b0c000ec2fc498d11e Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle <rffontenelle@users.noreply.github.com> Date: Tue, 2 May 2023 03:34:44 -0300 Subject: [PATCH 36/43] GH-103484: Fix redirected permanently URLs (#104001) Co-authored-by: Hugo van Kemenade <hugovk@users.noreply.github.com> Co-authored-by: Oleg Iarygin <dralife@yandex.ru> --- Doc/conf.py | 24 +++++++++++++++++++++--- Doc/faq/extending.rst | 2 +- Doc/faq/general.rst | 8 ++++---- Doc/faq/programming.rst | 2 +- Doc/howto/pyporting.rst | 2 +- Doc/library/asyncio-eventloop.rst | 2 +- Doc/library/hashlib.rst | 12 ++++++------ Doc/library/http.client.rst | 2 +- Doc/library/importlib.metadata.rst | 2 +- Doc/library/os.rst | 2 +- Doc/library/plistlib.rst | 2 +- Doc/library/resource.rst | 2 +- Doc/library/select.rst | 2 +- Doc/library/ssl.rst | 2 +- Doc/library/statistics.rst | 2 +- Doc/library/struct.rst | 2 +- Doc/library/sys.rst | 2 +- Doc/library/unittest.mock-examples.rst | 2 +- Doc/library/unittest.rst | 2 +- Doc/library/xmlrpc.client.rst | 2 +- Doc/library/zipfile.rst | 2 +- Doc/reference/datamodel.rst | 2 +- Doc/reference/introduction.rst | 2 +- Doc/using/cmdline.rst | 2 +- Doc/using/mac.rst | 2 +- Doc/using/windows.rst | 2 +- Doc/whatsnew/2.0.rst | 2 +- Doc/whatsnew/2.1.rst | 2 +- Doc/whatsnew/2.2.rst | 4 ++-- Doc/whatsnew/2.3.rst | 6 +++--- Doc/whatsnew/2.4.rst | 2 +- Doc/whatsnew/2.6.rst | 4 ++-- Doc/whatsnew/3.11.rst | 8 ++++---- Doc/whatsnew/3.2.rst | 6 +++--- Doc/whatsnew/3.3.rst | 2 +- Doc/whatsnew/3.5.rst | 4 ++-- Doc/whatsnew/3.6.rst | 2 +- Doc/whatsnew/3.8.rst | 2 +- Doc/whatsnew/3.9.rst | 2 +- Misc/NEWS.d/3.7.0a1.rst | 2 +- 40 files changed, 78 insertions(+), 60 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index cef2a0e2837f6a..6a3c01cd91a77b 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -264,11 +264,29 @@ linkcheck_allowed_redirects = { # bpo-NNNN -> BPO -> GH Issues - r'https://bugs.python.org/issue\?@action=redirect&bpo=\d+': 'https://github.com/python/cpython/issues/\d+', + r'https://bugs.python.org/issue\?@action=redirect&bpo=\d+': r'https://github.com/python/cpython/issues/\d+', # GH-NNNN used to refer to pull requests - r'https://github.com/python/cpython/issues/\d+': 'https://github.com/python/cpython/pull/\d+', + r'https://github.com/python/cpython/issues/\d+': r'https://github.com/python/cpython/pull/\d+', # :source:`something` linking files in the repository - r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*' + r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*', + # Intentional HTTP use at Misc/NEWS.d/3.5.0a1.rst + r'http://www.python.org/$': 'https://www.python.org/$', + # Used in license page, keep as is + r'https://www.zope.org/': r'https://www.zope.dev/', + # Microsoft's redirects to learn.microsoft.com + r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*', + r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*', + r'https://go.microsoft.com/fwlink/\?LinkID=\d+': 'https://learn.microsoft.com/.*', + # Language redirects + r'https://toml.io': 'https://toml.io/en/', + r'https://www.redhat.com': 'https://www.redhat.com/en', + # Other redirects + r'https://www.boost.org/libs/.+': r'https://www.boost.org/doc/libs/\d_\d+_\d/.+', + r'https://support.microsoft.com/en-us/help/\d+': 'https://support.microsoft.com/en-us/topic/.+', + r'https://perf.wiki.kernel.org$': 'https://perf.wiki.kernel.org/index.php/Main_Page', + r'https://www.sqlite.org': 'https://www.sqlite.org/index.html', + r'https://mitpress.mit.edu/sicp$': 'https://mitpress.mit.edu/9780262510875/structure-and-interpretation-of-computer-programs/', + r'https://www.python.org/psf/': 'https://www.python.org/psf-landing/', } linkcheck_anchors_ignore = [ diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 07282639e4f9b4..bc3080f60ee237 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -42,7 +42,7 @@ on what you're trying to do. .. XXX make sure these all work `Cython <https://cython.org>`_ and its relative `Pyrex -<https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/>`_ are compilers +<https://www.csse.canterbury.ac.nz/greg.ewing/python/Pyrex/>`_ are compilers that accept a slightly modified form of Python and generate the corresponding C code. Cython and Pyrex make it possible to write an extension without having to learn Python's C API. diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 6256deb5797c89..a9b2622e02ef3b 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -54,8 +54,8 @@ commercial use, to sell copies of Python in source or binary form (modified or unmodified), or to sell products that incorporate Python in some form. We would still like to know about all commercial use of Python, of course. -See `the PSF license page <https://www.python.org/psf/license/>`_ to find further -explanations and a link to the full text of the license. +See `the license page <https://docs.python.org/3/license.html>`_ to find further +explanations and the full text of the PSF License. The Python logo is trademarked, and in certain cases permission is required to use it. Consult `the Trademark Usage Policy @@ -215,7 +215,7 @@ every day, and Usenet readers are often more able to cope with this volume. Announcements of new software releases and events can be found in comp.lang.python.announce, a low-traffic moderated list that receives about five postings per day. It's available as `the python-announce mailing list -<https://mail.python.org/mailman/listinfo/python-announce-list>`_. +<https://mail.python.org/mailman3/lists/python-announce-list.python.org/>`_. More info about other mailing lists and newsgroups can be found at https://www.python.org/community/lists/. @@ -352,7 +352,7 @@ titled "Python X.Y Release Schedule", where X.Y is a version that hasn't been publicly released yet. New development is discussed on `the python-dev mailing list -<https://mail.python.org/mailman/listinfo/python-dev/>`_. +<https://mail.python.org/mailman3/lists/python-dev.python.org/>`_. Is it reasonable to propose incompatible changes to Python? diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 38f9b171618b26..ab5618db84f77e 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -61,7 +61,7 @@ Yes. `Pyflakes <https://github.com/PyCQA/pyflakes>`_ do basic checking that will help you catch bugs sooner. -Static type checkers such as `Mypy <http://mypy-lang.org/>`_, +Static type checkers such as `Mypy <https://mypy-lang.org/>`_, `Pyre <https://pyre-check.org/>`_, and `Pytype <https://github.com/google/pytype>`_ can check type hints in Python source code. diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index add1c11be534e3..baea3e85c3b84b 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -438,7 +438,7 @@ to make sure everything functions as expected in both versions of Python. .. _Futurize: https://python-future.org/automatic_conversion.html .. _importlib2: https://pypi.org/project/importlib2 .. _Modernize: https://python-modernize.readthedocs.io/ -.. _mypy: http://mypy-lang.org/ +.. _mypy: https://mypy-lang.org/ .. _Porting to Python 3: http://python3porting.com/ .. _Pylint: https://pypi.org/project/pylint diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index e982cc166a3f2d..8d0022cc66daac 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -529,7 +529,7 @@ Opening network connections specifies requirements for algorithms that reduce this user-visible delay and provides an algorithm. - For more information: https://tools.ietf.org/html/rfc6555 + For more information: https://datatracker.ietf.org/doc/html/rfc6555 .. versionchanged:: 3.11 diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index f8d10c0c295c7a..6275f96f7d4d19 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -432,7 +432,7 @@ Constructor functions also accept the following tree hashing parameters: :alt: Explanation of tree mode parameters. See section 2.10 in `BLAKE2 specification -<https://blake2.net/blake2_20130129.pdf>`_ for comprehensive review of tree +<https://www.blake2.net/blake2_20130129.pdf>`_ for comprehensive review of tree hashing. @@ -619,7 +619,7 @@ on the hash function used in digital signatures. by the signer. (`NIST SP-800-106 "Randomized Hashing for Digital Signatures" - <https://csrc.nist.gov/publications/detail/sp/800-106/final>`_) + <https://csrc.nist.gov/publications/detail/sp/800-106/archive/2009-02-25>`_) In BLAKE2 the salt is processed as a one-time input to the hash function during initialization, rather than as an input to each compression function. @@ -628,7 +628,7 @@ initialization, rather than as an input to each compression function. *Salted hashing* (or just hashing) with BLAKE2 or any other general-purpose cryptographic hash function, such as SHA-256, is not suitable for hashing - passwords. See `BLAKE2 FAQ <https://blake2.net/#qa>`_ for more + passwords. See `BLAKE2 FAQ <https://www.blake2.net/#qa>`_ for more information. .. @@ -764,9 +764,9 @@ Domain Dedication 1.0 Universal: * *Alexandr Sokolovskiy* -.. _BLAKE2: https://blake2.net +.. _BLAKE2: https://www.blake2.net .. _HMAC: https://en.wikipedia.org/wiki/Hash-based_message_authentication_code -.. _BLAKE: https://131002.net/blake/ +.. _BLAKE: https://web.archive.org/web/20200918190133/https://131002.net/blake/ .. _SHA-3: https://en.wikipedia.org/wiki/NIST_hash_function_competition .. _ChaCha: https://cr.yp.to/chacha.html .. _pyblake2: https://pythonhosted.org/pyblake2/ @@ -782,7 +782,7 @@ Domain Dedication 1.0 Universal: Module :mod:`base64` Another way to encode binary hashes for non-binary environments. - https://blake2.net + https://www.blake2.net Official BLAKE2 website. https://csrc.nist.gov/csrc/media/publications/fips/180/2/archive/2002-08-01/documents/fips180-2.pdf diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 38821b32c91cf1..abdc6b447a8b61 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -354,7 +354,7 @@ HTTPConnection Objects the CONNECT request. As HTTP/1.1 is used for HTTP CONNECT tunnelling request, `as per the RFC - <https://tools.ietf.org/html/rfc7231#section-4.3.6>`_, a HTTP ``Host:`` + <https://datatracker.ietf.org/doc/html/rfc7231#section-4.3.6>`_, a HTTP ``Host:`` header must be provided, matching the authority-form of the request target provided as the destination for the CONNECT request. If a HTTP ``Host:`` header is not provided via the headers argument, one is generated and diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index b306d5f55a714f..3097bcf47b627f 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -178,7 +178,7 @@ The "selectable" entry points were introduced in ``importlib_metadata`` no parameters and always returned a dictionary of entry points, keyed by group. With ``importlib_metadata`` 5.0 and Python 3.12, ``entry_points`` always returns an ``EntryPoints`` object. See -`backports.entry_points_selectable <https://pypi.org/project/backports.entry_points_selectable>`_ +`backports.entry_points_selectable <https://pypi.org/project/backports.entry-points-selectable>`_ for compatibility options. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 50e951c631fa88..76623c6305432a 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4593,7 +4593,7 @@ written in Python, such as a mail server's external command delivery program. :attr:`!children_system`, and :attr:`!elapsed` in that order. See the Unix manual page - :manpage:`times(2)` and `times(3) <https://www.freebsd.org/cgi/man.cgi?time(3)>`_ manual page on Unix or `the GetProcessTimes MSDN + :manpage:`times(2)` and `times(3) <https://man.freebsd.org/cgi/man.cgi?time(3)>`_ manual page on Unix or `the GetProcessTimes MSDN <https://docs.microsoft.com/windows/win32/api/processthreadsapi/nf-processthreadsapi-getprocesstimes>`_ on Windows. On Windows, only :attr:`!user` and :attr:`!system` are known; the other attributes are zero. diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 7aad15ec91a0ac..732ef3536863cc 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -46,7 +46,7 @@ or :class:`datetime.datetime` objects. .. seealso:: - `PList manual page <https://developer.apple.com/library/content/documentation/Cocoa/Conceptual/PropertyLists/>`_ + `PList manual page <https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/PropertyLists/>`_ Apple's documentation of the file format. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index e7bf45d7d569fa..a5324c82c63484 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -244,7 +244,7 @@ platform. used by all of this user id's processes. This limit is enforced only if bit 1 of the vm.overcommit sysctl is set. Please see - `tuning(7) <https://www.freebsd.org/cgi/man.cgi?query=tuning&sektion=7>`__ + `tuning(7) <https://man.freebsd.org/cgi/man.cgi?query=tuning&sektion=7>`__ for a complete description of this sysctl. .. availability:: FreeBSD. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 2890706bab729c..b0891b0c8f584a 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -505,7 +505,7 @@ Kqueue Objects Kevent Objects -------------- -https://www.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 .. attribute:: kevent.ident diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 4b60b7c643b62c..18a6c5ab4858a4 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1719,7 +1719,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.3 .. seealso:: - `SSL/TLS & Perfect Forward Secrecy <https://vincent.bernat.im/en/blog/2011-ssl-perfect-forward-secrecy>`_ + `SSL/TLS & Perfect Forward Secrecy <https://vincent.bernat.ch/en/blog/2011-ssl-perfect-forward-secrecy>`_ Vincent Bernat. .. method:: SSLContext.wrap_socket(sock, server_side=False, \ diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index f934b0e0319dca..395b324c860389 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -22,7 +22,7 @@ This module provides functions for calculating mathematical statistics of numeric (:class:`~numbers.Real`-valued) data. The module is not intended to be a competitor to third-party libraries such -as `NumPy <https://numpy.org>`_, `SciPy <https://www.scipy.org/>`_, or +as `NumPy <https://numpy.org>`_, `SciPy <https://scipy.org/>`_, or proprietary full-featured statistics packages aimed at professional statisticians such as Minitab, SAS and Matlab. It is aimed at the level of graphing and scientific calculators. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 9c0e32ba16bf68..78fd6e397ae635 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -602,4 +602,4 @@ The :mod:`struct` module also defines the following type: .. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_754-2008_revision -.. _IETF RFC 1700: https://tools.ietf.org/html/rfc1700 +.. _IETF RFC 1700: https://datatracker.ietf.org/doc/html/rfc1700 diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 7c0e85142e7716..95ad243bdde398 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -792,7 +792,7 @@ always available. additional garbage collector overhead if the object is managed by the garbage collector. - See `recursive sizeof recipe <https://code.activestate.com/recipes/577504>`_ + See `recursive sizeof recipe <https://code.activestate.com/recipes/577504/>`_ for an example of using :func:`getsizeof` recursively to find the size of containers and all their contents. diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index f9a207bad6903f..895b9f9f07671b 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -1074,7 +1074,7 @@ subclass. Sometimes this is inconvenient. For example, `one user <https://code.google.com/archive/p/mock/issues/105>`_ is subclassing mock to created a `Twisted adaptor -<https://twistedmatrix.com/documents/11.0.0/api/twisted.python.components.html>`_. +<https://twisted.org/documents/11.0.0/api/twisted.python.components.html>`_. Having this applied to attributes too actually causes errors. ``Mock`` (in all its flavours) uses a method called ``_get_child_mock`` to create diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index c70153dfcd69e1..b26e6c0e6bc024 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -72,7 +72,7 @@ test runner a GUI tool for test discovery and execution. This is intended largely for ease of use for those new to unit testing. For production environments it is recommended that tests be driven by a continuous integration system such as - `Buildbot <https://buildbot.net/>`_, `Jenkins <https://jenkins.io/>`_, + `Buildbot <https://buildbot.net/>`_, `Jenkins <https://www.jenkins.io/>`_, `GitHub Actions <https://github.com/features/actions>`_, or `AppVeyor <https://www.appveyor.com/>`_. diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index bd2c49a6edab7f..146c4fd768233b 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -161,7 +161,7 @@ between conformable Python objects and XML on the wire. .. seealso:: - `XML-RPC HOWTO <https://www.tldp.org/HOWTO/XML-RPC-HOWTO/index.html>`_ + `XML-RPC HOWTO <https://tldp.org/HOWTO/XML-RPC-HOWTO/index.html>`_ A good description of XML-RPC operation and client software in several languages. Contains pretty much everything an XML-RPC client developer needs to know. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 6f4826cb065c64..45f3d340bd82d3 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -128,7 +128,7 @@ The module defines the following items: Documentation on the ZIP file format by Phil Katz, the creator of the format and algorithms used. - `Info-ZIP Home Page <http://www.info-zip.org/>`_ + `Info-ZIP Home Page <https://infozip.sourceforge.net/>`_ Information about the Info-ZIP project's ZIP archive programs and development libraries. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 55431f1951e50d..c35bf4016a28d2 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1564,7 +1564,7 @@ Basic customization This is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict insertion, O(n\ :sup:`2`) complexity. See - http://www.ocert.org/advisories/ocert-2011-003.html for details. + http://ocert.org/advisories/ocert-2011-003.html for details. Changing hash values affects the iteration order of sets. Python has never made guarantees about this ordering diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index 914a11556c94e6..81f0a5c5d43883 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -74,7 +74,7 @@ PyPy and a Just in Time compiler. One of the goals of the project is to encourage experimentation with the language itself by making it easier to modify the interpreter (since it is written in Python). Additional information is - available on `the PyPy project's home page <https://pypy.org/>`_. + available on `the PyPy project's home page <https://www.pypy.org/>`_. Each of these implementations varies in some way from the language as documented in this manual, or introduces specific information beyond what's covered in the diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b35e8454fa2a1a..9d4042ce5a7e8a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -370,7 +370,7 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict construction, O(n\ :sup:`2`) complexity. See - http://www.ocert.org/advisories/ocert-2011-003.html for details. + http://ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 9ae0270eaee7ab..69cd5c92d884d0 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -66,7 +66,7 @@ number of standard Unix command line editors, :program:`vim` and :program:`BBEdit` or :program:`TextWrangler` from Bare Bones Software (see http://www.barebones.com/products/bbedit/index.html) are good choices, as is :program:`TextMate` (see https://macromates.com/). Other editors include -:program:`Gvim` (https://macvim-dev.github.io/macvim/) and :program:`Aquamacs` +:program:`Gvim` (https://macvim.org/macvim/) and :program:`Aquamacs` (http://aquamacs.org/). To run your script from the Terminal window you must make sure that diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 380950eb507ffb..43e3c72f3e1cde 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -541,7 +541,7 @@ Besides the standard CPython distribution, there are modified packages including additional functionality. The following is a list of popular versions and their key features: -`ActivePython <https://www.activestate.com/activepython/>`_ +`ActivePython <https://www.activestate.com/products/python/>`_ Installer with multi-platform compatibility, documentation, PyWin32 `Anaconda <https://www.anaconda.com/download/>`_ diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index 4bcb2acae1e640..0eefefd863a68f 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -933,7 +933,7 @@ using it:: parser.parse( 'hamlet.xml' ) For more information, consult the Python documentation, or the XML HOWTO at -http://pyxml.sourceforge.net/topics/howto/xml-howto.html. +https://pyxml.sourceforge.net/topics/howto/xml-howto.html. DOM Support diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index 0136de58774038..676da702b39693 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -613,7 +613,7 @@ New and Improved Modules framework based on running embedded examples in docstrings and comparing the results against the expected output. PyUnit, contributed by Steve Purcell, is a unit testing framework inspired by JUnit, which was in turn an adaptation of - Kent Beck's Smalltalk testing framework. See http://pyunit.sourceforge.net/ for + Kent Beck's Smalltalk testing framework. See https://pyunit.sourceforge.net/ for more information about PyUnit. * The :mod:`difflib` module contains a class, :class:`SequenceMatcher`, which diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 0c3bfda1933957..82aff0be1ed3b3 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -632,10 +632,10 @@ queen threatens another) and the Knight's Tour (a route that takes a knight to every square of an $NxN$ chessboard without visiting any square twice). The idea of generators comes from other programming languages, especially Icon -(https://www.cs.arizona.edu/icon/), where the idea of generators is central. In +(https://www2.cs.arizona.edu/icon/), where the idea of generators is central. In Icon, every expression and function call behaves like a generator. One example from "An Overview of the Icon Programming Language" at -https://www.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks +https://www2.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks like:: sentence := "Store it in the neighboring harbor" diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index c6e2003e92f1b3..af489d7cb45c2a 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -218,10 +218,10 @@ queen threatens another) and the Knight's Tour (a route that takes a knight to every square of an $NxN$ chessboard without visiting any square twice). The idea of generators comes from other programming languages, especially Icon -(https://www.cs.arizona.edu/icon/), where the idea of generators is central. In +(https://www2.cs.arizona.edu/icon/), where the idea of generators is central. In Icon, every expression and function call behaves like a generator. One example from "An Overview of the Icon Programming Language" at -https://www.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks +https://www2.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks like:: sentence := "Store it in the neighboring harbor" @@ -1332,7 +1332,7 @@ complete list of changes, or look through the CVS logs for all the details. (Contributed by Kevin O'Connor.) * The IDLE integrated development environment has been updated using the code - from the IDLEfork project (http://idlefork.sourceforge.net). The most notable feature is + from the IDLEfork project (https://idlefork.sourceforge.net). The most notable feature is that the code being developed is now executed in a subprocess, meaning that there's no longer any need for manual ``reload()`` operations. IDLE's core code has been incorporated into the standard library as the :mod:`idlelib` package. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 63e819876ce310..98dc83fe935d5e 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -756,7 +756,7 @@ API that perform ASCII-only conversions, ignoring the locale setting: :c:expr:`double` to an ASCII string. The code for these functions came from the GLib library -(https://developer.gnome.org/glib/stable/), whose developers kindly +(https://developer-old.gnome.org/glib/2.26/), whose developers kindly relicensed the relevant functions and donated them to the Python Software Foundation. The :mod:`locale` module can now change the numeric locale, letting extensions such as GTK+ produce the correct results. diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 4ee2aacb108a36..84bb651e68eed5 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -1433,7 +1433,7 @@ one, :func:`math.trunc`, that's been backported to Python 2.6. `Scheme's numerical tower <https://www.gnu.org/software/guile/manual/html_node/Numerical-Tower.html#Numerical-Tower>`__, from the Guile manual. - `Scheme's number datatypes <https://schemers.org/Documents/Standards/R5RS/HTML/r5rs-Z-H-9.html#%_sec_6.2>`__ from the R5RS Scheme specification. + `Scheme's number datatypes <https://conservatory.scheme.org/schemers/Documents/Standards/R5RS/HTML/r5rs-Z-H-9.html#%_sec_6.2>`__ from the R5RS Scheme specification. The :mod:`fractions` Module @@ -2363,7 +2363,7 @@ changes, or look through the Subversion logs for all the details. negotiation itself. (Patch contributed by Bill Fenner; :issue:`829951`.) -* The :mod:`socket` module now supports TIPC (http://tipc.sourceforge.net/), +* The :mod:`socket` module now supports TIPC (https://tipc.sourceforge.net/), a high-performance non-IP-based protocol designed for use in clustered environments. TIPC addresses are 4- or 5-tuples. (Contributed by Alberto Bertogli; :issue:`1646`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 687719a260a61c..6b591d5e184ea7 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -220,7 +220,7 @@ The copy of the :ref:`launcher` included with Python 3.11 has been significantly updated. It now supports company/tag syntax as defined in :pep:`514` using the ``-V:<company>/<tag>`` argument instead of the limited ``-<major>.<minor>``. This allows launching distributions other than ``PythonCore``, -the one hosted on `python.org <https://python.org>`_. +the one hosted on `python.org <https://www.python.org>`_. When using ``-V:`` selectors, either company or tag can be omitted, but all installs will be searched. For example, ``-V:OtherPython/`` will select the @@ -2481,7 +2481,7 @@ Porting to Python 3.11 #endif Or use the `pythoncapi_compat project - <https://github.com/python/pythoncapi_compat>`__ to get these two + <https://github.com/python/pythoncapi-compat>`__ to get these two functions on older Python versions. * Changes of the :c:type:`PyThreadState` structure members: @@ -2533,8 +2533,8 @@ Porting to Python 3.11 } #endif - Or use `the pythoncapi_compat project - <https://github.com/python/pythoncapi_compat>`__ to get these functions + Or use `the pythoncapi-compat project + <https://github.com/python/pythoncapi-compat>`__ to get these functions on old Python functions. * Distributors are encouraged to build Python with the optimized Blake2 diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 1b1455b72b9291..8dbe2a1d828b4e 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -785,8 +785,8 @@ functools (Contributed by Raymond Hettinger and incorporating design ideas from Jim Baker, Miki Tebeka, and Nick Coghlan; see `recipe 498245 - <https://code.activestate.com/recipes/498245>`_\, `recipe 577479 - <https://code.activestate.com/recipes/577479>`_\, :issue:`10586`, and + <https://code.activestate.com/recipes/498245/>`_\, `recipe 577479 + <https://code.activestate.com/recipes/577479/>`_\, :issue:`10586`, and :issue:`10593`.) * The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute @@ -2603,7 +2603,7 @@ Also, there were a number of updates to the Mac OS X build, see for details. For users running a 32/64-bit build, there is a known problem with the default Tcl/Tk on Mac OS X 10.6. Accordingly, we recommend installing an updated alternative such as -`ActiveState Tcl/Tk 8.5.9 <https://www.activestate.com/activetcl/downloads>`_\. +`ActiveState Tcl/Tk 8.5.9 <https://web.archive.org/web/20101208191259/https://www.activestate.com/activetcl/downloads>`_\. See https://www.python.org/download/mac/tcltk/ for additional details. Porting to Python 3.2 diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 9e8d42469b019c..5b6c3dcd45c6f5 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1893,7 +1893,7 @@ socket * The :class:`~socket.socket` class now supports the PF_RDS protocol family (https://en.wikipedia.org/wiki/Reliable_Datagram_Sockets and - https://oss.oracle.com/projects/rds/). + `https://oss.oracle.com/projects/rds <https://web.archive.org/web/20130115155505/https://oss.oracle.com/projects/rds/>`__). * The :class:`~socket.socket` class now supports the ``PF_SYSTEM`` protocol family on OS X. (Contributed by Michael Goderbauer in :issue:`13777`.) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index f872579ef546f5..14b6425cea699e 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -425,7 +425,7 @@ are declared in the annotations:: While these annotations are available at runtime through the usual :attr:`__annotations__` attribute, *no automatic type checking happens at runtime*. Instead, it is assumed that a separate off-line type checker -(e.g. `mypy <http://mypy-lang.org>`_) will be used for on-demand +(e.g. `mypy <https://mypy-lang.org>`_) will be used for on-demand source code analysis. The type system supports unions, generic types, and a special type @@ -2212,7 +2212,7 @@ for details.) The :c:member:`PyTypeObject.tp_finalize` slot is now part of the stable ABI. Windows builds now require Microsoft Visual C++ 14.0, which -is available as part of `Visual Studio 2015 <https://www.visualstudio.com/>`_. +is available as part of `Visual Studio 2015 <https://visualstudio.microsoft.com/en/vs/older-downloads/#visual-studio-2015-and-other-products>`_. Extension modules now include a platform information tag in their filename on some platforms (the tag is optional, and CPython will import extensions without diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index e4294c88b58572..c7faaebfed62b3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -238,7 +238,7 @@ and the ``__annotations__`` attribute. and Guido van Rossum. Implemented by Ivan Levkivskyi. Tools that use or will use the new syntax: - `mypy <http://www.mypy-lang.org/>`_, + `mypy <https://www.mypy-lang.org/>`_, `pytype <https://github.com/google/pytype>`_, PyCharm, etc. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 37a6cf24e54562..85e088b64acb2d 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -2229,7 +2229,7 @@ The benchmarks were measured on an `Intel® Core™ i7-4960HQ processor <https://ark.intel.com/content/www/us/en/ark/products/76088/intel-core-i7-4960hq-processor-6m-cache-up-to-3-80-ghz.html>`_ running the macOS 64-bit builds found at -`python.org <https://www.python.org/downloads/mac-osx/>`_. +`python.org <https://www.python.org/downloads/macos/>`_. The benchmark script displays timings in nanoseconds. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index e974ee3a3f73ed..fd86db96302356 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -849,7 +849,7 @@ in nanoseconds. The benchmarks were measured on an `Intel® Core™ i7-4960HQ processor <https://ark.intel.com/content/www/us/en/ark/products/76088/intel-core-i7-4960hq-processor-6m-cache-up-to-3-80-ghz.html>`_ running the macOS 64-bit builds found at -`python.org <https://www.python.org/downloads/mac-osx/>`_. +`python.org <https://www.python.org/downloads/macos/>`_. Deprecated diff --git a/Misc/NEWS.d/3.7.0a1.rst b/Misc/NEWS.d/3.7.0a1.rst index 9bada1b76be7a8..ef93454784b77f 100644 --- a/Misc/NEWS.d/3.7.0a1.rst +++ b/Misc/NEWS.d/3.7.0a1.rst @@ -6255,7 +6255,7 @@ Fix python-gdb.py didn't support new dict implementation. .. section: Tools/Demos The pybench and pystone microbenchmark have been removed from Tools. Please -use the new Python benchmark suite https://github.com/python/performance +use the new Python benchmark suite https://github.com/python/pyperformance which is more reliable and includes a portable version of pybench working on Python 2 and Python 3. From f6314b92dcfc8ca6ff3fd150814f85448db69165 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak <felisiak.mariusz@gmail.com> Date: Tue, 2 May 2023 09:37:57 +0200 Subject: [PATCH 37/43] gh-102997: Update Windows installer to SQLite 3.41.2. (#102999) --- .../next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst | 1 + PCbuild/get_externals.bat | 2 +- PCbuild/python.props | 2 +- PCbuild/readme.txt | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst diff --git a/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst b/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst new file mode 100644 index 00000000000000..c8f7259aecba6f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst @@ -0,0 +1 @@ +Update Windows installer to use SQLite 3.41.2. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 128241393f9f09..30ee873af9af24 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1t -set libraries=%libraries% sqlite-3.40.1.0 +set libraries=%libraries% sqlite-3.41.2.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tix-8.4.3.6 diff --git a/PCbuild/python.props b/PCbuild/python.props index 7994fbe7cd5e0b..29add07795f900 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -68,7 +68,7 @@ <Import Project="$(ExternalProps)" Condition="$(ExternalProps) != '' and Exists('$(ExternalProps)')" /> <PropertyGroup> - <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.40.1.0\</sqlite3Dir> + <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.41.2.0\</sqlite3Dir> <bz2Dir Condition="$(bz2Dir) == ''">$(ExternalsDir)bzip2-1.0.8\</bz2Dir> <lzmaDir Condition="$(lzmaDir) == ''">$(ExternalsDir)xz-5.2.5\</lzmaDir> <libffiDir Condition="$(libffiDir) == ''">$(ExternalsDir)libffi-3.4.4\</libffiDir> diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 4c799b64c461c1..9df56685b76a87 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -188,7 +188,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.40.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.41.2, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter From 87223f32aba872cfebde6fbe38673799eb79f248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jurica=20Bradari=C4=87?= <jbradaric@users.noreply.github.com> Date: Tue, 2 May 2023 13:38:46 +0200 Subject: [PATCH 38/43] gh-103743: Add PyUnstable_Object_GC_NewWithExtraData (GH-103744) Co-authored-by: Petr Viktorin <encukou@gmail.com> Co-authored-by: Erlend E. Aasland <erlend.aasland@protonmail.com> --- Doc/c-api/gcsupport.rst | 21 +++- Include/cpython/objimpl.h | 3 + Lib/test/test_capi/test_misc.py | 14 +++ ...-04-24-10-31-59.gh-issue-103743.2xYA1K.rst | 2 + Modules/_testcapimodule.c | 105 +++++++++++++++++- Modules/gcmodule.c | 13 +++ 6 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index cb5d64a50487fe..c3260a21bc7f8b 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -59,12 +59,31 @@ rules: Analogous to :c:func:`PyObject_New` but for container objects with the :const:`Py_TPFLAGS_HAVE_GC` flag set. - .. c:function:: TYPE* PyObject_GC_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size) Analogous to :c:func:`PyObject_NewVar` but for container objects with the :const:`Py_TPFLAGS_HAVE_GC` flag set. +.. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size) + + Analogous to :c:func:`PyObject_GC_New` but allocates *extra_size* + bytes at the end of the object (at offset + :c:member:`~PyTypeObject.tp_basicsize`). + The allocated memory is initialized to zeros, + except for the :c:type:`Python object header <PyObject>`. + + The extra data will be deallocated with the object, but otherwise it is + not managed by Python. + + .. warning:: + The function is marked as unstable because the final mechanism + for reserving extra data after an instance is not yet decided. + For allocating a variable number of fields, prefer using + :c:type:`PyVarObject` and :c:member:`~PyTypeObject.tp_itemsize` + instead. + + .. versionadded:: 3.12 + .. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 0b038d31080be9..5a8cdd57c7845b 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -90,3 +90,6 @@ PyAPI_FUNC(int) PyObject_IS_GC(PyObject *obj); PyAPI_FUNC(int) PyType_SUPPORTS_WEAKREFS(PyTypeObject *type); PyAPI_FUNC(PyObject **) PyObject_GET_WEAKREFS_LISTPTR(PyObject *op); + +PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *, + size_t); diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 9470cf12a7d1c4..9d5d1ca6e7dce2 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1043,6 +1043,20 @@ class dictsub(dict): ... # dict subclasses must work self.assertEqual(_testcapi.function_get_kw_defaults(some), None) self.assertEqual(some.__kwdefaults__, None) + def test_unstable_gc_new_with_extra_data(self): + class Data(_testcapi.ObjExtraData): + __slots__ = ('x', 'y') + + d = Data() + d.x = 10 + d.y = 20 + d.extra = 30 + self.assertEqual(d.x, 10) + self.assertEqual(d.y, 20) + self.assertEqual(d.extra, 30) + del d.extra + self.assertIsNone(d.extra) + class TestPendingCalls(unittest.TestCase): diff --git a/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst b/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst new file mode 100644 index 00000000000000..d074350ed3ebbe --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnstable_Object_GC_NewWithExtraData` function that can be used to +allocate additional memory after an object for data not managed by Python. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c1892f6fa0a4b8..a5d23b1b3d50ec 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3363,7 +3363,7 @@ test_gc_visit_objects_basic(PyObject *Py_UNUSED(self), } state.target = obj; state.found = 0; - + PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state); Py_DECREF(obj); if (!state.found) { @@ -3400,6 +3400,98 @@ test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self), Py_RETURN_NONE; } +typedef struct { + PyObject_HEAD +} ObjExtraData; + +static PyObject * +obj_extra_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + size_t extra_size = sizeof(PyObject *); + PyObject *obj = PyUnstable_Object_GC_NewWithExtraData(type, extra_size); + if (obj == NULL) { + return PyErr_NoMemory(); + } + PyObject_GC_Track(obj); + return obj; +} + +static PyObject ** +obj_extra_data_get_extra_storage(PyObject *self) +{ + return (PyObject **)((char *)self + Py_TYPE(self)->tp_basicsize); +} + +static PyObject * +obj_extra_data_get(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + PyObject *value = *extra_storage; + if (!value) { + Py_RETURN_NONE; + } + return Py_NewRef(value); +} + +static int +obj_extra_data_set(PyObject *self, PyObject *newval, void *Py_UNUSED(ignored)) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + Py_CLEAR(*extra_storage); + if (newval) { + *extra_storage = Py_NewRef(newval); + } + return 0; +} + +static PyGetSetDef obj_extra_data_getset[] = { + {"extra", (getter)obj_extra_data_get, (setter)obj_extra_data_set, NULL}, + {NULL} +}; + +static int +obj_extra_data_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + PyObject *value = *extra_storage; + Py_VISIT(value); + return 0; +} + +static int +obj_extra_data_clear(PyObject *self) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + Py_CLEAR(*extra_storage); + return 0; +} + +static void +obj_extra_data_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + obj_extra_data_clear(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot ObjExtraData_Slots[] = { + {Py_tp_getset, obj_extra_data_getset}, + {Py_tp_dealloc, obj_extra_data_dealloc}, + {Py_tp_traverse, obj_extra_data_traverse}, + {Py_tp_clear, obj_extra_data_clear}, + {Py_tp_new, obj_extra_data_new}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec ObjExtraData_TypeSpec = { + .name = "_testcapi.ObjExtraData", + .basicsize = sizeof(ObjExtraData), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .slots = ObjExtraData_Slots, +}; struct atexit_data { int called; @@ -4124,6 +4216,17 @@ PyInit__testcapi(void) Py_INCREF(&MethStatic_Type); PyModule_AddObject(m, "MethStatic", (PyObject *)&MethStatic_Type); + PyObject *ObjExtraData_Type = PyType_FromModuleAndSpec( + m, &ObjExtraData_TypeSpec, NULL); + if (ObjExtraData_Type == 0) { + return NULL; + } + int ret = PyModule_AddType(m, (PyTypeObject*)ObjExtraData_Type); + Py_DECREF(&ObjExtraData_Type); + if (ret < 0) { + return NULL; + } + PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX)); PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN)); PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX)); diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 3fd5f4cd70e832..8a4d1a4398281e 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2367,6 +2367,19 @@ _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) return op; } +PyObject * +PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(_PyObject_SIZE(tp) + extra_size, presize); + if (op == NULL) { + return NULL; + } + memset(op, 0, _PyObject_SIZE(tp) + extra_size); + _PyObject_Init(op, tp); + return op; +} + PyVarObject * _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) { From d81ca7ec029ba05084751c8df64292bb48f4f30f Mon Sep 17 00:00:00 2001 From: Dong-hee Na <donghee.na@python.org> Date: Wed, 3 May 2023 00:05:30 +0900 Subject: [PATCH 39/43] gh-84436: Add integration C API tests for immortal objects (gh-103962) --- Lib/test/test_capi/test_immortal.py | 16 ++++++++++ Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/immortal.c | 47 +++++++++++++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 3 ++ PCbuild/_testcapi.vcxproj | 1 + 6 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_capi/test_immortal.py create mode 100644 Modules/_testcapi/immortal.c diff --git a/Lib/test/test_capi/test_immortal.py b/Lib/test/test_capi/test_immortal.py new file mode 100644 index 00000000000000..ef5d32b7f01935 --- /dev/null +++ b/Lib/test/test_capi/test_immortal.py @@ -0,0 +1,16 @@ +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +class TestCAPI(unittest.TestCase): + def test_immortal_builtins(self): + _testcapi.test_immortal_builtins() + + def test_immortal_small_ints(self): + _testcapi.test_immortal_small_ints() + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index fe1b9f8f5380c1..a90c1e96ef0231 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c _testcapi/immortal.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c new file mode 100644 index 00000000000000..10e1733d08a9ea --- /dev/null +++ b/Modules/_testcapi/immortal.c @@ -0,0 +1,47 @@ +#include "parts.h" + +int verify_immortality(PyObject *object) +{ + assert(_Py_IsImmortal(object)); + Py_ssize_t old_count = Py_REFCNT(object); + for (int j = 0; j < 10000; j++) { + Py_DECREF(object); + } + Py_ssize_t current_count = Py_REFCNT(object); + return old_count == current_count; +} + +static PyObject * +test_immortal_builtins(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *objects[] = {Py_True, Py_False, Py_None, Py_Ellipsis}; + Py_ssize_t n = Py_ARRAY_LENGTH(objects); + for (Py_ssize_t i = 0; i < n; i++) { + assert(verify_immortality(objects[i])); + } + Py_RETURN_NONE; +} + +static PyObject * +test_immortal_small_ints(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + for (int i = -5; i <= 256; i++) { + assert(verify_immortality(PyLong_FromLong(i))); + } + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"test_immortal_builtins", test_immortal_builtins, METH_NOARGS}, + {"test_immortal_small_ints", test_immortal_small_ints, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Immortal(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 60ec81dad2ba9e..4d2d6832a827ae 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -39,6 +39,7 @@ int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_Immortal(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a5d23b1b3d50ec..30b8b6c6b3a87b 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4313,6 +4313,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_PyOS(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Immortal(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 439cd687fda61d..21941247eb9692 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -110,6 +110,7 @@ <ClCompile Include="..\Modules\_testcapi\exceptions.c" /> <ClCompile Include="..\Modules\_testcapi\code.c" /> <ClCompile Include="..\Modules\_testcapi\pyos.c" /> + <ClCompile Include="..\Modules\_testcapi\immortal.c" /> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc" /> From 8611e7bf5ceace998fefcbf26ab1c5d5bc8a0e2a Mon Sep 17 00:00:00 2001 From: Barney Gale <barney.gale@gmail.com> Date: Tue, 2 May 2023 19:08:19 +0100 Subject: [PATCH 40/43] GH-103525: Improve exception message from `pathlib.PurePath()` (GH-103526) Check that arguments are strings before calling `os.path.join()`. Also improve performance of `PurePath(PurePath(...))` while we're in the area: we now use the *unnormalized* string path of such arguments. Co-authored-by: Terry Jan Reedy <tjreedy@udel.edu> --- Lib/pathlib.py | 37 ++++++++++++------- Lib/test/test_pathlib.py | 4 +- ...-04-13-19-43-15.gh-issue-103525.uY4VYg.rst | 2 + 3 files changed, 27 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index f43f01ef41a97f..8eb08949fa9b43 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -300,18 +300,27 @@ def __reduce__(self): return (self.__class__, self.parts) def __init__(self, *args): - if not args: - path = '' - elif len(args) == 1: - path = os.fspath(args[0]) + paths = [] + for arg in args: + if isinstance(arg, PurePath): + path = arg._raw_path + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + if len(paths) == 0: + self._raw_path = '' + elif len(paths) == 1: + self._raw_path = paths[0] else: - path = self._flavour.join(*args) - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - self._raw_path = path + self._raw_path = self._flavour.join(*paths) @classmethod def _parse_path(cls, path): @@ -620,7 +629,7 @@ def joinpath(self, *args): paths) or a totally different path (if one of the arguments is anchored). """ - return self.__class__(self._raw_path, *args) + return self.__class__(self, *args) def __truediv__(self, key): try: @@ -630,7 +639,7 @@ def __truediv__(self, key): def __rtruediv__(self, key): try: - return type(self)(key, self._raw_path) + return type(self)(key, self) except TypeError: return NotImplemented @@ -864,7 +873,7 @@ def absolute(self): cwd = self._flavour.abspath(self.drive) else: cwd = os.getcwd() - return type(self)(cwd, self._raw_path) + return type(self)(cwd, self) def resolve(self, strict=False): """ diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 76cfadeedcea84..8b5b61a818bbbc 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -81,9 +81,9 @@ def test_bytes(self): r"where __fspath__ returns a str, not 'bytes'") with self.assertRaisesRegex(TypeError, message): P(b'a') - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, message): P(b'a', 'b') - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, message): P('a', b'b') with self.assertRaises(TypeError): P('a').joinpath(b'b') diff --git a/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst b/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst new file mode 100644 index 00000000000000..1414cb07dd9155 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst @@ -0,0 +1,2 @@ +Fix misleading exception message when mixed ``str`` and ``bytes`` arguments +are supplied to :class:`pathlib.PurePath` and :class:`~pathlib.Path`. From 587f2f018051049cf5d9de3e12ed5aa7644404dc Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 2 May 2023 11:13:47 -0700 Subject: [PATCH 41/43] gh-65022: Fix description of tuple return value in copyreg (#103892) --- Doc/library/copyreg.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 2107215c0c1967..afc3e66f0bf7ac 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -29,7 +29,7 @@ Such constructors may be factory functions or class instances. Declares that *function* should be used as a "reduction" function for objects of type *type*. *function* must return either a string or a tuple - containing two or five elements. See the :attr:`~pickle.Pickler.dispatch_table` + containing between two and six elements. See the :attr:`~pickle.Pickler.dispatch_table` for more details on the interface of *function*. The *constructor_ob* parameter is a legacy feature and is now ignored, but if From 1f5384434dce013b5dcf7e7ea3ec5312d13bba72 Mon Sep 17 00:00:00 2001 From: Prince Roshan <princekrroshan01@gmail.com> Date: Wed, 3 May 2023 01:43:31 +0530 Subject: [PATCH 42/43] gh-103822: [Calendar] change return value to enum for day and month APIs (GH-103827) --- Lib/calendar.py | 3 +-- .../Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst diff --git a/Lib/calendar.py b/Lib/calendar.py index bbd4fea3b88ca4..ea56f12ccc41d0 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -83,7 +83,6 @@ class Day(IntEnum): SUNDAY = 6 - # Number of days per month (except for February in leap years) mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] @@ -156,7 +155,7 @@ def weekday(year, month, day): """Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31).""" if not datetime.MINYEAR <= year <= datetime.MAXYEAR: year = 2000 + year % 400 - return datetime.date(year, month, day).weekday() + return Day(datetime.date(year, month, day).weekday()) def monthrange(year, month): diff --git a/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst b/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst new file mode 100644 index 00000000000000..3daf9cc093807b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst @@ -0,0 +1 @@ +Update the return type of ``weekday`` to the newly added Day attribute From 47770a1e91d096fd1c689eb0c78b0f9e76b43639 Mon Sep 17 00:00:00 2001 From: Barney Gale <barney.gale@gmail.com> Date: Tue, 2 May 2023 22:51:18 +0100 Subject: [PATCH 43/43] GH-104104: Optimize `pathlib.Path.glob()` by avoiding repeated calls to `os.path.normcase()` (GH-104105) Use `re.IGNORECASE` to implement case-insensitive matching. This restores behaviour from before GH-31691. --- Lib/pathlib.py | 25 +++++++++++-------- ...-05-02-21-05-30.gh-issue-104104.9tjplT.rst | 2 ++ 2 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 8eb08949fa9b43..61e7f3e4430cae 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -59,6 +59,9 @@ def _is_wildcard_pattern(pat): # be looked up directly as a file. return "*" in pat or "?" in pat or "[" in pat +def _is_case_sensitive(flavour): + return flavour.normcase('Aa') == 'Aa' + # # Globbing helpers # @@ -100,15 +103,14 @@ def select_from(self, parent_path): is_dir = path_cls.is_dir exists = path_cls.exists scandir = path_cls._scandir - normcase = path_cls._flavour.normcase if not is_dir(parent_path): return iter([]) - return self._select_from(parent_path, is_dir, exists, scandir, normcase) + return self._select_from(parent_path, is_dir, exists, scandir) class _TerminatingSelector: - def _select_from(self, parent_path, is_dir, exists, scandir, normcase): + def _select_from(self, parent_path, is_dir, exists, scandir): yield parent_path @@ -118,11 +120,11 @@ def __init__(self, name, child_parts, flavour): self.name = name _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir, normcase): + def _select_from(self, parent_path, is_dir, exists, scandir): try: path = parent_path._make_child_relpath(self.name) if (is_dir if self.dironly else exists)(path): - for p in self.successor._select_from(path, is_dir, exists, scandir, normcase): + for p in self.successor._select_from(path, is_dir, exists, scandir): yield p except PermissionError: return @@ -131,10 +133,11 @@ def _select_from(self, parent_path, is_dir, exists, scandir, normcase): class _WildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): - self.match = re.compile(fnmatch.translate(flavour.normcase(pat))).fullmatch + flags = re.NOFLAG if _is_case_sensitive(flavour) else re.IGNORECASE + self.match = re.compile(fnmatch.translate(pat), flags=flags).fullmatch _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir, normcase): + def _select_from(self, parent_path, is_dir, exists, scandir): try: # We must close the scandir() object before proceeding to # avoid exhausting file descriptors when globbing deep trees. @@ -153,9 +156,9 @@ def _select_from(self, parent_path, is_dir, exists, scandir, normcase): raise continue name = entry.name - if self.match(normcase(name)): + if self.match(name): path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, is_dir, exists, scandir, normcase): + for p in self.successor._select_from(path, is_dir, exists, scandir): yield p except PermissionError: return @@ -187,13 +190,13 @@ def _iterate_directories(self, parent_path, is_dir, scandir): except PermissionError: return - def _select_from(self, parent_path, is_dir, exists, scandir, normcase): + def _select_from(self, parent_path, is_dir, exists, scandir): try: yielded = set() try: successor_select = self.successor._select_from for starting_point in self._iterate_directories(parent_path, is_dir, scandir): - for p in successor_select(starting_point, is_dir, exists, scandir, normcase): + for p in successor_select(starting_point, is_dir, exists, scandir): if p not in yielded: yield p yielded.add(p) diff --git a/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst b/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst new file mode 100644 index 00000000000000..935a0e2a2bff18 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst @@ -0,0 +1,2 @@ +Improve performance of :meth:`pathlib.Path.glob` by using +:data:`re.IGNORECASE` to implement case-insensitive matching.