diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 4984170f0d17ff..f4d0411504a832 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -21,6 +21,7 @@ configure* @erlend-aasland @corona10
**/*context* @1st1
**/*genobject* @markshannon
**/*hamt* @1st1
+**/*jit* @brandtbucher
Objects/set* @rhettinger
Objects/dict* @methane @markshannon
Objects/typevarobject.c @JelleZijlstra
@@ -37,7 +38,6 @@ Python/ast_opt.c @isidentical
Python/bytecodes.c @markshannon @gvanrossum
Python/optimizer*.c @markshannon @gvanrossum
Lib/test/test_patma.py @brandtbucher
-Lib/test/test_peepholer.py @brandtbucher
Lib/test/test_type_*.py @JelleZijlstra
Lib/test/test_capi/test_misc.py @markshannon @gvanrossum
Tools/c-analyzer/ @ericsnowcurrently
@@ -45,6 +45,30 @@ Tools/c-analyzer/ @ericsnowcurrently
# dbm
**/*dbm* @corona10 @erlend-aasland @serhiy-storchaka
+# runtime state/lifecycle
+**/*pylifecycle* @ericsnowcurrently
+**/*pystate* @ericsnowcurrently
+**/*preconfig* @ericsnowcurrently
+**/*initconfig* @ericsnowcurrently
+**/*pathconfig* @ericsnowcurrently
+**/*sysmodule* @ericsnowcurrently
+**/*bltinmodule* @ericsnowcurrently
+**/*gil* @ericsnowcurrently
+Include/internal/pycore_runtime.h @ericsnowcurrently
+Include/internal/pycore_interp.h @ericsnowcurrently
+Include/internal/pycore_tstate.h @ericsnowcurrently
+Include/internal/pycore_*_state.h @ericsnowcurrently
+Include/internal/pycore_*_init.h @ericsnowcurrently
+Include/internal/pycore_atexit.h @ericsnowcurrently
+Include/internal/pycore_freelist.h @ericsnowcurrently
+Include/internal/pycore_global_objects.h @ericsnowcurrently
+Include/internal/pycore_obmalloc.h @ericsnowcurrently
+Include/internal/pycore_pymem.h @ericsnowcurrently
+Modules/main.c @ericsnowcurrently
+Programs/_bootstrap_python.c @ericsnowcurrently
+Programs/python.c @ericsnowcurrently
+Tools/build/generate_global_objects.py @ericsnowcurrently
+
# Exceptions
Lib/traceback.py @iritkatriel
Lib/test/test_except*.py @iritkatriel
@@ -79,7 +103,20 @@ Modules/_hacl/** @gpshead
# Import (including importlib).
**/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw
/Python/import.c @kumaraditya303
-**/*importlib/resources/* @jaraco @warsaw @FFY00
+Python/dynload_*.c @ericsnowcurrently
+**/*freeze* @ericsnowcurrently
+**/*frozen* @ericsnowcurrently
+**/*modsupport* @ericsnowcurrently
+**/*modulefinder* @ericsnowcurrently
+**/*moduleobject* @ericsnowcurrently
+**/*multiphase* @ericsnowcurrently
+**/*pkgutil* @ericsnowcurrently
+**/*pythonrun* @ericsnowcurrently
+**/*runpy* @ericsnowcurrently
+**/*singlephase* @ericsnowcurrently
+Lib/test/test_module/ @ericsnowcurrently
+Doc/c-api/module.rst @ericsnowcurrently
+**/*importlib/resources/* @jaraco @warsaw @FFY00
**/importlib/metadata/* @jaraco @warsaw
# Dates and times
@@ -198,6 +235,8 @@ Doc/c-api/stable.rst @encukou
Doc/howto/clinic.rst @erlend-aasland
# Subinterpreters
+**/*interpreteridobject.* @ericsnowcurrently
+**/*crossinterp* @ericsnowcurrently
Lib/test/support/interpreters/ @ericsnowcurrently
Modules/_xx*interp*module.c @ericsnowcurrently
Lib/test/test_interpreters/ @ericsnowcurrently
diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml
new file mode 100644
index 00000000000000..e137fd21b0a0dd
--- /dev/null
+++ b/.github/workflows/jit.yml
@@ -0,0 +1,112 @@
+name: JIT
+on:
+ pull_request:
+ paths: '**jit**'
+ push:
+ paths: '**jit**'
+ workflow_dispatch:
+jobs:
+ jit:
+ name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }})
+ runs-on: ${{ matrix.runner }}
+ strategy:
+ fail-fast: false
+ matrix:
+ target:
+ - i686-pc-windows-msvc/msvc
+ - x86_64-pc-windows-msvc/msvc
+ - x86_64-apple-darwin/clang
+ - x86_64-unknown-linux-gnu/gcc
+ - x86_64-unknown-linux-gnu/clang
+ - aarch64-unknown-linux-gnu/gcc
+ - aarch64-unknown-linux-gnu/clang
+ debug:
+ - true
+ - false
+ llvm:
+ - 16
+ include:
+ - target: i686-pc-windows-msvc/msvc
+ architecture: Win32
+ runner: windows-latest
+ compiler: msvc
+ - target: x86_64-pc-windows-msvc/msvc
+ architecture: x64
+ runner: windows-latest
+ compiler: msvc
+ - target: x86_64-apple-darwin/clang
+ architecture: x86_64
+ runner: macos-latest
+ compiler: clang
+ exclude: test_embed
+ - target: x86_64-unknown-linux-gnu/gcc
+ architecture: x86_64
+ runner: ubuntu-latest
+ compiler: gcc
+ - target: x86_64-unknown-linux-gnu/clang
+ architecture: x86_64
+ runner: ubuntu-latest
+ compiler: clang
+ - target: aarch64-unknown-linux-gnu/gcc
+ architecture: aarch64
+ runner: ubuntu-latest
+ compiler: gcc
+ # These fail because of emulation, not because of the JIT:
+ exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv
+ - target: aarch64-unknown-linux-gnu/clang
+ architecture: aarch64
+ runner: ubuntu-latest
+ compiler: clang
+ # These fail because of emulation, not because of the JIT:
+ exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv
+ env:
+ CC: ${{ matrix.compiler }}
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Windows
+ if: runner.os == 'Windows'
+ run: |
+ choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+ ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
+ ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+
+ - name: macOS
+ if: runner.os == 'macOS'
+ run: |
+ brew install llvm@${{ matrix.llvm }}
+ export SDKROOT="$(xcrun --show-sdk-path)"
+ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+ make all --jobs 3
+ ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+
+ - name: Native Linux
+ if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
+ run: |
+ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+ export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+ make all --jobs 4
+ ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+ - name: Emulated Linux
+ if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
+ run: |
+ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+ export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+ ./configure --prefix="$(pwd)/../build"
+ make install --jobs 4
+ make clean --jobs 4
+ export HOST=${{ matrix.architecture }}-linux-gnu
+ sudo apt install --yes "gcc-$HOST" qemu-user
+ ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }}
+ ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }}
+ export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}"
+ export CPP="$CC --preprocess"
+ export HOSTRUNNER=qemu-${{ matrix.architecture }}
+ export QEMU_LD_PREFIX="/usr/$HOST"
+ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+ make all --jobs 4
+ ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
index 11928e72b9b43a..b766785de405d2 100644
--- a/.github/workflows/mypy.yml
+++ b/.github/workflows/mypy.yml
@@ -12,6 +12,7 @@ on:
- "Tools/build/generate_sbom.py"
- "Tools/cases_generator/**"
- "Tools/clinic/**"
+ - "Tools/jit/**"
- "Tools/peg_generator/**"
- "Tools/requirements-dev.txt"
- "Tools/wasm/**"
@@ -38,6 +39,7 @@ jobs:
"Tools/build/",
"Tools/cases_generator",
"Tools/clinic",
+ "Tools/jit",
"Tools/peg_generator",
"Tools/wasm",
]
diff --git a/.gitignore b/.gitignore
index c424a894c2a6e0..18eb2a9f0632ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -126,6 +126,7 @@ Tools/unicode/data/
# hendrikmuhs/ccache-action@v1
/.ccache
/cross-build/
+/jit_stencils.h
/platform
/profile-clean-stamp
/profile-run-stamp
diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst
index c5350123dfdfdc..4aaf3905e81c8a 100644
--- a/Doc/c-api/conversion.rst
+++ b/Doc/c-api/conversion.rst
@@ -48,6 +48,42 @@ The return value (*rv*) for these functions should be interpreted as follows:
The following functions provide locale-independent string to number conversions.
+.. c:function:: unsigned long PyOS_strtoul(const char *str, char **ptr, int base)
+
+ Convert the initial part of the string in ``str`` to an :c:expr:`unsigned
+ long` value according to the given ``base``, which must be between ``2`` and
+ ``36`` inclusive, or be the special value ``0``.
+
+ Leading white space and case of characters are ignored. If ``base`` is zero
+ it looks for a leading ``0b``, ``0o`` or ``0x`` to tell which base. If
+ these are absent it defaults to ``10``. Base must be 0 or between 2 and 36
+ (inclusive). If ``ptr`` is non-``NULL`` it will contain a pointer to the
+ end of the scan.
+
+ If the converted value falls out of range of corresponding return type,
+ range error occurs (:c:data:`errno` is set to :c:macro:`!ERANGE`) and
+ :c:macro:`!ULONG_MAX` is returned. If no conversion can be performed, ``0``
+ is returned.
+
+ See also the Unix man page :manpage:`strtoul(3)`.
+
+ .. versionadded:: 3.2
+
+
+.. c:function:: long PyOS_strtol(const char *str, char **ptr, int base)
+
+ Convert the initial part of the string in ``str`` to an :c:expr:`long` value
+ according to the given ``base``, which must be between ``2`` and ``36``
+ inclusive, or be the special value ``0``.
+
+ Same as :c:func:`PyOS_strtoul`, but return a :c:expr:`long` value instead
+ and :c:macro:`LONG_MAX` on overflows.
+
+ See also the Unix man page :manpage:`strtol(3)`.
+
+ .. versionadded:: 3.2
+
+
.. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception)
Convert a string ``s`` to a :c:expr:`double`, raising a Python
diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst
index 0a03841e467cad..d3a78c588454e8 100644
--- a/Doc/c-api/file.rst
+++ b/Doc/c-api/file.rst
@@ -65,9 +65,10 @@ the :mod:`io` APIs instead.
Overrides the normal behavior of :func:`io.open_code` to pass its parameter
through the provided handler.
- The handler is a function of type:
+ The *handler* is a function of type:
- .. c:type:: Py_OpenCodeHookFunction
+ .. c:namespace:: NULL
+ .. c:type:: PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *)
Equivalent of :c:expr:`PyObject *(\*)(PyObject *path,
void *userData)`, where *path* is guaranteed to be
diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst
index 2aa43318e7a455..f6038032805259 100644
--- a/Doc/c-api/memoryview.rst
+++ b/Doc/c-api/memoryview.rst
@@ -20,6 +20,17 @@ any other object.
read/write, otherwise it may be either read-only or read/write at the
discretion of the exporter.
+
+.. c:macro:: PyBUF_READ
+
+ Flag to request a readonly buffer.
+
+
+.. c:macro:: PyBUF_WRITE
+
+ Flag to request a writable buffer.
+
+
.. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags)
Create a memoryview object using *mem* as the underlying buffer.
@@ -41,6 +52,8 @@ any other object.
original memory. Otherwise, a copy is made and the memoryview points to a
new bytes object.
+ *buffertype* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`.
+
.. c:function:: int PyMemoryView_Check(PyObject *obj)
diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst
index 4f656779c80b1a..12476412799a4f 100644
--- a/Doc/c-api/object.rst
+++ b/Doc/c-api/object.rst
@@ -229,12 +229,8 @@ Object Protocol
.. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid)
Compare the values of *o1* and *o2* using the operation specified by *opid*,
- which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`,
- :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``,
- ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error,
- ``0`` if the result is false, ``1`` otherwise. This is the equivalent of the
- Python expression ``o1 op o2``, where ``op`` is the operator corresponding to
- *opid*.
+ like :c:func:`PyObject_RichCompare`, but returns ``-1`` on error, ``0`` if
+ the result is false, ``1`` otherwise.
.. note::
If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool`
diff --git a/Doc/conf.py b/Doc/conf.py
index 458954370debe2..c2d57696aeeaa3 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -6,7 +6,9 @@
# The contents of this file are pickled, so don't put values in the namespace
# that aren't pickleable (module imports are okay, they're removed automatically).
-import sys, os, time
+import os
+import sys
+import time
sys.path.append(os.path.abspath('tools/extensions'))
sys.path.append(os.path.abspath('includes'))
@@ -55,7 +57,7 @@
# General substitutions.
project = 'Python'
-copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y')
+copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation"
# We look for the Include/patchlevel.h file in the current Python source tree
# and replace the values accordingly.
@@ -89,22 +91,32 @@
nitpick_ignore = [
# Standard C functions
('c:func', 'calloc'),
+ ('c:func', 'ctime'),
('c:func', 'dlopen'),
('c:func', 'exec'),
('c:func', 'fcntl'),
('c:func', 'fork'),
('c:func', 'free'),
+ ('c:func', 'gettimeofday'),
('c:func', 'gmtime'),
+ ('c:func', 'grantpt'),
+ ('c:func', 'localeconv'),
('c:func', 'localtime'),
('c:func', 'main'),
('c:func', 'malloc'),
+ ('c:func', 'mktime'),
+ ('c:func', 'posix_openpt'),
('c:func', 'printf'),
+ ('c:func', 'ptsname'),
+ ('c:func', 'ptsname_r'),
('c:func', 'realloc'),
('c:func', 'snprintf'),
('c:func', 'sprintf'),
('c:func', 'stat'),
+ ('c:func', 'strftime'),
('c:func', 'system'),
('c:func', 'time'),
+ ('c:func', 'unlockpt'),
('c:func', 'vsnprintf'),
# Standard C types
('c:type', 'FILE'),
@@ -292,6 +304,9 @@
'root_include_title': False # We use the version switcher instead.
}
+if os.getenv("READTHEDOCS"):
+ html_theme_options["hosted_on"] = 'Read the Docs'
+
# Override stylesheet fingerprinting for Windows CHM htmlhelp to fix GH-91207
# https://github.com/python/cpython/issues/91207
if any('htmlhelp' in arg for arg in sys.argv):
@@ -300,7 +315,7 @@
print("It may be removed in the future\n")
# Short title used e.g. for HTML tags.
-html_short_title = '%s Documentation' % release
+html_short_title = f'{release} Documentation'
# Deployment preview information
# (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html)
@@ -349,12 +364,9 @@
latex_engine = 'xelatex'
-# Get LaTeX to handle Unicode correctly
latex_elements = {
-}
-
-# Additional stuff for the LaTeX preamble.
-latex_elements['preamble'] = r'''
+ # For the LaTeX preamble.
+ 'preamble': r'''
\authoraddress{
\sphinxstrong{Python Software Foundation}\\
Email: \sphinxemail{docs@python.org}
@@ -362,13 +374,12 @@
\let\Verbatim=\OriginalVerbatim
\let\endVerbatim=\endOriginalVerbatim
\setcounter{tocdepth}{2}
-'''
-
-# The paper size ('letter' or 'a4').
-latex_elements['papersize'] = 'a4'
-
-# The font size ('10pt', '11pt' or '12pt').
-latex_elements['pointsize'] = '10pt'
+''',
+ # The paper size ('letter' or 'a4').
+ 'papersize': 'a4',
+ # The font size ('10pt', '11pt' or '12pt').
+ 'pointsize': '10pt',
+}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
@@ -431,9 +442,9 @@
# Regexes to find C items in the source files.
coverage_c_regexes = {
- 'cfunction': (r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)'),
- 'data': (r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)'),
- 'macro': (r'^#define ([^_][\w_]+)\(.*\)[\s|\\]'),
+ 'cfunction': r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)',
+ 'data': r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)',
+ 'macro': r'^#define ([^_][\w_]+)\(.*\)[\s|\\]',
}
# The coverage checker will ignore all C items whose names match these regexes
diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat
index 811b1bd84d2417..da28a2be60bc1b 100644
--- a/Doc/data/stable_abi.dat
+++ b/Doc/data/stable_abi.dat
@@ -220,6 +220,7 @@ var,PyExc_GeneratorExit,3.2,,
var,PyExc_IOError,3.2,,
var,PyExc_ImportError,3.2,,
var,PyExc_ImportWarning,3.2,,
+var,PyExc_IncompleteInputError,3.13,,
var,PyExc_IndentationError,3.2,,
var,PyExc_IndexError,3.2,,
var,PyExc_InterruptedError,3.7,,
diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst
index d261e4a4f338a5..762f8b4695b3dd 100644
--- a/Doc/library/__future__.rst
+++ b/Doc/library/__future__.rst
@@ -8,20 +8,68 @@
--------------
-:mod:`__future__` is a real module, and serves three purposes:
+Imports of the form ``from __future__ import feature`` are called
+:ref:`future statements <future>`. These are special-cased by the Python compiler
+to allow the use of new Python features in modules containing the future statement
+before the release in which the feature becomes standard.
+
+While these future statements are given additional special meaning by the
+Python compiler, they are still executed like any other import statement and
+the :mod:`__future__` exists and is handled by the import system the same way
+any other Python module would be. This design serves three purposes:
* To avoid confusing existing tools that analyze import statements and expect to
find the modules they're importing.
-* To ensure that :ref:`future statements <future>` run under releases prior to
- 2.1 at least yield runtime exceptions (the import of :mod:`__future__` will
- fail, because there was no module of that name prior to 2.1).
-
* To document when incompatible changes were introduced, and when they will be
--- or were --- made mandatory. This is a form of executable documentation, and
can be inspected programmatically via importing :mod:`__future__` and examining
its contents.
+* To ensure that :ref:`future statements <future>` run under releases prior to
+ Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__`
+ will fail, because there was no module of that name prior to 2.1).
+
+Module Contents
+---------------
+
+No feature description will ever be deleted from :mod:`__future__`. Since its
+introduction in Python 2.1 the following features have found their way into the
+language using this mechanism:
+
++------------------+-------------+--------------+---------------------------------------------+
+| feature | optional in | mandatory in | effect |
++==================+=============+==============+=============================================+
+| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: |
+| | | | *Statically Nested Scopes* |
++------------------+-------------+--------------+---------------------------------------------+
+| generators | 2.2.0a1 | 2.3 | :pep:`255`: |
+| | | | *Simple Generators* |
++------------------+-------------+--------------+---------------------------------------------+
+| division | 2.2.0a2 | 3.0 | :pep:`238`: |
+| | | | *Changing the Division Operator* |
++------------------+-------------+--------------+---------------------------------------------+
+| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: |
+| | | | *Imports: Multi-Line and Absolute/Relative* |
++------------------+-------------+--------------+---------------------------------------------+
+| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: |
+| | | | *The "with" Statement* |
++------------------+-------------+--------------+---------------------------------------------+
+| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: |
+| | | | *Make print a function* |
++------------------+-------------+--------------+---------------------------------------------+
+| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: |
+| | | | *Bytes literals in Python 3000* |
++------------------+-------------+--------------+---------------------------------------------+
+| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: |
+| | | | *StopIteration handling inside generators* |
++------------------+-------------+--------------+---------------------------------------------+
+| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: |
+| | | | *Postponed evaluation of annotations* |
++------------------+-------------+--------------+---------------------------------------------+
+
+.. XXX Adding a new entry? Remember to update simple_stmts.rst, too.
+
.. _future-classes:
.. class:: _Feature
@@ -65,43 +113,6 @@
dynamically compiled code. This flag is stored in the :attr:`_Feature.compiler_flag`
attribute on :class:`_Feature` instances.
-No feature description will ever be deleted from :mod:`__future__`. Since its
-introduction in Python 2.1 the following features have found their way into the
-language using this mechanism:
-
-+------------------+-------------+--------------+---------------------------------------------+
-| feature | optional in | mandatory in | effect |
-+==================+=============+==============+=============================================+
-| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: |
-| | | | *Statically Nested Scopes* |
-+------------------+-------------+--------------+---------------------------------------------+
-| generators | 2.2.0a1 | 2.3 | :pep:`255`: |
-| | | | *Simple Generators* |
-+------------------+-------------+--------------+---------------------------------------------+
-| division | 2.2.0a2 | 3.0 | :pep:`238`: |
-| | | | *Changing the Division Operator* |
-+------------------+-------------+--------------+---------------------------------------------+
-| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: |
-| | | | *Imports: Multi-Line and Absolute/Relative* |
-+------------------+-------------+--------------+---------------------------------------------+
-| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: |
-| | | | *The "with" Statement* |
-+------------------+-------------+--------------+---------------------------------------------+
-| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: |
-| | | | *Make print a function* |
-+------------------+-------------+--------------+---------------------------------------------+
-| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: |
-| | | | *Bytes literals in Python 3000* |
-+------------------+-------------+--------------+---------------------------------------------+
-| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: |
-| | | | *StopIteration handling inside generators* |
-+------------------+-------------+--------------+---------------------------------------------+
-| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: |
-| | | | *Postponed evaluation of annotations* |
-+------------------+-------------+--------------+---------------------------------------------+
-
-.. XXX Adding a new entry? Remember to update simple_stmts.rst, too.
-
.. [1]
``from __future__ import annotations`` was previously scheduled to
become mandatory in Python 3.10, but the Python Steering Council
@@ -115,3 +126,6 @@ language using this mechanism:
:ref:`future`
How the compiler treats future imports.
+
+ :pep:`236` - Back to the __future__
+ The original proposal for the __future__ mechanism.
diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst
index 3dbef69580d9b3..43a8bd2d7cd133 100644
--- a/Doc/library/atexit.rst
+++ b/Doc/library/atexit.rst
@@ -4,8 +4,8 @@
.. module:: atexit
:synopsis: Register and execute cleanup functions.
-.. moduleauthor:: Skip Montanaro <skip@pobox.com>
-.. sectionauthor:: Skip Montanaro <skip@pobox.com>
+.. moduleauthor:: Skip Montanaro <skip.montanaro@gmail.com>
+.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com>
--------------
diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst
index 6586f539a8da4f..c4dcf5641d6066 100644
--- a/Doc/library/calendar.rst
+++ b/Doc/library/calendar.rst
@@ -586,10 +586,16 @@ The following options are accepted:
or as an HTML document.
+.. option:: --first-weekday WEEKDAY, -f WEEKDAY
+
+ The weekday to start each week.
+ Must be a number between 0 (Monday) and 6 (Sunday).
+ Defaults to 0.
+
+
.. option:: year
The year to print the calendar for.
- Must be a number between 1 and 9999.
Defaults to the current year.
diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst
index 7a5589e68b3052..07f38f5690bb54 100644
--- a/Doc/library/csv.rst
+++ b/Doc/library/csv.rst
@@ -4,7 +4,7 @@
.. module:: csv
:synopsis: Write and read tabular data to and from delimited files.
-.. sectionauthor:: Skip Montanaro <skip@pobox.com>
+.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com>
**Source code:** :source:`Lib/csv.py`
diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst
index cde147d1cbb501..4ada69d63abada 100644
--- a/Doc/library/dataclasses.rst
+++ b/Doc/library/dataclasses.rst
@@ -141,7 +141,7 @@ Module contents
then :func:`dataclass` *may* add an implicit :meth:`~object.__hash__` method.
Although not recommended, you can force :func:`dataclass` to create a
:meth:`~object.__hash__` method with ``unsafe_hash=True``. This might be the case
- if your class is logically immutable but can nonetheless be mutated.
+ if your class is logically immutable but can still be mutated.
This is a specialized use case and should be considered carefully.
Here are the rules governing implicit creation of a :meth:`~object.__hash__`
@@ -538,8 +538,8 @@ that has to be called, it is common to call this method in a
class Rectangle:
def __init__(self, height, width):
- self.height = height
- self.width = width
+ self.height = height
+ self.width = width
@dataclass
class Square(Rectangle):
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index b36f8c19cd6040..4ff049c8709289 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -14,7 +14,7 @@
.. XXX what order should the types be discussed in?
-The :mod:`datetime` module supplies classes for manipulating dates and times.
+The :mod:`!datetime` module supplies classes for manipulating dates and times.
While date and time arithmetic is supported, the focus of the implementation is
on efficient attribute extraction for output formatting and manipulation.
@@ -70,7 +70,7 @@ These :class:`tzinfo` objects capture information about the offset from UTC
time, the time zone name, and whether daylight saving time is in effect.
Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is
-supplied by the :mod:`datetime` module. The :class:`timezone` class can
+supplied by the :mod:`!datetime` module. The :class:`timezone` class can
represent simple timezones with fixed offsets from UTC, such as UTC itself or
North American EST and EDT timezones. Supporting timezones at deeper levels of
detail is up to the application. The rules for time adjustment across the
@@ -80,7 +80,7 @@ standard suitable for every application aside from UTC.
Constants
---------
-The :mod:`datetime` module exports the following constants:
+The :mod:`!datetime` module exports the following constants:
.. data:: MINYEAR
@@ -631,7 +631,7 @@ Notes:
date2.toordinal()``. Date comparison raises :exc:`TypeError` if
the other comparand isn't also a :class:`date` object. However,
``NotImplemented`` is returned instead if the other comparand has a
- :meth:`timetuple` attribute. This hook gives other kinds of date objects a
+ :attr:`~date.timetuple` attribute. This hook gives other kinds of date objects a
chance at implementing mixed-type comparison. If not, when a :class:`date`
object is compared to an object of a different type, :exc:`TypeError` is raised
unless the comparison is ``==`` or ``!=``. The latter cases return
@@ -1215,7 +1215,7 @@ Supported operations:
object addresses, datetime comparison normally raises :exc:`TypeError` if the
other comparand isn't also a :class:`.datetime` object. However,
``NotImplemented`` is returned instead if the other comparand has a
- :meth:`timetuple` attribute. This hook gives other kinds of date objects a
+ :attr:`~.datetime.timetuple` attribute. This hook gives other kinds of date objects a
chance at implementing mixed-type comparison. If not, when a :class:`.datetime`
object is compared to an object of a different type, :exc:`TypeError` is raised
unless the comparison is ``==`` or ``!=``. The latter cases return
@@ -1255,7 +1255,7 @@ Instance methods:
``tzinfo=None`` can be specified to create a naive datetime from an aware
datetime with no conversion of date and time data.
- :class:`datetime` objects are also supported by generic function
+ :class:`.datetime` objects are also supported by generic function
:func:`copy.replace`.
.. versionchanged:: 3.6
@@ -1347,22 +1347,22 @@ Instance methods:
where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1``
is the day number within the current year starting with ``1`` for January
- 1st. The :attr:`tm_isdst` flag of the result is set according to the
+ 1st. The :attr:`~time.struct_time.tm_isdst` flag of the result is set according to the
:meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns
- ``None``, :attr:`tm_isdst` is set to ``-1``; else if :meth:`dst` returns a
- non-zero value, :attr:`tm_isdst` is set to ``1``; else :attr:`tm_isdst` is
+ ``None``, :attr:`!tm_isdst` is set to ``-1``; else if :meth:`dst` returns a
+ non-zero value, :attr:`!tm_isdst` is set to ``1``; else :attr:`!tm_isdst` is
set to ``0``.
.. method:: datetime.utctimetuple()
If :class:`.datetime` instance *d* is naive, this is the same as
- ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what
+ ``d.timetuple()`` except that :attr:`~.time.struct_time.tm_isdst` is forced to 0 regardless of what
``d.dst()`` returns. DST is never in effect for a UTC time.
If *d* is aware, *d* is normalized to UTC time, by subtracting
``d.utcoffset()``, and a :class:`time.struct_time` for the
- normalized time is returned. :attr:`tm_isdst` is forced to 0. Note
+ normalized time is returned. :attr:`!tm_isdst` is forced to 0. Note
that an :exc:`OverflowError` may be raised if *d*.year was
``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year
boundary.
@@ -1550,7 +1550,7 @@ Instance methods:
Examples of Usage: :class:`.datetime`
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Examples of working with :class:`~datetime.datetime` objects:
+Examples of working with :class:`.datetime` objects:
.. doctest::
@@ -1678,7 +1678,7 @@ Usage of ``KabulTz`` from above::
:class:`.time` Objects
----------------------
-A :class:`time` object represents a (local) time of day, independent of any particular
+A :class:`.time` object represents a (local) time of day, independent of any particular
day, and subject to adjustment via a :class:`tzinfo` object.
.. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0)
@@ -1761,9 +1761,9 @@ is aware, :exc:`TypeError` is raised if an order comparison is attempted. For eq
comparisons, naive instances are never equal to aware instances.
If both comparands are aware, and have
-the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is
+the same :attr:`~.time.tzinfo` attribute, the common :attr:`!tzinfo` attribute is
ignored and the base times are compared. If both comparands are aware and
-have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by
+have different :attr:`!tzinfo` attributes, the comparands are first adjusted by
subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order
to stop mixed-type comparisons from falling back to the default comparison by
object address, when a :class:`.time` object is compared to an object of a
@@ -1771,7 +1771,7 @@ different type, :exc:`TypeError` is raised unless the comparison is ``==`` or
``!=``. The latter cases return :const:`False` or :const:`True`, respectively.
.. versionchanged:: 3.3
- Equality comparisons between aware and naive :class:`~datetime.time` instances
+ Equality comparisons between aware and naive :class:`.time` instances
don't raise :exc:`TypeError`.
In Boolean contexts, a :class:`.time` object is always considered to be true.
@@ -1836,7 +1836,7 @@ Instance methods:
``tzinfo=None`` can be specified to create a naive :class:`.time` from an
aware :class:`.time`, without conversion of the time data.
- :class:`time` objects are also supported by generic function
+ :class:`.time` objects are also supported by generic function
:func:`copy.replace`.
.. versionchanged:: 3.6
@@ -1981,7 +1981,7 @@ Examples of working with a :class:`.time` object::
You need to derive a concrete subclass, and (at least)
supply implementations of the standard :class:`tzinfo` methods needed by the
- :class:`.datetime` methods you use. The :mod:`datetime` module provides
+ :class:`.datetime` methods you use. The :mod:`!datetime` module provides
:class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can
represent timezones with fixed offset from UTC such as UTC itself or North
American EST and EDT.
@@ -1994,7 +1994,7 @@ Examples of working with a :class:`.time` object::
A concrete subclass of :class:`tzinfo` may need to implement the following
methods. Exactly which methods are needed depends on the uses made of aware
- :mod:`datetime` objects. If in doubt, simply implement all of them.
+ :mod:`!datetime` objects. If in doubt, simply implement all of them.
.. method:: tzinfo.utcoffset(dt)
@@ -2035,7 +2035,7 @@ Examples of working with a :class:`.time` object::
already been added to the UTC offset returned by :meth:`utcoffset`, so there's
no need to consult :meth:`dst` unless you're interested in obtaining DST info
separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo`
- attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag
+ attribute's :meth:`dst` method to determine how the :attr:`~time.struct_time.tm_isdst` flag
should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for
DST changes when crossing time zones.
@@ -2051,7 +2051,7 @@ Examples of working with a :class:`.time` object::
relies on this, but cannot detect violations; it's the programmer's
responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee
this, it may be able to override the default implementation of
- :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless.
+ :meth:`tzinfo.fromutc` to work correctly with :meth:`~.datetime.astimezone` regardless.
Most implementations of :meth:`dst` will probably look like one of these two::
@@ -2080,7 +2080,7 @@ Examples of working with a :class:`.time` object::
.. method:: tzinfo.tzname(dt)
Return the time zone name corresponding to the :class:`.datetime` object *dt*, as
- a string. Nothing about string names is defined by the :mod:`datetime` module,
+ a string. Nothing about string names is defined by the :mod:`!datetime` module,
and there's no requirement that it mean anything in particular. For example,
"GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all
valid replies. Return ``None`` if a string name isn't known. Note that this is
@@ -2128,7 +2128,7 @@ There is one more :class:`tzinfo` method that a subclass may wish to override:
different years. An example of a time zone the default :meth:`fromutc`
implementation may not handle correctly in all cases is one where the standard
offset (from UTC) depends on the specific date and time passed, which can happen
- for political reasons. The default implementations of :meth:`astimezone` and
+ for political reasons. The default implementations of :meth:`~.datetime.astimezone` and
:meth:`fromutc` may not produce the result you want if the result is one of the
hours straddling the moment the standard offset changes.
@@ -2194,10 +2194,10 @@ hour that can't be spelled unambiguously in local wall time: the last hour of
daylight time. In Eastern, that's times of the form 5:MM UTC on the day
daylight time ends. The local wall clock leaps from 1:59 (daylight time) back
to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous.
-:meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC
+:meth:`~.datetime.astimezone` mimics the local clock's behavior by mapping two adjacent UTC
hours into the same local hour then. In the Eastern example, UTC times of the
form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times
-have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1.
+have the :attr:`~.datetime.fold` attribute set to 0 and the later times have it set to 1.
For example, at the Fall back transition of 2016, we get::
>>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc)
@@ -2212,10 +2212,10 @@ For example, at the Fall back transition of 2016, we get::
07:00:00 UTC = 02:00:00 EST 0
Note that the :class:`.datetime` instances that differ only by the value of the
-:attr:`~datetime.fold` attribute are considered equal in comparisons.
+:attr:`~.datetime.fold` attribute are considered equal in comparisons.
Applications that can't bear wall-time ambiguities should explicitly check the
-value of the :attr:`~datetime.fold` attribute or avoid using hybrid
+value of the :attr:`~.datetime.fold` attribute or avoid using hybrid
:class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`,
or any other fixed-offset :class:`tzinfo` subclass (such as a class representing
only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)).
@@ -2223,7 +2223,7 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)).
.. seealso::
:mod:`zoneinfo`
- The :mod:`datetime` module has a basic :class:`timezone` class (for
+ The :mod:`!datetime` module has a basic :class:`timezone` class (for
handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc`
attribute (a UTC timezone instance).
@@ -2241,7 +2241,7 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)).
.. _datetime-timezone:
:class:`timezone` Objects
---------------------------
+-------------------------
The :class:`timezone` class is a subclass of :class:`tzinfo`, each
instance of which represents a timezone defined by a fixed offset from
@@ -2316,8 +2316,8 @@ Class attributes:
.. _strftime-strptime-behavior:
-:meth:`strftime` and :meth:`strptime` Behavior
-----------------------------------------------
+:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior
+--------------------------------------------------------------------
:class:`date`, :class:`.datetime`, and :class:`.time` objects all support a
``strftime(format)`` method, to create a string representing the time under the
@@ -2327,8 +2327,8 @@ Conversely, the :meth:`datetime.strptime` class method creates a
:class:`.datetime` object from a string representing a date and time and a
corresponding format string.
-The table below provides a high-level comparison of :meth:`strftime`
-versus :meth:`strptime`:
+The table below provides a high-level comparison of :meth:`~.datetime.strftime`
+versus :meth:`~.datetime.strptime`:
+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+
| | ``strftime`` | ``strptime`` |
@@ -2345,8 +2345,8 @@ versus :meth:`strptime`:
.. _format-codes:
-:meth:`strftime` and :meth:`strptime` Format Codes
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
These methods accept format codes that can be used to parse and format dates::
@@ -2485,13 +2485,13 @@ convenience. These parameters all correspond to ISO 8601 date values.
| | naive). | -03:07:12.345216 | |
+-----------+--------------------------------+------------------------+-------+
-These may not be available on all platforms when used with the :meth:`strftime`
+These may not be available on all platforms when used with the :meth:`~.datetime.strftime`
method. The ISO 8601 year and ISO 8601 week directives are not interchangeable
-with the year and week number directives above. Calling :meth:`strptime` with
+with the year and week number directives above. Calling :meth:`~.datetime.strptime` with
incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`.
The full set of format codes supported varies across platforms, because Python
-calls the platform C library's :func:`strftime` function, and platform
+calls the platform C library's :c:func:`strftime` function, and platform
variations are common. To see the full set of format codes supported on your
platform, consult the :manpage:`strftime(3)` documentation. There are also
differences between platforms in handling of unsupported format specifiers.
@@ -2507,9 +2507,9 @@ Technical Detail
Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's
``time.strftime(fmt, d.timetuple())`` although not all objects support a
-:meth:`timetuple` method.
+:meth:`~date.timetuple` method.
-For the :meth:`datetime.strptime` class method, the default value is
+For the :meth:`.datetime.strptime` class method, the default value is
``1900-01-01T00:00:00.000``: any components not specified in the format string
will be pulled from the default value. [#]_
@@ -2522,7 +2522,7 @@ information, which are supported in ``datetime.strptime`` but are discarded by
``time.strptime``.
For :class:`.time` objects, the format codes for year, month, and day should not
-be used, as :class:`time` objects have no such values. If they're used anyway,
+be used, as :class:`!time` objects have no such values. If they're used anyway,
``1900`` is substituted for the year, and ``1`` for the month and day.
For :class:`date` objects, the format codes for hours, minutes, seconds, and
@@ -2544,27 +2544,27 @@ Notes:
contain non-ASCII characters.
(2)
- The :meth:`strptime` method can parse years in the full [1, 9999] range, but
+ The :meth:`~.datetime.strptime` method can parse years in the full [1, 9999] range, but
years < 1000 must be zero-filled to 4-digit width.
.. versionchanged:: 3.2
- In previous versions, :meth:`strftime` method was restricted to
+ In previous versions, :meth:`~.datetime.strftime` method was restricted to
years >= 1900.
.. versionchanged:: 3.3
- In version 3.2, :meth:`strftime` method was restricted to
+ In version 3.2, :meth:`~.datetime.strftime` method was restricted to
years >= 1000.
(3)
- When used with the :meth:`strptime` method, the ``%p`` directive only affects
+ When used with the :meth:`~.datetime.strptime` method, the ``%p`` directive only affects
the output hour field if the ``%I`` directive is used to parse the hour.
(4)
- Unlike the :mod:`time` module, the :mod:`datetime` module does not support
+ Unlike the :mod:`time` module, the :mod:`!datetime` module does not support
leap seconds.
(5)
- When used with the :meth:`strptime` method, the ``%f`` directive
+ When used with the :meth:`~.datetime.strptime` method, the ``%f`` directive
accepts from one to six digits and zero pads on the right. ``%f`` is
an extension to the set of format characters in the C standard (but
implemented separately in datetime objects, and therefore always
@@ -2577,7 +2577,7 @@ Notes:
For an aware object:
``%z``
- :meth:`utcoffset` is transformed into a string of the form
+ :meth:`~.datetime.utcoffset` is transformed into a string of the form
``±HHMM[SS[.ffffff]]``, where ``HH`` is a 2-digit string giving the number
of UTC offset hours, ``MM`` is a 2-digit string giving the number of UTC
offset minutes, ``SS`` is a 2-digit string giving the number of UTC offset
@@ -2585,14 +2585,14 @@ Notes:
offset microseconds. The ``ffffff`` part is omitted when the offset is a
whole number of seconds and both the ``ffffff`` and the ``SS`` part is
omitted when the offset is a whole number of minutes. For example, if
- :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is
+ :meth:`~.datetime.utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is
replaced with the string ``'-0330'``.
.. versionchanged:: 3.7
The UTC offset is not restricted to a whole number of minutes.
.. versionchanged:: 3.7
- When the ``%z`` directive is provided to the :meth:`strptime` method,
+ When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method,
the UTC offsets can have a colon as a separator between hours, minutes
and seconds.
For example, ``'+01:00:00'`` will be parsed as an offset of one hour.
@@ -2603,11 +2603,11 @@ Notes:
hours, minutes and seconds.
``%Z``
- In :meth:`strftime`, ``%Z`` is replaced by an empty string if
- :meth:`tzname` returns ``None``; otherwise ``%Z`` is replaced by the
+ In :meth:`~.datetime.strftime`, ``%Z`` is replaced by an empty string if
+ :meth:`~.datetime.tzname` returns ``None``; otherwise ``%Z`` is replaced by the
returned value, which must be a string.
- :meth:`strptime` only accepts certain values for ``%Z``:
+ :meth:`~.datetime.strptime` only accepts certain values for ``%Z``:
1. any value in ``time.tzname`` for your machine's locale
2. the hard-coded values ``UTC`` and ``GMT``
@@ -2617,23 +2617,23 @@ Notes:
invalid values.
.. versionchanged:: 3.2
- When the ``%z`` directive is provided to the :meth:`strptime` method, an
+ When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, an
aware :class:`.datetime` object will be produced. The ``tzinfo`` of the
result will be set to a :class:`timezone` instance.
(7)
- When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used
+ When used with the :meth:`~.datetime.strptime` method, ``%U`` and ``%W`` are only used
in calculations when the day of the week and the calendar year (``%Y``)
are specified.
(8)
Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the
day of the week and the ISO year (``%G``) are specified in a
- :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not
+ :meth:`~.datetime.strptime` format string. Also note that ``%G`` and ``%Y`` are not
interchangeable.
(9)
- When used with the :meth:`strptime` method, the leading zero is optional
+ When used with the :meth:`~.datetime.strptime` method, the leading zero is optional
for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``,
``%W``, and ``%V``. Format ``%y`` does require a leading zero.
diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst
index 37ced603a13f3a..8dc8bc0ca72aab 100644
--- a/Doc/library/dbm.rst
+++ b/Doc/library/dbm.rst
@@ -63,41 +63,33 @@ the Oracle Berkeley DB.
.. function:: open(file, flag='r', mode=0o666)
- Open the database file *file* and return a corresponding object.
+ Open a database and return the corresponding database object.
- If the database file already exists, the :func:`whichdb` function is used to
- determine its type and the appropriate module is used; if it does not exist,
- the first module listed above that can be imported is used.
+ :param file:
+ The database file to open.
- The optional *flag* argument can be:
+ If the database file already exists, the :func:`whichdb` function is used to
+ determine its type and the appropriate module is used; if it does not exist,
+ the first submodule listed above that can be imported is used.
+ :type file: :term:`path-like object`
- .. csv-table::
- :header: "Value", "Meaning"
-
- ``'r'`` (default), |flag_r|
- ``'w'``, |flag_w|
- ``'c'``, |flag_c|
- ``'n'``, |flag_n|
+ :param str flag:
+ * ``'r'`` (default), |flag_r|
+ * ``'w'``, |flag_w|
+ * ``'c'``, |flag_c|
+ * ``'n'``, |flag_n|
- The optional *mode* argument is the Unix mode of the file, used only when the
- database has to be created. It defaults to octal ``0o666`` (and will be
- modified by the prevailing umask).
+ :param int mode:
+ The Unix file access mode of the file (default: octal ``0o666``),
+ used only when the database has to be created.
.. versionchanged:: 3.11
*file* accepts a :term:`path-like object`.
-
-The object returned by :func:`open` supports the same basic functionality as a
+The object returned by :func:`~dbm.open` supports the same basic functionality as a
:class:`dict`; keys and their corresponding values can be stored, retrieved, and
deleted, and the :keyword:`in` operator and the :meth:`!keys` method are
-available, as well as :meth:`!get` and :meth:`!setdefault`.
-
-.. versionchanged:: 3.2
- :meth:`!get` and :meth:`!setdefault` are now available in all database modules.
-
-.. versionchanged:: 3.8
- Deleting a key from a read-only database raises database module specific error
- instead of :exc:`KeyError`.
+available, as well as :meth:`!get` and :meth:`!setdefault` methods.
Key and values are always stored as :class:`bytes`. This means that when
strings are used they are implicitly converted to the default encoding before
@@ -106,9 +98,17 @@ being stored.
These objects also support being used in a :keyword:`with` statement, which
will automatically close them when done.
+.. versionchanged:: 3.2
+ :meth:`!get` and :meth:`!setdefault` methods are now available for all
+ :mod:`dbm` backends.
+
.. versionchanged:: 3.4
Added native support for the context management protocol to the objects
- returned by :func:`.open`.
+ returned by :func:`~dbm.open`.
+
+.. versionchanged:: 3.8
+ Deleting a key from a read-only database raises a database module specific exception
+ instead of :exc:`KeyError`.
The following example records some hostnames and a corresponding title, and
then prints out the contents of the database::
@@ -399,13 +399,14 @@ This module can be used with the "classic" NDBM interface or the
--------------
-The :mod:`dbm.dumb` module provides a persistent dictionary-like interface which
-is written entirely in Python. Unlike other modules such as :mod:`dbm.gnu` no
-external library is required. As with other persistent mappings, the keys and
-values are always stored as bytes.
-
-The module defines the following:
+The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like
+interface which is written entirely in Python.
+Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no
+external library is required.
+As with other :mod:`dbm` backends,
+the keys and values are always stored as :class:`bytes`.
+The :mod:`!dbm.dumb` module defines the following:
.. exception:: error
@@ -413,26 +414,33 @@ The module defines the following:
raised for general mapping errors like specifying an incorrect key.
-.. function:: open(filename[, flag[, mode]])
+.. function:: open(filename, flag="c", mode=0o666)
- Open a ``dumbdbm`` database and return a dumbdbm object. The *filename* argument is
- the basename of the database file (without any specific extensions). When a
- dumbdbm database is created, files with :file:`.dat` and :file:`.dir` extensions
- are created.
+ Open a :mod:`!dbm.dumb` database.
+ The returned database object behaves similar to a :term:`mapping`,
+ in addition to providing :meth:`~dumbdbm.sync` and :meth:`~dumbdbm.close`
+ methods.
- The optional *flag* argument can be:
+ :param filename:
+ The basename of the database file (without extensions).
+ A new database creates the following files:
- .. csv-table::
- :header: "Value", "Meaning"
+ - :file:`{filename}.dat`
+ - :file:`{filename}.dir`
+ :type database: :term:`path-like object`
- ``'r'``, |flag_r|
- ``'w'``, |flag_w|
- ``'c'`` (default), |flag_c|
- ``'n'``, |flag_n|
+ :param str flag:
+ .. csv-table::
+ :header: "Value", "Meaning"
- The optional *mode* argument is the Unix mode of the file, used only when the
- database has to be created. It defaults to octal ``0o666`` (and will be modified
- by the prevailing umask).
+ ``'r'``, |flag_r|
+ ``'w'``, |flag_w|
+ ``'c'`` (default), |flag_c|
+ ``'n'``, |flag_n|
+
+ :param int mode:
+ The Unix file access mode of the file (default: ``0o666``),
+ used only when the database has to be created.
.. warning::
It is possible to crash the Python interpreter when loading a database
@@ -440,20 +448,18 @@ The module defines the following:
Python's AST compiler.
.. versionchanged:: 3.5
- :func:`.open` always creates a new database when the flag has the value
- ``'n'``.
+ :func:`open` always creates a new database when *flag* is ``'n'``.
.. versionchanged:: 3.8
- A database opened with flags ``'r'`` is now read-only. Opening with
- flags ``'r'`` and ``'w'`` no longer creates a database if it does not
- exist.
+ A database opened read-only if *flag* is ``'r'``.
+ A database is not created if it does not exist if *flag* is ``'r'`` or ``'w'``.
.. versionchanged:: 3.11
- Accepts :term:`path-like object` for filename.
+ *filename* accepts a :term:`path-like object`.
In addition to the methods provided by the
- :class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects
- provide the following methods:
+ :class:`collections.abc.MutableMapping` class,
+ the following methods are provided:
.. method:: dumbdbm.sync()
@@ -462,5 +468,5 @@ The module defines the following:
.. method:: dumbdbm.close()
- Close the ``dumbdbm`` database.
+ Close the database.
diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst
index d7c0d203d191f8..dc0dd3b9eebde6 100644
--- a/Doc/library/email.mime.rst
+++ b/Doc/library/email.mime.rst
@@ -28,7 +28,7 @@ make things easier.
Here are the classes:
-.. currentmodule:: email.mime.base
+.. module:: email.mime.base
.. class:: MIMEBase(_maintype, _subtype, *, policy=compat32, **_params)
@@ -58,7 +58,7 @@ Here are the classes:
Added *policy* keyword-only parameter.
-.. currentmodule:: email.mime.nonmultipart
+.. module:: email.mime.nonmultipart
.. class:: MIMENonMultipart()
@@ -72,7 +72,7 @@ Here are the classes:
is called, a :exc:`~email.errors.MultipartConversionError` exception is raised.
-.. currentmodule:: email.mime.multipart
+.. module:: email.mime.multipart
.. class:: MIMEMultipart(_subtype='mixed', boundary=None, _subparts=None, \
*, policy=compat32, **_params)
@@ -104,7 +104,7 @@ Here are the classes:
.. versionchanged:: 3.6
Added *policy* keyword-only parameter.
-.. currentmodule:: email.mime.application
+.. module:: email.mime.application
.. class:: MIMEApplication(_data, _subtype='octet-stream', \
_encoder=email.encoders.encode_base64, \
@@ -135,7 +135,7 @@ Here are the classes:
.. versionchanged:: 3.6
Added *policy* keyword-only parameter.
-.. currentmodule:: email.mime.audio
+.. module:: email.mime.audio
.. class:: MIMEAudio(_audiodata, _subtype=None, \
_encoder=email.encoders.encode_base64, \
@@ -169,7 +169,7 @@ Here are the classes:
.. versionchanged:: 3.6
Added *policy* keyword-only parameter.
-.. currentmodule:: email.mime.image
+.. module:: email.mime.image
.. class:: MIMEImage(_imagedata, _subtype=None, \
_encoder=email.encoders.encode_base64, \
@@ -205,7 +205,7 @@ Here are the classes:
.. versionchanged:: 3.6
Added *policy* keyword-only parameter.
-.. currentmodule:: email.mime.message
+.. module:: email.mime.message
.. class:: MIMEMessage(_msg, _subtype='rfc822', *, policy=compat32)
@@ -225,7 +225,7 @@ Here are the classes:
.. versionchanged:: 3.6
Added *policy* keyword-only parameter.
-.. currentmodule:: email.mime.text
+.. module:: email.mime.text
.. class:: MIMEText(_text, _subtype='plain', _charset=None, *, policy=compat32)
diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst
index 1f774e64b0eae3..d5c868def3b64f 100644
--- a/Doc/library/imaplib.rst
+++ b/Doc/library/imaplib.rst
@@ -531,7 +531,7 @@ An :class:`IMAP4` instance has the following methods:
allowed creation of such tags, and popular IMAP servers, such as Gmail,
accept and produce such flags. There are non-Python programs which also
create such tags. Although it is an RFC violation and IMAP clients and
- servers are supposed to be strict, imaplib nonetheless continues to allow
+ servers are supposed to be strict, imaplib still continues to allow
such tags to be created for backward compatibility reasons, and as of
Python 3.6, handles them if they are sent from the server, since this
improves real-world compatibility.
diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst
index fa5b273093f583..a613548c9e518e 100644
--- a/Doc/library/mailbox.rst
+++ b/Doc/library/mailbox.rst
@@ -1136,8 +1136,8 @@ When a :class:`!MaildirMessage` instance is created based upon a
leading "From " or trailing newline. For convenience, *time_* may be
specified and will be formatted appropriately and appended to *from_*. If
*time_* is specified, it should be a :class:`time.struct_time` instance, a
- tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use
- :meth:`time.gmtime`).
+ tuple suitable for passing to :func:`time.strftime`, or ``True`` (to use
+ :func:`time.gmtime`).
.. method:: get_flags()
@@ -1508,8 +1508,8 @@ When a :class:`!BabylMessage` instance is created based upon an
leading "From " or trailing newline. For convenience, *time_* may be
specified and will be formatted appropriately and appended to *from_*. If
*time_* is specified, it should be a :class:`time.struct_time` instance, a
- tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use
- :meth:`time.gmtime`).
+ tuple suitable for passing to :func:`time.strftime`, or ``True`` (to use
+ :func:`time.gmtime`).
.. method:: get_flags()
diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst
index 3cab7a260df008..34bc76b231de92 100644
--- a/Doc/library/os.path.rst
+++ b/Doc/library/os.path.rst
@@ -326,6 +326,28 @@ the :mod:`glob` module.)
.. versionadded:: 3.12
+.. function:: isreserved(path)
+
+ Return ``True`` if *path* is a reserved pathname on the current system.
+
+ On Windows, reserved filenames include those that end with a space or dot;
+ those that contain colons (i.e. file streams such as "name:stream"),
+ wildcard characters (i.e. ``'*?"<>'``), pipe, or ASCII control characters;
+ as well as DOS device names such as "NUL", "CON", "CONIN$", "CONOUT$",
+ "AUX", "PRN", "COM1", and "LPT1".
+
+ .. note::
+
+ This function approximates rules for reserved paths on most Windows
+ systems. These rules change over time in various Windows releases.
+ This function may be updated in future Python releases as changes to
+ the rules become broadly available.
+
+ .. availability:: Windows.
+
+ .. versionadded:: 3.13
+
+
.. function:: join(path, *paths)
Join one or more path segments intelligently. The return value is the
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index 0008ec6a40c76f..cc9f3e75a80c51 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -1122,6 +1122,20 @@ as internal buffering of data.
.. versionchanged:: 3.12
Added support for pipes on Windows.
+
+.. function:: grantpt(fd, /)
+
+ Grant access to the slave pseudo-terminal device associated with the
+ master pseudo-terminal device to which the file descriptor *fd* refers.
+ The file descriptor *fd* is not closed upon failure.
+
+ Calls the C standard library function :c:func:`grantpt`.
+
+ .. availability:: Unix, not Emscripten, not WASI.
+
+ .. versionadded:: 3.13
+
+
.. function:: isatty(fd, /)
Return ``True`` if the file descriptor *fd* is open and connected to a
@@ -1429,6 +1443,23 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo
.. versionadded:: 3.3
+.. function:: posix_openpt(oflag, /)
+
+ Open and return a file descriptor for a master pseudo-terminal device.
+
+ Calls the C standard library function :c:func:`posix_openpt`. The *oflag*
+ argument is used to set file status flags and file access modes as
+ specified in the manual page of :c:func:`posix_openpt` of your system.
+
+ The returned file descriptor is :ref:`non-inheritable <fd_inheritance>`.
+ If the value :data:`O_CLOEXEC` is available on the system, it is added to
+ *oflag*.
+
+ .. availability:: Unix, not Emscripten, not WASI.
+
+ .. versionadded:: 3.13
+
+
.. function:: preadv(fd, buffers, offset, flags=0, /)
Read from a file descriptor *fd* at a position of *offset* into mutable
@@ -1486,6 +1517,21 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo
.. versionadded:: 3.7
+.. function:: ptsname(fd, /)
+
+ Return the name of the slave pseudo-terminal device associated with the
+ master pseudo-terminal device to which the file descriptor *fd* refers.
+ The file descriptor *fd* is not closed upon failure.
+
+ Calls the reentrant C standard library function :c:func:`ptsname_r` if
+ it is available; otherwise, the C standard library function
+ :c:func:`ptsname`, which is not guaranteed to be thread-safe, is called.
+
+ .. availability:: Unix, not Emscripten, not WASI.
+
+ .. versionadded:: 3.13
+
+
.. function:: pwrite(fd, str, offset, /)
Write the bytestring in *str* to file descriptor *fd* at position of
@@ -1738,6 +1784,19 @@ or `the MSDN <https://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windo
.. availability:: Unix.
+.. function:: unlockpt(fd, /)
+
+ Unlock the slave pseudo-terminal device associated with the master
+ pseudo-terminal device to which the file descriptor *fd* refers.
+ The file descriptor *fd* is not closed upon failure.
+
+ Calls the C standard library function :c:func:`unlockpt`.
+
+ .. availability:: Unix, not Emscripten, not WASI.
+
+ .. versionadded:: 3.13
+
+
.. function:: write(fd, str, /)
Write the bytestring in *str* to file descriptor *fd*.
diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst
index 2f4ff4efec47f8..f94b6fb3805684 100644
--- a/Doc/library/pathlib.rst
+++ b/Doc/library/pathlib.rst
@@ -535,14 +535,13 @@ Pure paths provide the following methods and properties:
reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`,
``False`` is always returned.
- >>> PureWindowsPath('nul').is_reserved()
- True
- >>> PurePosixPath('nul').is_reserved()
- False
-
- File system calls on reserved paths can fail mysteriously or have
- unintended effects.
+ .. versionchanged:: 3.13
+ Windows path names that contain a colon, or end with a dot or a space,
+ are considered reserved. UNC paths may be reserved.
+ .. deprecated-removed:: 3.13 3.15
+ This method is deprecated; use :func:`os.path.isreserved` to detect
+ reserved paths on Windows.
.. method:: PurePath.joinpath(*pathsegments)
@@ -1039,9 +1038,8 @@ call fails (for example because the path doesn't exist).
The *follow_symlinks* parameter was added.
.. versionchanged:: 3.13
- Emits :exc:`FutureWarning` if the pattern ends with "``**``". In a
- future Python release, patterns with this ending will match both files
- and directories. Add a trailing slash to match only directories.
+ Return files and directories if *pattern* ends with "``**``". In
+ previous versions, only directories were returned.
.. versionchanged:: 3.13
The *pattern* parameter accepts a :term:`path-like object`.
diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst
index 1adafcaa02eab9..54c6d9f3b32b1a 100644
--- a/Doc/library/readline.rst
+++ b/Doc/library/readline.rst
@@ -5,7 +5,7 @@
:platform: Unix
:synopsis: GNU readline support for Python.
-.. sectionauthor:: Skip Montanaro <skip@pobox.com>
+.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com>
--------------
diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst
index 34a738a7f1c41f..2134293a0bb0de 100644
--- a/Doc/library/tarfile.rst
+++ b/Doc/library/tarfile.rst
@@ -673,6 +673,7 @@ be finalized; only the internally used file object will be closed. See the
.. attribute:: TarFile.pax_headers
+ :type: dict
A dictionary containing key-value pairs of pax global headers.
@@ -838,26 +839,31 @@ A ``TarInfo`` object has the following public data attributes:
attribute.
.. attribute:: TarInfo.chksum
+ :type: int
Header checksum.
.. attribute:: TarInfo.devmajor
+ :type: int
Device major number.
.. attribute:: TarInfo.devminor
+ :type: int
Device minor number.
.. attribute:: TarInfo.offset
+ :type: int
The tar header starts here.
.. attribute:: TarInfo.offset_data
+ :type: int
The file's data starts here.
diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst
index f063e463753e0b..b5a49d9c592387 100644
--- a/Doc/library/urllib.robotparser.rst
+++ b/Doc/library/urllib.robotparser.rst
@@ -5,7 +5,7 @@
:synopsis: Load a robots.txt file and answer questions about
fetchability of other URLs.
-.. sectionauthor:: Skip Montanaro <skip@pobox.com>
+.. sectionauthor:: Skip Montanaro <skip.montanaro@gmail.com>
**Source code:** :source:`Lib/urllib/robotparser.py`
diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index 87ebdc1ca1c9c6..50e0f97a6534af 100644
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -1890,8 +1890,9 @@ the unpacking.
.. index:: pair: trailing; comma
-The trailing comma is required only to create a single tuple (a.k.a. a
-*singleton*); it is optional in all other cases. A single expression without a
+A trailing comma is required only to create a one-item tuple,
+such as ``1,``; it is optional in all other cases.
+A single expression without a
trailing comma doesn't create a tuple, but rather yields the value of that
expression. (To create an empty tuple, use an empty pair of parentheses:
``()``.)
diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst
index a7beeea29b4556..f8c9724114da9e 100644
--- a/Doc/reference/import.rst
+++ b/Doc/reference/import.rst
@@ -327,14 +327,15 @@ modules, and one that knows how to import modules from an :term:`import path`
finders replaced :meth:`!find_module`, which
is now deprecated. While it will continue to work without change, the
import machinery will try it only if the finder does not implement
- ``find_spec()``.
+ :meth:`~importlib.abc.MetaPathFinder.find_spec`.
.. versionchanged:: 3.10
Use of :meth:`!find_module` by the import system
now raises :exc:`ImportWarning`.
.. versionchanged:: 3.12
- ``find_module()`` has been removed. Use :meth:`find_spec` instead.
+ :meth:`!find_module` has been removed.
+ Use :meth:`~importlib.abc.MetaPathFinder.find_spec` instead.
Loading
@@ -812,7 +813,7 @@ attributes on package objects are also used. These provide additional ways
that the import machinery can be customized.
:data:`sys.path` contains a list of strings providing search locations for
-modules and packages. It is initialized from the :data:`PYTHONPATH`
+modules and packages. It is initialized from the :envvar:`PYTHONPATH`
environment variable and various other installation- and
implementation-specific defaults. Entries in :data:`sys.path` can name
directories on the file system, zip files, and potentially other "locations"
diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore
index d56a44ad09a6f8..bba4fe0d5f2425 100644
--- a/Doc/tools/.nitignore
+++ b/Doc/tools/.nitignore
@@ -9,7 +9,6 @@ Doc/c-api/gcsupport.rst
Doc/c-api/init.rst
Doc/c-api/init_config.rst
Doc/c-api/intro.rst
-Doc/c-api/memoryview.rst
Doc/c-api/module.rst
Doc/c-api/stable.rst
Doc/c-api/sys.rst
@@ -27,13 +26,11 @@ Doc/library/asyncio-subprocess.rst
Doc/library/bdb.rst
Doc/library/collections.rst
Doc/library/csv.rst
-Doc/library/datetime.rst
Doc/library/dbm.rst
Doc/library/decimal.rst
Doc/library/email.charset.rst
Doc/library/email.compat32-message.rst
Doc/library/email.errors.rst
-Doc/library/email.mime.rst
Doc/library/email.parser.rst
Doc/library/email.policy.rst
Doc/library/enum.rst
@@ -88,12 +85,10 @@ Doc/library/xmlrpc.server.rst
Doc/library/zlib.rst
Doc/reference/compound_stmts.rst
Doc/reference/datamodel.rst
-Doc/reference/import.rst
Doc/tutorial/datastructures.rst
Doc/using/windows.rst
Doc/whatsnew/2.0.rst
Doc/whatsnew/2.1.rst
-Doc/whatsnew/2.2.rst
Doc/whatsnew/2.4.rst
Doc/whatsnew/2.5.rst
Doc/whatsnew/2.6.rst
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
index df8b07c6118599..53c95ca1a05c9b 100644
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -623,7 +623,7 @@ Setting the environment variable ``TERM`` to ``dumb`` will disable color.
If the environment variable ``FORCE_COLOR`` is set, then color will be
enabled regardless of the value of TERM. This is useful on CI systems which
-aren’t terminals but can none-the-less display ANSI escape sequences.
+aren’t terminals but can still display ANSI escape sequences.
If the environment variable ``NO_COLOR`` is set, Python will disable all color
in the output. This takes precedence over ``FORCE_COLOR``.
diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst
index 6efc23a82de923..968bd7a126bdf0 100644
--- a/Doc/whatsnew/2.2.rst
+++ b/Doc/whatsnew/2.2.rst
@@ -53,9 +53,9 @@ A long time ago I wrote a web page listing flaws in Python's design. One of the
most significant flaws was that it's impossible to subclass Python types
implemented in C. In particular, it's not possible to subclass built-in types,
so you can't just subclass, say, lists in order to add a single useful method to
-them. The :mod:`UserList` module provides a class that supports all of the
+them. The :mod:`!UserList` module provides a class that supports all of the
methods of lists and that can be subclassed further, but there's lots of C code
-that expects a regular Python list and won't accept a :class:`UserList`
+that expects a regular Python list and won't accept a :class:`!UserList`
instance.
Python 2.2 fixes this, and in the process adds some exciting new capabilities.
@@ -69,7 +69,7 @@ A brief summary:
* It's also possible to automatically call methods on accessing or setting an
instance attribute by using a new mechanism called :dfn:`properties`. Many uses
- of :meth:`__getattr__` can be rewritten to use properties instead, making the
+ of :meth:`!__getattr__` can be rewritten to use properties instead, making the
resulting code simpler and faster. As a small side benefit, attributes can now
have docstrings, too.
@@ -120,7 +120,7 @@ added so if no built-in type is suitable, you can just subclass
This means that :keyword:`class` statements that don't have any base classes are
always classic classes in Python 2.2. (Actually you can also change this by
-setting a module-level variable named :attr:`__metaclass__` --- see :pep:`253`
+setting a module-level variable named :attr:`!__metaclass__` --- see :pep:`253`
for the details --- but it's easier to just subclass :class:`object`.)
The type objects for the built-in types are available as built-ins, named using
@@ -134,8 +134,8 @@ type objects that behave as factories when called. ::
123
To make the set of types complete, new type objects such as :func:`dict` and
-:func:`file` have been added. Here's a more interesting example, adding a
-:meth:`lock` method to file objects::
+:func:`!file` have been added. Here's a more interesting example, adding a
+:meth:`!lock` method to file objects::
class LockableFile(file):
def lock (self, operation, length=0, start=0, whence=0):
@@ -146,7 +146,7 @@ To make the set of types complete, new type objects such as :func:`dict` and
The now-obsolete :mod:`!posixfile` module contained a class that emulated all of
a file object's methods and also added a :meth:`!lock` method, but this class
couldn't be passed to internal functions that expected a built-in file,
-something which is possible with our new :class:`LockableFile`.
+something which is possible with our new :class:`!LockableFile`.
Descriptors
@@ -154,11 +154,11 @@ Descriptors
In previous versions of Python, there was no consistent way to discover what
attributes and methods were supported by an object. There were some informal
-conventions, such as defining :attr:`__members__` and :attr:`__methods__`
+conventions, such as defining :attr:`!__members__` and :attr:`!__methods__`
attributes that were lists of names, but often the author of an extension type
or a class wouldn't bother to define them. You could fall back on inspecting
the :attr:`~object.__dict__` of an object, but when class inheritance or an arbitrary
-:meth:`__getattr__` hook were in use this could still be inaccurate.
+:meth:`!__getattr__` hook were in use this could still be inaccurate.
The one big idea underlying the new class model is that an API for describing
the attributes of an object using :dfn:`descriptors` has been formalized.
@@ -171,7 +171,7 @@ attributes of their own:
* :attr:`~definition.__name__` is the attribute's name.
-* :attr:`__doc__` is the attribute's docstring.
+* :attr:`!__doc__` is the attribute's docstring.
* ``__get__(object)`` is a method that retrieves the attribute value from
*object*.
@@ -186,7 +186,7 @@ are::
descriptor = obj.__class__.x
descriptor.__get__(obj)
-For methods, :meth:`descriptor.__get__` returns a temporary object that's
+For methods, :meth:`!descriptor.__get__` returns a temporary object that's
callable, and wraps up the instance and the method to be called on it. This is
also why static methods and class methods are now possible; they have
descriptors that wrap up just the method, or the method and the class. As a
@@ -204,7 +204,7 @@ methods are defined like this::
...
g = classmethod(g)
-The :func:`staticmethod` function takes the function :func:`f`, and returns it
+The :func:`staticmethod` function takes the function :func:`!f`, and returns it
wrapped up in a descriptor so it can be stored in the class object. You might
expect there to be special syntax for creating such methods (``def static f``,
``defstatic f()``, or something like that) but no such syntax has been defined
@@ -232,10 +232,10 @@ like this::
f = eiffelmethod(f, pre_f, post_f)
-Note that a person using the new :func:`eiffelmethod` doesn't have to understand
+Note that a person using the new :func:`!eiffelmethod` doesn't have to understand
anything about descriptors. This is why I think the new features don't increase
the basic complexity of the language. There will be a few wizards who need to
-know about it in order to write :func:`eiffelmethod` or the ZODB or whatever,
+know about it in order to write :func:`!eiffelmethod` or the ZODB or whatever,
but most users will just write code on top of the resulting libraries and ignore
the implementation details.
@@ -263,10 +263,10 @@ from :pep:`253` by Guido van Rossum)::
The lookup rule for classic classes is simple but not very smart; the base
classes are searched depth-first, going from left to right. A reference to
-:meth:`D.save` will search the classes :class:`D`, :class:`B`, and then
-:class:`A`, where :meth:`save` would be found and returned. :meth:`C.save`
-would never be found at all. This is bad, because if :class:`C`'s :meth:`save`
-method is saving some internal state specific to :class:`C`, not calling it will
+:meth:`!D.save` will search the classes :class:`!D`, :class:`!B`, and then
+:class:`!A`, where :meth:`!save` would be found and returned. :meth:`!C.save`
+would never be found at all. This is bad, because if :class:`!C`'s :meth:`!save`
+method is saving some internal state specific to :class:`!C`, not calling it will
result in that state never getting saved.
New-style classes follow a different algorithm that's a bit more complicated to
@@ -276,22 +276,22 @@ produces more useful results for really complicated inheritance graphs.)
#. List all the base classes, following the classic lookup rule and include a
class multiple times if it's visited repeatedly. In the above example, the list
- of visited classes is [:class:`D`, :class:`B`, :class:`A`, :class:`C`,
- :class:`A`].
+ of visited classes is [:class:`!D`, :class:`!B`, :class:`!A`, :class:`!C`,
+ :class:`!A`].
#. Scan the list for duplicated classes. If any are found, remove all but one
occurrence, leaving the *last* one in the list. In the above example, the list
- becomes [:class:`D`, :class:`B`, :class:`C`, :class:`A`] after dropping
+ becomes [:class:`!D`, :class:`!B`, :class:`!C`, :class:`!A`] after dropping
duplicates.
-Following this rule, referring to :meth:`D.save` will return :meth:`C.save`,
+Following this rule, referring to :meth:`!D.save` will return :meth:`!C.save`,
which is the behaviour we're after. This lookup rule is the same as the one
followed by Common Lisp. A new built-in function, :func:`super`, provides a way
to get at a class's superclasses without having to reimplement Python's
algorithm. The most commonly used form will be ``super(class, obj)``, which
returns a bound superclass object (not the actual class object). This form
will be used in methods to call a method in the superclass; for example,
-:class:`D`'s :meth:`save` method would look like this::
+:class:`!D`'s :meth:`!save` method would look like this::
class D (B,C):
def save (self):
@@ -309,7 +309,7 @@ Attribute Access
----------------
A fair number of sophisticated Python classes define hooks for attribute access
-using :meth:`__getattr__`; most commonly this is done for convenience, to make
+using :meth:`~object.__getattr__`; most commonly this is done for convenience, to make
code more readable by automatically mapping an attribute access such as
``obj.parent`` into a method call such as ``obj.get_parent``. Python 2.2 adds
some new ways of controlling attribute access.
@@ -321,22 +321,22 @@ instance's dictionary.
New-style classes also support a new method,
``__getattribute__(attr_name)``. The difference between the two methods is
-that :meth:`__getattribute__` is *always* called whenever any attribute is
-accessed, while the old :meth:`__getattr__` is only called if ``foo`` isn't
+that :meth:`~object.__getattribute__` is *always* called whenever any attribute is
+accessed, while the old :meth:`~object.__getattr__` is only called if ``foo`` isn't
found in the instance's dictionary.
However, Python 2.2's support for :dfn:`properties` will often be a simpler way
-to trap attribute references. Writing a :meth:`__getattr__` method is
+to trap attribute references. Writing a :meth:`!__getattr__` method is
complicated because to avoid recursion you can't use regular attribute accesses
inside them, and instead have to mess around with the contents of
-:attr:`~object.__dict__`. :meth:`__getattr__` methods also end up being called by Python
-when it checks for other methods such as :meth:`__repr__` or :meth:`__coerce__`,
+:attr:`~object.__dict__`. :meth:`~object.__getattr__` methods also end up being called by Python
+when it checks for other methods such as :meth:`~object.__repr__` or :meth:`!__coerce__`,
and so have to be written with this in mind. Finally, calling a function on
every attribute access results in a sizable performance loss.
:class:`property` is a new built-in type that packages up three functions that
get, set, or delete an attribute, and a docstring. For example, if you want to
-define a :attr:`size` attribute that's computed, but also settable, you could
+define a :attr:`!size` attribute that's computed, but also settable, you could
write::
class C(object):
@@ -355,9 +355,9 @@ write::
"Storage size of this instance")
That is certainly clearer and easier to write than a pair of
-:meth:`__getattr__`/:meth:`__setattr__` methods that check for the :attr:`size`
+:meth:`!__getattr__`/:meth:`!__setattr__` methods that check for the :attr:`!size`
attribute and handle it specially while retrieving all other attributes from the
-instance's :attr:`~object.__dict__`. Accesses to :attr:`size` are also the only ones
+instance's :attr:`~object.__dict__`. Accesses to :attr:`!size` are also the only ones
which have to perform the work of calling a function, so references to other
attributes run at their usual speed.
@@ -447,7 +447,7 @@ an iterator for the object *obj*, while ``iter(C, sentinel)`` returns an
iterator that will invoke the callable object *C* until it returns *sentinel* to
signal that the iterator is done.
-Python classes can define an :meth:`__iter__` method, which should create and
+Python classes can define an :meth:`!__iter__` method, which should create and
return a new iterator for the object; if the object is its own iterator, this
method can just return ``self``. In particular, iterators will usually be their
own iterators. Extension types implemented in C can implement a :c:member:`~PyTypeObject.tp_iter`
@@ -478,7 +478,7 @@ there are no more values to be returned, calling :meth:`next` should raise the
In 2.2, Python's :keyword:`for` statement no longer expects a sequence; it
expects something for which :func:`iter` will return an iterator. For backward
compatibility and convenience, an iterator is automatically constructed for
-sequences that don't implement :meth:`__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so
+sequences that don't implement :meth:`!__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so
``for i in [1,2,3]`` will still work. Wherever the Python interpreter loops
over a sequence, it's been changed to use the iterator protocol. This means you
can do things like this::
@@ -510,8 +510,8 @@ Iterator support has been added to some of Python's basic types. Calling
Oct 10
That's just the default behaviour. If you want to iterate over keys, values, or
-key/value pairs, you can explicitly call the :meth:`iterkeys`,
-:meth:`itervalues`, or :meth:`iteritems` methods to get an appropriate iterator.
+key/value pairs, you can explicitly call the :meth:`!iterkeys`,
+:meth:`!itervalues`, or :meth:`!iteritems` methods to get an appropriate iterator.
In a minor related change, the :keyword:`in` operator now works on dictionaries,
so ``key in dict`` is now equivalent to ``dict.has_key(key)``.
@@ -580,7 +580,7 @@ allowed inside the :keyword:`!try` block of a
:keyword:`try`...\ :keyword:`finally` statement; read :pep:`255` for a full
explanation of the interaction between :keyword:`!yield` and exceptions.)
-Here's a sample usage of the :func:`generate_ints` generator::
+Here's a sample usage of the :func:`!generate_ints` generator::
>>> gen = generate_ints(3)
>>> gen
@@ -641,7 +641,7 @@ like::
sentence := "Store it in the neighboring harbor"
if (i := find("or", sentence)) > 5 then write(i)
-In Icon the :func:`find` function returns the indexes at which the substring
+In Icon the :func:`!find` function returns the indexes at which the substring
"or" is found: 3, 23, 33. In the :keyword:`if` statement, ``i`` is first
assigned a value of 3, but 3 is less than 5, so the comparison fails, and Icon
retries it with the second value of 23. 23 is greater than 5, so the comparison
@@ -671,7 +671,7 @@ PEP 237: Unifying Long Integers and Integers
In recent versions, the distinction between regular integers, which are 32-bit
values on most machines, and long integers, which can be of arbitrary size, was
becoming an annoyance. For example, on platforms that support files larger than
-``2**32`` bytes, the :meth:`tell` method of file objects has to return a long
+``2**32`` bytes, the :meth:`!tell` method of file objects has to return a long
integer. However, there were various bits of Python that expected plain integers
and would raise an error if a long integer was provided instead. For example,
in Python 1.5, only regular integers could be used as a slice index, and
@@ -752,7 +752,7 @@ Here are the changes 2.2 introduces:
0.5. Without the ``__future__`` statement, ``/`` still means classic division.
The default meaning of ``/`` will not change until Python 3.0.
-* Classes can define methods called :meth:`__truediv__` and :meth:`__floordiv__`
+* Classes can define methods called :meth:`~object.__truediv__` and :meth:`~object.__floordiv__`
to overload the two division operators. At the C level, there are also slots in
the :c:type:`PyNumberMethods` structure so extension types can define the two
operators.
@@ -785,17 +785,17 @@ support.)
When built to use UCS-4 (a "wide Python"), the interpreter can natively handle
Unicode characters from U+000000 to U+110000, so the range of legal values for
-the :func:`unichr` function is expanded accordingly. Using an interpreter
+the :func:`!unichr` function is expanded accordingly. Using an interpreter
compiled to use UCS-2 (a "narrow Python"), values greater than 65535 will still
-cause :func:`unichr` to raise a :exc:`ValueError` exception. This is all
+cause :func:`!unichr` to raise a :exc:`ValueError` exception. This is all
described in :pep:`261`, "Support for 'wide' Unicode characters"; consult it for
further details.
Another change is simpler to explain. Since their introduction, Unicode strings
-have supported an :meth:`encode` method to convert the string to a selected
+have supported an :meth:`!encode` method to convert the string to a selected
encoding such as UTF-8 or Latin-1. A symmetric ``decode([*encoding*])``
method has been added to 8-bit strings (though not to Unicode strings) in 2.2.
-:meth:`decode` assumes that the string is in the specified encoding and decodes
+:meth:`!decode` assumes that the string is in the specified encoding and decodes
it, returning whatever is returned by the codec.
Using this new feature, codecs have been added for tasks not directly related to
@@ -819,10 +819,10 @@ encoding, and compression with the :mod:`zlib` module::
>>> "sheesh".encode('rot-13')
'furrfu'
-To convert a class instance to Unicode, a :meth:`__unicode__` method can be
-defined by a class, analogous to :meth:`__str__`.
+To convert a class instance to Unicode, a :meth:`!__unicode__` method can be
+defined by a class, analogous to :meth:`!__str__`.
-:meth:`encode`, :meth:`decode`, and :meth:`__unicode__` were implemented by
+:meth:`!encode`, :meth:`!decode`, and :meth:`!__unicode__` were implemented by
Marc-André Lemburg. The changes to support using UCS-4 internally were
implemented by Fredrik Lundh and Martin von Löwis.
@@ -859,7 +859,7 @@ doesn't work::
return g(value-1) + 1
...
-The function :func:`g` will always raise a :exc:`NameError` exception, because
+The function :func:`!g` will always raise a :exc:`NameError` exception, because
the binding of the name ``g`` isn't in either its local namespace or in the
module-level namespace. This isn't much of a problem in practice (how often do
you recursively define interior functions like this?), but this also made using
@@ -915,7 +915,7 @@ To make the preceding explanation a bit clearer, here's an example::
Line 4 containing the ``exec`` statement is a syntax error, since
``exec`` would define a new local variable named ``x`` whose value should
-be accessed by :func:`g`.
+be accessed by :func:`!g`.
This shouldn't be much of a limitation, since ``exec`` is rarely used in
most Python code (and when it is used, it's often a sign of a poor design
@@ -933,7 +933,7 @@ anyway).
New and Improved Modules
========================
-* The :mod:`xmlrpclib` module was contributed to the standard library by Fredrik
+* The :mod:`!xmlrpclib` module was contributed to the standard library by Fredrik
Lundh, providing support for writing XML-RPC clients. XML-RPC is a simple
remote procedure call protocol built on top of HTTP and XML. For example, the
following snippet retrieves a list of RSS channels from the O'Reilly Network,
@@ -956,7 +956,7 @@ New and Improved Modules
# 'description': 'A utility which converts HTML to XSL FO.',
# 'title': 'html2fo 0.3 (Default)'}, ... ]
- The :mod:`SimpleXMLRPCServer` module makes it easy to create straightforward
+ The :mod:`!SimpleXMLRPCServer` module makes it easy to create straightforward
XML-RPC servers. See http://xmlrpc.scripting.com/ for more information about XML-RPC.
* The new :mod:`hmac` module implements the HMAC algorithm described by
@@ -964,9 +964,9 @@ New and Improved Modules
* Several functions that originally returned lengthy tuples now return
pseudo-sequences that still behave like tuples but also have mnemonic attributes such
- as memberst_mtime or :attr:`tm_year`. The enhanced functions include
- :func:`stat`, :func:`fstat`, :func:`statvfs`, and :func:`fstatvfs` in the
- :mod:`os` module, and :func:`localtime`, :func:`gmtime`, and :func:`strptime` in
+ as :attr:`!memberst_mtime` or :attr:`!tm_year`. The enhanced functions include
+ :func:`~os.stat`, :func:`~os.fstat`, :func:`~os.statvfs`, and :func:`~os.fstatvfs` in the
+ :mod:`os` module, and :func:`~time.localtime`, :func:`~time.gmtime`, and :func:`~time.strptime` in
the :mod:`time` module.
For example, to obtain a file's size using the old tuples, you'd end up writing
@@ -999,7 +999,7 @@ New and Improved Modules
underlying the :mod:`re` module. For example, the :func:`re.sub` and
:func:`re.split` functions have been rewritten in C. Another contributed patch
speeds up certain Unicode character ranges by a factor of two, and a new
- :meth:`finditer` method that returns an iterator over all the non-overlapping
+ :meth:`~re.finditer` method that returns an iterator over all the non-overlapping
matches in a given string. (SRE is maintained by Fredrik Lundh. The
BIGCHARSET patch was contributed by Martin von Löwis.)
@@ -1012,33 +1012,33 @@ New and Improved Modules
new extensions: the NAMESPACE extension defined in :rfc:`2342`, SORT, GETACL and
SETACL. (Contributed by Anthony Baxter and Michel Pelletier.)
-* The :mod:`rfc822` module's parsing of email addresses is now compliant with
+* The :mod:`!rfc822` module's parsing of email addresses is now compliant with
:rfc:`2822`, an update to :rfc:`822`. (The module's name is *not* going to be
changed to ``rfc2822``.) A new package, :mod:`email`, has also been added for
parsing and generating e-mail messages. (Contributed by Barry Warsaw, and
arising out of his work on Mailman.)
-* The :mod:`difflib` module now contains a new :class:`Differ` class for
+* The :mod:`difflib` module now contains a new :class:`!Differ` class for
producing human-readable lists of changes (a "delta") between two sequences of
- lines of text. There are also two generator functions, :func:`ndiff` and
- :func:`restore`, which respectively return a delta from two sequences, or one of
+ lines of text. There are also two generator functions, :func:`!ndiff` and
+ :func:`!restore`, which respectively return a delta from two sequences, or one of
the original sequences from a delta. (Grunt work contributed by David Goodger,
from ndiff.py code by Tim Peters who then did the generatorization.)
-* New constants :const:`ascii_letters`, :const:`ascii_lowercase`, and
- :const:`ascii_uppercase` were added to the :mod:`string` module. There were
- several modules in the standard library that used :const:`string.letters` to
+* New constants :const:`!ascii_letters`, :const:`!ascii_lowercase`, and
+ :const:`!ascii_uppercase` were added to the :mod:`string` module. There were
+ several modules in the standard library that used :const:`!string.letters` to
mean the ranges A-Za-z, but that assumption is incorrect when locales are in
- use, because :const:`string.letters` varies depending on the set of legal
+ use, because :const:`!string.letters` varies depending on the set of legal
characters defined by the current locale. The buggy modules have all been fixed
- to use :const:`ascii_letters` instead. (Reported by an unknown person; fixed by
+ to use :const:`!ascii_letters` instead. (Reported by an unknown person; fixed by
Fred L. Drake, Jr.)
* The :mod:`mimetypes` module now makes it easier to use alternative MIME-type
- databases by the addition of a :class:`MimeTypes` class, which takes a list of
+ databases by the addition of a :class:`~mimetypes.MimeTypes` class, which takes a list of
filenames to be parsed. (Contributed by Fred L. Drake, Jr.)
-* A :class:`Timer` class was added to the :mod:`threading` module that allows
+* A :class:`~threading.Timer` class was added to the :mod:`threading` module that allows
scheduling an activity to happen at some future time. (Contributed by Itamar
Shtull-Trauring.)
@@ -1114,7 +1114,7 @@ code, none of the changes described here will affect you very much.
* Two new wrapper functions, :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf`
were added to provide cross-platform implementations for the relatively new
:c:func:`snprintf` and :c:func:`vsnprintf` C lib APIs. In contrast to the standard
- :c:func:`sprintf` and :c:func:`vsprintf` functions, the Python versions check the
+ :c:func:`sprintf` and :c:func:`!vsprintf` functions, the Python versions check the
bounds of the buffer used to protect against buffer overruns. (Contributed by
M.-A. Lemburg.)
@@ -1212,12 +1212,12 @@ Some of the more notable changes are:
* The :file:`Tools/scripts/ftpmirror.py` script now parses a :file:`.netrc`
file, if you have one. (Contributed by Mike Romberg.)
-* Some features of the object returned by the :func:`xrange` function are now
+* Some features of the object returned by the :func:`!xrange` function are now
deprecated, and trigger warnings when they're accessed; they'll disappear in
- Python 2.3. :class:`xrange` objects tried to pretend they were full sequence
+ Python 2.3. :class:`!xrange` objects tried to pretend they were full sequence
types by supporting slicing, sequence multiplication, and the :keyword:`in`
operator, but these features were rarely used and therefore buggy. The
- :meth:`tolist` method and the :attr:`start`, :attr:`stop`, and :attr:`step`
+ :meth:`!tolist` method and the :attr:`!start`, :attr:`!stop`, and :attr:`!step`
attributes are also being deprecated. At the C level, the fourth argument to
the :c:func:`!PyRange_New` function, ``repeat``, has also been deprecated.
diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst
index 241d58720399af..524967b4524234 100644
--- a/Doc/whatsnew/2.7.rst
+++ b/Doc/whatsnew/2.7.rst
@@ -2130,7 +2130,7 @@ Changes to Python's build process and to the C API include:
only the filename, function name, and first line number are required.
This is useful for extension modules that are attempting to
construct a more useful traceback stack. Previously such
- extensions needed to call :c:func:`PyCode_New`, which had many
+ extensions needed to call :c:func:`!PyCode_New`, which had many
more arguments. (Added by Jeffrey Yasskin.)
* New function: :c:func:`PyErr_NewExceptionWithDoc` creates a new
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index cb646a54df3607..4f4c1de8d8d596 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -672,7 +672,7 @@ enum
* Changed :meth:`Enum.__format__() <enum.Enum.__format__>` (the default for
:func:`format`, :meth:`str.format` and :term:`f-string`\s) to always produce
- the same result as :meth:`Enum.__str__()`: for enums inheriting from
+ the same result as :meth:`Enum.__str__() <enum.Enum.__str__>`: for enums inheriting from
:class:`~enum.ReprEnum` it will be the member's value; for all other enums
it will be the enum and member name (e.g. ``Color.RED``).
@@ -1604,7 +1604,7 @@ raw, adaptive bytecode containing quickened data.
New opcodes
-----------
-* :opcode:`ASYNC_GEN_WRAP`, :opcode:`RETURN_GENERATOR` and :opcode:`SEND`,
+* :opcode:`!ASYNC_GEN_WRAP`, :opcode:`RETURN_GENERATOR` and :opcode:`SEND`,
used in generators and co-routines.
* :opcode:`COPY_FREE_VARS`,
@@ -1615,7 +1615,7 @@ New opcodes
* :opcode:`MAKE_CELL`, to create :ref:`cell-objects`.
-* :opcode:`CHECK_EG_MATCH` and :opcode:`PREP_RERAISE_STAR`,
+* :opcode:`CHECK_EG_MATCH` and :opcode:`!PREP_RERAISE_STAR`,
to handle the :ref:`new exception groups and except* <whatsnew311-pep654>`
added in :pep:`654`.
@@ -1630,38 +1630,38 @@ New opcodes
Replaced opcodes
----------------
-+------------------------------------+-----------------------------------+-----------------------------------------+
-| Replaced Opcode(s) | New Opcode(s) | Notes |
-+====================================+===================================+=========================================+
-| | :opcode:`!BINARY_*` | :opcode:`BINARY_OP` | Replaced all numeric binary/in-place |
-| | :opcode:`!INPLACE_*` | | opcodes with a single opcode |
-+------------------------------------+-----------------------------------+-----------------------------------------+
-| | :opcode:`!CALL_FUNCTION` | | :opcode:`CALL` | Decouples argument shifting for methods |
-| | :opcode:`!CALL_FUNCTION_KW` | | :opcode:`KW_NAMES` | from handling of keyword arguments; |
-| | :opcode:`!CALL_METHOD` | | :opcode:`PRECALL` | allows better specialization of calls |
-| | | :opcode:`PUSH_NULL` | |
-+------------------------------------+-----------------------------------+-----------------------------------------+
-| | :opcode:`!DUP_TOP` | | :opcode:`COPY` | Stack manipulation instructions |
-| | :opcode:`!DUP_TOP_TWO` | | :opcode:`SWAP` | |
-| | :opcode:`!ROT_TWO` | | |
-| | :opcode:`!ROT_THREE` | | |
-| | :opcode:`!ROT_FOUR` | | |
-| | :opcode:`!ROT_N` | | |
-+------------------------------------+-----------------------------------+-----------------------------------------+
-| | :opcode:`!JUMP_IF_NOT_EXC_MATCH` | | :opcode:`CHECK_EXC_MATCH` | Now performs check but doesn't jump |
-+------------------------------------+-----------------------------------+-----------------------------------------+
-| | :opcode:`!JUMP_ABSOLUTE` | | :opcode:`JUMP_BACKWARD` | See [#bytecode-jump]_; |
-| | :opcode:`!POP_JUMP_IF_FALSE` | | :opcode:`POP_JUMP_BACKWARD_IF_* | ``TRUE``, ``FALSE``, |
-| | :opcode:`!POP_JUMP_IF_TRUE` | <POP_JUMP_BACKWARD_IF_TRUE>` | ``NONE`` and ``NOT_NONE`` variants |
-| | | :opcode:`POP_JUMP_FORWARD_IF_* | for each direction |
-| | <POP_JUMP_FORWARD_IF_TRUE>` | |
-+------------------------------------+-----------------------------------+-----------------------------------------+
-| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup |
-| | :opcode:`!SETUP_ASYNC_WITH` | | |
-+------------------------------------+-----------------------------------+-----------------------------------------+
++------------------------------------+------------------------------------+-----------------------------------------+
+| Replaced Opcode(s) | New Opcode(s) | Notes |
++====================================+====================================+=========================================+
+| | :opcode:`!BINARY_*` | :opcode:`BINARY_OP` | Replaced all numeric binary/in-place |
+| | :opcode:`!INPLACE_*` | | opcodes with a single opcode |
++------------------------------------+------------------------------------+-----------------------------------------+
+| | :opcode:`!CALL_FUNCTION` | | :opcode:`CALL` | Decouples argument shifting for methods |
+| | :opcode:`!CALL_FUNCTION_KW` | | :opcode:`!KW_NAMES` | from handling of keyword arguments; |
+| | :opcode:`!CALL_METHOD` | | :opcode:`!PRECALL` | allows better specialization of calls |
+| | | :opcode:`PUSH_NULL` | |
++------------------------------------+------------------------------------+-----------------------------------------+
+| | :opcode:`!DUP_TOP` | | :opcode:`COPY` | Stack manipulation instructions |
+| | :opcode:`!DUP_TOP_TWO` | | :opcode:`SWAP` | |
+| | :opcode:`!ROT_TWO` | | |
+| | :opcode:`!ROT_THREE` | | |
+| | :opcode:`!ROT_FOUR` | | |
+| | :opcode:`!ROT_N` | | |
++------------------------------------+------------------------------------+-----------------------------------------+
+| | :opcode:`!JUMP_IF_NOT_EXC_MATCH` | | :opcode:`CHECK_EXC_MATCH` | Now performs check but doesn't jump |
++------------------------------------+------------------------------------+-----------------------------------------+
+| | :opcode:`!JUMP_ABSOLUTE` | | :opcode:`JUMP_BACKWARD` | See [#bytecode-jump]_; |
+| | :opcode:`!POP_JUMP_IF_FALSE` | | :opcode:`!POP_JUMP_BACKWARD_IF_*`| ``TRUE``, ``FALSE``, |
+| | :opcode:`!POP_JUMP_IF_TRUE` | | :opcode:`!POP_JUMP_FORWARD_IF_*` | ``NONE`` and ``NOT_NONE`` variants |
+| | | for each direction |
+| | | |
++------------------------------------+------------------------------------+-----------------------------------------+
+| | :opcode:`!SETUP_WITH` | :opcode:`BEFORE_WITH` | :keyword:`with` block setup |
+| | :opcode:`!SETUP_ASYNC_WITH` | | |
++------------------------------------+------------------------------------+-----------------------------------------+
.. [#bytecode-jump] All jump opcodes are now relative, including the
- existing :opcode:`JUMP_IF_TRUE_OR_POP` and :opcode:`JUMP_IF_FALSE_OR_POP`.
+ existing :opcode:`!JUMP_IF_TRUE_OR_POP` and :opcode:`!JUMP_IF_FALSE_OR_POP`.
The argument is now an offset from the current instruction
rather than an absolute location.
@@ -1789,13 +1789,13 @@ Standard Library
and will be removed in a future Python version,
due to not supporting resources located within package subdirectories:
- * :func:`importlib.resources.contents`
- * :func:`importlib.resources.is_resource`
- * :func:`importlib.resources.open_binary`
- * :func:`importlib.resources.open_text`
- * :func:`importlib.resources.read_binary`
- * :func:`importlib.resources.read_text`
- * :func:`importlib.resources.path`
+ * :func:`!importlib.resources.contents`
+ * :func:`!importlib.resources.is_resource`
+ * :func:`!importlib.resources.open_binary`
+ * :func:`!importlib.resources.open_text`
+ * :func:`!importlib.resources.read_binary`
+ * :func:`!importlib.resources.read_text`
+ * :func:`!importlib.resources.path`
* The :func:`locale.getdefaultlocale` function is deprecated and will be
removed in Python 3.15. Use :func:`locale.setlocale`,
@@ -1803,7 +1803,7 @@ Standard Library
:func:`locale.getlocale` functions instead.
(Contributed by Victor Stinner in :gh:`90817`.)
-* The :func:`locale.resetlocale` function is deprecated and will be
+* The :func:`!locale.resetlocale` function is deprecated and will be
removed in Python 3.13. Use ``locale.setlocale(locale.LC_ALL, "")`` instead.
(Contributed by Victor Stinner in :gh:`90817`.)
@@ -1967,7 +1967,7 @@ Removed C APIs are :ref:`listed separately <whatsnew311-c-api-removed>`.
(Contributed by Victor Stinner in :issue:`45085`.)
-* Removed the :mod:`distutils` ``bdist_msi`` command deprecated in Python 3.9.
+* Removed the :mod:`!distutils` ``bdist_msi`` command deprecated in Python 3.9.
Use ``bdist_wheel`` (wheel packages) instead.
(Contributed by Hugo van Kemenade in :issue:`45124`.)
@@ -2295,7 +2295,7 @@ Porting to Python 3.11
as its second parameter, instead of ``PyFrameObject*``.
See :pep:`523` for more details of how to use this function pointer type.
-* :c:func:`PyCode_New` and :c:func:`PyCode_NewWithPosOnlyArgs` now take
+* :c:func:`!PyCode_New` and :c:func:`!PyCode_NewWithPosOnlyArgs` now take
an additional ``exception_table`` argument.
Using these functions should be avoided, if at all possible.
To get a custom code object: create a code object using the compiler,
@@ -2402,7 +2402,7 @@ Porting to Python 3.11
been included directly, consider including ``Python.h`` instead.
(Contributed by Victor Stinner in :issue:`35134`.)
-* The :c:func:`PyUnicode_CHECK_INTERNED` macro has been excluded from the
+* The :c:func:`!PyUnicode_CHECK_INTERNED` macro has been excluded from the
limited C API. It was never usable there, because it used internal structures
which are not available in the limited C API.
(Contributed by Victor Stinner in :issue:`46007`.)
@@ -2465,7 +2465,7 @@ Porting to Python 3.11
Debuggers that accessed the :attr:`~frame.f_locals` directly *must* call
:c:func:`PyFrame_GetLocals` instead. They no longer need to call
- :c:func:`PyFrame_FastToLocalsWithError` or :c:func:`PyFrame_LocalsToFast`,
+ :c:func:`!PyFrame_FastToLocalsWithError` or :c:func:`!PyFrame_LocalsToFast`,
in fact they should not call those functions. The necessary updating of the
frame is now managed by the virtual machine.
@@ -2604,8 +2604,8 @@ and will be removed in Python 3.12.
* :c:func:`!PyUnicode_GET_DATA_SIZE`
* :c:func:`!PyUnicode_GET_SIZE`
* :c:func:`!PyUnicode_GetSize`
-* :c:func:`PyUnicode_IS_COMPACT`
-* :c:func:`PyUnicode_IS_READY`
+* :c:func:`!PyUnicode_IS_COMPACT`
+* :c:func:`!PyUnicode_IS_READY`
* :c:func:`PyUnicode_READY`
* :c:func:`!PyUnicode_WSTR_LENGTH`
* :c:func:`!_PyUnicode_AsUnicode`
@@ -2660,7 +2660,7 @@ Removed
(Contributed by Victor Stinner in :issue:`45474`.)
* Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never
- worked since the :c:type:`PyWeakReference` structure is opaque in the
+ worked since the :c:type:`!PyWeakReference` structure is opaque in the
limited C API.
(Contributed by Victor Stinner in :issue:`35134`.)
@@ -2701,4 +2701,30 @@ Removed
(Contributed by Inada Naoki in :issue:`44029`.)
+Notable changes in 3.11.4
+=========================
+
+tarfile
+-------
+
+* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`,
+ have a new a *filter* argument that allows limiting tar features than may be
+ surprising or dangerous, such as creating files outside the destination
+ directory.
+ See :ref:`tarfile-extraction-filter` for details.
+ In Python 3.12, use without the *filter* argument will show a
+ :exc:`DeprecationWarning`.
+ In Python 3.14, the default will switch to ``'data'``.
+ (Contributed by Petr Viktorin in :pep:`706`.)
+
+
+Notable changes in 3.11.5
+=========================
+
+OpenSSL
+-------
+
+* Windows builds and macOS installers from python.org now use OpenSSL 3.0.
+
+
.. _libb2: https://www.blake2.net/
diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
index 77b12f9284ba0d..100312a5940b79 100644
--- a/Doc/whatsnew/3.12.rst
+++ b/Doc/whatsnew/3.12.rst
@@ -690,7 +690,7 @@ csv
* Add :const:`csv.QUOTE_NOTNULL` and :const:`csv.QUOTE_STRINGS` flags to
provide finer grained control of ``None`` and empty strings by
- :class:`csv.writer` objects.
+ :class:`~csv.reader` and :class:`~csv.writer` objects.
dis
---
diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
index ac1e07e0f8319e..57000effb34fea 100644
--- a/Doc/whatsnew/3.13.rst
+++ b/Doc/whatsnew/3.13.rst
@@ -146,14 +146,6 @@ New Modules
Improved Modules
================
-ast
----
-
-* :func:`ast.parse` now accepts an optional argument ``optimize``
- which is passed on to the :func:`compile` built-in. This makes it
- possible to obtain an optimized ``AST``.
- (Contributed by Irit Katriel in :gh:`108113`).
-
array
-----
@@ -161,6 +153,14 @@ array
It can be used instead of ``'u'`` type code, which is deprecated.
(Contributed by Inada Naoki in :gh:`80480`.)
+ast
+---
+
+* :func:`ast.parse` now accepts an optional argument ``optimize``
+ which is passed on to the :func:`compile` built-in. This makes it
+ possible to obtain an optimized ``AST``.
+ (Contributed by Irit Katriel in :gh:`108113`.)
+
asyncio
-------
@@ -180,6 +180,13 @@ copy
any user classes which define the :meth:`!__replace__` method.
(Contributed by Serhiy Storchaka in :gh:`108751`.)
+dbm
+---
+
+* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items
+ from the database.
+ (Contributed by Donghee Na in :gh:`107122`.)
+
dis
---
@@ -216,7 +223,7 @@ email
parameter to these two functions: use ``strict=False`` to get the old
behavior, accept malformed inputs.
``getattr(email.utils, 'supports_strict_parsing', False)`` can be use to
- check if the *strict* paramater is available.
+ check if the *strict* parameter is available.
(Contributed by Thomas Dwyer and Victor Stinner for :gh:`102988` to improve
the CVE-2023-27043 fix.)
@@ -226,7 +233,7 @@ fractions
* Formatting for objects of type :class:`fractions.Fraction` now supports
the standard format specification mini-language rules for fill, alignment,
sign handling, minimum width and grouping. (Contributed by Mark Dickinson
- in :gh:`111320`)
+ in :gh:`111320`.)
glob
----
@@ -300,17 +307,17 @@ os
the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option
:option:`-X cpu_count <-X>`. This option is useful for users who need to limit
CPU resources of a container system without having to modify the container (application code).
- (Contributed by Donghee Na in :gh:`109595`)
+ (Contributed by Donghee Na in :gh:`109595`.)
* Add support of :func:`os.lchmod` and the *follow_symlinks* argument
in :func:`os.chmod` on Windows.
Note that the default value of *follow_symlinks* in :func:`!os.lchmod` is
``False`` on Windows.
- (Contributed by Serhiy Storchaka in :gh:`59616`)
+ (Contributed by Serhiy Storchaka in :gh:`59616`.)
* Add support of :func:`os.fchmod` and a file descriptor
in :func:`os.chmod` on Windows.
- (Contributed by Serhiy Storchaka in :gh:`113191`)
+ (Contributed by Serhiy Storchaka in :gh:`113191`.)
* :func:`os.posix_spawn` now accepts ``env=None``, which makes the newly spawned
process use the current process environment.
@@ -324,6 +331,9 @@ os
os.path
-------
+* Add :func:`os.path.isreserved` to check if a path is reserved on the current
+ system. This function is only available on Windows.
+ (Contributed by Barney Gale in :gh:`88569`.)
* On Windows, :func:`os.path.isabs` no longer considers paths starting with
exactly one (back)slash to be absolute.
(Contributed by Barney Gale and Jon Foster in :gh:`44626`.)
@@ -350,6 +360,11 @@ pathlib
(Contributed by Barney Gale in :gh:`77609` and :gh:`105793`, and
Kamil Turek in :gh:`107962`).
+* Return files and directories from :meth:`pathlib.Path.glob` and
+ :meth:`~pathlib.Path.rglob` when given a pattern that ends with "``**``". In
+ earlier versions, only directories were returned.
+ (Contributed by Barney Gale in :gh:`70303`).
+
pdb
---
@@ -357,7 +372,7 @@ pdb
the new ``exceptions [exc_number]`` command for Pdb. (Contributed by Matthias
Bussonnier in :gh:`106676`.)
-* Expressions/Statements whose prefix is a pdb command are now correctly
+* Expressions/statements whose prefix is a pdb command are now correctly
identified and executed.
(Contributed by Tian Gao in :gh:`108464`.)
@@ -487,28 +502,69 @@ Deprecated
Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``.
(Contributed by Victor Stinner in :gh:`105733`.)
+* :mod:`decimal`: Deprecate non-standard format specifier "N" for
+ :class:`decimal.Decimal`.
+ It was not documented and only supported in the C implementation.
+ (Contributed by Serhiy Storchaka in :gh:`89902`.)
+
+* :mod:`dis`: The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check
+ membership in :data:`~dis.hasarg` instead.
+ (Contributed by Irit Katriel in :gh:`109319`.)
+
* :mod:`getopt` and :mod:`optparse` modules: They are now
- :term:`soft deprecated`: the :mod:`argparse` should be used for new projects.
+ :term:`soft deprecated`: the :mod:`argparse` module should be used for new projects.
Previously, the :mod:`optparse` module was already deprecated, its removal
was not scheduled, and no warnings was emitted: so there is no change in
practice.
(Contributed by Victor Stinner in :gh:`106535`.)
+* :mod:`gettext`: Emit deprecation warning for non-integer numbers in
+ :mod:`gettext` functions and methods that consider plural forms even if the
+ translation was not found.
+ (Contributed by Serhiy Storchaka in :gh:`88434`.)
+
* :mod:`http.server`: :class:`http.server.CGIHTTPRequestHandler` now emits a
- :exc:`DeprecationWarning` as it will be removed in 3.15. Process based CGI
- http servers have been out of favor for a very long time. This code was
+ :exc:`DeprecationWarning` as it will be removed in 3.15. Process-based CGI
+ HTTP servers have been out of favor for a very long time. This code was
outdated, unmaintained, and rarely used. It has a high potential for both
security and functionality bugs. This includes removal of the ``--cgi``
flag to the ``python -m http.server`` command line in 3.15.
+* :mod:`pathlib`:
+ :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for
+ removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved
+ paths on Windows.
+
+* :mod:`pydoc`: Deprecate undocumented :func:`!pydoc.ispackage` function.
+ (Contributed by Zackery Spytz in :gh:`64020`.)
+
+* :mod:`sqlite3`: Passing more than one positional argument to
+ :func:`sqlite3.connect` and the :class:`sqlite3.Connection` constructor is
+ deprecated. The remaining parameters will become keyword-only in Python 3.15.
+
+ Deprecate passing name, number of arguments, and the callable as keyword
+ arguments for the following :class:`sqlite3.Connection` APIs:
+
+ * :meth:`~sqlite3.Connection.create_function`
+ * :meth:`~sqlite3.Connection.create_aggregate`
+
+ Deprecate passing the callback callable by keyword for the following
+ :class:`sqlite3.Connection` APIs:
+
+ * :meth:`~sqlite3.Connection.set_authorizer`
+ * :meth:`~sqlite3.Connection.set_progress_handler`
+ * :meth:`~sqlite3.Connection.set_trace_callback`
+
+ The affected parameters will become positional-only in Python 3.15.
+
+ (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.)
+
* :mod:`sys`: :func:`sys._enablelegacywindowsfsencoding` function.
- Replace it with :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable.
+ Replace it with the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable.
(Contributed by Inada Naoki in :gh:`73427`.)
-* :mod:`traceback`:
-
- * The field *exc_type* of :class:`traceback.TracebackException` is
- deprecated. Use *exc_type_str* instead.
+* :mod:`traceback`: The field *exc_type* of :class:`traceback.TracebackException`
+ is deprecated. Use *exc_type_str* instead.
* :mod:`typing`:
@@ -544,39 +600,6 @@ Deprecated
They will be removed in Python 3.15.
(Contributed by Victor Stinner in :gh:`105096`.)
-* Passing more than one positional argument to :func:`sqlite3.connect` and the
- :class:`sqlite3.Connection` constructor is deprecated. The remaining
- parameters will become keyword-only in Python 3.15.
-
- Deprecate passing name, number of arguments, and the callable as keyword
- arguments, for the following :class:`sqlite3.Connection` APIs:
-
- * :meth:`~sqlite3.Connection.create_function`
- * :meth:`~sqlite3.Connection.create_aggregate`
-
- Deprecate passing the callback callable by keyword for the following
- :class:`sqlite3.Connection` APIs:
-
- * :meth:`~sqlite3.Connection.set_authorizer`
- * :meth:`~sqlite3.Connection.set_progress_handler`
- * :meth:`~sqlite3.Connection.set_trace_callback`
-
- The affected parameters will become positional-only in Python 3.15.
-
- (Contributed by Erlend E. Aasland in :gh:`107948` and :gh:`108278`.)
-
-* The ``dis.HAVE_ARGUMENT`` separator is deprecated. Check membership
- in :data:`~dis.hasarg` instead.
- (Contributed by Irit Katriel in :gh:`109319`.)
-
-* Deprecate non-standard format specifier "N" for :class:`decimal.Decimal`.
- It was not documented and only supported in the C implementation.
- (Contributed by Serhiy Storchaka in :gh:`89902`.)
-
-* Emit deprecation warning for non-integer numbers in :mod:`gettext` functions
- and methods that consider plural forms even if the translation was not found.
- (Contributed by Serhiy Storchaka in :gh:`88434`.)
-
* Calling :meth:`frame.clear` on a suspended frame raises :exc:`RuntimeError`
(as has always been the case for an executing frame).
(Contributed by Irit Katriel in :gh:`79932`.)
@@ -587,9 +610,6 @@ Deprecated
coroutine.
(Contributed by Irit Katriel in :gh:`81137`.)
-* Deprecate undocumented :func:`!pydoc.ispackage` function.
- (Contributed by Zackery Spytz in :gh:`64020`.)
-
Pending Removal in Python 3.14
------------------------------
@@ -651,11 +671,11 @@ Pending Removal in Python 3.14
:func:`~multiprocessing.set_start_method` APIs to explicitly specify when
your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`.
-* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to`,
+* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and
:meth:`~pathlib.PurePath.relative_to`: passing additional arguments is
deprecated.
-* :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader`
+* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader`
now raise :exc:`DeprecationWarning`;
use :func:`importlib.util.find_spec` instead.
(Contributed by Nikita Sobolev in :gh:`97850`.)
@@ -712,6 +732,18 @@ Pending Removal in Python 3.15
:func:`locale.getlocale()` instead.
(Contributed by Hugo van Kemenade in :gh:`111187`.)
+* :mod:`pathlib`:
+ :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for
+ removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved
+ paths on Windows.
+
+* :mod:`threading`:
+ Passing any arguments to :func:`threading.RLock` is now deprecated.
+ C version allows any numbers of args and kwargs,
+ but they are just ignored. Python version does not allow any arguments.
+ All arguments will be removed from :func:`threading.RLock` in Python 3.15.
+ (Contributed by Nikita Sobolev in :gh:`102029`.)
+
* :class:`typing.NamedTuple`:
* The undocumented keyword argument syntax for creating NamedTuple classes
@@ -737,12 +769,6 @@ Pending Removal in Python 3.15
They will be removed in Python 3.15.
(Contributed by Victor Stinner in :gh:`105096`.)
-* Passing any arguments to :func:`threading.RLock` is now deprecated.
- C version allows any numbers of args and kwargs,
- but they are just ignored. Python version does not allow any arguments.
- All arguments will be removed from :func:`threading.RLock` in Python 3.15.
- (Contributed by Nikita Sobolev in :gh:`102029`.)
-
Pending Removal in Python 3.16
------------------------------
@@ -789,6 +815,9 @@ although there is currently no date scheduled for their removal.
:data:`calendar.FEBRUARY`.
(Contributed by Prince Roshan in :gh:`103636`.)
+* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method
+ instead.
+
* :mod:`datetime`:
* :meth:`~datetime.datetime.utcnow`:
@@ -824,11 +853,13 @@ although there is currently no date scheduled for their removal.
underscore.
(Contributed by Serhiy Storchaka in :gh:`91760`.)
+* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules.
+
* :mod:`ssl` options and protocols:
* :class:`ssl.SSLContext` without protocol argument is deprecated.
* :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and
- :meth:`!~ssl.SSLContext.selected_npn_protocol` are deprecated: use ALPN
+ :meth:`!selected_npn_protocol` are deprecated: use ALPN
instead.
* ``ssl.OP_NO_SSL*`` options
* ``ssl.OP_NO_TLS*`` options
@@ -841,13 +872,6 @@ although there is currently no date scheduled for their removal.
* ``ssl.TLSVersion.TLSv1``
* ``ssl.TLSVersion.TLSv1_1``
-* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules.
-
-* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method
- instead.
-
-* :class:`typing.Text` (:gh:`92332`).
-
* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and
ignored.
@@ -862,14 +886,10 @@ although there is currently no date scheduled for their removal.
* :meth:`!threading.currentThread`: use :meth:`threading.current_thread`.
* :meth:`!threading.activeCount`: use :meth:`threading.active_count`.
-* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value
- that is not None from a test case.
-
-* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and
- :class:`~urllib.request.FancyURLopener` style of invoking requests is
- deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods.
+* :class:`typing.Text` (:gh:`92332`).
-* :func:`!urllib.parse.to_bytes`.
+* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value
+ that is not ``None`` from a test case.
* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead
@@ -883,6 +903,11 @@ although there is currently no date scheduled for their removal.
* ``splittype()``
* ``splituser()``
* ``splitvalue()``
+ * ``to_bytes()``
+
+* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and
+ :class:`~urllib.request.FancyURLopener` style of invoking requests is
+ deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods.
* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial
writes.
@@ -1178,10 +1203,10 @@ Changes in the Python API
* Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`,
:c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`,
:c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and
- :c:func:`PySys_GetObject`, which clear all errors occurred during calling
- the function, report now them using :func:`sys.unraisablehook`.
- You can consider to replace these functions with other functions as
- recomended in the documentation.
+ :c:func:`PySys_GetObject`, which clear all errors which occurred when calling
+ them, now report them using :func:`sys.unraisablehook`.
+ You may replace them with other functions as
+ recommended in the documentation.
(Contributed by Serhiy Storchaka in :gh:`106672`.)
* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to
@@ -1190,7 +1215,7 @@ Changes in the Python API
* The :mod:`threading` module now expects the :mod:`!_thread` module to have
an ``_is_main_interpreter`` attribute. It is a function with no
- arguments that returns ``True`` if the current interpreter is the
+ arguments that return ``True`` if the current interpreter is the
main interpreter.
Any library or application that provides a custom ``_thread`` module
@@ -1201,6 +1226,11 @@ Changes in the Python API
* :class:`mailbox.Maildir` now ignores files with a leading dot.
(Contributed by Zackery Spytz in :gh:`65559`.)
+* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both
+ files and directories if a pattern that ends with "``**``" is given, rather
+ than directories only. Users may add a trailing slash to match only
+ directories.
+
Build Changes
=============
@@ -1213,7 +1243,7 @@ Build Changes
(Contributed by Erlend Aasland in :gh:`105875`.)
* Python built with :file:`configure` :option:`--with-trace-refs` (tracing
- references) is now ABI compatible with Python release build and
+ references) is now ABI compatible with the Python release build and
:ref:`debug build <debug-build>`.
(Contributed by Victor Stinner in :gh:`108634`.)
@@ -1240,7 +1270,7 @@ New Features
(Contributed by Inada Naoki in :gh:`104922`.)
* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and
- :c:func:`PyArg_VaParseTupleAndKeywords` has now type :c:expr:`char * const *`
+ :c:func:`PyArg_VaParseTupleAndKeywords` now has type :c:expr:`char * const *`
in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`.
It makes these functions compatible with arguments of type
:c:expr:`const char * const *`, :c:expr:`const char **` or
@@ -1297,14 +1327,14 @@ New Features
always steals a reference to the value.
(Contributed by Serhiy Storchaka in :gh:`86493`.)
-* Added :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef`
+* Add :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef`
functions: similar to :c:func:`PyDict_GetItemWithError` but returning a
:term:`strong reference` instead of a :term:`borrowed reference`. Moreover,
these functions return -1 on error and so checking ``PyErr_Occurred()`` is
not needed.
(Contributed by Victor Stinner in :gh:`106004`.)
-* Added :c:func:`PyDict_ContainsString` function: same as
+* Add :c:func:`PyDict_ContainsString` function: same as
:c:func:`PyDict_Contains`, but *key* is specified as a :c:expr:`const char*`
UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`.
(Contributed by Victor Stinner in :gh:`108314`.)
@@ -1362,7 +1392,7 @@ New Features
(Contributed by Victor Stinner in :gh:`85283`.)
* Add :c:func:`PyErr_FormatUnraisable` function: similar to
- :c:func:`PyErr_WriteUnraisable`, but allow to customize the warning mesage.
+ :c:func:`PyErr_WriteUnraisable`, but allow customizing the warning message.
(Contributed by Serhiy Storchaka in :gh:`108082`.)
* Add :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions: similar to
@@ -1372,7 +1402,7 @@ New Features
* Add :c:func:`PyDict_Pop` and :c:func:`PyDict_PopString` functions: remove a
key from a dictionary and optionally return the removed value. This is
similar to :meth:`dict.pop`, but without the default value and not raising
- :exc:`KeyError` if the key missing.
+ :exc:`KeyError` if the key is missing.
(Contributed by Stefan Behnel and Victor Stinner in :gh:`111262`.)
* Add :c:func:`Py_HashPointer` function to hash a pointer.
@@ -1485,7 +1515,7 @@ Removed
-------
* Removed chained :class:`classmethod` descriptors (introduced in
- :issue:`19072`). This can no longer be used to wrap other descriptors
+ :gh:`63272`). This can no longer be used to wrap other descriptors
such as :class:`property`. The core design of this feature was flawed
and caused a number of downstream problems. To "pass-through" a
:class:`classmethod`, consider using the :attr:`!__wrapped__`
@@ -1499,14 +1529,14 @@ Removed
add ``cc @vstinner`` to the issue to notify Victor Stinner.
(Contributed by Victor Stinner in :gh:`106320`.)
-* Remove functions deprecated in Python 3.9.
+* Remove functions deprecated in Python 3.9:
* ``PyEval_CallObject()``, ``PyEval_CallObjectWithKeywords()``: use
:c:func:`PyObject_CallNoArgs` or :c:func:`PyObject_Call` instead.
Warning: :c:func:`PyObject_Call` positional arguments must be a
- :class:`tuple` and must not be *NULL*, keyword arguments must be a
- :class:`dict` or *NULL*, whereas removed functions checked arguments type
- and accepted *NULL* positional and keyword arguments.
+ :class:`tuple` and must not be ``NULL``, keyword arguments must be a
+ :class:`dict` or ``NULL``, whereas removed functions checked arguments type
+ and accepted ``NULL`` positional and keyword arguments.
To replace ``PyEval_CallObjectWithKeywords(func, NULL, kwargs)`` with
:c:func:`PyObject_Call`, pass an empty tuple as positional arguments using
:c:func:`PyTuple_New(0) <PyTuple_New>`.
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index d373fa163ff737..b041e592d61ed1 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -754,8 +754,8 @@ datetime
--------
Added new alternate constructors :meth:`datetime.date.fromisocalendar` and
-:meth:`datetime.datetime.fromisocalendar`, which construct :class:`date` and
-:class:`datetime` objects respectively from ISO year, week number, and weekday;
+:meth:`datetime.datetime.fromisocalendar`, which construct :class:`~datetime.date` and
+:class:`~datetime.datetime` objects respectively from ISO year, week number, and weekday;
these are the inverse of each class's ``isocalendar`` method.
(Contributed by Paul Ganssle in :issue:`36004`.)
@@ -1623,8 +1623,8 @@ Build and C API Changes
allocation or deallocation may need to be adjusted.
(Contributed by Eddie Elizondo in :issue:`35810`.)
-* The new function :c:func:`PyCode_NewWithPosOnlyArgs` allows to create
- code objects like :c:func:`PyCode_New`, but with an extra *posonlyargcount*
+* The new function :c:func:`!PyCode_NewWithPosOnlyArgs` allows to create
+ code objects like :c:func:`!PyCode_New`, but with an extra *posonlyargcount*
parameter for indicating the number of positional-only arguments.
(Contributed by Pablo Galindo in :issue:`37221`.)
diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h
index 8ade1b164681f9..49f5e8d6d1a0d6 100644
--- a/Include/cpython/listobject.h
+++ b/Include/cpython/listobject.h
@@ -29,7 +29,11 @@ typedef struct {
static inline Py_ssize_t PyList_GET_SIZE(PyObject *op) {
PyListObject *list = _PyList_CAST(op);
+#ifdef Py_GIL_DISABLED
+ return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(list)->ob_size));
+#else
return Py_SIZE(list);
+#endif
}
#define PyList_GET_SIZE(op) PyList_GET_SIZE(_PyObject_CAST(op))
diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h
index 96e829f8fbe97d..ecf3cae4cbc3f1 100644
--- a/Include/cpython/optimizer.h
+++ b/Include/cpython/optimizer.h
@@ -39,6 +39,8 @@ typedef struct {
typedef struct _PyExecutorObject {
PyObject_VAR_HEAD
_PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */
+ void *jit_code;
+ size_t jit_size;
_PyUOpInstruction trace[1];
} _PyExecutorObject;
diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h
index d96870e9197bbf..b4e1f8cf1e320b 100644
--- a/Include/internal/pycore_dict.h
+++ b/Include/internal/pycore_dict.h
@@ -209,8 +209,14 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
#define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS)
#define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1)
+#ifdef Py_GIL_DISABLED
+#define DICT_NEXT_VERSION(INTERP) \
+ (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
+
+#else
#define DICT_NEXT_VERSION(INTERP) \
((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT)
+#endif
void
_PyDict_SendEvent(int watcher_bits,
diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h
index dfb12839affedf..b91d2bc066b783 100644
--- a/Include/internal/pycore_freelist.h
+++ b/Include/internal/pycore_freelist.h
@@ -103,13 +103,13 @@ struct _Py_object_stack_state {
};
typedef struct _Py_freelist_state {
- struct _Py_float_state float_state;
- struct _Py_tuple_state tuple_state;
- struct _Py_list_state list_state;
- struct _Py_slice_state slice_state;
- struct _Py_context_state context_state;
- struct _Py_async_gen_state async_gen_state;
- struct _Py_object_stack_state object_stack_state;
+ struct _Py_float_state floats;
+ struct _Py_tuple_state tuples;
+ struct _Py_list_state lists;
+ struct _Py_slice_state slices;
+ struct _Py_context_state contexts;
+ struct _Py_async_gen_state async_gens;
+ struct _Py_object_stack_state object_stacks;
} _PyFreeListState;
#ifdef __cplusplus
diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h
index e92707051c12b7..dd09ff40f39fe6 100644
--- a/Include/internal/pycore_global_objects_fini_generated.h
+++ b/Include/internal/pycore_global_objects_fini_generated.h
@@ -876,6 +876,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default));
@@ -945,6 +946,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fix_imports));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flags));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flush));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param));
@@ -975,6 +977,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(headers));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint));
@@ -1049,6 +1052,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxmem));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsplit));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxvalue));
@@ -1058,11 +1062,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metaclass));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module_globals));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(modules));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(month));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mro));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mycmp));
@@ -1167,6 +1174,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(selectors));
@@ -1243,6 +1251,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(type_params));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tz));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzinfo));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzname));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uid));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unlink));
diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h
index eb60b80c964d42..79d6509abcdfd9 100644
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -365,6 +365,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(d)
STRUCT_FOR_ID(data)
STRUCT_FOR_ID(database)
+ STRUCT_FOR_ID(day)
STRUCT_FOR_ID(decode)
STRUCT_FOR_ID(decoder)
STRUCT_FOR_ID(default)
@@ -434,6 +435,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(fix_imports)
STRUCT_FOR_ID(flags)
STRUCT_FOR_ID(flush)
+ STRUCT_FOR_ID(fold)
STRUCT_FOR_ID(follow_symlinks)
STRUCT_FOR_ID(format)
STRUCT_FOR_ID(from_param)
@@ -464,6 +466,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(headers)
STRUCT_FOR_ID(hi)
STRUCT_FOR_ID(hook)
+ STRUCT_FOR_ID(hour)
STRUCT_FOR_ID(id)
STRUCT_FOR_ID(ident)
STRUCT_FOR_ID(identity_hint)
@@ -538,6 +541,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(max_length)
STRUCT_FOR_ID(maxdigits)
STRUCT_FOR_ID(maxevents)
+ STRUCT_FOR_ID(maxlen)
STRUCT_FOR_ID(maxmem)
STRUCT_FOR_ID(maxsplit)
STRUCT_FOR_ID(maxvalue)
@@ -547,11 +551,14 @@ struct _Py_global_strings {
STRUCT_FOR_ID(metaclass)
STRUCT_FOR_ID(metadata)
STRUCT_FOR_ID(method)
+ STRUCT_FOR_ID(microsecond)
+ STRUCT_FOR_ID(minute)
STRUCT_FOR_ID(mod)
STRUCT_FOR_ID(mode)
STRUCT_FOR_ID(module)
STRUCT_FOR_ID(module_globals)
STRUCT_FOR_ID(modules)
+ STRUCT_FOR_ID(month)
STRUCT_FOR_ID(mro)
STRUCT_FOR_ID(msg)
STRUCT_FOR_ID(mycmp)
@@ -656,6 +663,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(salt)
STRUCT_FOR_ID(sched_priority)
STRUCT_FOR_ID(scheduler)
+ STRUCT_FOR_ID(second)
STRUCT_FOR_ID(seek)
STRUCT_FOR_ID(seekable)
STRUCT_FOR_ID(selectors)
@@ -732,6 +740,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(type)
STRUCT_FOR_ID(type_params)
STRUCT_FOR_ID(tz)
+ STRUCT_FOR_ID(tzinfo)
STRUCT_FOR_ID(tzname)
STRUCT_FOR_ID(uid)
STRUCT_FOR_ID(unlink)
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index 662a18d93f329d..04e75940dcb573 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -203,7 +203,17 @@ struct _is {
struct _mimalloc_interp_state mimalloc;
#endif
- struct _obmalloc_state obmalloc;
+ // Per-interpreter state for the obmalloc allocator. For the main
+ // interpreter and for all interpreters that don't have their
+ // own obmalloc state, this points to the static structure in
+ // obmalloc.c obmalloc_state_main. For other interpreters, it is
+ // heap allocated by _PyMem_init_obmalloc() and freed when the
+ // interpreter structure is freed. In the case of a heap allocated
+ // obmalloc state, it is not safe to hold on to or use memory after
+ // the interpreter is freed. The obmalloc state corresponding to
+ // that allocated memory is gone. See free_obmalloc_arenas() for
+ // more comments.
+ struct _obmalloc_state *obmalloc;
PyObject *audit_hooks;
PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS];
diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h
new file mode 100644
index 00000000000000..0b71eb6f758ac6
--- /dev/null
+++ b/Include/internal/pycore_jit.h
@@ -0,0 +1,25 @@
+#ifndef Py_INTERNAL_JIT_H
+#define Py_INTERNAL_JIT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+#ifdef _Py_JIT
+
+typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate);
+
+int _PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length);
+void _PyJIT_Free(_PyExecutorObject *executor);
+
+#endif // _Py_JIT
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // !Py_INTERNAL_JIT_H
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
index 4e52ffc77c5956..e32ea2f528940a 100644
--- a/Include/internal/pycore_object.h
+++ b/Include/internal/pycore_object.h
@@ -178,7 +178,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
}
_Py_DECREF_STAT_INC();
#ifdef Py_REF_DEBUG
- _Py_DEC_REFTOTAL(_PyInterpreterState_GET());
+ _Py_DEC_REFTOTAL(PyInterpreterState_Get());
#endif
if (--op->ob_refcnt != 0) {
assert(op->ob_refcnt > 0);
@@ -199,7 +199,7 @@ _Py_DECREF_NO_DEALLOC(PyObject *op)
}
_Py_DECREF_STAT_INC();
#ifdef Py_REF_DEBUG
- _Py_DEC_REFTOTAL(_PyInterpreterState_GET());
+ _Py_DEC_REFTOTAL(PyInterpreterState_Get());
#endif
op->ob_refcnt--;
#ifdef Py_DEBUG
diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h
index 17572dba65487d..9140d8f08f0af1 100644
--- a/Include/internal/pycore_obmalloc.h
+++ b/Include/internal/pycore_obmalloc.h
@@ -686,6 +686,8 @@ extern Py_ssize_t _Py_GetGlobalAllocatedBlocks(void);
_Py_GetGlobalAllocatedBlocks()
extern Py_ssize_t _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *);
extern void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *);
+extern int _PyMem_init_obmalloc(PyInterpreterState *interp);
+extern bool _PyMem_obmalloc_state_on_heap(PyInterpreterState *interp);
#ifdef WITH_PYMALLOC
diff --git a/Include/internal/pycore_obmalloc_init.h b/Include/internal/pycore_obmalloc_init.h
index 8ee72ff2d4126f..e6811b7aeca73c 100644
--- a/Include/internal/pycore_obmalloc_init.h
+++ b/Include/internal/pycore_obmalloc_init.h
@@ -59,13 +59,6 @@ extern "C" {
.dump_debug_stats = -1, \
}
-#define _obmalloc_state_INIT(obmalloc) \
- { \
- .pools = { \
- .used = _obmalloc_pools_INIT(obmalloc.pools), \
- }, \
- }
-
#ifdef __cplusplus
}
diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h
index b4806ab09fd145..0a5c92bb84b524 100644
--- a/Include/internal/pycore_runtime_init.h
+++ b/Include/internal/pycore_runtime_init.h
@@ -155,7 +155,6 @@ extern PyTypeObject _PyExc_MemoryError;
{ \
.id_refcount = -1, \
.imports = IMPORTS_INIT, \
- .obmalloc = _obmalloc_state_INIT(INTERP.obmalloc), \
.ceval = { \
.recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
}, \
diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h
index 9b39de1d69c6c7..f3c55acfb3c282 100644
--- a/Include/internal/pycore_runtime_init_generated.h
+++ b/Include/internal/pycore_runtime_init_generated.h
@@ -874,6 +874,7 @@ extern "C" {
INIT_ID(d), \
INIT_ID(data), \
INIT_ID(database), \
+ INIT_ID(day), \
INIT_ID(decode), \
INIT_ID(decoder), \
INIT_ID(default), \
@@ -943,6 +944,7 @@ extern "C" {
INIT_ID(fix_imports), \
INIT_ID(flags), \
INIT_ID(flush), \
+ INIT_ID(fold), \
INIT_ID(follow_symlinks), \
INIT_ID(format), \
INIT_ID(from_param), \
@@ -973,6 +975,7 @@ extern "C" {
INIT_ID(headers), \
INIT_ID(hi), \
INIT_ID(hook), \
+ INIT_ID(hour), \
INIT_ID(id), \
INIT_ID(ident), \
INIT_ID(identity_hint), \
@@ -1047,6 +1050,7 @@ extern "C" {
INIT_ID(max_length), \
INIT_ID(maxdigits), \
INIT_ID(maxevents), \
+ INIT_ID(maxlen), \
INIT_ID(maxmem), \
INIT_ID(maxsplit), \
INIT_ID(maxvalue), \
@@ -1056,11 +1060,14 @@ extern "C" {
INIT_ID(metaclass), \
INIT_ID(metadata), \
INIT_ID(method), \
+ INIT_ID(microsecond), \
+ INIT_ID(minute), \
INIT_ID(mod), \
INIT_ID(mode), \
INIT_ID(module), \
INIT_ID(module_globals), \
INIT_ID(modules), \
+ INIT_ID(month), \
INIT_ID(mro), \
INIT_ID(msg), \
INIT_ID(mycmp), \
@@ -1165,6 +1172,7 @@ extern "C" {
INIT_ID(salt), \
INIT_ID(sched_priority), \
INIT_ID(scheduler), \
+ INIT_ID(second), \
INIT_ID(seek), \
INIT_ID(seekable), \
INIT_ID(selectors), \
@@ -1241,6 +1249,7 @@ extern "C" {
INIT_ID(type), \
INIT_ID(type_params), \
INIT_ID(tz), \
+ INIT_ID(tzinfo), \
INIT_ID(tzname), \
INIT_ID(uid), \
INIT_ID(unlink), \
diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h
index 898d386f4cfd05..2e9572382fe033 100644
--- a/Include/internal/pycore_unicodeobject_generated.h
+++ b/Include/internal/pycore_unicodeobject_generated.h
@@ -936,6 +936,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
string = &_Py_ID(database);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
+ string = &_Py_ID(day);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ _PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(decode);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
@@ -1143,6 +1146,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
string = &_Py_ID(flush);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
+ string = &_Py_ID(fold);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ _PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(follow_symlinks);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
@@ -1233,6 +1239,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
string = &_Py_ID(hook);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
+ string = &_Py_ID(hour);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ _PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(id);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
@@ -1455,6 +1464,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
string = &_Py_ID(maxevents);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
+ string = &_Py_ID(maxlen);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ _PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(maxmem);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
@@ -1482,6 +1494,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
string = &_Py_ID(method);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
+ string = &_Py_ID(microsecond);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ _PyUnicode_InternInPlace(interp, &string);
+ string = &_Py_ID(minute);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ _PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(mod);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
@@ -1497,6 +1515,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
string = &_Py_ID(modules);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
+ string = &_Py_ID(month);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ _PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(mro);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
@@ -1809,6 +1830,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
string = &_Py_ID(scheduler);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
+ string = &_Py_ID(second);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ _PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(seek);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
@@ -2037,6 +2061,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
string = &_Py_ID(tz);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
+ string = &_Py_ID(tzinfo);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ _PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(tzname);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
diff --git a/Include/pyerrors.h b/Include/pyerrors.h
index 5d0028c116e2d8..68d7985dac8876 100644
--- a/Include/pyerrors.h
+++ b/Include/pyerrors.h
@@ -108,6 +108,7 @@ PyAPI_DATA(PyObject *) PyExc_NotImplementedError;
PyAPI_DATA(PyObject *) PyExc_SyntaxError;
PyAPI_DATA(PyObject *) PyExc_IndentationError;
PyAPI_DATA(PyObject *) PyExc_TabError;
+PyAPI_DATA(PyObject *) PyExc_IncompleteInputError;
PyAPI_DATA(PyObject *) PyExc_ReferenceError;
PyAPI_DATA(PyObject *) PyExc_SystemError;
PyAPI_DATA(PyObject *) PyExc_SystemExit;
diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py
index e1c56d140bef7d..f322b1f6653f6a 100644
--- a/Lib/asyncio/taskgroups.py
+++ b/Lib/asyncio/taskgroups.py
@@ -132,7 +132,7 @@ async def __aexit__(self, et, exc, tb):
# Propagate CancelledError if there is one, except if there
# are other errors -- those have priority.
- if propagate_cancellation_error and not self._errors:
+ if propagate_cancellation_error is not None and not self._errors:
raise propagate_cancellation_error
if et is not None and not issubclass(et, exceptions.CancelledError):
diff --git a/Lib/calendar.py b/Lib/calendar.py
index 03469d8ac96bcd..833ce331b14a0c 100644
--- a/Lib/calendar.py
+++ b/Lib/calendar.py
@@ -734,10 +734,15 @@ def main(args=None):
choices=("text", "html"),
help="output type (text or html)"
)
+ parser.add_argument(
+ "-f", "--first-weekday",
+ type=int, default=0,
+ help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)"
+ )
parser.add_argument(
"year",
nargs='?', type=int,
- help="year number (1-9999)"
+ help="year number"
)
parser.add_argument(
"month",
@@ -761,6 +766,7 @@ def main(args=None):
cal = LocaleHTMLCalendar(locale=locale)
else:
cal = HTMLCalendar()
+ cal.setfirstweekday(options.first_weekday)
encoding = options.encoding
if encoding is None:
encoding = sys.getdefaultencoding()
@@ -775,6 +781,7 @@ def main(args=None):
cal = LocaleTextCalendar(locale=locale)
else:
cal = TextCalendar()
+ cal.setfirstweekday(options.first_weekday)
optdict = dict(w=options.width, l=options.lines)
if options.month is None:
optdict["c"] = options.spacing
diff --git a/Lib/codeop.py b/Lib/codeop.py
index 91146be2c438e2..6ad60e7f85098d 100644
--- a/Lib/codeop.py
+++ b/Lib/codeop.py
@@ -65,9 +65,10 @@ def _maybe_compile(compiler, source, filename, symbol):
try:
compiler(source + "\n", filename, symbol)
return None
+ except IncompleteInputError as e:
+ return None
except SyntaxError as e:
- if "incomplete input" in str(e):
- return None
+ pass
# fallthrough
return compiler(source, filename, symbol, incomplete_input=False)
diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index ffaffdb8b3d0aa..ca843e11eeb83d 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -190,16 +190,6 @@ def _on_queue_feeder_error(self, e, obj):
super()._on_queue_feeder_error(e, obj)
-def _get_chunks(*iterables, chunksize):
- """ Iterates over zip()ed iterables in chunks. """
- it = zip(*iterables)
- while True:
- chunk = tuple(itertools.islice(it, chunksize))
- if not chunk:
- return
- yield chunk
-
-
def _process_chunk(fn, chunk):
""" Processes a chunk of an iterable passed to map.
@@ -847,7 +837,7 @@ def map(self, fn, *iterables, timeout=None, chunksize=1):
raise ValueError("chunksize must be >= 1.")
results = super().map(partial(_process_chunk, fn),
- _get_chunks(*iterables, chunksize=chunksize),
+ itertools.batched(zip(*iterables), chunksize),
timeout=timeout)
return _chain_from_iterable_of_lists(results)
diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py
index febe411355d6be..0f1bf8e4253ec4 100644
--- a/Lib/email/_parseaddr.py
+++ b/Lib/email/_parseaddr.py
@@ -13,7 +13,7 @@
'quote',
]
-import time, calendar
+import time
SPACE = ' '
EMPTYSTRING = ''
@@ -194,6 +194,9 @@ def mktime_tz(data):
# No zone info, so localtime is better assumption than GMT
return time.mktime(data[:8] + (-1,))
else:
+ # Delay the import, since mktime_tz is rarely used
+ import calendar
+
t = calendar.timegm(data)
return t - data[9]
diff --git a/Lib/email/policy.py b/Lib/email/policy.py
index 611deb50bb5290..8816c84ed175a7 100644
--- a/Lib/email/policy.py
+++ b/Lib/email/policy.py
@@ -210,8 +210,15 @@ def _fold(self, name, value, refold_binary=False):
self.refold_source == 'long' and
(lines and len(lines[0])+len(name)+2 > maxlen or
any(len(x) > maxlen for x in lines[1:])))
- if refold or refold_binary and _has_surrogates(value):
+
+ if not refold:
+ if not self.utf8:
+ refold = not value.isascii()
+ elif refold_binary:
+ refold = _has_surrogates(value)
+ if refold:
return self.header_factory(name, ''.join(lines)).fold(policy=self)
+
return name + ': ' + self.linesep.join(lines) + self.linesep
diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py
index a09bf3201e1fb7..80ee125cfd4ed3 100644
--- a/Lib/ensurepip/__init__.py
+++ b/Lib/ensurepip/__init__.py
@@ -1,78 +1,64 @@
-import collections
import os
-import os.path
import subprocess
import sys
import sysconfig
import tempfile
+from contextlib import nullcontext
from importlib import resources
+from pathlib import Path
+from shutil import copy2
__all__ = ["version", "bootstrap"]
-_PACKAGE_NAMES = ('pip',)
_PIP_VERSION = "23.3.2"
-_PROJECTS = [
- ("pip", _PIP_VERSION, "py3"),
-]
-
-# Packages bundled in ensurepip._bundled have wheel_name set.
-# Packages from WHEEL_PKG_DIR have wheel_path set.
-_Package = collections.namedtuple('Package',
- ('version', 'wheel_name', 'wheel_path'))
# Directory of system wheel packages. Some Linux distribution packaging
# policies recommend against bundling dependencies. For example, Fedora
# installs wheel packages in the /usr/share/python-wheels/ directory and don't
# install the ensurepip._bundled package.
-_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR')
+if (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None:
+ _WHEEL_PKG_DIR = Path(_pkg_dir).resolve()
+else:
+ _WHEEL_PKG_DIR = None
+
+def _find_wheel_pkg_dir_pip():
+ if _WHEEL_PKG_DIR is None:
+ # NOTE: The compile-time `WHEEL_PKG_DIR` is unset so there is no place
+ # NOTE: for looking up the wheels.
+ return None
-def _find_packages(path):
- packages = {}
+ dist_matching_wheels = _WHEEL_PKG_DIR.glob('pip-*.whl')
try:
- filenames = os.listdir(path)
- except OSError:
- # Ignore: path doesn't exist or permission error
- filenames = ()
- # Make the code deterministic if a directory contains multiple wheel files
- # of the same package, but don't attempt to implement correct version
- # comparison since this case should not happen.
- filenames = sorted(filenames)
- for filename in filenames:
- # filename is like 'pip-21.2.4-py3-none-any.whl'
- if not filename.endswith(".whl"):
- continue
- for name in _PACKAGE_NAMES:
- prefix = name + '-'
- if filename.startswith(prefix):
- break
- else:
- continue
-
- # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl'
- version = filename.removeprefix(prefix).partition('-')[0]
- wheel_path = os.path.join(path, filename)
- packages[name] = _Package(version, None, wheel_path)
- return packages
-
-
-def _get_packages():
- global _PACKAGES, _WHEEL_PKG_DIR
- if _PACKAGES is not None:
- return _PACKAGES
-
- packages = {}
- for name, version, py_tag in _PROJECTS:
- wheel_name = f"{name}-{version}-{py_tag}-none-any.whl"
- packages[name] = _Package(version, wheel_name, None)
- if _WHEEL_PKG_DIR:
- dir_packages = _find_packages(_WHEEL_PKG_DIR)
- # only used the wheel package directory if all packages are found there
- if all(name in dir_packages for name in _PACKAGE_NAMES):
- packages = dir_packages
- _PACKAGES = packages
- return packages
-_PACKAGES = None
+ last_matching_dist_wheel = sorted(dist_matching_wheels)[-1]
+ except IndexError:
+ # NOTE: `WHEEL_PKG_DIR` does not contain any wheel files for `pip`.
+ return None
+
+ return nullcontext(last_matching_dist_wheel)
+
+
+def _get_pip_whl_path_ctx():
+ # Prefer pip from the wheel package directory, if present.
+ if (alternative_pip_wheel_path := _find_wheel_pkg_dir_pip()) is not None:
+ return alternative_pip_wheel_path
+
+ return resources.as_file(
+ resources.files('ensurepip')
+ / '_bundled'
+ / f'pip-{_PIP_VERSION}-py3-none-any.whl'
+ )
+
+
+def _get_pip_version():
+ with _get_pip_whl_path_ctx() as bundled_wheel_path:
+ wheel_name = bundled_wheel_path.name
+ return (
+ # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl'
+ wheel_name.
+ removeprefix('pip-').
+ partition('-')[0]
+ )
def _run_pip(args, additional_paths=None):
@@ -105,7 +91,7 @@ def version():
"""
Returns a string specifying the bundled version of pip.
"""
- return _get_packages()['pip'].version
+ return _get_pip_version()
def _disable_pip_configuration_settings():
@@ -167,24 +153,10 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
with tempfile.TemporaryDirectory() as tmpdir:
# Put our bundled wheels into a temporary directory and construct the
# additional paths that need added to sys.path
- additional_paths = []
- for name, package in _get_packages().items():
- if package.wheel_name:
- # Use bundled wheel package
- wheel_name = package.wheel_name
- wheel_path = resources.files("ensurepip") / "_bundled" / wheel_name
- whl = wheel_path.read_bytes()
- else:
- # Use the wheel package directory
- with open(package.wheel_path, "rb") as fp:
- whl = fp.read()
- wheel_name = os.path.basename(package.wheel_path)
-
- filename = os.path.join(tmpdir, wheel_name)
- with open(filename, "wb") as fp:
- fp.write(whl)
-
- additional_paths.append(filename)
+ tmpdir_path = Path(tmpdir)
+ with _get_pip_whl_path_ctx() as bundled_wheel_path:
+ tmp_wheel_path = tmpdir_path / bundled_wheel_path.name
+ copy2(bundled_wheel_path, tmp_wheel_path)
# Construct the arguments to be passed to the pip command
args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir]
@@ -197,7 +169,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
if verbosity:
args += ["-" + "v" * verbosity]
- return _run_pip([*args, *_PACKAGE_NAMES], additional_paths)
+ return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)])
+
def _uninstall_helper(*, verbosity=0):
"""Helper to support a clean default uninstall process on Windows
@@ -227,7 +200,7 @@ def _uninstall_helper(*, verbosity=0):
if verbosity:
args += ["-" + "v" * verbosity]
- return _run_pip([*args, *reversed(_PACKAGE_NAMES)])
+ return _run_pip([*args, "pip"])
def _main(argv=None):
diff --git a/Lib/fileinput.py b/Lib/fileinput.py
index 1b25f28f3d3432..3dba3d2fbfa967 100644
--- a/Lib/fileinput.py
+++ b/Lib/fileinput.py
@@ -53,7 +53,7 @@
sequence must be accessed in strictly sequential order; sequence
access and readline() cannot be mixed.
-Optional in-place filtering: if the keyword argument inplace=1 is
+Optional in-place filtering: if the keyword argument inplace=True is
passed to input() or to the FileInput constructor, the file is moved
to a backup file and standard output is directed to the input file.
This makes it possible to write a filter that rewrites its input file
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index d942045f3de666..6d6292f9559253 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -53,7 +53,7 @@ def _new_module(name):
# For a list that can have a weakref to it.
class _List(list):
- pass
+ __slots__ = ("__weakref__",)
# Copied from weakref.py with some simplifications and modifications unique to
diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py
index 7b142e786e829e..c612fbefee2e80 100644
--- a/Lib/importlib/metadata/__init__.py
+++ b/Lib/importlib/metadata/__init__.py
@@ -1,7 +1,6 @@
import os
import re
import abc
-import csv
import sys
import json
import email
@@ -478,6 +477,10 @@ def make_file(name, hash=None, size_str=None):
@pass_none
def make_files(lines):
+ # Delay csv import, since Distribution.files is not as widely used
+ # as other parts of importlib.metadata
+ import csv
+
return starmap(make_file, csv.reader(lines))
@pass_none
diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py
index eb7e020d1edfc0..684b58d5548f91 100644
--- a/Lib/logging/__init__.py
+++ b/Lib/logging/__init__.py
@@ -1493,7 +1493,7 @@ def debug(self, msg, *args, **kwargs):
To pass exception information, use the keyword argument exc_info with
a true value, e.g.
- logger.debug("Houston, we have a %s", "thorny problem", exc_info=1)
+ logger.debug("Houston, we have a %s", "thorny problem", exc_info=True)
"""
if self.isEnabledFor(DEBUG):
self._log(DEBUG, msg, args, **kwargs)
@@ -1505,7 +1505,7 @@ def info(self, msg, *args, **kwargs):
To pass exception information, use the keyword argument exc_info with
a true value, e.g.
- logger.info("Houston, we have a %s", "notable problem", exc_info=1)
+ logger.info("Houston, we have a %s", "notable problem", exc_info=True)
"""
if self.isEnabledFor(INFO):
self._log(INFO, msg, args, **kwargs)
@@ -1517,7 +1517,7 @@ def warning(self, msg, *args, **kwargs):
To pass exception information, use the keyword argument exc_info with
a true value, e.g.
- logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1)
+ logger.warning("Houston, we have a %s", "bit of a problem", exc_info=True)
"""
if self.isEnabledFor(WARNING):
self._log(WARNING, msg, args, **kwargs)
@@ -1529,7 +1529,7 @@ def error(self, msg, *args, **kwargs):
To pass exception information, use the keyword argument exc_info with
a true value, e.g.
- logger.error("Houston, we have a %s", "major problem", exc_info=1)
+ logger.error("Houston, we have a %s", "major problem", exc_info=True)
"""
if self.isEnabledFor(ERROR):
self._log(ERROR, msg, args, **kwargs)
@@ -1547,7 +1547,7 @@ def critical(self, msg, *args, **kwargs):
To pass exception information, use the keyword argument exc_info with
a true value, e.g.
- logger.critical("Houston, we have a %s", "major disaster", exc_info=1)
+ logger.critical("Houston, we have a %s", "major disaster", exc_info=True)
"""
if self.isEnabledFor(CRITICAL):
self._log(CRITICAL, msg, args, **kwargs)
@@ -1565,7 +1565,7 @@ def log(self, level, msg, *args, **kwargs):
To pass exception information, use the keyword argument exc_info with
a true value, e.g.
- logger.log(level, "We have a %s", "mysterious problem", exc_info=1)
+ logger.log(level, "We have a %s", "mysterious problem", exc_info=True)
"""
if not isinstance(level, int):
if raiseExceptions:
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
index 9840b7b0aeba88..e7f1322e4ba3d9 100644
--- a/Lib/logging/handlers.py
+++ b/Lib/logging/handlers.py
@@ -1586,6 +1586,7 @@ def stop(self):
Note that if you don't call this before your application exits, there
may be some records still left on the queue, which won't be processed.
"""
- self.enqueue_sentinel()
- self._thread.join()
- self._thread = None
+ if self._thread: # see gh-114706 - allow calling this more than once
+ self.enqueue_sentinel()
+ self._thread.join()
+ self._thread = None
diff --git a/Lib/ntpath.py b/Lib/ntpath.py
index aa0e018eb668c2..e7cbfe17ecb3c8 100644
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -26,8 +26,8 @@
__all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext",
"basename","dirname","commonprefix","getsize","getmtime",
"getatime","getctime", "islink","exists","lexists","isdir","isfile",
- "ismount", "expanduser","expandvars","normpath","abspath",
- "curdir","pardir","sep","pathsep","defpath","altsep",
+ "ismount","isreserved","expanduser","expandvars","normpath",
+ "abspath","curdir","pardir","sep","pathsep","defpath","altsep",
"extsep","devnull","realpath","supports_unicode_filenames","relpath",
"samefile", "sameopenfile", "samestat", "commonpath", "isjunction"]
@@ -330,6 +330,42 @@ def ismount(path):
return False
+_reserved_chars = frozenset(
+ {chr(i) for i in range(32)} |
+ {'"', '*', ':', '<', '>', '?', '|', '/', '\\'}
+)
+
+_reserved_names = frozenset(
+ {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} |
+ {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} |
+ {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'}
+)
+
+def isreserved(path):
+ """Return true if the pathname is reserved by the system."""
+ # Refer to "Naming Files, Paths, and Namespaces":
+ # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+ path = os.fsdecode(splitroot(path)[2]).replace(altsep, sep)
+ return any(_isreservedname(name) for name in reversed(path.split(sep)))
+
+def _isreservedname(name):
+ """Return true if the filename is reserved by the system."""
+ # Trailing dots and spaces are reserved.
+ if name.endswith(('.', ' ')) and name not in ('.', '..'):
+ return True
+ # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved.
+ # ASCII control characters (0-31) are reserved.
+ # Colon is reserved for file streams (e.g. "name:stream[:type]").
+ if _reserved_chars.intersection(name):
+ return True
+ # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules
+ # are complex and vary across Windows versions. On the side of
+ # caution, return True for names that may not be reserved.
+ if name.partition('.')[0].rstrip(' ').upper() in _reserved_names:
+ return True
+ return False
+
+
# Expand paths beginning with '~' or '~user'.
# '~' means $HOME; '~user' means that user's home directory.
# If the path doesn't begin with '~', or if the user or $HOME is unknown,
diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py
index eee82ef26bc7e7..4447f98e3e8689 100644
--- a/Lib/pathlib/__init__.py
+++ b/Lib/pathlib/__init__.py
@@ -33,15 +33,6 @@
]
-# Reference for Windows paths can be found at
-# https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file .
-_WIN_RESERVED_NAMES = frozenset(
- {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} |
- {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} |
- {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'}
-)
-
-
class _PathParents(Sequence):
"""This object provides sequence-like access to the logical ancestors
of a path. Don't try to construct it yourself."""
@@ -433,18 +424,13 @@ def is_absolute(self):
def is_reserved(self):
"""Return True if the path contains one of the special names reserved
by the system, if any."""
- if self.pathmod is not ntpath or not self.name:
- return False
-
- # NOTE: the rules for reserved names seem somewhat complicated
- # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not
- # exist). We err on the side of caution and return True for paths
- # which are not considered reserved by Windows.
- if self.drive.startswith('\\\\'):
- # UNC paths are never reserved.
- return False
- name = self.name.partition('.')[0].partition(':')[0].rstrip(' ')
- return name.upper() in _WIN_RESERVED_NAMES
+ msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled "
+ "for removal in Python 3.15. Use os.path.isreserved() to "
+ "detect reserved paths on Windows.")
+ warnings.warn(msg, DeprecationWarning, stacklevel=2)
+ if self.pathmod is ntpath:
+ return self.pathmod.isreserved(self)
+ return False
def as_uri(self):
"""Return the path as a URI."""
@@ -479,14 +465,6 @@ def _pattern_stack(self):
elif pattern[-1] in (self.pathmod.sep, self.pathmod.altsep):
# GH-65238: pathlib doesn't preserve trailing slash. Add it back.
parts.append('')
- elif parts[-1] == '**':
- # GH-70303: '**' only matches directories. Add trailing slash.
- warnings.warn(
- "Pattern ending '**' will match files and directories in a "
- "future Python release. Add a trailing slash to match only "
- "directories and remove this warning.",
- FutureWarning, 4)
- parts.append('')
parts.reverse()
return parts
diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py
index 6303a18680befc..580d631cbf3b53 100644
--- a/Lib/pathlib/_abc.py
+++ b/Lib/pathlib/_abc.py
@@ -299,10 +299,13 @@ def with_suffix(self, suffix):
has no suffix, add given suffix. If the given suffix is an empty
string, remove the suffix from the path.
"""
+ stem = self.stem
if not suffix:
- return self.with_name(self.stem)
+ return self.with_name(stem)
+ elif not stem:
+ raise ValueError(f"{self!r} has an empty name")
elif suffix.startswith('.') and len(suffix) > 1:
- return self.with_name(self.stem + suffix)
+ return self.with_name(stem + suffix)
else:
raise ValueError(f"Invalid suffix {suffix!r}")
@@ -771,7 +774,7 @@ def glob(self, pattern, *, case_sensitive=None, follow_symlinks=None):
filter_paths = False
deduplicate_paths = False
sep = self.pathmod.sep
- paths = iter([self.joinpath('')] if self.is_dir() else [])
+ paths = iter([self] if self.is_dir() else [])
while stack:
part = stack.pop()
if part in specials:
diff --git a/Lib/platform.py b/Lib/platform.py
index 75aa55510858fd..b56472235ee9e4 100755
--- a/Lib/platform.py
+++ b/Lib/platform.py
@@ -752,6 +752,8 @@ def architecture(executable=sys.executable, bits='', linkage=''):
# Linkage
if 'ELF' in fileout:
linkage = 'ELF'
+ elif 'Mach-O' in fileout:
+ linkage = "Mach-O"
elif 'PE' in fileout:
# E.g. Windows uses this format
if 'Windows' in fileout:
diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py
index ce4a11b119c900..de7d0da560a1c7 100644
--- a/Lib/test/audit-tests.py
+++ b/Lib/test/audit-tests.py
@@ -487,7 +487,13 @@ def hook(event, args):
print(event, args[0])
sys.addaudithook(hook)
- _wmi.exec_query("SELECT * FROM Win32_OperatingSystem")
+ try:
+ _wmi.exec_query("SELECT * FROM Win32_OperatingSystem")
+ except WindowsError as e:
+ # gh-112278: WMI may be slow response when first called, but we still
+ # get the audit event, so just ignore the timeout
+ if e.winerror != 258:
+ raise
def test_syslog():
import syslog
diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt
index 1eca123be0fecb..217ee15d4c8af5 100644
--- a/Lib/test/exception_hierarchy.txt
+++ b/Lib/test/exception_hierarchy.txt
@@ -44,6 +44,7 @@ BaseException
├── StopAsyncIteration
├── StopIteration
├── SyntaxError
+ │ └── IncompleteInputError
│ └── IndentationError
│ └── TabError
├── SystemError
diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py
index 69fef5945ae66f..21a4cb586ff665 100644
--- a/Lib/test/test_csv.py
+++ b/Lib/test/test_csv.py
@@ -392,10 +392,26 @@ def test_read_quoting(self):
# will this fail where locale uses comma for decimals?
self._read_test([',3,"5",7.3, 9'], [['', 3, '5', 7.3, 9]],
quoting=csv.QUOTE_NONNUMERIC)
+ self._read_test([',3,"5",7.3, 9'], [[None, '3', '5', '7.3', ' 9']],
+ quoting=csv.QUOTE_NOTNULL)
+ self._read_test([',3,"5",7.3, 9'], [[None, 3, '5', 7.3, 9]],
+ quoting=csv.QUOTE_STRINGS)
+
+ self._read_test([',,"",'], [['', '', '', '']])
+ self._read_test([',,"",'], [['', '', '', '']],
+ quoting=csv.QUOTE_NONNUMERIC)
+ self._read_test([',,"",'], [[None, None, '', None]],
+ quoting=csv.QUOTE_NOTNULL)
+ self._read_test([',,"",'], [[None, None, '', None]],
+ quoting=csv.QUOTE_STRINGS)
+
self._read_test(['"a\nb", 7'], [['a\nb', ' 7']])
self.assertRaises(ValueError, self._read_test,
['abc,3'], [[]],
quoting=csv.QUOTE_NONNUMERIC)
+ self.assertRaises(ValueError, self._read_test,
+ ['abc,3'], [[]],
+ quoting=csv.QUOTE_STRINGS)
self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@')
self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0')
@@ -403,6 +419,15 @@ def test_read_skipinitialspace(self):
self._read_test(['no space, space, spaces,\ttab'],
[['no space', 'space', 'spaces', '\ttab']],
skipinitialspace=True)
+ self._read_test([' , , '],
+ [['', '', '']],
+ skipinitialspace=True)
+ self._read_test([' , , '],
+ [[None, None, None]],
+ skipinitialspace=True, quoting=csv.QUOTE_NOTNULL)
+ self._read_test([' , , '],
+ [[None, None, None]],
+ skipinitialspace=True, quoting=csv.QUOTE_STRINGS)
def test_read_bigfield(self):
# This exercises the buffer realloc functionality and field size
diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py
index 7a5fe62b467372..1423bc61c7f690 100644
--- a/Lib/test/test_decimal.py
+++ b/Lib/test/test_decimal.py
@@ -41,6 +41,7 @@
darwin_malloc_err_warning, is_emscripten)
from test.support.import_helper import import_fresh_module
from test.support import threading_helper
+from test.support import warnings_helper
import random
import inspect
import threading
@@ -1237,7 +1238,14 @@ def test_deprecated_N_format(self):
else:
self.assertRaises(ValueError, format, h, 'N')
self.assertRaises(ValueError, format, h, '010.3N')
-
+ with warnings_helper.check_no_warnings(self):
+ self.assertEqual(format(h, 'N>10.3'), 'NN6.63E-34')
+ self.assertEqual(format(h, 'N>10.3n'), 'NN6.63e-34')
+ self.assertEqual(format(h, 'N>10.3e'), 'N6.626e-34')
+ self.assertEqual(format(h, 'N>10.3f'), 'NNNNN0.000')
+ self.assertRaises(ValueError, format, h, '>Nf')
+ self.assertRaises(ValueError, format, h, '10Nf')
+ self.assertRaises(ValueError, format, h, 'Nx')
@run_with_locale('LC_ALL', 'ps_AF')
def test_wide_char_separator_decimal_point(self):
diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
index fd0af9b30a0a71..beeab6cb7f254c 100644
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -4734,6 +4734,20 @@ class X(object):
with self.assertRaises(AttributeError):
del X.__abstractmethods__
+ def test_gh55664(self):
+ # gh-55664: issue a warning when the
+ # __dict__ of a class contains non-string keys
+ with self.assertWarnsRegex(RuntimeWarning, 'MyClass'):
+ MyClass = type('MyClass', (), {1: 2})
+
+ class meta(type):
+ def __new__(mcls, name, bases, ns):
+ ns[1] = 2
+ return super().__new__(mcls, name, bases, ns)
+
+ with self.assertWarnsRegex(RuntimeWarning, 'MyClass'):
+ MyClass = meta('MyClass', (), {})
+
def test_proxy_call(self):
class FakeStr:
__class__ = str
@@ -5151,7 +5165,8 @@ class Base2(object):
mykey = 'from Base2'
mykey2 = 'from Base2'
- X = type('X', (Base,), {MyKey(): 5})
+ with self.assertWarnsRegex(RuntimeWarning, 'X'):
+ X = type('X', (Base,), {MyKey(): 5})
# mykey is read from Base
self.assertEqual(X.mykey, 'from Base')
# mykey2 is read from Base2 because MyKey.__eq__ has set __bases__
diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py
index e87c275549406d..c6b9c80efe1b54 100644
--- a/Lib/test/test_email/test_policy.py
+++ b/Lib/test/test_email/test_policy.py
@@ -135,6 +135,23 @@ def test_policy_addition(self):
for attr, value in expected.items():
self.assertEqual(getattr(added, attr), value)
+ def test_fold_utf8(self):
+ expected_ascii = 'Subject: =?utf-8?q?=C3=A1?=\n'
+ expected_utf8 = 'Subject: á\n'
+
+ msg = email.message.EmailMessage()
+ s = 'á'
+ msg['Subject'] = s
+
+ p_ascii = email.policy.default.clone()
+ p_utf8 = email.policy.default.clone(utf8=True)
+
+ self.assertEqual(p_ascii.fold('Subject', msg['Subject']), expected_ascii)
+ self.assertEqual(p_utf8.fold('Subject', msg['Subject']), expected_utf8)
+
+ self.assertEqual(p_ascii.fold('Subject', s), expected_ascii)
+ self.assertEqual(p_utf8.fold('Subject', s), expected_utf8)
+
def test_fold_zero_max_line_length(self):
expected = 'Subject: =?utf-8?q?=C3=A1?=\n'
diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py
index 69ab2a4feaa938..a4b36a90d8815e 100644
--- a/Lib/test/test_ensurepip.py
+++ b/Lib/test/test_ensurepip.py
@@ -6,6 +6,8 @@
import test.support
import unittest
import unittest.mock
+from importlib.resources.abc import Traversable
+from pathlib import Path
import ensurepip
import ensurepip._uninstall
@@ -20,41 +22,35 @@ def test_version(self):
# Test version()
with tempfile.TemporaryDirectory() as tmpdir:
self.touch(tmpdir, "pip-1.2.3b1-py2.py3-none-any.whl")
- with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None),
- unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)):
+ with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', Path(tmpdir)):
self.assertEqual(ensurepip.version(), '1.2.3b1')
- def test_get_packages_no_dir(self):
- # Test _get_packages() without a wheel package directory
- with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None),
- unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None)):
- packages = ensurepip._get_packages()
-
- # when bundled wheel packages are used, we get _PIP_VERSION
+ def test_version_no_dir(self):
+ # Test version() without a wheel package directory
+ with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None):
+ # when the bundled pip wheel is used, we get _PIP_VERSION
self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version())
- # use bundled wheel packages
- self.assertIsNotNone(packages['pip'].wheel_name)
+ def test_selected_wheel_path_no_dir(self):
+ pip_filename = f'pip-{ensurepip._PIP_VERSION}-py3-none-any.whl'
+ with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None):
+ with ensurepip._get_pip_whl_path_ctx() as bundled_wheel_path:
+ self.assertEqual(pip_filename, bundled_wheel_path.name)
- def test_get_packages_with_dir(self):
- # Test _get_packages() with a wheel package directory
+ def test_selected_wheel_path_with_dir(self):
+ # Test _get_pip_whl_path_ctx() with a wheel package directory
pip_filename = "pip-20.2.2-py2.py3-none-any.whl"
with tempfile.TemporaryDirectory() as tmpdir:
self.touch(tmpdir, pip_filename)
- # not used, make sure that it's ignored
+ # not used, make sure that they're ignored
+ self.touch(tmpdir, "pip-1.2.3-py2.py3-none-any.whl")
self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl")
+ self.touch(tmpdir, "pip-script.py")
- with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None),
- unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)):
- packages = ensurepip._get_packages()
-
- self.assertEqual(packages['pip'].version, '20.2.2')
- self.assertEqual(packages['pip'].wheel_path,
- os.path.join(tmpdir, pip_filename))
-
- # wheel package is ignored
- self.assertEqual(sorted(packages), ['pip'])
+ with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', Path(tmpdir)):
+ with ensurepip._get_pip_whl_path_ctx() as bundled_wheel_path:
+ self.assertEqual(pip_filename, bundled_wheel_path.name)
class EnsurepipMixin:
@@ -69,7 +65,7 @@ def setUp(self):
real_devnull = os.devnull
os_patch = unittest.mock.patch("ensurepip.os")
patched_os = os_patch.start()
- # But expose os.listdir() used by _find_packages()
+ # But expose os.listdir() used by _find_wheel_pkg_dir_pip()
patched_os.listdir = os.listdir
self.addCleanup(os_patch.stop)
patched_os.devnull = real_devnull
diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py
index 786d9186634305..b3ad41d2588c4c 100644
--- a/Lib/test/test_fileinput.py
+++ b/Lib/test/test_fileinput.py
@@ -151,7 +151,7 @@ def test_buffer_sizes(self):
print('6. Inplace')
savestdout = sys.stdout
try:
- fi = FileInput(files=(t1, t2, t3, t4), inplace=1, encoding="utf-8")
+ fi = FileInput(files=(t1, t2, t3, t4), inplace=True, encoding="utf-8")
for line in fi:
line = line[:-1].upper()
print(line)
@@ -256,7 +256,7 @@ def test_detached_stdin_binary_mode(self):
def test_file_opening_hook(self):
try:
# cannot use openhook and inplace mode
- fi = FileInput(inplace=1, openhook=lambda f, m: None)
+ fi = FileInput(inplace=True, openhook=lambda f, m: None)
self.fail("FileInput should raise if both inplace "
"and openhook arguments are given")
except ValueError:
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
index 908e242b85f5e7..888523227c2ac4 100644
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -4089,6 +4089,7 @@ def test_queue_listener(self):
self.que_logger.critical(self.next_message())
finally:
listener.stop()
+ listener.stop() # gh-114706 - ensure no crash if called again
self.assertTrue(handler.matches(levelno=logging.WARNING, message='1'))
self.assertTrue(handler.matches(levelno=logging.ERROR, message='2'))
self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='3'))
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index aefcb98f1c30eb..9cb03e3cd5de8d 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -981,6 +981,62 @@ def test_ismount(self):
self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$"))
self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\"))
+ def test_isreserved(self):
+ self.assertFalse(ntpath.isreserved(''))
+ self.assertFalse(ntpath.isreserved('.'))
+ self.assertFalse(ntpath.isreserved('..'))
+ self.assertFalse(ntpath.isreserved('/'))
+ self.assertFalse(ntpath.isreserved('/foo/bar'))
+ # A name that ends with a space or dot is reserved.
+ self.assertTrue(ntpath.isreserved('foo.'))
+ self.assertTrue(ntpath.isreserved('foo '))
+ # ASCII control characters are reserved.
+ self.assertTrue(ntpath.isreserved('\foo'))
+ # Wildcard characters, colon, and pipe are reserved.
+ self.assertTrue(ntpath.isreserved('foo*bar'))
+ self.assertTrue(ntpath.isreserved('foo?bar'))
+ self.assertTrue(ntpath.isreserved('foo"bar'))
+ self.assertTrue(ntpath.isreserved('foo<bar'))
+ self.assertTrue(ntpath.isreserved('foo>bar'))
+ self.assertTrue(ntpath.isreserved('foo:bar'))
+ self.assertTrue(ntpath.isreserved('foo|bar'))
+ # Case-insensitive DOS-device names are reserved.
+ self.assertTrue(ntpath.isreserved('nul'))
+ self.assertTrue(ntpath.isreserved('aux'))
+ self.assertTrue(ntpath.isreserved('prn'))
+ self.assertTrue(ntpath.isreserved('con'))
+ self.assertTrue(ntpath.isreserved('conin$'))
+ self.assertTrue(ntpath.isreserved('conout$'))
+ # COM/LPT + 1-9 or + superscript 1-3 are reserved.
+ self.assertTrue(ntpath.isreserved('COM1'))
+ self.assertTrue(ntpath.isreserved('LPT9'))
+ self.assertTrue(ntpath.isreserved('com\xb9'))
+ self.assertTrue(ntpath.isreserved('com\xb2'))
+ self.assertTrue(ntpath.isreserved('lpt\xb3'))
+ # DOS-device name matching ignores characters after a dot or
+ # a colon and also ignores trailing spaces.
+ self.assertTrue(ntpath.isreserved('NUL.txt'))
+ self.assertTrue(ntpath.isreserved('PRN '))
+ self.assertTrue(ntpath.isreserved('AUX .txt'))
+ self.assertTrue(ntpath.isreserved('COM1:bar'))
+ self.assertTrue(ntpath.isreserved('LPT9 :bar'))
+ # DOS-device names are only matched at the beginning
+ # of a path component.
+ self.assertFalse(ntpath.isreserved('bar.com9'))
+ self.assertFalse(ntpath.isreserved('bar.lpt9'))
+ # The entire path is checked, except for the drive.
+ self.assertTrue(ntpath.isreserved('c:/bar/baz/NUL'))
+ self.assertTrue(ntpath.isreserved('c:/NUL/bar/baz'))
+ self.assertFalse(ntpath.isreserved('//./NUL'))
+ # Bytes are supported.
+ self.assertFalse(ntpath.isreserved(b''))
+ self.assertFalse(ntpath.isreserved(b'.'))
+ self.assertFalse(ntpath.isreserved(b'..'))
+ self.assertFalse(ntpath.isreserved(b'/'))
+ self.assertFalse(ntpath.isreserved(b'/foo/bar'))
+ self.assertTrue(ntpath.isreserved(b'foo.'))
+ self.assertTrue(ntpath.isreserved(b'nul'))
+
def assertEqualCI(self, s1, s2):
"""Assert that two strings are equal ignoring case differences."""
self.assertEqual(s1.lower(), s2.lower())
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index ed1f304c6c8cac..ed79a2c24ef30b 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -3129,10 +3129,9 @@ def cleanup():
if support.verbose:
print(" without access:", stat2)
- # We cannot get st_dev/st_ino, so ensure those are 0 or else our test
- # is not set up correctly
- self.assertEqual(0, stat2.st_dev)
- self.assertEqual(0, stat2.st_ino)
+ # We may not get st_dev/st_ino, so ensure those are 0 or match
+ self.assertIn(stat2.st_dev, (0, stat1.st_dev))
+ self.assertIn(stat2.st_ino, (0, stat1.st_ino))
# st_mode and st_size should match (for a normal file, at least)
self.assertEqual(stat1.st_mode, stat2.st_mode)
@@ -4537,13 +4536,46 @@ def test_dup2(self):
self.assertEqual(os.dup2(fd, fd3, inheritable=False), fd3)
self.assertFalse(os.get_inheritable(fd3))
- @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()")
+@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()")
+class PseudoterminalTests(unittest.TestCase):
+ def open_pty(self):
+ """Open a pty fd-pair, and schedule cleanup for it"""
+ main_fd, second_fd = os.openpty()
+ self.addCleanup(os.close, main_fd)
+ self.addCleanup(os.close, second_fd)
+ return main_fd, second_fd
+
def test_openpty(self):
- master_fd, slave_fd = os.openpty()
- self.addCleanup(os.close, master_fd)
- self.addCleanup(os.close, slave_fd)
- self.assertEqual(os.get_inheritable(master_fd), False)
- self.assertEqual(os.get_inheritable(slave_fd), False)
+ main_fd, second_fd = self.open_pty()
+ self.assertEqual(os.get_inheritable(main_fd), False)
+ self.assertEqual(os.get_inheritable(second_fd), False)
+
+ @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()")
+ @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR")
+ @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY")
+ def test_open_via_ptsname(self):
+ main_fd, second_fd = self.open_pty()
+ second_path = os.ptsname(main_fd)
+ reopened_second_fd = os.open(second_path, os.O_RDWR|os.O_NOCTTY)
+ self.addCleanup(os.close, reopened_second_fd)
+ os.write(reopened_second_fd, b'foo')
+ self.assertEqual(os.read(main_fd, 3), b'foo')
+
+ @unittest.skipUnless(hasattr(os, 'posix_openpt'), "need os.posix_openpt()")
+ @unittest.skipUnless(hasattr(os, 'grantpt'), "need os.grantpt()")
+ @unittest.skipUnless(hasattr(os, 'unlockpt'), "need os.unlockpt()")
+ @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()")
+ @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR")
+ @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY")
+ def test_posix_pty_functions(self):
+ mother_fd = os.posix_openpt(os.O_RDWR|os.O_NOCTTY)
+ self.addCleanup(os.close, mother_fd)
+ os.grantpt(mother_fd)
+ os.unlockpt(mother_fd)
+ son_path = os.ptsname(mother_fd)
+ son_fd = os.open(son_path, os.O_RDWR|os.O_NOCTTY)
+ self.addCleanup(os.close, son_fd)
+ self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd))
@unittest.skipUnless(hasattr(os, 'spawnl'), "need os.openpty()")
def test_pipe_spawnl(self):
diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py
index bdbe92369639ef..3bee9b8c762b80 100644
--- a/Lib/test/test_pathlib/test_pathlib.py
+++ b/Lib/test/test_pathlib/test_pathlib.py
@@ -19,6 +19,7 @@
from test.support import os_helper
from test.support.os_helper import TESTFN, FakePath
from test.test_pathlib import test_pathlib_abc
+from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows, needs_symlinks
try:
import grp, pwd
@@ -26,11 +27,6 @@
grp = pwd = None
-only_nt = unittest.skipIf(os.name != 'nt',
- 'test requires a Windows-compatible system')
-only_posix = unittest.skipIf(os.name == 'nt',
- 'test requires a POSIX-compatible system')
-
root_in_posix = False
if hasattr(os, 'geteuid'):
root_in_posix = (os.geteuid() == 0)
@@ -238,8 +234,10 @@ def test_eq_common(self):
self.assertNotEqual(P(), {})
self.assertNotEqual(P(), int)
- def test_equivalences(self):
- for k, tuples in self.equivalences.items():
+ def test_equivalences(self, equivalences=None):
+ if equivalences is None:
+ equivalences = self.equivalences
+ for k, tuples in equivalences.items():
canon = k.replace('/', self.sep)
posix = k.replace(self.sep, '/')
if canon != posix:
@@ -329,13 +327,6 @@ def test_with_stem_empty(self):
self.assertRaises(ValueError, P('a/b').with_stem, '')
self.assertRaises(ValueError, P('a/b').with_stem, '.')
- def test_with_suffix_empty(self):
- # Path doesn't have a "filename" component.
- P = self.cls
- self.assertRaises(ValueError, P('').with_suffix, '.gz')
- self.assertRaises(ValueError, P('.').with_suffix, '.gz')
- self.assertRaises(ValueError, P('/').with_suffix, '.gz')
-
def test_relative_to_several_args(self):
P = self.cls
p = P('a/b')
@@ -349,16 +340,19 @@ def test_is_relative_to_several_args(self):
with self.assertWarns(DeprecationWarning):
p.is_relative_to('a', 'b')
+ def test_is_reserved_deprecated(self):
+ P = self.cls
+ p = P('a/b')
+ with self.assertWarns(DeprecationWarning):
+ p.is_reserved()
+
def test_match_empty(self):
P = self.cls
self.assertRaises(ValueError, P('a').match, '')
self.assertRaises(ValueError, P('a').match, '.')
-
-class PurePosixPathTest(PurePathTest):
- cls = pathlib.PurePosixPath
-
- def test_parse_path(self):
+ @needs_posix
+ def test_parse_path_posix(self):
check = self._check_parse_path
# Collapsing of excess leading slashes, except for the double-slash
# special case.
@@ -370,25 +364,21 @@ def test_parse_path(self):
check('c:\\a', '', '', ['c:\\a',])
check('\\a', '', '', ['\\a',])
- def test_root(self):
- P = self.cls
- self.assertEqual(P('/a/b').root, '/')
- self.assertEqual(P('///a/b').root, '/')
- # POSIX special case for two leading slashes.
- self.assertEqual(P('//a/b').root, '//')
-
- def test_eq(self):
+ @needs_posix
+ def test_eq_posix(self):
P = self.cls
self.assertNotEqual(P('a/b'), P('A/b'))
self.assertEqual(P('/a'), P('///a'))
self.assertNotEqual(P('/a'), P('//a'))
- def test_as_uri(self):
+ @needs_posix
+ def test_as_uri_posix(self):
P = self.cls
self.assertEqual(P('/').as_uri(), 'file:///')
self.assertEqual(P('/a/b.c').as_uri(), 'file:///a/b.c')
self.assertEqual(P('/a/b%#c').as_uri(), 'file:///a/b%25%23c')
+ @needs_posix
def test_as_uri_non_ascii(self):
from urllib.parse import quote_from_bytes
P = self.cls
@@ -399,61 +389,14 @@ def test_as_uri_non_ascii(self):
self.assertEqual(P('/a/b\xe9').as_uri(),
'file:///a/b' + quote_from_bytes(os.fsencode('\xe9')))
- def test_match(self):
- P = self.cls
- self.assertFalse(P('A.py').match('a.PY'))
-
- def test_is_absolute(self):
- P = self.cls
- self.assertFalse(P().is_absolute())
- self.assertFalse(P('a').is_absolute())
- self.assertFalse(P('a/b/').is_absolute())
- self.assertTrue(P('/').is_absolute())
- self.assertTrue(P('/a').is_absolute())
- self.assertTrue(P('/a/b/').is_absolute())
- self.assertTrue(P('//a').is_absolute())
- self.assertTrue(P('//a/b').is_absolute())
-
- def test_is_reserved(self):
- P = self.cls
- self.assertIs(False, P('').is_reserved())
- self.assertIs(False, P('/').is_reserved())
- self.assertIs(False, P('/foo/bar').is_reserved())
- self.assertIs(False, P('/dev/con/PRN/NUL').is_reserved())
-
- def test_join(self):
- P = self.cls
- p = P('//a')
- pp = p.joinpath('b')
- self.assertEqual(pp, P('//a/b'))
- pp = P('/a').joinpath('//c')
- self.assertEqual(pp, P('//c'))
- pp = P('//a').joinpath('/c')
- self.assertEqual(pp, P('/c'))
-
- def test_div(self):
- # Basically the same as joinpath().
- P = self.cls
- p = P('//a')
- pp = p / 'b'
- self.assertEqual(pp, P('//a/b'))
- pp = P('/a') / '//c'
- self.assertEqual(pp, P('//c'))
- pp = P('//a') / '/c'
- self.assertEqual(pp, P('/c'))
-
+ @needs_posix
def test_parse_windows_path(self):
P = self.cls
p = P('c:', 'a', 'b')
pp = P(pathlib.PureWindowsPath('c:\\a\\b'))
self.assertEqual(p, pp)
-
-class PureWindowsPathTest(PurePathTest):
- cls = pathlib.PureWindowsPath
-
- equivalences = PurePathTest.equivalences.copy()
- equivalences.update({
+ windows_equivalences = {
'./a:b': [ ('./a:b',) ],
'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ],
'c:/a': [
@@ -464,9 +407,14 @@ class PureWindowsPathTest(PurePathTest):
'//a/b/c': [
('//a/b', 'c'), ('//a/b/', 'c'),
],
- })
+ }
+
+ @needs_windows
+ def test_equivalences_windows(self):
+ self.test_equivalences(self.windows_equivalences)
- def test_parse_path(self):
+ @needs_windows
+ def test_parse_path_windows(self):
check = self._check_parse_path
# First part is anchored.
check('c:', 'c:', '', [])
@@ -514,31 +462,8 @@ def test_parse_path(self):
check('D:a/c:b', 'D:', '', ['a', 'c:b'])
check('D:/a/c:b', 'D:', '\\', ['a', 'c:b'])
- def test_str(self):
- p = self.cls('a/b/c')
- self.assertEqual(str(p), 'a\\b\\c')
- p = self.cls('c:/a/b/c')
- self.assertEqual(str(p), 'c:\\a\\b\\c')
- p = self.cls('//a/b')
- self.assertEqual(str(p), '\\\\a\\b\\')
- p = self.cls('//a/b/c')
- self.assertEqual(str(p), '\\\\a\\b\\c')
- p = self.cls('//a/b/c/d')
- self.assertEqual(str(p), '\\\\a\\b\\c\\d')
-
- def test_str_subclass(self):
- self._check_str_subclass('.\\a:b')
- self._check_str_subclass('c:')
- self._check_str_subclass('c:a')
- self._check_str_subclass('c:a\\b.txt')
- self._check_str_subclass('c:\\')
- self._check_str_subclass('c:\\a')
- self._check_str_subclass('c:\\a\\b.txt')
- self._check_str_subclass('\\\\some\\share')
- self._check_str_subclass('\\\\some\\share\\a')
- self._check_str_subclass('\\\\some\\share\\a\\b.txt')
-
- def test_eq(self):
+ @needs_windows
+ def test_eq_windows(self):
P = self.cls
self.assertEqual(P('c:a/b'), P('c:a/b'))
self.assertEqual(P('c:a/b'), P('c:', 'a', 'b'))
@@ -551,7 +476,8 @@ def test_eq(self):
self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b'))
self.assertEqual(P('\u0130'), P('i\u0307'))
- def test_as_uri(self):
+ @needs_windows
+ def test_as_uri_windows(self):
P = self.cls
with self.assertRaises(ValueError):
P('/a/b').as_uri()
@@ -567,34 +493,8 @@ def test_as_uri(self):
self.assertEqual(P('//some/share/a/b%#c\xe9').as_uri(),
'file://some/share/a/b%25%23c%C3%A9')
- def test_match(self):
- P = self.cls
- # Absolute patterns.
- self.assertTrue(P('c:/b.py').match('*:/*.py'))
- self.assertTrue(P('c:/b.py').match('c:/*.py'))
- self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive
- self.assertFalse(P('b.py').match('/*.py'))
- self.assertFalse(P('b.py').match('c:*.py'))
- self.assertFalse(P('b.py').match('c:/*.py'))
- self.assertFalse(P('c:b.py').match('/*.py'))
- self.assertFalse(P('c:b.py').match('c:/*.py'))
- self.assertFalse(P('/b.py').match('c:*.py'))
- self.assertFalse(P('/b.py').match('c:/*.py'))
- # UNC patterns.
- self.assertTrue(P('//some/share/a.py').match('//*/*/*.py'))
- self.assertTrue(P('//some/share/a.py').match('//some/share/*.py'))
- self.assertFalse(P('//other/share/a.py').match('//some/share/*.py'))
- self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py'))
- # Case-insensitivity.
- self.assertTrue(P('B.py').match('b.PY'))
- self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY'))
- self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY'))
- # Path anchor doesn't match pattern anchor
- self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/'
- self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:'
- self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/'
-
- def test_ordering_common(self):
+ @needs_windows
+ def test_ordering_windows(self):
# Case-insensitivity.
def assertOrderedEqual(a, b):
self.assertLessEqual(a, b)
@@ -611,511 +511,13 @@ def assertOrderedEqual(a, b):
self.assertFalse(p < q)
self.assertFalse(p > q)
- def test_parts(self):
- P = self.cls
- p = P('c:a/b')
- parts = p.parts
- self.assertEqual(parts, ('c:', 'a', 'b'))
- p = P('c:/a/b')
- parts = p.parts
- self.assertEqual(parts, ('c:\\', 'a', 'b'))
- p = P('//a/b/c/d')
- parts = p.parts
- self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd'))
-
- def test_parent(self):
- # Anchored
- P = self.cls
- p = P('z:a/b/c')
- self.assertEqual(p.parent, P('z:a/b'))
- self.assertEqual(p.parent.parent, P('z:a'))
- self.assertEqual(p.parent.parent.parent, P('z:'))
- self.assertEqual(p.parent.parent.parent.parent, P('z:'))
- p = P('z:/a/b/c')
- self.assertEqual(p.parent, P('z:/a/b'))
- self.assertEqual(p.parent.parent, P('z:/a'))
- self.assertEqual(p.parent.parent.parent, P('z:/'))
- self.assertEqual(p.parent.parent.parent.parent, P('z:/'))
- p = P('//a/b/c/d')
- self.assertEqual(p.parent, P('//a/b/c'))
- self.assertEqual(p.parent.parent, P('//a/b'))
- self.assertEqual(p.parent.parent.parent, P('//a/b'))
-
- def test_parents(self):
- # Anchored
- P = self.cls
- p = P('z:a/b/')
- par = p.parents
- self.assertEqual(len(par), 2)
- self.assertEqual(par[0], P('z:a'))
- self.assertEqual(par[1], P('z:'))
- self.assertEqual(par[0:1], (P('z:a'),))
- self.assertEqual(par[:-1], (P('z:a'),))
- self.assertEqual(par[:2], (P('z:a'), P('z:')))
- self.assertEqual(par[1:], (P('z:'),))
- self.assertEqual(par[::2], (P('z:a'),))
- self.assertEqual(par[::-1], (P('z:'), P('z:a')))
- self.assertEqual(list(par), [P('z:a'), P('z:')])
- with self.assertRaises(IndexError):
- par[2]
- p = P('z:/a/b/')
- par = p.parents
- self.assertEqual(len(par), 2)
- self.assertEqual(par[0], P('z:/a'))
- self.assertEqual(par[1], P('z:/'))
- self.assertEqual(par[0:1], (P('z:/a'),))
- self.assertEqual(par[0:-1], (P('z:/a'),))
- self.assertEqual(par[:2], (P('z:/a'), P('z:/')))
- self.assertEqual(par[1:], (P('z:/'),))
- self.assertEqual(par[::2], (P('z:/a'),))
- self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),))
- self.assertEqual(list(par), [P('z:/a'), P('z:/')])
- with self.assertRaises(IndexError):
- par[2]
- p = P('//a/b/c/d')
- par = p.parents
- self.assertEqual(len(par), 2)
- self.assertEqual(par[0], P('//a/b/c'))
- self.assertEqual(par[1], P('//a/b'))
- self.assertEqual(par[0:1], (P('//a/b/c'),))
- self.assertEqual(par[0:-1], (P('//a/b/c'),))
- self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b')))
- self.assertEqual(par[1:], (P('//a/b'),))
- self.assertEqual(par[::2], (P('//a/b/c'),))
- self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c')))
- self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')])
- with self.assertRaises(IndexError):
- par[2]
-
- def test_drive(self):
- P = self.cls
- self.assertEqual(P('c:').drive, 'c:')
- self.assertEqual(P('c:a/b').drive, 'c:')
- self.assertEqual(P('c:/').drive, 'c:')
- self.assertEqual(P('c:/a/b/').drive, 'c:')
- self.assertEqual(P('//a/b').drive, '\\\\a\\b')
- self.assertEqual(P('//a/b/').drive, '\\\\a\\b')
- self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b')
- self.assertEqual(P('./c:a').drive, '')
-
- def test_root(self):
- P = self.cls
- self.assertEqual(P('c:').root, '')
- self.assertEqual(P('c:a/b').root, '')
- self.assertEqual(P('c:/').root, '\\')
- self.assertEqual(P('c:/a/b/').root, '\\')
- self.assertEqual(P('//a/b').root, '\\')
- self.assertEqual(P('//a/b/').root, '\\')
- self.assertEqual(P('//a/b/c/d').root, '\\')
-
- def test_anchor(self):
- P = self.cls
- self.assertEqual(P('c:').anchor, 'c:')
- self.assertEqual(P('c:a/b').anchor, 'c:')
- self.assertEqual(P('c:/').anchor, 'c:\\')
- self.assertEqual(P('c:/a/b/').anchor, 'c:\\')
- self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\')
- self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\')
- self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\')
-
- def test_name(self):
- P = self.cls
- self.assertEqual(P('c:').name, '')
- self.assertEqual(P('c:/').name, '')
- self.assertEqual(P('c:a/b').name, 'b')
- self.assertEqual(P('c:/a/b').name, 'b')
- self.assertEqual(P('c:a/b.py').name, 'b.py')
- self.assertEqual(P('c:/a/b.py').name, 'b.py')
- self.assertEqual(P('//My.py/Share.php').name, '')
- self.assertEqual(P('//My.py/Share.php/a/b').name, 'b')
-
- def test_suffix(self):
- P = self.cls
- self.assertEqual(P('c:').suffix, '')
- self.assertEqual(P('c:/').suffix, '')
- self.assertEqual(P('c:a/b').suffix, '')
- self.assertEqual(P('c:/a/b').suffix, '')
- self.assertEqual(P('c:a/b.py').suffix, '.py')
- self.assertEqual(P('c:/a/b.py').suffix, '.py')
- self.assertEqual(P('c:a/.hgrc').suffix, '')
- self.assertEqual(P('c:/a/.hgrc').suffix, '')
- self.assertEqual(P('c:a/.hg.rc').suffix, '.rc')
- self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc')
- self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz')
- self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz')
- self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '')
- self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '')
- self.assertEqual(P('//My.py/Share.php').suffix, '')
- self.assertEqual(P('//My.py/Share.php/a/b').suffix, '')
-
- def test_suffixes(self):
- P = self.cls
- self.assertEqual(P('c:').suffixes, [])
- self.assertEqual(P('c:/').suffixes, [])
- self.assertEqual(P('c:a/b').suffixes, [])
- self.assertEqual(P('c:/a/b').suffixes, [])
- self.assertEqual(P('c:a/b.py').suffixes, ['.py'])
- self.assertEqual(P('c:/a/b.py').suffixes, ['.py'])
- self.assertEqual(P('c:a/.hgrc').suffixes, [])
- self.assertEqual(P('c:/a/.hgrc').suffixes, [])
- self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc'])
- self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc'])
- self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz'])
- self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz'])
- self.assertEqual(P('//My.py/Share.php').suffixes, [])
- self.assertEqual(P('//My.py/Share.php/a/b').suffixes, [])
- self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, [])
- self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, [])
-
- def test_stem(self):
- P = self.cls
- self.assertEqual(P('c:').stem, '')
- self.assertEqual(P('c:.').stem, '')
- self.assertEqual(P('c:..').stem, '..')
- self.assertEqual(P('c:/').stem, '')
- self.assertEqual(P('c:a/b').stem, 'b')
- self.assertEqual(P('c:a/b.py').stem, 'b')
- self.assertEqual(P('c:a/.hgrc').stem, '.hgrc')
- self.assertEqual(P('c:a/.hg.rc').stem, '.hg')
- self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar')
- self.assertEqual(P('c:a/Some name. Ending with a dot.').stem,
- 'Some name. Ending with a dot.')
-
- def test_with_name(self):
- P = self.cls
- self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml'))
- self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml'))
- self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml'))
- self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml'))
- self.assertRaises(ValueError, P('c:').with_name, 'd.xml')
- self.assertRaises(ValueError, P('c:/').with_name, 'd.xml')
- self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml')
- self.assertEqual(str(P('a').with_name('d:')), '.\\d:')
- self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e')
- self.assertEqual(P('c:a/b').with_name('d:'), P('c:a/d:'))
- self.assertEqual(P('c:a/b').with_name('d:e'), P('c:a/d:e'))
- self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e')
- self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share')
-
- def test_with_stem(self):
- P = self.cls
- self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d'))
- self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d'))
- self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d'))
- self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d'))
- self.assertRaises(ValueError, P('c:').with_stem, 'd')
- self.assertRaises(ValueError, P('c:/').with_stem, 'd')
- self.assertRaises(ValueError, P('//My/Share').with_stem, 'd')
- self.assertEqual(str(P('a').with_stem('d:')), '.\\d:')
- self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e')
- self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:'))
- self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e'))
- self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e')
- self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share')
-
- def test_with_suffix(self):
- P = self.cls
- self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz'))
- self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz'))
- self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz'))
- self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz'))
- # Path doesn't have a "filename" component.
- self.assertRaises(ValueError, P('').with_suffix, '.gz')
- self.assertRaises(ValueError, P('.').with_suffix, '.gz')
- self.assertRaises(ValueError, P('/').with_suffix, '.gz')
- self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz')
- # Invalid suffix.
- self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz')
- self.assertRaises(ValueError, P('c:a/b').with_suffix, '/')
- self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\')
- self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:')
- self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz')
- self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz')
- self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz')
- self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d')
- self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d')
- self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d')
- self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d')
-
- def test_relative_to(self):
- P = self.cls
- p = P('C:Foo/Bar')
- self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar'))
- self.assertEqual(p.relative_to('c:'), P('Foo/Bar'))
- self.assertEqual(p.relative_to(P('c:foO')), P('Bar'))
- self.assertEqual(p.relative_to('c:foO'), P('Bar'))
- self.assertEqual(p.relative_to('c:foO/'), P('Bar'))
- self.assertEqual(p.relative_to(P('c:foO/baR')), P())
- self.assertEqual(p.relative_to('c:foO/baR'), P())
- self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar'))
- self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar'))
- self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar'))
- self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar'))
- self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar'))
- self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P())
- self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P())
- self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..'))
- self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar'))
- self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar'))
- # Unrelated paths.
- self.assertRaises(ValueError, p.relative_to, P())
- self.assertRaises(ValueError, p.relative_to, '')
- self.assertRaises(ValueError, p.relative_to, P('d:'))
- self.assertRaises(ValueError, p.relative_to, P('/'))
- self.assertRaises(ValueError, p.relative_to, P('Foo'))
- self.assertRaises(ValueError, p.relative_to, P('/Foo'))
- self.assertRaises(ValueError, p.relative_to, P('C:/Foo'))
- self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz'))
- self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz'))
- self.assertRaises(ValueError, p.relative_to, P(), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, '', walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True)
- p = P('C:/Foo/Bar')
- self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar'))
- self.assertEqual(p.relative_to('c:/'), P('Foo/Bar'))
- self.assertEqual(p.relative_to(P('c:/foO')), P('Bar'))
- self.assertEqual(p.relative_to('c:/foO'), P('Bar'))
- self.assertEqual(p.relative_to('c:/foO/'), P('Bar'))
- self.assertEqual(p.relative_to(P('c:/foO/baR')), P())
- self.assertEqual(p.relative_to('c:/foO/baR'), P())
- self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar'))
- self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar'))
- self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar'))
- self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar'))
- self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar'))
- self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P())
- self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P())
- self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar'))
- self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..'))
- self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar'))
- # Unrelated paths.
- self.assertRaises(ValueError, p.relative_to, 'c:')
- self.assertRaises(ValueError, p.relative_to, P('c:'))
- self.assertRaises(ValueError, p.relative_to, P('C:/Baz'))
- self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz'))
- self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz'))
- self.assertRaises(ValueError, p.relative_to, P('C:Foo'))
- self.assertRaises(ValueError, p.relative_to, P('d:'))
- self.assertRaises(ValueError, p.relative_to, P('d:/'))
- self.assertRaises(ValueError, p.relative_to, P('/'))
- self.assertRaises(ValueError, p.relative_to, P('/Foo'))
- self.assertRaises(ValueError, p.relative_to, P('//C/Foo'))
- self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True)
- # UNC paths.
- p = P('//Server/Share/Foo/Bar')
- self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar'))
- self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar'))
- self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar'))
- self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar'))
- self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar'))
- self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar'))
- self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P())
- self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P())
- self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar'))
- self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar'))
- self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar'))
- self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar'))
- self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar'))
- self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar'))
- self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P())
- self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P())
- self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar'))
- self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar'))
- # Unrelated paths.
- self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'))
- self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'))
- self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'))
- self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'))
- self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True)
- self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True)
-
- def test_is_relative_to(self):
- P = self.cls
- p = P('C:Foo/Bar')
- self.assertTrue(p.is_relative_to(P('c:')))
- self.assertTrue(p.is_relative_to('c:'))
- self.assertTrue(p.is_relative_to(P('c:foO')))
- self.assertTrue(p.is_relative_to('c:foO'))
- self.assertTrue(p.is_relative_to('c:foO/'))
- self.assertTrue(p.is_relative_to(P('c:foO/baR')))
- self.assertTrue(p.is_relative_to('c:foO/baR'))
- # Unrelated paths.
- self.assertFalse(p.is_relative_to(P()))
- self.assertFalse(p.is_relative_to(''))
- self.assertFalse(p.is_relative_to(P('d:')))
- self.assertFalse(p.is_relative_to(P('/')))
- self.assertFalse(p.is_relative_to(P('Foo')))
- self.assertFalse(p.is_relative_to(P('/Foo')))
- self.assertFalse(p.is_relative_to(P('C:/Foo')))
- self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz')))
- self.assertFalse(p.is_relative_to(P('C:Foo/Baz')))
- p = P('C:/Foo/Bar')
- self.assertTrue(p.is_relative_to(P('c:/')))
- self.assertTrue(p.is_relative_to(P('c:/foO')))
- self.assertTrue(p.is_relative_to('c:/foO/'))
- self.assertTrue(p.is_relative_to(P('c:/foO/baR')))
- self.assertTrue(p.is_relative_to('c:/foO/baR'))
- # Unrelated paths.
- self.assertFalse(p.is_relative_to('c:'))
- self.assertFalse(p.is_relative_to(P('C:/Baz')))
- self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz')))
- self.assertFalse(p.is_relative_to(P('C:/Foo/Baz')))
- self.assertFalse(p.is_relative_to(P('C:Foo')))
- self.assertFalse(p.is_relative_to(P('d:')))
- self.assertFalse(p.is_relative_to(P('d:/')))
- self.assertFalse(p.is_relative_to(P('/')))
- self.assertFalse(p.is_relative_to(P('/Foo')))
- self.assertFalse(p.is_relative_to(P('//C/Foo')))
- # UNC paths.
- p = P('//Server/Share/Foo/Bar')
- self.assertTrue(p.is_relative_to(P('//sErver/sHare')))
- self.assertTrue(p.is_relative_to('//sErver/sHare'))
- self.assertTrue(p.is_relative_to('//sErver/sHare/'))
- self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo')))
- self.assertTrue(p.is_relative_to('//sErver/sHare/Foo'))
- self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/'))
- self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar')))
- self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar'))
- # Unrelated paths.
- self.assertFalse(p.is_relative_to(P('/Server/Share/Foo')))
- self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo')))
- self.assertFalse(p.is_relative_to(P('//z/Share/Foo')))
- self.assertFalse(p.is_relative_to(P('//Server/z/Foo')))
-
- def test_is_absolute(self):
- P = self.cls
- # Under NT, only paths with both a drive and a root are absolute.
- self.assertFalse(P().is_absolute())
- self.assertFalse(P('a').is_absolute())
- self.assertFalse(P('a/b/').is_absolute())
- self.assertFalse(P('/').is_absolute())
- self.assertFalse(P('/a').is_absolute())
- self.assertFalse(P('/a/b/').is_absolute())
- self.assertFalse(P('c:').is_absolute())
- self.assertFalse(P('c:a').is_absolute())
- self.assertFalse(P('c:a/b/').is_absolute())
- self.assertTrue(P('c:/').is_absolute())
- self.assertTrue(P('c:/a').is_absolute())
- self.assertTrue(P('c:/a/b/').is_absolute())
- # UNC paths are absolute by definition.
- self.assertTrue(P('//').is_absolute())
- self.assertTrue(P('//a').is_absolute())
- self.assertTrue(P('//a/b').is_absolute())
- self.assertTrue(P('//a/b/').is_absolute())
- self.assertTrue(P('//a/b/c').is_absolute())
- self.assertTrue(P('//a/b/c/d').is_absolute())
- self.assertTrue(P('//?/UNC/').is_absolute())
- self.assertTrue(P('//?/UNC/spam').is_absolute())
-
- def test_join(self):
- P = self.cls
- p = P('C:/a/b')
- pp = p.joinpath('x/y')
- self.assertEqual(pp, P('C:/a/b/x/y'))
- pp = p.joinpath('/x/y')
- self.assertEqual(pp, P('C:/x/y'))
- # Joining with a different drive => the first path is ignored, even
- # if the second path is relative.
- pp = p.joinpath('D:x/y')
- self.assertEqual(pp, P('D:x/y'))
- pp = p.joinpath('D:/x/y')
- self.assertEqual(pp, P('D:/x/y'))
- pp = p.joinpath('//host/share/x/y')
- self.assertEqual(pp, P('//host/share/x/y'))
- # Joining with the same drive => the first path is appended to if
- # the second path is relative.
- pp = p.joinpath('c:x/y')
- self.assertEqual(pp, P('C:/a/b/x/y'))
- pp = p.joinpath('c:/x/y')
- self.assertEqual(pp, P('C:/x/y'))
- # Joining with files with NTFS data streams => the filename should
- # not be parsed as a drive letter
- pp = p.joinpath(P('./d:s'))
- self.assertEqual(pp, P('C:/a/b/d:s'))
- pp = p.joinpath(P('./dd:s'))
- self.assertEqual(pp, P('C:/a/b/dd:s'))
- pp = p.joinpath(P('E:d:s'))
- self.assertEqual(pp, P('E:d:s'))
- # Joining onto a UNC path with no root
- pp = P('//').joinpath('server')
- self.assertEqual(pp, P('//server'))
- pp = P('//server').joinpath('share')
- self.assertEqual(pp, P('//server/share'))
- pp = P('//./BootPartition').joinpath('Windows')
- self.assertEqual(pp, P('//./BootPartition/Windows'))
-
- def test_div(self):
- # Basically the same as joinpath().
- P = self.cls
- p = P('C:/a/b')
- self.assertEqual(p / 'x/y', P('C:/a/b/x/y'))
- self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y'))
- self.assertEqual(p / '/x/y', P('C:/x/y'))
- self.assertEqual(p / '/x' / 'y', P('C:/x/y'))
- # Joining with a different drive => the first path is ignored, even
- # if the second path is relative.
- self.assertEqual(p / 'D:x/y', P('D:x/y'))
- self.assertEqual(p / 'D:' / 'x/y', P('D:x/y'))
- self.assertEqual(p / 'D:/x/y', P('D:/x/y'))
- self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y'))
- self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y'))
- # Joining with the same drive => the first path is appended to if
- # the second path is relative.
- self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y'))
- self.assertEqual(p / 'c:/x/y', P('C:/x/y'))
- # Joining with files with NTFS data streams => the filename should
- # not be parsed as a drive letter
- self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s'))
- self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s'))
- self.assertEqual(p / P('E:d:s'), P('E:d:s'))
-
- def test_is_reserved(self):
- P = self.cls
- self.assertIs(False, P('').is_reserved())
- self.assertIs(False, P('/').is_reserved())
- self.assertIs(False, P('/foo/bar').is_reserved())
- # UNC paths are never reserved.
- self.assertIs(False, P('//my/share/nul/con/aux').is_reserved())
- # Case-insensitive DOS-device names are reserved.
- self.assertIs(True, P('nul').is_reserved())
- self.assertIs(True, P('aux').is_reserved())
- self.assertIs(True, P('prn').is_reserved())
- self.assertIs(True, P('con').is_reserved())
- self.assertIs(True, P('conin$').is_reserved())
- self.assertIs(True, P('conout$').is_reserved())
- # COM/LPT + 1-9 or + superscript 1-3 are reserved.
- self.assertIs(True, P('COM1').is_reserved())
- self.assertIs(True, P('LPT9').is_reserved())
- self.assertIs(True, P('com\xb9').is_reserved())
- self.assertIs(True, P('com\xb2').is_reserved())
- self.assertIs(True, P('lpt\xb3').is_reserved())
- # DOS-device name mataching ignores characters after a dot or
- # a colon and also ignores trailing spaces.
- self.assertIs(True, P('NUL.txt').is_reserved())
- self.assertIs(True, P('PRN ').is_reserved())
- self.assertIs(True, P('AUX .txt').is_reserved())
- self.assertIs(True, P('COM1:bar').is_reserved())
- self.assertIs(True, P('LPT9 :bar').is_reserved())
- # DOS-device names are only matched at the beginning
- # of a path component.
- self.assertIs(False, P('bar.com9').is_reserved())
- self.assertIs(False, P('bar.lpt9').is_reserved())
- # Only the last path component matters.
- self.assertIs(True, P('c:/baz/con/NUL').is_reserved())
- self.assertIs(False, P('c:/NUL/con/baz').is_reserved())
+
+class PurePosixPathTest(PurePathTest):
+ cls = pathlib.PurePosixPath
+
+
+class PureWindowsPathTest(PurePathTest):
+ cls = pathlib.PureWindowsPath
class PurePathSubclassTest(PurePathTest):
@@ -1304,7 +706,7 @@ def test_chmod(self):
self.assertEqual(p.stat().st_mode, new_mode)
# On Windows, os.chmod does not follow symlinks (issue #15411)
- @only_posix
+ @needs_posix
@os_helper.skip_unless_working_chmod
def test_chmod_follow_symlinks_true(self):
p = self.cls(self.base) / 'linkA'
@@ -1573,7 +975,7 @@ def test_mkdir_exist_ok_root(self):
self.cls('/').resolve().mkdir(exist_ok=True)
self.cls('/').resolve().mkdir(parents=True, exist_ok=True)
- @only_nt # XXX: not sure how to test this on POSIX.
+ @needs_windows # XXX: not sure how to test this on POSIX.
def test_mkdir_with_unknown_drive(self):
for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA':
p = self.cls(d + ':\\')
@@ -1638,9 +1040,8 @@ def my_mkdir(path, mode=0o777):
self.assertNotIn(str(p12), concurrently_created)
self.assertTrue(p.exists())
+ @needs_symlinks
def test_symlink_to(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
P = self.cls(self.base)
target = P / 'fileA'
# Symlinking a path target.
@@ -1824,6 +1225,8 @@ def test_glob_empty_pattern(self):
list(p.glob(''))
with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'):
list(p.glob('.'))
+ with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'):
+ list(p.glob('./'))
def test_glob_many_open_files(self):
depth = 30
@@ -1855,18 +1258,6 @@ def test_glob_above_recursion_limit(self):
with set_recursion_limit(recursion_limit):
list(base.glob('**/'))
- def test_glob_recursive_no_trailing_slash(self):
- P = self.cls
- p = P(self.base)
- with self.assertWarns(FutureWarning):
- p.glob('**')
- with self.assertWarns(FutureWarning):
- p.glob('*/**')
- with self.assertWarns(FutureWarning):
- p.rglob('**')
- with self.assertWarns(FutureWarning):
- p.rglob('*/**')
-
def test_glob_pathlike(self):
P = self.cls
p = P(self.base)
@@ -1883,12 +1274,8 @@ def test_rglob_pathlike(self):
self.assertEqual(expect, set(p.rglob(P(pattern))))
self.assertEqual(expect, set(p.rglob(FakePath(pattern))))
-
-@only_posix
-class PosixPathTest(PathTest, PurePosixPathTest):
- cls = pathlib.PosixPath
-
- def test_absolute(self):
+ @needs_posix
+ def test_absolute_posix(self):
P = self.cls
self.assertEqual(str(P('/').absolute()), '/')
self.assertEqual(str(P('/a').absolute()), '/a')
@@ -1903,6 +1290,7 @@ def test_absolute(self):
is_emscripten or is_wasi,
"umask is not implemented on Emscripten/WASI."
)
+ @needs_posix
def test_open_mode(self):
old_mask = os.umask(0)
self.addCleanup(os.umask, old_mask)
@@ -1917,6 +1305,7 @@ def test_open_mode(self):
st = os.stat(self.pathmod.join(self.base, 'other_new_file'))
self.assertEqual(stat.S_IMODE(st.st_mode), 0o644)
+ @needs_posix
def test_resolve_root(self):
current_directory = os.getcwd()
try:
@@ -1930,6 +1319,7 @@ def test_resolve_root(self):
is_emscripten or is_wasi,
"umask is not implemented on Emscripten/WASI."
)
+ @needs_posix
def test_touch_mode(self):
old_mask = os.umask(0)
self.addCleanup(os.umask, old_mask)
@@ -1945,27 +1335,12 @@ def test_touch_mode(self):
st = os.stat(self.pathmod.join(self.base, 'masked_new_file'))
self.assertEqual(stat.S_IMODE(st.st_mode), 0o750)
- def test_glob(self):
- P = self.cls
- p = P(self.base)
- given = set(p.glob("FILEa"))
- expect = set() if not os_helper.fs_is_case_insensitive(self.base) else given
- self.assertEqual(given, expect)
- self.assertEqual(set(p.glob("FILEa*")), set())
-
- def test_rglob(self):
- P = self.cls
- p = P(self.base, "dirC")
- given = set(p.rglob("FILEd"))
- expect = set() if not os_helper.fs_is_case_insensitive(self.base) else given
- self.assertEqual(given, expect)
- self.assertEqual(set(p.rglob("FILEd*")), set())
-
@unittest.skipUnless(hasattr(pwd, 'getpwall'),
'pwd module does not expose getpwall()')
@unittest.skipIf(sys.platform == "vxworks",
"no home directory on VxWorks")
- def test_expanduser(self):
+ @needs_posix
+ def test_expanduser_posix(self):
P = self.cls
import_helper.import_module('pwd')
import pwd
@@ -2020,6 +1395,7 @@ def test_expanduser(self):
@unittest.skipIf(sys.platform != "darwin",
"Bad file descriptor in /dev/fd affects only macOS")
+ @needs_posix
def test_handling_bad_descriptor(self):
try:
file_descriptors = list(pathlib.Path('/dev/fd').rglob("*"))[3:]
@@ -2041,7 +1417,8 @@ def test_handling_bad_descriptor(self):
self.fail("Bad file descriptor not handled.")
raise
- def test_from_uri(self):
+ @needs_posix
+ def test_from_uri_posix(self):
P = self.cls
self.assertEqual(P.from_uri('file:/foo/bar'), P('/foo/bar'))
self.assertEqual(P.from_uri('file://foo/bar'), P('//foo/bar'))
@@ -2054,17 +1431,14 @@ def test_from_uri(self):
self.assertRaises(ValueError, P.from_uri, 'file:foo/bar')
self.assertRaises(ValueError, P.from_uri, 'http://foo/bar')
- def test_from_uri_pathname2url(self):
+ @needs_posix
+ def test_from_uri_pathname2url_posix(self):
P = self.cls
self.assertEqual(P.from_uri('file:' + pathname2url('/foo/bar')), P('/foo/bar'))
self.assertEqual(P.from_uri('file:' + pathname2url('//foo/bar')), P('//foo/bar'))
-
-@only_nt
-class WindowsPathTest(PathTest, PureWindowsPathTest):
- cls = pathlib.WindowsPath
-
- def test_absolute(self):
+ @needs_windows
+ def test_absolute_windows(self):
P = self.cls
# Simple absolute paths.
@@ -2109,23 +1483,8 @@ def test_absolute(self):
self.assertEqual(str(P(other_drive).absolute()), other_cwd)
self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo')
- def test_glob(self):
- P = self.cls
- p = P(self.base)
- self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") })
- self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") })
- self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") })
- self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"})
- self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"})
-
- def test_rglob(self):
- P = self.cls
- p = P(self.base, "dirC")
- self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") })
- self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") })
- self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"})
-
- def test_expanduser(self):
+ @needs_windows
+ def test_expanduser_windows(self):
P = self.cls
with os_helper.EnvironmentVarGuard() as env:
env.pop('HOME', None)
@@ -2178,7 +1537,8 @@ def check():
env['HOME'] = 'C:\\Users\\eve'
check()
- def test_from_uri(self):
+ @needs_windows
+ def test_from_uri_windows(self):
P = self.cls
# DOS drive paths
self.assertEqual(P.from_uri('file:c:/path/to/file'), P('c:/path/to/file'))
@@ -2199,22 +1559,35 @@ def test_from_uri(self):
self.assertRaises(ValueError, P.from_uri, 'file:foo/bar')
self.assertRaises(ValueError, P.from_uri, 'http://foo/bar')
- def test_from_uri_pathname2url(self):
+ @needs_windows
+ def test_from_uri_pathname2url_windows(self):
P = self.cls
self.assertEqual(P.from_uri('file:' + pathname2url(r'c:\path\to\file')), P('c:/path/to/file'))
self.assertEqual(P.from_uri('file:' + pathname2url(r'\\server\path\to\file')), P('//server/path/to/file'))
- def test_owner(self):
+ @needs_windows
+ def test_owner_windows(self):
P = self.cls
with self.assertRaises(pathlib.UnsupportedOperation):
P('c:/').owner()
- def test_group(self):
+ @needs_windows
+ def test_group_windows(self):
P = self.cls
with self.assertRaises(pathlib.UnsupportedOperation):
P('c:/').group()
+@unittest.skipIf(os.name == 'nt', 'test requires a POSIX-compatible system')
+class PosixPathTest(PathTest, PurePosixPathTest):
+ cls = pathlib.PosixPath
+
+
+@unittest.skipIf(os.name != 'nt', 'test requires a Windows-compatible system')
+class WindowsPathTest(PathTest, PureWindowsPathTest):
+ cls = pathlib.WindowsPath
+
+
class PathSubclassTest(PathTest):
class cls(pathlib.Path):
pass
diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py
index 364f776dbb1413..0e12182c162c14 100644
--- a/Lib/test/test_pathlib/test_pathlib_abc.py
+++ b/Lib/test/test_pathlib/test_pathlib_abc.py
@@ -11,6 +11,27 @@
from test.support.os_helper import TESTFN
+_tests_needing_posix = set()
+_tests_needing_windows = set()
+_tests_needing_symlinks = set()
+
+
+def needs_posix(fn):
+ """Decorator that marks a test as requiring a POSIX-flavoured path class."""
+ _tests_needing_posix.add(fn.__name__)
+ return fn
+
+def needs_windows(fn):
+ """Decorator that marks a test as requiring a Windows-flavoured path class."""
+ _tests_needing_windows.add(fn.__name__)
+ return fn
+
+def needs_symlinks(fn):
+ """Decorator that marks a test as requiring a path class that supports symlinks."""
+ _tests_needing_symlinks.add(fn.__name__)
+ return fn
+
+
class UnsupportedOperationTest(unittest.TestCase):
def test_is_notimplemented(self):
self.assertTrue(issubclass(UnsupportedOperation, NotImplementedError))
@@ -115,6 +136,11 @@ class DummyPurePathTest(unittest.TestCase):
base = f'/this/path/kills/fascists/{TESTFN}'
def setUp(self):
+ name = self.id().split('.')[-1]
+ if name in _tests_needing_posix and self.cls.pathmod is not posixpath:
+ self.skipTest('requires POSIX-flavoured path class')
+ if name in _tests_needing_windows and self.cls.pathmod is posixpath:
+ self.skipTest('requires Windows-flavoured path class')
p = self.cls('a')
self.pathmod = p.pathmod
self.sep = self.pathmod.sep
@@ -148,6 +174,19 @@ def test_str_subclass_common(self):
self._check_str_subclass('a/b.txt')
self._check_str_subclass('/a/b.txt')
+ @needs_windows
+ def test_str_subclass_windows(self):
+ self._check_str_subclass('.\\a:b')
+ self._check_str_subclass('c:')
+ self._check_str_subclass('c:a')
+ self._check_str_subclass('c:a\\b.txt')
+ self._check_str_subclass('c:\\')
+ self._check_str_subclass('c:\\a')
+ self._check_str_subclass('c:\\a\\b.txt')
+ self._check_str_subclass('\\\\some\\share')
+ self._check_str_subclass('\\\\some\\share\\a')
+ self._check_str_subclass('\\\\some\\share\\a\\b.txt')
+
def test_with_segments_common(self):
class P(self.cls):
def __init__(self, *pathsegments, session_id):
@@ -180,6 +219,55 @@ def test_join_common(self):
pp = p.joinpath('/c')
self.assertEqual(pp, P('/c'))
+ @needs_posix
+ def test_join_posix(self):
+ P = self.cls
+ p = P('//a')
+ pp = p.joinpath('b')
+ self.assertEqual(pp, P('//a/b'))
+ pp = P('/a').joinpath('//c')
+ self.assertEqual(pp, P('//c'))
+ pp = P('//a').joinpath('/c')
+ self.assertEqual(pp, P('/c'))
+
+ @needs_windows
+ def test_join_windows(self):
+ P = self.cls
+ p = P('C:/a/b')
+ pp = p.joinpath('x/y')
+ self.assertEqual(pp, P('C:/a/b/x/y'))
+ pp = p.joinpath('/x/y')
+ self.assertEqual(pp, P('C:/x/y'))
+ # Joining with a different drive => the first path is ignored, even
+ # if the second path is relative.
+ pp = p.joinpath('D:x/y')
+ self.assertEqual(pp, P('D:x/y'))
+ pp = p.joinpath('D:/x/y')
+ self.assertEqual(pp, P('D:/x/y'))
+ pp = p.joinpath('//host/share/x/y')
+ self.assertEqual(pp, P('//host/share/x/y'))
+ # Joining with the same drive => the first path is appended to if
+ # the second path is relative.
+ pp = p.joinpath('c:x/y')
+ self.assertEqual(pp, P('C:/a/b/x/y'))
+ pp = p.joinpath('c:/x/y')
+ self.assertEqual(pp, P('C:/x/y'))
+ # Joining with files with NTFS data streams => the filename should
+ # not be parsed as a drive letter
+ pp = p.joinpath(P('./d:s'))
+ self.assertEqual(pp, P('C:/a/b/d:s'))
+ pp = p.joinpath(P('./dd:s'))
+ self.assertEqual(pp, P('C:/a/b/dd:s'))
+ pp = p.joinpath(P('E:d:s'))
+ self.assertEqual(pp, P('E:d:s'))
+ # Joining onto a UNC path with no root
+ pp = P('//').joinpath('server')
+ self.assertEqual(pp, P('//server'))
+ pp = P('//server').joinpath('share')
+ self.assertEqual(pp, P('//server/share'))
+ pp = P('//./BootPartition').joinpath('Windows')
+ self.assertEqual(pp, P('//./BootPartition/Windows'))
+
def test_div_common(self):
# Basically the same as joinpath().
P = self.cls
@@ -196,6 +284,44 @@ def test_div_common(self):
pp = p/ '/c'
self.assertEqual(pp, P('/c'))
+ @needs_posix
+ def test_div_posix(self):
+ # Basically the same as joinpath().
+ P = self.cls
+ p = P('//a')
+ pp = p / 'b'
+ self.assertEqual(pp, P('//a/b'))
+ pp = P('/a') / '//c'
+ self.assertEqual(pp, P('//c'))
+ pp = P('//a') / '/c'
+ self.assertEqual(pp, P('/c'))
+
+ @needs_windows
+ def test_div_windows(self):
+ # Basically the same as joinpath().
+ P = self.cls
+ p = P('C:/a/b')
+ self.assertEqual(p / 'x/y', P('C:/a/b/x/y'))
+ self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y'))
+ self.assertEqual(p / '/x/y', P('C:/x/y'))
+ self.assertEqual(p / '/x' / 'y', P('C:/x/y'))
+ # Joining with a different drive => the first path is ignored, even
+ # if the second path is relative.
+ self.assertEqual(p / 'D:x/y', P('D:x/y'))
+ self.assertEqual(p / 'D:' / 'x/y', P('D:x/y'))
+ self.assertEqual(p / 'D:/x/y', P('D:/x/y'))
+ self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y'))
+ self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y'))
+ # Joining with the same drive => the first path is appended to if
+ # the second path is relative.
+ self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y'))
+ self.assertEqual(p / 'c:/x/y', P('C:/x/y'))
+ # Joining with files with NTFS data streams => the filename should
+ # not be parsed as a drive letter
+ self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s'))
+ self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s'))
+ self.assertEqual(p / P('E:d:s'), P('E:d:s'))
+
def _check_str(self, expected, args):
p = self.cls(*args)
self.assertEqual(str(p), expected.replace('/', self.sep))
@@ -206,6 +332,19 @@ def test_str_common(self):
self._check_str(pathstr, (pathstr,))
# Other tests for str() are in test_equivalences().
+ @needs_windows
+ def test_str_windows(self):
+ p = self.cls('a/b/c')
+ self.assertEqual(str(p), 'a\\b\\c')
+ p = self.cls('c:/a/b/c')
+ self.assertEqual(str(p), 'c:\\a\\b\\c')
+ p = self.cls('//a/b')
+ self.assertEqual(str(p), '\\\\a\\b\\')
+ p = self.cls('//a/b/c')
+ self.assertEqual(str(p), '\\\\a\\b\\c')
+ p = self.cls('//a/b/c/d')
+ self.assertEqual(str(p), '\\\\a\\b\\c\\d')
+
def test_as_posix_common(self):
P = self.cls
for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'):
@@ -261,6 +400,39 @@ def test_match_common(self):
self.assertFalse(P('').match('**'))
self.assertFalse(P('').match('**/*'))
+ @needs_posix
+ def test_match_posix(self):
+ P = self.cls
+ self.assertFalse(P('A.py').match('a.PY'))
+
+ @needs_windows
+ def test_match_windows(self):
+ P = self.cls
+ # Absolute patterns.
+ self.assertTrue(P('c:/b.py').match('*:/*.py'))
+ self.assertTrue(P('c:/b.py').match('c:/*.py'))
+ self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive
+ self.assertFalse(P('b.py').match('/*.py'))
+ self.assertFalse(P('b.py').match('c:*.py'))
+ self.assertFalse(P('b.py').match('c:/*.py'))
+ self.assertFalse(P('c:b.py').match('/*.py'))
+ self.assertFalse(P('c:b.py').match('c:/*.py'))
+ self.assertFalse(P('/b.py').match('c:*.py'))
+ self.assertFalse(P('/b.py').match('c:/*.py'))
+ # UNC patterns.
+ self.assertTrue(P('//some/share/a.py').match('//*/*/*.py'))
+ self.assertTrue(P('//some/share/a.py').match('//some/share/*.py'))
+ self.assertFalse(P('//other/share/a.py').match('//some/share/*.py'))
+ self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py'))
+ # Case-insensitivity.
+ self.assertTrue(P('B.py').match('b.PY'))
+ self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY'))
+ self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY'))
+ # Path anchor doesn't match pattern anchor
+ self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/'
+ self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:'
+ self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/'
+
def test_full_match_common(self):
P = self.cls
# Simple relative pattern.
@@ -346,6 +518,19 @@ def test_parts_common(self):
parts = p.parts
self.assertEqual(parts, (sep, 'a', 'b'))
+ @needs_windows
+ def test_parts_windows(self):
+ P = self.cls
+ p = P('c:a/b')
+ parts = p.parts
+ self.assertEqual(parts, ('c:', 'a', 'b'))
+ p = P('c:/a/b')
+ parts = p.parts
+ self.assertEqual(parts, ('c:\\', 'a', 'b'))
+ p = P('//a/b/c/d')
+ parts = p.parts
+ self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd'))
+
def test_parent_common(self):
# Relative
P = self.cls
@@ -361,6 +546,25 @@ def test_parent_common(self):
self.assertEqual(p.parent.parent.parent, P('/'))
self.assertEqual(p.parent.parent.parent.parent, P('/'))
+ @needs_windows
+ def test_parent_windows(self):
+ # Anchored
+ P = self.cls
+ p = P('z:a/b/c')
+ self.assertEqual(p.parent, P('z:a/b'))
+ self.assertEqual(p.parent.parent, P('z:a'))
+ self.assertEqual(p.parent.parent.parent, P('z:'))
+ self.assertEqual(p.parent.parent.parent.parent, P('z:'))
+ p = P('z:/a/b/c')
+ self.assertEqual(p.parent, P('z:/a/b'))
+ self.assertEqual(p.parent.parent, P('z:/a'))
+ self.assertEqual(p.parent.parent.parent, P('z:/'))
+ self.assertEqual(p.parent.parent.parent.parent, P('z:/'))
+ p = P('//a/b/c/d')
+ self.assertEqual(p.parent, P('//a/b/c'))
+ self.assertEqual(p.parent.parent, P('//a/b'))
+ self.assertEqual(p.parent.parent.parent, P('//a/b'))
+
def test_parents_common(self):
# Relative
P = self.cls
@@ -408,12 +612,71 @@ def test_parents_common(self):
with self.assertRaises(IndexError):
par[3]
+ @needs_windows
+ def test_parents_windows(self):
+ # Anchored
+ P = self.cls
+ p = P('z:a/b/')
+ par = p.parents
+ self.assertEqual(len(par), 2)
+ self.assertEqual(par[0], P('z:a'))
+ self.assertEqual(par[1], P('z:'))
+ self.assertEqual(par[0:1], (P('z:a'),))
+ self.assertEqual(par[:-1], (P('z:a'),))
+ self.assertEqual(par[:2], (P('z:a'), P('z:')))
+ self.assertEqual(par[1:], (P('z:'),))
+ self.assertEqual(par[::2], (P('z:a'),))
+ self.assertEqual(par[::-1], (P('z:'), P('z:a')))
+ self.assertEqual(list(par), [P('z:a'), P('z:')])
+ with self.assertRaises(IndexError):
+ par[2]
+ p = P('z:/a/b/')
+ par = p.parents
+ self.assertEqual(len(par), 2)
+ self.assertEqual(par[0], P('z:/a'))
+ self.assertEqual(par[1], P('z:/'))
+ self.assertEqual(par[0:1], (P('z:/a'),))
+ self.assertEqual(par[0:-1], (P('z:/a'),))
+ self.assertEqual(par[:2], (P('z:/a'), P('z:/')))
+ self.assertEqual(par[1:], (P('z:/'),))
+ self.assertEqual(par[::2], (P('z:/a'),))
+ self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),))
+ self.assertEqual(list(par), [P('z:/a'), P('z:/')])
+ with self.assertRaises(IndexError):
+ par[2]
+ p = P('//a/b/c/d')
+ par = p.parents
+ self.assertEqual(len(par), 2)
+ self.assertEqual(par[0], P('//a/b/c'))
+ self.assertEqual(par[1], P('//a/b'))
+ self.assertEqual(par[0:1], (P('//a/b/c'),))
+ self.assertEqual(par[0:-1], (P('//a/b/c'),))
+ self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b')))
+ self.assertEqual(par[1:], (P('//a/b'),))
+ self.assertEqual(par[::2], (P('//a/b/c'),))
+ self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c')))
+ self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')])
+ with self.assertRaises(IndexError):
+ par[2]
+
def test_drive_common(self):
P = self.cls
self.assertEqual(P('a/b').drive, '')
self.assertEqual(P('/a/b').drive, '')
self.assertEqual(P('').drive, '')
+ @needs_windows
+ def test_drive_windows(self):
+ P = self.cls
+ self.assertEqual(P('c:').drive, 'c:')
+ self.assertEqual(P('c:a/b').drive, 'c:')
+ self.assertEqual(P('c:/').drive, 'c:')
+ self.assertEqual(P('c:/a/b/').drive, 'c:')
+ self.assertEqual(P('//a/b').drive, '\\\\a\\b')
+ self.assertEqual(P('//a/b/').drive, '\\\\a\\b')
+ self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b')
+ self.assertEqual(P('./c:a').drive, '')
+
def test_root_common(self):
P = self.cls
sep = self.sep
@@ -422,6 +685,24 @@ def test_root_common(self):
self.assertEqual(P('/').root, sep)
self.assertEqual(P('/a/b').root, sep)
+ @needs_posix
+ def test_root_posix(self):
+ P = self.cls
+ self.assertEqual(P('/a/b').root, '/')
+ # POSIX special case for two leading slashes.
+ self.assertEqual(P('//a/b').root, '//')
+
+ @needs_windows
+ def test_root_windows(self):
+ P = self.cls
+ self.assertEqual(P('c:').root, '')
+ self.assertEqual(P('c:a/b').root, '')
+ self.assertEqual(P('c:/').root, '\\')
+ self.assertEqual(P('c:/a/b/').root, '\\')
+ self.assertEqual(P('//a/b').root, '\\')
+ self.assertEqual(P('//a/b/').root, '\\')
+ self.assertEqual(P('//a/b/c/d').root, '\\')
+
def test_anchor_common(self):
P = self.cls
sep = self.sep
@@ -430,6 +711,17 @@ def test_anchor_common(self):
self.assertEqual(P('/').anchor, sep)
self.assertEqual(P('/a/b').anchor, sep)
+ @needs_windows
+ def test_anchor_windows(self):
+ P = self.cls
+ self.assertEqual(P('c:').anchor, 'c:')
+ self.assertEqual(P('c:a/b').anchor, 'c:')
+ self.assertEqual(P('c:/').anchor, 'c:\\')
+ self.assertEqual(P('c:/a/b/').anchor, 'c:\\')
+ self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\')
+ self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\')
+ self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\')
+
def test_name_empty(self):
P = self.cls
self.assertEqual(P('').name, '')
@@ -444,6 +736,18 @@ def test_name_common(self):
self.assertEqual(P('a/b.py').name, 'b.py')
self.assertEqual(P('/a/b.py').name, 'b.py')
+ @needs_windows
+ def test_name_windows(self):
+ P = self.cls
+ self.assertEqual(P('c:').name, '')
+ self.assertEqual(P('c:/').name, '')
+ self.assertEqual(P('c:a/b').name, 'b')
+ self.assertEqual(P('c:/a/b').name, 'b')
+ self.assertEqual(P('c:a/b.py').name, 'b.py')
+ self.assertEqual(P('c:/a/b.py').name, 'b.py')
+ self.assertEqual(P('//My.py/Share.php').name, '')
+ self.assertEqual(P('//My.py/Share.php/a/b').name, 'b')
+
def test_suffix_common(self):
P = self.cls
self.assertEqual(P('').suffix, '')
@@ -464,6 +768,26 @@ def test_suffix_common(self):
self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '')
self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '')
+ @needs_windows
+ def test_suffix_windows(self):
+ P = self.cls
+ self.assertEqual(P('c:').suffix, '')
+ self.assertEqual(P('c:/').suffix, '')
+ self.assertEqual(P('c:a/b').suffix, '')
+ self.assertEqual(P('c:/a/b').suffix, '')
+ self.assertEqual(P('c:a/b.py').suffix, '.py')
+ self.assertEqual(P('c:/a/b.py').suffix, '.py')
+ self.assertEqual(P('c:a/.hgrc').suffix, '')
+ self.assertEqual(P('c:/a/.hgrc').suffix, '')
+ self.assertEqual(P('c:a/.hg.rc').suffix, '.rc')
+ self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc')
+ self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz')
+ self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz')
+ self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '')
+ self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '')
+ self.assertEqual(P('//My.py/Share.php').suffix, '')
+ self.assertEqual(P('//My.py/Share.php/a/b').suffix, '')
+
def test_suffixes_common(self):
P = self.cls
self.assertEqual(P('').suffixes, [])
@@ -483,6 +807,26 @@ def test_suffixes_common(self):
self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, [])
self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, [])
+ @needs_windows
+ def test_suffixes_windows(self):
+ P = self.cls
+ self.assertEqual(P('c:').suffixes, [])
+ self.assertEqual(P('c:/').suffixes, [])
+ self.assertEqual(P('c:a/b').suffixes, [])
+ self.assertEqual(P('c:/a/b').suffixes, [])
+ self.assertEqual(P('c:a/b.py').suffixes, ['.py'])
+ self.assertEqual(P('c:/a/b.py').suffixes, ['.py'])
+ self.assertEqual(P('c:a/.hgrc').suffixes, [])
+ self.assertEqual(P('c:/a/.hgrc').suffixes, [])
+ self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc'])
+ self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc'])
+ self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz'])
+ self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz'])
+ self.assertEqual(P('//My.py/Share.php').suffixes, [])
+ self.assertEqual(P('//My.py/Share.php/a/b').suffixes, [])
+ self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, [])
+ self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, [])
+
def test_stem_empty(self):
P = self.cls
self.assertEqual(P('').stem, '')
@@ -500,6 +844,20 @@ def test_stem_common(self):
self.assertEqual(P('a/Some name. Ending with a dot.').stem,
'Some name. Ending with a dot.')
+ @needs_windows
+ def test_stem_windows(self):
+ P = self.cls
+ self.assertEqual(P('c:').stem, '')
+ self.assertEqual(P('c:.').stem, '')
+ self.assertEqual(P('c:..').stem, '..')
+ self.assertEqual(P('c:/').stem, '')
+ self.assertEqual(P('c:a/b').stem, 'b')
+ self.assertEqual(P('c:a/b.py').stem, 'b')
+ self.assertEqual(P('c:a/.hgrc').stem, '.hgrc')
+ self.assertEqual(P('c:a/.hg.rc').stem, '.hg')
+ self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar')
+ self.assertEqual(P('c:a/Some name. Ending with a dot.').stem,
+ 'Some name. Ending with a dot.')
def test_with_name_common(self):
P = self.cls
self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml'))
@@ -509,6 +867,23 @@ def test_with_name_common(self):
self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml'))
self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml'))
+ @needs_windows
+ def test_with_name_windows(self):
+ P = self.cls
+ self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml'))
+ self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml'))
+ self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml'))
+ self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml'))
+ self.assertRaises(ValueError, P('c:').with_name, 'd.xml')
+ self.assertRaises(ValueError, P('c:/').with_name, 'd.xml')
+ self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml')
+ self.assertEqual(str(P('a').with_name('d:')), '.\\d:')
+ self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e')
+ self.assertEqual(P('c:a/b').with_name('d:'), P('c:a/d:'))
+ self.assertEqual(P('c:a/b').with_name('d:e'), P('c:a/d:e'))
+ self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e')
+ self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share')
+
def test_with_name_empty(self):
P = self.cls
self.assertEqual(P('').with_name('d.xml'), P('d.xml'))
@@ -533,6 +908,23 @@ def test_with_stem_common(self):
self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d'))
self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d'))
+ @needs_windows
+ def test_with_stem_windows(self):
+ P = self.cls
+ self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d'))
+ self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d'))
+ self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d'))
+ self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d'))
+ self.assertRaises(ValueError, P('c:').with_stem, 'd')
+ self.assertRaises(ValueError, P('c:/').with_stem, 'd')
+ self.assertRaises(ValueError, P('//My/Share').with_stem, 'd')
+ self.assertEqual(str(P('a').with_stem('d:')), '.\\d:')
+ self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e')
+ self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:'))
+ self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e'))
+ self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e')
+ self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share')
+
def test_with_stem_empty(self):
P = self.cls
self.assertEqual(P('').with_stem('d'), P('d'))
@@ -557,12 +949,36 @@ def test_with_suffix_common(self):
self.assertEqual(P('a/b.py').with_suffix(''), P('a/b'))
self.assertEqual(P('/a/b').with_suffix(''), P('/a/b'))
+ @needs_windows
+ def test_with_suffix_windows(self):
+ P = self.cls
+ self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz'))
+ self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz'))
+ self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz'))
+ self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz'))
+ # Path doesn't have a "filename" component.
+ self.assertRaises(ValueError, P('').with_suffix, '.gz')
+ self.assertRaises(ValueError, P('.').with_suffix, '.gz')
+ self.assertRaises(ValueError, P('/').with_suffix, '.gz')
+ self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz')
+ # Invalid suffix.
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '/')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d')
+
def test_with_suffix_empty(self):
P = self.cls
# Path doesn't have a "filename" component.
- self.assertEqual(P('').with_suffix('.gz'), P('.gz'))
- self.assertEqual(P('.').with_suffix('.gz'), P('..gz'))
- self.assertEqual(P('/').with_suffix('.gz'), P('/.gz'))
+ self.assertRaises(ValueError, P('').with_suffix, '.gz')
+ self.assertRaises(ValueError, P('/').with_suffix, '.gz')
def test_with_suffix_seps(self):
P = self.cls
@@ -651,6 +1067,112 @@ def test_relative_to_common(self):
self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True)
self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True)
+ @needs_windows
+ def test_relative_to_windows(self):
+ P = self.cls
+ p = P('C:Foo/Bar')
+ self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('c:'), P('Foo/Bar'))
+ self.assertEqual(p.relative_to(P('c:foO')), P('Bar'))
+ self.assertEqual(p.relative_to('c:foO'), P('Bar'))
+ self.assertEqual(p.relative_to('c:foO/'), P('Bar'))
+ self.assertEqual(p.relative_to(P('c:foO/baR')), P())
+ self.assertEqual(p.relative_to('c:foO/baR'), P())
+ self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar'))
+ self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar'))
+ self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar'))
+ self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar'))
+ self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P())
+ self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P())
+ self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..'))
+ self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar'))
+ self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar'))
+ # Unrelated paths.
+ self.assertRaises(ValueError, p.relative_to, P())
+ self.assertRaises(ValueError, p.relative_to, '')
+ self.assertRaises(ValueError, p.relative_to, P('d:'))
+ self.assertRaises(ValueError, p.relative_to, P('/'))
+ self.assertRaises(ValueError, p.relative_to, P('Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('C:/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz'))
+ self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz'))
+ self.assertRaises(ValueError, p.relative_to, P(), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, '', walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True)
+ p = P('C:/Foo/Bar')
+ self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('c:/'), P('Foo/Bar'))
+ self.assertEqual(p.relative_to(P('c:/foO')), P('Bar'))
+ self.assertEqual(p.relative_to('c:/foO'), P('Bar'))
+ self.assertEqual(p.relative_to('c:/foO/'), P('Bar'))
+ self.assertEqual(p.relative_to(P('c:/foO/baR')), P())
+ self.assertEqual(p.relative_to('c:/foO/baR'), P())
+ self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar'))
+ self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar'))
+ self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar'))
+ self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar'))
+ self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P())
+ self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P())
+ self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar'))
+ self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..'))
+ self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar'))
+ # Unrelated paths.
+ self.assertRaises(ValueError, p.relative_to, 'c:')
+ self.assertRaises(ValueError, p.relative_to, P('c:'))
+ self.assertRaises(ValueError, p.relative_to, P('C:/Baz'))
+ self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz'))
+ self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz'))
+ self.assertRaises(ValueError, p.relative_to, P('C:Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('d:'))
+ self.assertRaises(ValueError, p.relative_to, P('d:/'))
+ self.assertRaises(ValueError, p.relative_to, P('/'))
+ self.assertRaises(ValueError, p.relative_to, P('/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('//C/Foo'))
+ self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True)
+ # UNC paths.
+ p = P('//Server/Share/Foo/Bar')
+ self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar'))
+ self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar'))
+ self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P())
+ self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P())
+ self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar'))
+ self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar'))
+ self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P())
+ self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P())
+ self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar'))
+ # Unrelated paths.
+ self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True)
+ self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True)
+
def test_is_relative_to_common(self):
P = self.cls
p = P('a/b')
@@ -683,6 +1205,98 @@ def test_is_relative_to_common(self):
self.assertFalse(p.is_relative_to(''))
self.assertFalse(p.is_relative_to(P('a')))
+ @needs_windows
+ def test_is_relative_to_windows(self):
+ P = self.cls
+ p = P('C:Foo/Bar')
+ self.assertTrue(p.is_relative_to(P('c:')))
+ self.assertTrue(p.is_relative_to('c:'))
+ self.assertTrue(p.is_relative_to(P('c:foO')))
+ self.assertTrue(p.is_relative_to('c:foO'))
+ self.assertTrue(p.is_relative_to('c:foO/'))
+ self.assertTrue(p.is_relative_to(P('c:foO/baR')))
+ self.assertTrue(p.is_relative_to('c:foO/baR'))
+ # Unrelated paths.
+ self.assertFalse(p.is_relative_to(P()))
+ self.assertFalse(p.is_relative_to(''))
+ self.assertFalse(p.is_relative_to(P('d:')))
+ self.assertFalse(p.is_relative_to(P('/')))
+ self.assertFalse(p.is_relative_to(P('Foo')))
+ self.assertFalse(p.is_relative_to(P('/Foo')))
+ self.assertFalse(p.is_relative_to(P('C:/Foo')))
+ self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz')))
+ self.assertFalse(p.is_relative_to(P('C:Foo/Baz')))
+ p = P('C:/Foo/Bar')
+ self.assertTrue(p.is_relative_to(P('c:/')))
+ self.assertTrue(p.is_relative_to(P('c:/foO')))
+ self.assertTrue(p.is_relative_to('c:/foO/'))
+ self.assertTrue(p.is_relative_to(P('c:/foO/baR')))
+ self.assertTrue(p.is_relative_to('c:/foO/baR'))
+ # Unrelated paths.
+ self.assertFalse(p.is_relative_to('c:'))
+ self.assertFalse(p.is_relative_to(P('C:/Baz')))
+ self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz')))
+ self.assertFalse(p.is_relative_to(P('C:/Foo/Baz')))
+ self.assertFalse(p.is_relative_to(P('C:Foo')))
+ self.assertFalse(p.is_relative_to(P('d:')))
+ self.assertFalse(p.is_relative_to(P('d:/')))
+ self.assertFalse(p.is_relative_to(P('/')))
+ self.assertFalse(p.is_relative_to(P('/Foo')))
+ self.assertFalse(p.is_relative_to(P('//C/Foo')))
+ # UNC paths.
+ p = P('//Server/Share/Foo/Bar')
+ self.assertTrue(p.is_relative_to(P('//sErver/sHare')))
+ self.assertTrue(p.is_relative_to('//sErver/sHare'))
+ self.assertTrue(p.is_relative_to('//sErver/sHare/'))
+ self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo')))
+ self.assertTrue(p.is_relative_to('//sErver/sHare/Foo'))
+ self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/'))
+ self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar')))
+ self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar'))
+ # Unrelated paths.
+ self.assertFalse(p.is_relative_to(P('/Server/Share/Foo')))
+ self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo')))
+ self.assertFalse(p.is_relative_to(P('//z/Share/Foo')))
+ self.assertFalse(p.is_relative_to(P('//Server/z/Foo')))
+
+ @needs_posix
+ def test_is_absolute_posix(self):
+ P = self.cls
+ self.assertFalse(P('').is_absolute())
+ self.assertFalse(P('a').is_absolute())
+ self.assertFalse(P('a/b/').is_absolute())
+ self.assertTrue(P('/').is_absolute())
+ self.assertTrue(P('/a').is_absolute())
+ self.assertTrue(P('/a/b/').is_absolute())
+ self.assertTrue(P('//a').is_absolute())
+ self.assertTrue(P('//a/b').is_absolute())
+
+ @needs_windows
+ def test_is_absolute_windows(self):
+ P = self.cls
+ # Under NT, only paths with both a drive and a root are absolute.
+ self.assertFalse(P().is_absolute())
+ self.assertFalse(P('a').is_absolute())
+ self.assertFalse(P('a/b/').is_absolute())
+ self.assertFalse(P('/').is_absolute())
+ self.assertFalse(P('/a').is_absolute())
+ self.assertFalse(P('/a/b/').is_absolute())
+ self.assertFalse(P('c:').is_absolute())
+ self.assertFalse(P('c:a').is_absolute())
+ self.assertFalse(P('c:a/b/').is_absolute())
+ self.assertTrue(P('c:/').is_absolute())
+ self.assertTrue(P('c:/a').is_absolute())
+ self.assertTrue(P('c:/a/b/').is_absolute())
+ # UNC paths are absolute by definition.
+ self.assertTrue(P('//').is_absolute())
+ self.assertTrue(P('//a').is_absolute())
+ self.assertTrue(P('//a/b').is_absolute())
+ self.assertTrue(P('//a/b/').is_absolute())
+ self.assertTrue(P('//a/b/c').is_absolute())
+ self.assertTrue(P('//a/b/c/d').is_absolute())
+ self.assertTrue(P('//?/UNC/').is_absolute())
+ self.assertTrue(P('//?/UNC/spam').is_absolute())
+
#
# Tests for the virtual classes.
@@ -888,6 +1502,9 @@ class DummyPathTest(DummyPurePathTest):
def setUp(self):
super().setUp()
+ name = self.id().split('.')[-1]
+ if name in _tests_needing_symlinks and not self.can_symlink:
+ self.skipTest('requires symlinks')
pathmod = self.cls.pathmod
p = self.cls(self.base)
p.mkdir(parents=True)
@@ -1045,9 +1662,8 @@ def test_iterdir(self):
expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop']
self.assertEqual(paths, { P(self.base, q) for q in expected })
+ @needs_symlinks
def test_iterdir_symlink(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
# __iter__ on a symlink to a directory.
P = self.cls
p = P(self.base, 'linkB')
@@ -1096,13 +1712,31 @@ def _check(glob, expected):
else:
_check(p.glob("*/"), ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"])
+ @needs_posix
+ def test_glob_posix(self):
+ P = self.cls
+ p = P(self.base)
+ given = set(p.glob("FILEa"))
+ expect = set()
+ self.assertEqual(given, expect)
+ self.assertEqual(set(p.glob("FILEa*")), set())
+
+ @needs_windows
+ def test_glob_windows(self):
+ P = self.cls
+ p = P(self.base)
+ self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") })
+ self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") })
+ self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") })
+ self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"})
+ self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"})
+
def test_glob_empty_pattern(self):
- def _check(glob, expected):
- self.assertEqual(set(glob), { P(self.base, q) for q in expected })
P = self.cls
p = P(self.base)
- _check(p.glob(""), [""])
- _check(p.glob("."), ["."])
+ self.assertEqual(list(p.glob("")), [p])
+ self.assertEqual(list(p.glob(".")), [p / "."])
+ self.assertEqual(list(p.glob("./")), [p / "./"])
def test_glob_case_sensitive(self):
P = self.cls
@@ -1116,9 +1750,8 @@ def _check(path, pattern, case_sensitive, expected):
_check(path, "dirb/file*", True, [])
_check(path, "dirb/file*", False, ["dirB/fileB"])
+ @needs_symlinks
def test_glob_follow_symlinks_common(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
def _check(path, glob, expected):
actual = {path for path in path.glob(glob, follow_symlinks=True)
if path.parts.count("linkD") <= 1} # exclude symlink loop.
@@ -1132,21 +1765,30 @@ def _check(path, glob, expected):
_check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"])
_check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"])
_check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."])
+ _check(p, "dir*/**", [
+ "dirA", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB",
+ "dirB", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB",
+ "dirC", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt",
+ "dirE"])
_check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/",
"dirC/", "dirC/dirD/", "dirE/"])
_check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..",
"dirB/linkD/..", "dirA/linkC/linkD/..",
"dirC/..", "dirC/dirD/..", "dirE/.."])
+ _check(p, "dir*/*/**", [
+ "dirA/linkC", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB",
+ "dirB/linkD", "dirB/linkD/fileB",
+ "dirC/dirD", "dirC/dirD/fileD"])
_check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"])
_check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..",
"dirB/linkD/..", "dirC/dirD/.."])
_check(p, "dir*/**/fileC", ["dirC/fileC"])
_check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"])
+ _check(p, "*/dirD/**", ["dirC/dirD", "dirC/dirD/fileD"])
_check(p, "*/dirD/**/", ["dirC/dirD/"])
+ @needs_symlinks
def test_glob_no_follow_symlinks_common(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
def _check(path, glob, expected):
actual = {path for path in path.glob(glob, follow_symlinks=False)}
self.assertEqual(actual, { P(self.base, q) for q in expected })
@@ -1159,12 +1801,20 @@ def _check(path, glob, expected):
_check(p, "*/fileB", ["dirB/fileB"])
_check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/"])
_check(p, "dir*/*/..", ["dirC/dirD/.."])
+ _check(p, "dir*/**", [
+ "dirA", "dirA/linkC",
+ "dirB", "dirB/fileB", "dirB/linkD",
+ "dirC", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt",
+ "dirE"])
_check(p, "dir*/**/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"])
_check(p, "dir*/**/..", ["dirA/..", "dirB/..", "dirC/..", "dirC/dirD/..", "dirE/.."])
+ _check(p, "dir*/*/**", ["dirC/dirD", "dirC/dirD/fileD"])
_check(p, "dir*/*/**/", ["dirC/dirD/"])
_check(p, "dir*/*/**/..", ["dirC/dirD/.."])
_check(p, "dir*/**/fileC", ["dirC/fileC"])
+ _check(p, "dir*/*/../dirD/**", ["dirC/dirD/../dirD", "dirC/dirD/../dirD/fileD"])
_check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"])
+ _check(p, "*/dirD/**", ["dirC/dirD", "dirC/dirD/fileD"])
_check(p, "*/dirD/**/", ["dirC/dirD/"])
def test_rglob_common(self):
@@ -1201,18 +1851,37 @@ def _check(glob, expected):
"dirC/dirD", "dirC/dirD/fileD"])
_check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"])
_check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"])
+ _check(p.rglob("dir*/**"), ["dirC/dirD", "dirC/dirD/fileD"])
_check(p.rglob("dir*/**/"), ["dirC/dirD/"])
_check(p.rglob("*/*"), ["dirC/dirD/fileD"])
_check(p.rglob("*/"), ["dirC/dirD/"])
_check(p.rglob(""), ["dirC/", "dirC/dirD/"])
+ _check(p.rglob("**"), [
+ "dirC", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"])
_check(p.rglob("**/"), ["dirC/", "dirC/dirD/"])
# gh-91616, a re module regression
_check(p.rglob("*.txt"), ["dirC/novel.txt"])
_check(p.rglob("*.*"), ["dirC/novel.txt"])
+ @needs_posix
+ def test_rglob_posix(self):
+ P = self.cls
+ p = P(self.base, "dirC")
+ given = set(p.rglob("FILEd"))
+ expect = set()
+ self.assertEqual(given, expect)
+ self.assertEqual(set(p.rglob("FILEd*")), set())
+
+ @needs_windows
+ def test_rglob_windows(self):
+ P = self.cls
+ p = P(self.base, "dirC")
+ self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") })
+ self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") })
+ self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"})
+
+ @needs_symlinks
def test_rglob_follow_symlinks_common(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
def _check(path, glob, expected):
actual = {path for path in path.rglob(glob, follow_symlinks=True)
if path.parts.count("linkD") <= 1} # exclude symlink loop.
@@ -1243,9 +1912,8 @@ def _check(path, glob, expected):
_check(p, "*.txt", ["dirC/novel.txt"])
_check(p, "*.*", ["dirC/novel.txt"])
+ @needs_symlinks
def test_rglob_no_follow_symlinks_common(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
def _check(path, glob, expected):
actual = {path for path in path.rglob(glob, follow_symlinks=False)}
self.assertEqual(actual, { P(self.base, q) for q in expected })
@@ -1269,10 +1937,9 @@ def _check(path, glob, expected):
_check(p, "*.txt", ["dirC/novel.txt"])
_check(p, "*.*", ["dirC/novel.txt"])
+ @needs_symlinks
def test_rglob_symlink_loop(self):
# Don't get fooled by symlink loops (Issue #26012).
- if not self.can_symlink:
- self.skipTest("symlinks required")
P = self.cls
p = P(self.base)
given = set(p.rglob('*'))
@@ -1302,10 +1969,9 @@ def test_glob_dotdot(self):
self.assertEqual(set(p.glob("xyzzy/..")), set())
self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)})
+ @needs_symlinks
def test_glob_permissions(self):
# See bpo-38894
- if not self.can_symlink:
- self.skipTest("symlinks required")
P = self.cls
base = P(self.base) / 'permissions'
base.mkdir()
@@ -1322,19 +1988,17 @@ def test_glob_permissions(self):
self.assertEqual(len(set(base.glob("*/fileC"))), 50)
self.assertEqual(len(set(base.glob("*/file*"))), 50)
+ @needs_symlinks
def test_glob_long_symlink(self):
# See gh-87695
- if not self.can_symlink:
- self.skipTest("symlinks required")
base = self.cls(self.base) / 'long_symlink'
base.mkdir()
bad_link = base / 'bad_link'
bad_link.symlink_to("bad" * 200)
self.assertEqual(sorted(base.glob('**/*')), [bad_link])
+ @needs_symlinks
def test_readlink(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
P = self.cls(self.base)
self.assertEqual((P / 'linkA').readlink(), self.cls('fileA'))
self.assertEqual((P / 'brokenLink').readlink(),
@@ -1358,9 +2022,8 @@ def _check_resolve(self, p, expected, strict=True):
# This can be used to check both relative and absolute resolutions.
_check_resolve_relative = _check_resolve_absolute = _check_resolve
+ @needs_symlinks
def test_resolve_common(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
P = self.cls
p = P(self.base, 'foo')
with self.assertRaises(OSError) as cm:
@@ -1419,10 +2082,9 @@ def test_resolve_common(self):
# resolves to 'dirB/..' first before resolving to parent of dirB.
self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False)
+ @needs_symlinks
def test_resolve_dot(self):
# See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/
- if not self.can_symlink:
- self.skipTest("symlinks required")
pathmod = self.pathmod
p = self.cls(self.base)
p.joinpath('0').symlink_to('.', target_is_directory=True)
@@ -1441,11 +2103,9 @@ def _check_symlink_loop(self, *args):
path.resolve(strict=True)
self.assertEqual(cm.exception.errno, errno.ELOOP)
+ @needs_posix
+ @needs_symlinks
def test_resolve_loop(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
- if self.cls.pathmod is not posixpath:
- self.skipTest("symlink loops work differently with concrete Windows paths")
# Loops with relative symlinks.
self.cls(self.base, 'linkX').symlink_to('linkX/inside')
self._check_symlink_loop(self.base, 'linkX')
@@ -1487,9 +2147,8 @@ def test_stat(self):
self.assertEqual(statA.st_dev, statC.st_dev)
# other attributes not used by pathlib.
+ @needs_symlinks
def test_stat_no_follow_symlinks(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
p = self.cls(self.base) / 'linkA'
st = p.stat()
self.assertNotEqual(st, p.stat(follow_symlinks=False))
@@ -1499,9 +2158,8 @@ def test_stat_no_follow_symlinks_nosymlink(self):
st = p.stat()
self.assertEqual(st, p.stat(follow_symlinks=False))
+ @needs_symlinks
def test_lstat(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
p = self.cls(self.base)/ 'linkA'
st = p.stat()
self.assertNotEqual(st, p.lstat())
@@ -1634,9 +2292,6 @@ def test_is_char_device_false(self):
self.assertIs((P / 'fileA\x00').is_char_device(), False)
def _check_complex_symlinks(self, link0_target):
- if not self.can_symlink:
- self.skipTest("symlinks required")
-
# Test solving a non-looping chain of symlinks (issue #19887).
pathmod = self.pathmod
P = self.cls(self.base)
@@ -1682,12 +2337,15 @@ def _check_complex_symlinks(self, link0_target):
finally:
os.chdir(old_path)
+ @needs_symlinks
def test_complex_symlinks_absolute(self):
self._check_complex_symlinks(self.base)
+ @needs_symlinks
def test_complex_symlinks_relative(self):
self._check_complex_symlinks('.')
+ @needs_symlinks
def test_complex_symlinks_relative_dot_dot(self):
self._check_complex_symlinks(self.pathmod.join('dirA', '..'))
@@ -1803,9 +2461,8 @@ def test_walk_bottom_up(self):
raise AssertionError(f"Unexpected path: {path}")
self.assertTrue(seen_testfn)
+ @needs_symlinks
def test_walk_follow_symlinks(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
self.setUpWalk()
walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
@@ -1816,9 +2473,8 @@ def test_walk_follow_symlinks(self):
else:
self.fail("Didn't follow symlink with follow_symlinks=True")
+ @needs_symlinks
def test_walk_symlink_location(self):
- if not self.can_symlink:
- self.skipTest("symlinks required")
self.setUpWalk()
# Tests whether symlinks end up in filenames or dirnames depending
# on the `follow_symlinks` argument.
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
index b2245ddf72f708..5e187e5189d117 100644
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -567,7 +567,8 @@ def test_exceptions(self):
RecursionError,
EncodingWarning,
BaseExceptionGroup,
- ExceptionGroup):
+ ExceptionGroup,
+ IncompleteInputError):
continue
if exc is not OSError and issubclass(exc, OSError):
self.assertEqual(reverse_mapping('builtins', name),
diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py
index f31a68c5d84e03..51e3a46d0df178 100644
--- a/Lib/test/test_pty.py
+++ b/Lib/test/test_pty.py
@@ -1,4 +1,5 @@
from test.support import verbose, reap_children
+from test.support.os_helper import TESTFN, unlink
from test.support.import_helper import import_module
# Skip these tests if termios or fcntl are not available
@@ -292,7 +293,26 @@ def test_master_read(self):
self.assertEqual(data, b"")
def test_spawn_doesnt_hang(self):
- pty.spawn([sys.executable, '-c', 'print("hi there")'])
+ self.addCleanup(unlink, TESTFN)
+ with open(TESTFN, 'wb') as f:
+ STDOUT_FILENO = 1
+ dup_stdout = os.dup(STDOUT_FILENO)
+ os.dup2(f.fileno(), STDOUT_FILENO)
+ buf = b''
+ def master_read(fd):
+ nonlocal buf
+ data = os.read(fd, 1024)
+ buf += data
+ return data
+ try:
+ pty.spawn([sys.executable, '-c', 'print("hi there")'],
+ master_read)
+ finally:
+ os.dup2(dup_stdout, STDOUT_FILENO)
+ os.close(dup_stdout)
+ self.assertEqual(buf, b'hi there\r\n')
+ with open(TESTFN, 'rb') as f:
+ self.assertEqual(f.read(), b'hi there\r\n')
class SmallPtyTests(unittest.TestCase):
"""These tests don't spawn children or hang."""
diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py
index 4976ac3642bbe4..054e7f0feb1a19 100644
--- a/Lib/test/test_stable_abi_ctypes.py
+++ b/Lib/test/test_stable_abi_ctypes.py
@@ -9,6 +9,13 @@
from _testcapi import get_feature_macros
feature_macros = get_feature_macros()
+
+# Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject
+# layout differences.
+# See https://github.com/python/cpython/issues/88299#issuecomment-1113366226
+if feature_macros['Py_TRACE_REFS']:
+ raise unittest.SkipTest("incompatible with Py_TRACE_REFS.")
+
ctypes_test = import_module('ctypes')
class TestStableABIAvailability(unittest.TestCase):
@@ -254,6 +261,7 @@ def test_windows_feature_macros(self):
"PyExc_IOError",
"PyExc_ImportError",
"PyExc_ImportWarning",
+ "PyExc_IncompleteInputError",
"PyExc_IndentationError",
"PyExc_IndexError",
"PyExc_InterruptedError",
@@ -441,7 +449,9 @@ def test_windows_feature_macros(self):
"PyModule_AddObjectRef",
"PyModule_AddStringConstant",
"PyModule_AddType",
+ "PyModule_Create2",
"PyModule_ExecDef",
+ "PyModule_FromDefAndSpec2",
"PyModule_GetDef",
"PyModule_GetDict",
"PyModule_GetFilename",
@@ -911,6 +921,13 @@ def test_windows_feature_macros(self):
"_Py_TrueStruct",
"_Py_VaBuildValue_SizeT",
)
+if feature_macros['HAVE_FORK']:
+ SYMBOL_NAMES += (
+ 'PyOS_AfterFork',
+ 'PyOS_AfterFork_Child',
+ 'PyOS_AfterFork_Parent',
+ 'PyOS_BeforeFork',
+ )
if feature_macros['MS_WINDOWS']:
SYMBOL_NAMES += (
'PyErr_SetExcFromWindowsErr',
@@ -926,17 +943,6 @@ def test_windows_feature_macros(self):
'PyUnicode_DecodeMBCSStateful',
'PyUnicode_EncodeCodePage',
)
-if feature_macros['HAVE_FORK']:
- SYMBOL_NAMES += (
- 'PyOS_AfterFork',
- 'PyOS_AfterFork_Child',
- 'PyOS_AfterFork_Parent',
- 'PyOS_BeforeFork',
- )
-if feature_macros['USE_STACKCHECK']:
- SYMBOL_NAMES += (
- 'PyOS_CheckStack',
- )
if feature_macros['PY_HAVE_THREAD_NATIVE_ID']:
SYMBOL_NAMES += (
'PyThread_get_thread_native_id',
@@ -946,14 +952,23 @@ def test_windows_feature_macros(self):
'_Py_NegativeRefcount',
'_Py_RefTotal',
)
+if feature_macros['Py_TRACE_REFS']:
+ SYMBOL_NAMES += (
+ )
+if feature_macros['USE_STACKCHECK']:
+ SYMBOL_NAMES += (
+ 'PyOS_CheckStack',
+ )
EXPECTED_FEATURE_MACROS = set(['HAVE_FORK',
'MS_WINDOWS',
'PY_HAVE_THREAD_NATIVE_ID',
'Py_REF_DEBUG',
+ 'Py_TRACE_REFS',
'USE_STACKCHECK'])
WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False,
'MS_WINDOWS': True,
'PY_HAVE_THREAD_NATIVE_ID': True,
'Py_REF_DEBUG': 'maybe',
+ 'Py_TRACE_REFS': 'maybe',
'USE_STACKCHECK': 'maybe'}
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 37a8b06987c710..fff3d3c4914e7a 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -433,6 +433,7 @@ PYTHON_OBJS= \
Python/initconfig.o \
Python/instrumentation.o \
Python/intrinsics.o \
+ Python/jit.o \
Python/legacy_tracing.o \
Python/lock.o \
Python/marshal.o \
@@ -1365,7 +1366,7 @@ regen-unicodedata:
regen-all: regen-cases regen-typeslots \
regen-token regen-ast regen-keyword regen-sre regen-frozen \
regen-pegen-metaparser regen-pegen regen-test-frozenmain \
- regen-test-levenshtein regen-global-objects regen-sbom
+ regen-test-levenshtein regen-global-objects regen-sbom regen-jit
@echo
@echo "Note: make regen-stdlib-module-names, make regen-limited-abi, "
@echo "make regen-configure and make regen-unicodedata should be run manually"
@@ -1846,6 +1847,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_initconfig.h \
$(srcdir)/Include/internal/pycore_interp.h \
$(srcdir)/Include/internal/pycore_intrinsics.h \
+ $(srcdir)/Include/internal/pycore_jit.h \
$(srcdir)/Include/internal/pycore_list.h \
$(srcdir)/Include/internal/pycore_llist.h \
$(srcdir)/Include/internal/pycore_lock.h \
@@ -2641,6 +2643,12 @@ config.status: $(srcdir)/configure
Python/asm_trampoline.o: $(srcdir)/Python/asm_trampoline.S
$(CC) -c $(PY_CORE_CFLAGS) -o $@ $<
+Python/jit.o: regen-jit
+
+.PHONY: regen-jit
+regen-jit:
+ @REGEN_JIT_COMMAND@
+
# Some make's put the object file in the current directory
.c.o:
$(CC) -c $(PY_CORE_CFLAGS) -o $@ $<
@@ -2733,6 +2741,7 @@ clean-retain-profile: pycremoval
-rm -f Python/deepfreeze/*.[co]
-rm -f Python/frozen_modules/*.h
-rm -f Python/frozen_modules/MANIFEST
+ -rm -f jit_stencils.h
-find build -type f -a ! -name '*.gc??' -exec rm -f {} ';'
-rm -f Include/pydtrace_probes.h
-rm -f profile-gen-stamp
diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst
index eb1456f1bcf353..a6b5fe54b391c5 100644
--- a/Misc/NEWS.d/3.11.0a2.rst
+++ b/Misc/NEWS.d/3.11.0a2.rst
@@ -1189,7 +1189,7 @@ context objects can now be disabled.
.. section: C API
Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never
-worked since the :c:type:`PyWeakReference` structure is opaque in the
+worked since the :c:type:`!PyWeakReference` structure is opaque in the
limited C API.
..
diff --git a/Misc/NEWS.d/3.11.0a4.rst b/Misc/NEWS.d/3.11.0a4.rst
index 5abacd8473f394..78b682f7a22cc6 100644
--- a/Misc/NEWS.d/3.11.0a4.rst
+++ b/Misc/NEWS.d/3.11.0a4.rst
@@ -161,7 +161,7 @@ faster due to reference-counting optimizations. Patch by Dennis Sweeney.
.. nonce: 7oGp-I
.. section: Core and Builtins
-:opcode:`PREP_RERAISE_STAR` no longer pushes ``lasti`` to the stack.
+:opcode:`!PREP_RERAISE_STAR` no longer pushes ``lasti`` to the stack.
..
@@ -170,7 +170,7 @@ faster due to reference-counting optimizations. Patch by Dennis Sweeney.
.. nonce: IKx4v6
.. section: Core and Builtins
-Remove :opcode:`POP_EXCEPT_AND_RERAISE` and replace it by an equivalent
+Remove :opcode:`!POP_EXCEPT_AND_RERAISE` and replace it by an equivalent
sequence of other opcodes.
..
@@ -1171,7 +1171,7 @@ Replaced deprecated usage of :c:func:`PyImport_ImportModuleNoBlock` with
.. nonce: sMgDLz
.. section: C API
-The :c:func:`PyUnicode_CHECK_INTERNED` macro has been excluded from the
+The :c:func:`!PyUnicode_CHECK_INTERNED` macro has been excluded from the
limited C API. It was never usable there, because it used internal
structures which are not available in the limited C API. Patch by Victor
Stinner.
diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst
index 2b50b7773492cb..2fdceef7746d4e 100644
--- a/Misc/NEWS.d/3.11.0a6.rst
+++ b/Misc/NEWS.d/3.11.0a6.rst
@@ -248,7 +248,7 @@ Don't un-adapt :opcode:`COMPARE_OP` when collecting specialization stats.
.. nonce: RX_AzJ
.. section: Core and Builtins
-Fix specialization stats gathering for :opcode:`PRECALL` instructions.
+Fix specialization stats gathering for :opcode:`!PRECALL` instructions.
..
diff --git a/Misc/NEWS.d/3.11.0a7.rst b/Misc/NEWS.d/3.11.0a7.rst
index 76699632db223a..ec99bd0294ceca 100644
--- a/Misc/NEWS.d/3.11.0a7.rst
+++ b/Misc/NEWS.d/3.11.0a7.rst
@@ -138,7 +138,7 @@ Replaced :opcode:`JUMP_ABSOLUTE` by the relative jump
.. nonce: SwrrFO
.. section: Core and Builtins
-:c:func:`PyFrame_FastToLocalsWithError` and :c:func:`PyFrame_LocalsToFast`
+:c:func:`!PyFrame_FastToLocalsWithError` and :c:func:`!PyFrame_LocalsToFast`
are no longer called during profiling nor tracing. C code can access the
``f_locals`` attribute of :c:type:`PyFrameObject` by calling
:c:func:`PyFrame_GetLocals`.
@@ -295,7 +295,7 @@ oparg) as an adaptive counter.
.. nonce: O12Pba
.. section: Core and Builtins
-Use inline caching for :opcode:`PRECALL` and :opcode:`CALL`, and remove the
+Use inline caching for :opcode:`!PRECALL` and :opcode:`CALL`, and remove the
internal machinery for managing the (now unused) non-inline caches.
..
diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst
index 2c30dc6e084bfb..f9296679655573 100644
--- a/Misc/NEWS.d/3.11.0b1.rst
+++ b/Misc/NEWS.d/3.11.0b1.rst
@@ -403,8 +403,8 @@ so this led to crashes. The problem is now fixed.
.. nonce: 6S_uoU
.. section: Core and Builtins
-Make opcodes :opcode:`JUMP_IF_TRUE_OR_POP` and
-:opcode:`JUMP_IF_FALSE_OR_POP` relative rather than absolute.
+Make opcodes :opcode:`!JUMP_IF_TRUE_OR_POP` and
+:opcode:`!JUMP_IF_FALSE_OR_POP` relative rather than absolute.
..
diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst
index ce2814bbe2e5ab..82faa5ad0b2031 100644
--- a/Misc/NEWS.d/3.12.0a4.rst
+++ b/Misc/NEWS.d/3.12.0a4.rst
@@ -13,8 +13,8 @@ Fix misleading default value in :func:`input`'s ``__text_signature__``.
.. nonce: cmGwxv
.. section: Core and Builtins
-Remove :opcode:`UNARY_POSITIVE`, :opcode:`ASYNC_GEN_WRAP` and
-:opcode:`LIST_TO_TUPLE`, replacing them with intrinsics.
+Remove :opcode:`!UNARY_POSITIVE`, :opcode:`!ASYNC_GEN_WRAP` and
+:opcode:`!LIST_TO_TUPLE`, replacing them with intrinsics.
..
diff --git a/Misc/NEWS.d/3.12.0a6.rst b/Misc/NEWS.d/3.12.0a6.rst
index 5bd600cd8b6fc0..cf28bdb9258820 100644
--- a/Misc/NEWS.d/3.12.0a6.rst
+++ b/Misc/NEWS.d/3.12.0a6.rst
@@ -170,7 +170,7 @@ all as not all platform C libraries generate an error.
.. section: Core and Builtins
Add :opcode:`CALL_INTRINSIC_2` and use it instead of
-:opcode:`PREP_RERAISE_STAR`.
+:opcode:`!PREP_RERAISE_STAR`.
..
diff --git a/Misc/NEWS.d/3.12.0a7.rst b/Misc/NEWS.d/3.12.0a7.rst
index f22050b0dc377b..a859be8a047456 100644
--- a/Misc/NEWS.d/3.12.0a7.rst
+++ b/Misc/NEWS.d/3.12.0a7.rst
@@ -24,7 +24,7 @@ Reduce the number of inline :opcode:`CACHE` entries for
.. nonce: PRkGca
.. section: Core and Builtins
-Removed :opcode:`JUMP_IF_FALSE_OR_POP` and :opcode:`JUMP_IF_TRUE_OR_POP`
+Removed :opcode:`!JUMP_IF_FALSE_OR_POP` and :opcode:`!JUMP_IF_TRUE_OR_POP`
instructions.
..
diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst
index 007a6ad4ffd4d4..211513d05d0040 100644
--- a/Misc/NEWS.d/3.12.0b1.rst
+++ b/Misc/NEWS.d/3.12.0b1.rst
@@ -1008,7 +1008,7 @@ Update the bundled copy of pip to version 23.1.2.
.. nonce: pst8iT
.. section: Library
-Make :mod:`dis` display the value of oparg of :opcode:`KW_NAMES`.
+Make :mod:`dis` display the value of oparg of :opcode:`!KW_NAMES`.
..
diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst
index 102bddcee5c5c2..d385b6a4504f97 100644
--- a/Misc/NEWS.d/3.13.0a1.rst
+++ b/Misc/NEWS.d/3.13.0a1.rst
@@ -2276,7 +2276,7 @@ creation.
.. nonce: m2H5Bk
.. section: Library
-Remove unnecessary extra ``__slots__`` in :py:class:`datetime`\'s pure
+Remove unnecessary extra ``__slots__`` in :class:`~datetime.datetime`\'s pure
python implementation to reduce memory size, as they are defined in the
superclass. Patch by James Hilton-Balfe
diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst
index 11b303e89ad04f..bd9061601fe190 100644
--- a/Misc/NEWS.d/3.8.0a1.rst
+++ b/Misc/NEWS.d/3.8.0a1.rst
@@ -3395,8 +3395,8 @@ Zackery Spytz.
.. nonce: S0Irst
.. section: Library
-Fix parsing non-ASCII identifiers in :mod:`lib2to3.pgen2.tokenize` (PEP
-3131).
+Fix parsing non-ASCII identifiers in :mod:`!lib2to3.pgen2.tokenize`
+(:pep:`3131`).
..
diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst
index 0772a0fed20652..66d7fc1f32e705 100644
--- a/Misc/NEWS.d/3.9.0a1.rst
+++ b/Misc/NEWS.d/3.9.0a1.rst
@@ -5715,8 +5715,8 @@ The :c:macro:`METH_FASTCALL` calling convention has been documented.
.. nonce: 4tClQT
.. section: C API
-The new function :c:func:`PyCode_NewWithPosOnlyArgs` allows to create code
-objects like :c:func:`PyCode_New`, but with an extra *posonlyargcount*
+The new function :c:func:`!PyCode_NewWithPosOnlyArgs` allows to create code
+objects like :c:func:`!PyCode_New`, but with an extra *posonlyargcount*
parameter for indicating the number of positonal-only arguments.
..
diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst
index b4594aade3b3ed..f0015ac54df307 100644
--- a/Misc/NEWS.d/3.9.0a5.rst
+++ b/Misc/NEWS.d/3.9.0a5.rst
@@ -1122,7 +1122,7 @@ a different condition than the GIL.
.. nonce: Nbl7lF
.. section: Tools/Demos
-Added support to fix ``getproxies`` in the :mod:`lib2to3.fixes.fix_urllib`
+Added support to fix ``getproxies`` in the :mod:`!lib2to3.fixes.fix_urllib`
module. Patch by José Roberto Meza Cabrera.
..
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst
new file mode 100644
index 00000000000000..438be985496650
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-06-19-09-00.gh-issue-55664.vYYl0V.rst
@@ -0,0 +1 @@
+Add warning when creating :class:`type` using a namespace dictionary with non-string keys. Patched by Daniel Urban and Furkan Onder.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst
new file mode 100644
index 00000000000000..90f49272218c96
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-22-13-21-39.gh-issue-113055.47xBMF.rst
@@ -0,0 +1,5 @@
+Make interp->obmalloc a pointer. For interpreters that share state with the
+main interpreter, this points to the same static memory structure. For
+interpreters with their own obmalloc state, it is heap allocated. Add
+free_obmalloc_arenas() which will free the obmalloc arenas and radix tree
+structures for interpreters with their own obmalloc state.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst
new file mode 100644
index 00000000000000..bdee4d645f61c8
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-24-03-25-28.gh-issue-113464.dvjQmA.rst
@@ -0,0 +1,4 @@
+Add an option (``--enable-experimental-jit`` for ``configure``-based builds
+or ``--experimental-jit`` for ``PCbuild``-based ones) to build an
+*experimental* just-in-time compiler, based on `copy-and-patch
+<https://fredrikbk.com/publications/copy-and-patch.pdf>`_
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst
new file mode 100644
index 00000000000000..e5f3d5ea0cea25
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-11-22-58-45.gh-issue-112050.hDuvDW.rst
@@ -0,0 +1 @@
+Convert :class:`collections.deque` to use Argument Clinic.
diff --git a/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst b/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst
new file mode 100644
index 00000000000000..31dd985bb5c3b6
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-07-31-01-24-40.gh-issue-88569.eU0--b.rst
@@ -0,0 +1,4 @@
+Add :func:`os.path.isreserved`, which identifies reserved pathnames such
+as "NUL", "AUX" and "CON". This function is only available on Windows.
+
+Deprecate :meth:`pathlib.PurePath.is_reserved`.
diff --git a/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst b/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst
new file mode 100644
index 00000000000000..0e54a1fe3c8a1c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-03-15-03-21-18.gh-issue-85984.Xaq6ZN.rst
@@ -0,0 +1,2 @@
+Add POSIX pseudo-terminal functions :func:`os.posix_openpt`,
+:func:`os.grantpt`, :func:`os.unlockpt`, and :func:`os.ptsname`.
diff --git a/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst b/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst
new file mode 100644
index 00000000000000..686f0311e80dcb
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-11-18-16-30-21.gh-issue-112240.YXS0tj.rst
@@ -0,0 +1,2 @@
+Add option to calendar module CLI to specify the weekday to start each week.
+Patch by Steven Ward.
diff --git a/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst b/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst
new file mode 100644
index 00000000000000..3e99d480139cbe
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-12-09-23-31-17.gh-issue-112919.S5k9QN.rst
@@ -0,0 +1,2 @@
+Speed-up :func:`datetime.datetime.replace`, :func:`datetime.date.replace` and
+:func:`datetime.time.replace`.
diff --git a/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst b/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst
new file mode 100644
index 00000000000000..7582603dcf95f5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-01-05-16-27-34.gh-issue-113732.fgDRXA.rst
@@ -0,0 +1,2 @@
+Fix support of :data:`~csv.QUOTE_NOTNULL` and :data:`~csv.QUOTE_STRINGS` in
+:func:`csv.reader`.
diff --git a/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst b/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst
new file mode 100644
index 00000000000000..f1c99c09d2dfe1
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-01-26-16-46-21.gh-issue-77749.NY_7TS.rst
@@ -0,0 +1,2 @@
+:meth:`email.policy.EmailPolicy.fold` now always encodes non-ASCII characters
+in headers if :attr:`~email.policy.EmailPolicy.utf8` is false.
diff --git a/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst b/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst
new file mode 100644
index 00000000000000..fb3382098853b3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-01-28-00-48-12.gh-issue-109653.vF4exe.rst
@@ -0,0 +1 @@
+Improve import time of :mod:`importlib.metadata` and :mod:`email.utils`.
diff --git a/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst b/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst
new file mode 100644
index 00000000000000..dedda24b481241
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-01-28-18-38-18.gh-issue-70303._Lt_pj.rst
@@ -0,0 +1,2 @@
+Return both files and directories from :meth:`pathlib.Path.glob` if a
+pattern ends with "``**``". Previously only directories were returned.
diff --git a/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst b/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst
new file mode 100644
index 00000000000000..2306af4a39dcf6
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-01-28-19-40-40.gh-issue-114678.kYKcJw.rst
@@ -0,0 +1,3 @@
+Ensure that deprecation warning for 'N' specifier in :class:`~decimal.Decimal`
+format is not raised for cases where 'N' appears in other places
+in the format specifier. Based on patch by Stefan Krah.
diff --git a/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst b/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst
new file mode 100644
index 00000000000000..abd296f8608518
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2024-01-23-11-35-26.gh-issue-114490.FrQOQ0.rst
@@ -0,0 +1 @@
+Add Mach-O linkage support for :func:`platform.architecture()`.
diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml
index 22b25dd0ec141f..ae19d25809ec86 100644
--- a/Misc/stable_abi.toml
+++ b/Misc/stable_abi.toml
@@ -78,6 +78,10 @@
[feature_macro.Py_REF_DEBUG]
doc = 'when Python is compiled in debug mode (with Py_REF_DEBUG)'
windows = 'maybe'
+[feature_macro.Py_TRACE_REFS]
+ # nb. This mode is not compatible with Stable ABI/Limited API.
+ doc = 'when Python is compiled with Py_TRACE_REFS'
+ windows = 'maybe'
# Mentioned in PEP 384:
@@ -2481,3 +2485,5 @@
[function._Py_SetRefcnt]
added = '3.13'
abi_only = true
+[data.PyExc_IncompleteInputError]
+ added = '3.13'
diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
index c8cd53de5e2262..ef77d34b10e47b 100644
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -44,8 +44,11 @@ find_module_state_by_def(PyTypeObject *type)
/*[clinic input]
module _collections
class _tuplegetter "_tuplegetterobject *" "clinic_state()->tuplegetter_type"
+class _collections.deque "dequeobject *" "clinic_state()->deque_type"
[clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7356042a89862e0e]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a033cc2a8476b3f1]*/
+
+typedef struct dequeobject dequeobject;
/* We can safely assume type to be the defining class,
* since tuplegetter is not a base type */
@@ -53,6 +56,12 @@ class _tuplegetter "_tuplegetterobject *" "clinic_state()->tuplegetter_type"
#include "clinic/_collectionsmodule.c.h"
#undef clinic_state
+/*[python input]
+class dequeobject_converter(self_converter):
+ type = "dequeobject *"
+[python start generated code]*/
+/*[python end generated code: output=da39a3ee5e6b4b0d input=b6ae4a3ff852be2f]*/
+
/* collections module implementation of a deque() datatype
Written and maintained by Raymond D. Hettinger <python@rcn.com>
*/
@@ -121,7 +130,7 @@ typedef struct BLOCK {
struct BLOCK *rightlink;
} block;
-typedef struct {
+struct dequeobject {
PyObject_VAR_HEAD
block *leftblock;
block *rightblock;
@@ -132,7 +141,7 @@ typedef struct {
Py_ssize_t numfreeblocks;
block *freeblocks[MAXFREEBLOCKS];
PyObject *weakreflist;
-} dequeobject;
+};
/* For debug builds, add error checking to track the endpoints
* in the chain of links. The goal is to make sure that link
@@ -219,8 +228,17 @@ deque_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
return (PyObject *)deque;
}
+/*[clinic input]
+_collections.deque.pop as deque_pop
+
+ deque: dequeobject
+
+Remove and return the rightmost element.
+[clinic start generated code]*/
+
static PyObject *
-deque_pop(dequeobject *deque, PyObject *unused)
+deque_pop_impl(dequeobject *deque)
+/*[clinic end generated code: output=2e5f7890c4251f07 input=eb6e6d020f877dec]*/
{
PyObject *item;
block *prevblock;
@@ -254,10 +272,17 @@ deque_pop(dequeobject *deque, PyObject *unused)
return item;
}
-PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element.");
+/*[clinic input]
+_collections.deque.popleft as deque_popleft
+
+ deque: dequeobject
+
+Remove and return the leftmost element.
+[clinic start generated code]*/
static PyObject *
-deque_popleft(dequeobject *deque, PyObject *unused)
+deque_popleft_impl(dequeobject *deque)
+/*[clinic end generated code: output=62b154897097ff68 input=acb41b9af50a9d9b]*/
{
PyObject *item;
block *prevblock;
@@ -292,8 +317,6 @@ deque_popleft(dequeobject *deque, PyObject *unused)
return item;
}
-PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element.");
-
/* The deque's size limit is d.maxlen. The limit can be zero or positive.
* If there is no limit, then d.maxlen == -1.
*
@@ -326,7 +349,7 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen)
deque->rightindex++;
deque->rightblock->data[deque->rightindex] = item;
if (NEEDS_TRIM(deque, maxlen)) {
- PyObject *olditem = deque_popleft(deque, NULL);
+ PyObject *olditem = deque_popleft_impl(deque);
Py_DECREF(olditem);
} else {
deque->state++;
@@ -334,16 +357,25 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen)
return 0;
}
+/*[clinic input]
+_collections.deque.append as deque_append
+
+ deque: dequeobject
+ item: object
+ /
+
+Add an element to the right side of the deque.
+[clinic start generated code]*/
+
static PyObject *
deque_append(dequeobject *deque, PyObject *item)
+/*[clinic end generated code: output=507b13efc4853ecc input=f112b83c380528e3]*/
{
if (deque_append_internal(deque, Py_NewRef(item), deque->maxlen) < 0)
return NULL;
Py_RETURN_NONE;
}
-PyDoc_STRVAR(append_doc, "Add an element to the right side of the deque.");
-
static inline int
deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen)
{
@@ -362,7 +394,7 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen)
deque->leftindex--;
deque->leftblock->data[deque->leftindex] = item;
if (NEEDS_TRIM(deque, deque->maxlen)) {
- PyObject *olditem = deque_pop(deque, NULL);
+ PyObject *olditem = deque_pop_impl(deque);
Py_DECREF(olditem);
} else {
deque->state++;
@@ -370,16 +402,25 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen)
return 0;
}
+/*[clinic input]
+_collections.deque.appendleft as deque_appendleft
+
+ deque: dequeobject
+ item: object
+ /
+
+Add an element to the left side of the deque.
+[clinic start generated code]*/
+
static PyObject *
deque_appendleft(dequeobject *deque, PyObject *item)
+/*[clinic end generated code: output=de0335a64800ffd8 input=bbdaa60a3e956062]*/
{
if (deque_appendleft_internal(deque, Py_NewRef(item), deque->maxlen) < 0)
return NULL;
Py_RETURN_NONE;
}
-PyDoc_STRVAR(appendleft_doc, "Add an element to the left side of the deque.");
-
static PyObject*
finalize_iterator(PyObject *it)
{
@@ -410,8 +451,19 @@ consume_iterator(PyObject *it)
return finalize_iterator(it);
}
+/*[clinic input]
+_collections.deque.extend as deque_extend
+
+ deque: dequeobject
+ iterable: object
+ /
+
+Extend the right side of the deque with elements from the iterable.
+[clinic start generated code]*/
+
static PyObject *
deque_extend(dequeobject *deque, PyObject *iterable)
+/*[clinic end generated code: output=a3a6e74d17063f8d input=cfebfd34d5383339]*/
{
PyObject *it, *item;
PyObject *(*iternext)(PyObject *);
@@ -454,11 +506,19 @@ deque_extend(dequeobject *deque, PyObject *iterable)
return finalize_iterator(it);
}
-PyDoc_STRVAR(extend_doc,
-"Extend the right side of the deque with elements from the iterable");
+/*[clinic input]
+_collections.deque.extendleft as deque_extendleft
+
+ deque: dequeobject
+ iterable: object
+ /
+
+Extend the left side of the deque with elements from the iterable.
+[clinic start generated code]*/
static PyObject *
deque_extendleft(dequeobject *deque, PyObject *iterable)
+/*[clinic end generated code: output=2dba946c50498c67 input=f4820e695a6f9416]*/
{
PyObject *it, *item;
PyObject *(*iternext)(PyObject *);
@@ -501,9 +561,6 @@ deque_extendleft(dequeobject *deque, PyObject *iterable)
return finalize_iterator(it);
}
-PyDoc_STRVAR(extendleft_doc,
-"Extend the left side of the deque with elements from the iterable");
-
static PyObject *
deque_inplace_concat(dequeobject *deque, PyObject *other)
{
@@ -517,8 +574,17 @@ deque_inplace_concat(dequeobject *deque, PyObject *other)
return (PyObject *)deque;
}
+/*[clinic input]
+_collections.deque.copy as deque_copy
+
+ deque: dequeobject
+
+Return a shallow copy of a deque.
+[clinic start generated code]*/
+
static PyObject *
-deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored))
+deque_copy_impl(dequeobject *deque)
+/*[clinic end generated code: output=6409b3d1ad2898b5 input=0e22f138bc1fcbee]*/
{
PyObject *result;
dequeobject *old_deque = (dequeobject *)deque;
@@ -537,7 +603,7 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored))
PyObject *item = old_deque->leftblock->data[old_deque->leftindex];
rv = deque_append(new_deque, item);
} else {
- rv = deque_extend(new_deque, deque);
+ rv = deque_extend(new_deque, (PyObject *)deque);
}
if (rv != NULL) {
Py_DECREF(rv);
@@ -547,7 +613,8 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored))
return NULL;
}
if (old_deque->maxlen < 0)
- result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), deque);
+ result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)),
+ (PyObject *)deque);
else
result = PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi",
deque, old_deque->maxlen, NULL);
@@ -561,7 +628,18 @@ deque_copy(PyObject *deque, PyObject *Py_UNUSED(ignored))
return result;
}
-PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque.");
+/*[clinic input]
+_collections.deque.__copy__ as deque___copy__ = _collections.deque.copy
+
+Return a shallow copy of a deque.
+[clinic start generated code]*/
+
+static PyObject *
+deque___copy___impl(dequeobject *deque)
+/*[clinic end generated code: output=7c5821504342bf23 input=fce05df783e7912b]*/
+{
+ return deque_copy_impl(deque);
+}
static PyObject *
deque_concat(dequeobject *deque, PyObject *other)
@@ -580,7 +658,7 @@ deque_concat(dequeobject *deque, PyObject *other)
return NULL;
}
- new_deque = deque_copy((PyObject *)deque, NULL);
+ new_deque = deque_copy_impl(deque);
if (new_deque == NULL)
return NULL;
result = deque_extend((dequeobject *)new_deque, other);
@@ -669,22 +747,29 @@ deque_clear(dequeobject *deque)
alternate_method:
while (Py_SIZE(deque)) {
- item = deque_pop(deque, NULL);
+ item = deque_pop_impl(deque);
assert (item != NULL);
Py_DECREF(item);
}
return 0;
}
+/*[clinic input]
+_collections.deque.clear as deque_clearmethod
+
+ deque: dequeobject
+
+Remove all elements from the deque.
+[clinic start generated code]*/
+
static PyObject *
-deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+deque_clearmethod_impl(dequeobject *deque)
+/*[clinic end generated code: output=79b2513e097615c1 input=20488eb932f89f9e]*/
{
deque_clear(deque);
Py_RETURN_NONE;
}
-PyDoc_STRVAR(clear_doc, "Remove all elements from the deque.");
-
static PyObject *
deque_inplace_repeat(dequeobject *deque, Py_ssize_t n)
{
@@ -768,7 +853,7 @@ deque_repeat(dequeobject *deque, Py_ssize_t n)
dequeobject *new_deque;
PyObject *rv;
- new_deque = (dequeobject *)deque_copy((PyObject *) deque, NULL);
+ new_deque = (dequeobject *)deque_copy_impl(deque);
if (new_deque == NULL)
return NULL;
rv = deque_inplace_repeat(new_deque, n);
@@ -925,36 +1010,36 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n)
return rv;
}
-static PyObject *
-deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs)
-{
- Py_ssize_t n=1;
+/*[clinic input]
+_collections.deque.rotate as deque_rotate
- if (!_PyArg_CheckPositional("deque.rotate", nargs, 0, 1)) {
- return NULL;
- }
- if (nargs) {
- PyObject *index = _PyNumber_Index(args[0]);
- if (index == NULL) {
- return NULL;
- }
- n = PyLong_AsSsize_t(index);
- Py_DECREF(index);
- if (n == -1 && PyErr_Occurred()) {
- return NULL;
- }
- }
+ deque: dequeobject
+ n: Py_ssize_t = 1
+ /
+Rotate the deque n steps to the right. If n is negative, rotates left.
+[clinic start generated code]*/
+
+static PyObject *
+deque_rotate_impl(dequeobject *deque, Py_ssize_t n)
+/*[clinic end generated code: output=96c2402a371eb15d input=d22070f49cc06c76]*/
+{
if (!_deque_rotate(deque, n))
Py_RETURN_NONE;
return NULL;
}
-PyDoc_STRVAR(rotate_doc,
-"Rotate the deque n steps to the right (default n=1). If n is negative, rotates left.");
+/*[clinic input]
+_collections.deque.reverse as deque_reverse
+
+ deque: dequeobject
+
+Reverse *IN PLACE*.
+[clinic start generated code]*/
static PyObject *
-deque_reverse(dequeobject *deque, PyObject *unused)
+deque_reverse_impl(dequeobject *deque)
+/*[clinic end generated code: output=bdeebc2cf8c1f064 input=f139787f406101c9]*/
{
block *leftblock = deque->leftblock;
block *rightblock = deque->rightblock;
@@ -991,11 +1076,19 @@ deque_reverse(dequeobject *deque, PyObject *unused)
Py_RETURN_NONE;
}
-PyDoc_STRVAR(reverse_doc,
-"D.reverse() -- reverse *IN PLACE*");
+/*[clinic input]
+_collections.deque.count as deque_count
+
+ deque: dequeobject
+ value as v: object
+ /
+
+Return number of occurrences of value.
+[clinic start generated code]*/
static PyObject *
deque_count(dequeobject *deque, PyObject *v)
+/*[clinic end generated code: output=7405d289d94d7b9b input=1892925260ff5d78]*/
{
block *b = deque->leftblock;
Py_ssize_t index = deque->leftindex;
@@ -1030,9 +1123,6 @@ deque_count(dequeobject *deque, PyObject *v)
return PyLong_FromSsize_t(count);
}
-PyDoc_STRVAR(count_doc,
-"D.count(value) -- return number of occurrences of value");
-
static int
deque_contains(dequeobject *deque, PyObject *v)
{
@@ -1071,22 +1161,33 @@ deque_len(dequeobject *deque)
return Py_SIZE(deque);
}
+/*[clinic input]
+@text_signature "($self, value, [start, [stop]])"
+_collections.deque.index as deque_index
+
+ deque: dequeobject
+ value as v: object
+ start: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='0') = NULL
+ stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='Py_SIZE(deque)') = NULL
+ /
+
+Return first index of value.
+
+Raises ValueError if the value is not present.
+[clinic start generated code]*/
+
static PyObject *
-deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs)
+deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start,
+ Py_ssize_t stop)
+/*[clinic end generated code: output=df45132753175ef9 input=140210c099830f64]*/
{
- Py_ssize_t i, n, start=0, stop=Py_SIZE(deque);
- PyObject *v, *item;
+ Py_ssize_t i, n;
+ PyObject *item;
block *b = deque->leftblock;
Py_ssize_t index = deque->leftindex;
size_t start_state = deque->state;
int cmp;
- if (!_PyArg_ParseStack(args, nargs, "O|O&O&:index", &v,
- _PyEval_SliceIndexNotNone, &start,
- _PyEval_SliceIndexNotNone, &stop)) {
- return NULL;
- }
-
if (start < 0) {
start += Py_SIZE(deque);
if (start < 0)
@@ -1138,10 +1239,6 @@ deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs)
return NULL;
}
-PyDoc_STRVAR(index_doc,
-"D.index(value, [start, [stop]]) -- return first index of value.\n"
-"Raises ValueError if the value is not present.");
-
/* insert(), remove(), and delitem() are implemented in terms of
rotate() for simplicity and reasonable performance near the end
points. If for some reason these methods become popular, it is not
@@ -1150,18 +1247,24 @@ PyDoc_STRVAR(index_doc,
boost (by moving each pointer only once instead of twice).
*/
+/*[clinic input]
+_collections.deque.insert as deque_insert
+
+ deque: dequeobject
+ index: Py_ssize_t
+ value: object
+ /
+
+Insert value before index.
+[clinic start generated code]*/
+
static PyObject *
-deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs)
+deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value)
+/*[clinic end generated code: output=ef4d2c15d5532b80 input=3e5c1c120d70c0e6]*/
{
- Py_ssize_t index;
Py_ssize_t n = Py_SIZE(deque);
- PyObject *value;
PyObject *rv;
- if (!_PyArg_ParseStack(args, nargs, "nO:insert", &index, &value)) {
- return NULL;
- }
-
if (deque->maxlen == Py_SIZE(deque)) {
PyErr_SetString(PyExc_IndexError, "deque already at its maximum size");
return NULL;
@@ -1184,12 +1287,6 @@ deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs)
Py_RETURN_NONE;
}
-PyDoc_STRVAR(insert_doc,
-"D.insert(index, object) -- insert object before index");
-
-PyDoc_STRVAR(remove_doc,
-"D.remove(value) -- remove first occurrence of value.");
-
static int
valid_index(Py_ssize_t i, Py_ssize_t limit)
{
@@ -1246,15 +1343,26 @@ deque_del_item(dequeobject *deque, Py_ssize_t i)
assert (i >= 0 && i < Py_SIZE(deque));
if (_deque_rotate(deque, -i))
return -1;
- item = deque_popleft(deque, NULL);
+ item = deque_popleft_impl(deque);
rv = _deque_rotate(deque, i);
assert (item != NULL);
Py_DECREF(item);
return rv;
}
+/*[clinic input]
+_collections.deque.remove as deque_remove
+
+ deque: dequeobject
+ value: object
+ /
+
+Remove first occurrence of value.
+[clinic start generated code]*/
+
static PyObject *
deque_remove(dequeobject *deque, PyObject *value)
+/*[clinic end generated code: output=49e1666d612fe911 input=d972f32d15990880]*/
{
PyObject *item;
block *b = deque->leftblock;
@@ -1375,8 +1483,17 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg)
return 0;
}
+/*[clinic input]
+_collections.deque.__reduce__ as deque___reduce__
+
+ deque: dequeobject
+
+Return state information for pickling.
+[clinic start generated code]*/
+
static PyObject *
-deque_reduce(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+deque___reduce___impl(dequeobject *deque)
+/*[clinic end generated code: output=cb85d9e0b7d2c5ad input=991a933a5bc7a526]*/
{
PyObject *state, *it;
@@ -1510,26 +1627,23 @@ deque_richcompare(PyObject *v, PyObject *w, int op)
return NULL;
}
+/*[clinic input]
+@text_signature "([iterable[, maxlen]])"
+_collections.deque.__init__ as deque_init
+
+ deque: dequeobject
+ iterable: object = NULL
+ maxlen as maxlenobj: object = NULL
+
+A list-like sequence optimized for data accesses near its endpoints.
+[clinic start generated code]*/
+
static int
-deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs)
+deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj)
+/*[clinic end generated code: output=7084a39d71218dcd input=5ebdffc48a2d27ae]*/
+
{
- PyObject *iterable = NULL;
- PyObject *maxlenobj = NULL;
Py_ssize_t maxlen = -1;
- char *kwlist[] = {"iterable", "maxlen", 0};
-
- if (kwdargs == NULL && PyTuple_GET_SIZE(args) <= 2) {
- if (PyTuple_GET_SIZE(args) > 0) {
- iterable = PyTuple_GET_ITEM(args, 0);
- }
- if (PyTuple_GET_SIZE(args) > 1) {
- maxlenobj = PyTuple_GET_ITEM(args, 1);
- }
- } else {
- if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist,
- &iterable, &maxlenobj))
- return -1;
- }
if (maxlenobj != NULL && maxlenobj != Py_None) {
maxlen = PyLong_AsSsize_t(maxlenobj);
if (maxlen == -1 && PyErr_Occurred())
@@ -1551,8 +1665,17 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs)
return 0;
}
+/*[clinic input]
+_collections.deque.__sizeof__ as deque___sizeof__
+
+ deque: dequeobject
+
+Return the size of the deque in memory, in bytes.
+[clinic start generated code]*/
+
static PyObject *
-deque_sizeof(dequeobject *deque, void *unused)
+deque___sizeof___impl(dequeobject *deque)
+/*[clinic end generated code: output=4d36e9fb4f30bbaf input=4e7c9a00c03c3290]*/
{
size_t res = _PyObject_SIZE(Py_TYPE(deque));
size_t blocks;
@@ -1563,9 +1686,6 @@ deque_sizeof(dequeobject *deque, void *unused)
return PyLong_FromSize_t(res);
}
-PyDoc_STRVAR(sizeof_doc,
-"D.__sizeof__() -- size of D in memory, in bytes");
-
static PyObject *
deque_get_maxlen(dequeobject *deque, void *Py_UNUSED(ignored))
{
@@ -1574,6 +1694,22 @@ deque_get_maxlen(dequeobject *deque, void *Py_UNUSED(ignored))
return PyLong_FromSsize_t(deque->maxlen);
}
+static PyObject *deque_reviter(dequeobject *deque);
+
+/*[clinic input]
+_collections.deque.__reversed__ as deque___reversed__
+
+ deque: dequeobject
+
+Return a reverse iterator over the deque.
+[clinic start generated code]*/
+
+static PyObject *
+deque___reversed___impl(dequeobject *deque)
+/*[clinic end generated code: output=3e7e7e715883cf2e input=3d494c25a6fe5c7e]*/
+{
+ return deque_reviter(deque);
+}
/* deque object ********************************************************/
@@ -1584,47 +1720,26 @@ static PyGetSetDef deque_getset[] = {
};
static PyObject *deque_iter(dequeobject *deque);
-static PyObject *deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored));
-PyDoc_STRVAR(reversed_doc,
- "D.__reversed__() -- return a reverse iterator over the deque");
static PyMethodDef deque_methods[] = {
- {"append", (PyCFunction)deque_append,
- METH_O, append_doc},
- {"appendleft", (PyCFunction)deque_appendleft,
- METH_O, appendleft_doc},
- {"clear", (PyCFunction)deque_clearmethod,
- METH_NOARGS, clear_doc},
- {"__copy__", deque_copy,
- METH_NOARGS, copy_doc},
- {"copy", deque_copy,
- METH_NOARGS, copy_doc},
- {"count", (PyCFunction)deque_count,
- METH_O, count_doc},
- {"extend", (PyCFunction)deque_extend,
- METH_O, extend_doc},
- {"extendleft", (PyCFunction)deque_extendleft,
- METH_O, extendleft_doc},
- {"index", _PyCFunction_CAST(deque_index),
- METH_FASTCALL, index_doc},
- {"insert", _PyCFunction_CAST(deque_insert),
- METH_FASTCALL, insert_doc},
- {"pop", (PyCFunction)deque_pop,
- METH_NOARGS, pop_doc},
- {"popleft", (PyCFunction)deque_popleft,
- METH_NOARGS, popleft_doc},
- {"__reduce__", (PyCFunction)deque_reduce,
- METH_NOARGS, reduce_doc},
- {"remove", (PyCFunction)deque_remove,
- METH_O, remove_doc},
- {"__reversed__", (PyCFunction)deque_reviter,
- METH_NOARGS, reversed_doc},
- {"reverse", (PyCFunction)deque_reverse,
- METH_NOARGS, reverse_doc},
- {"rotate", _PyCFunction_CAST(deque_rotate),
- METH_FASTCALL, rotate_doc},
- {"__sizeof__", (PyCFunction)deque_sizeof,
- METH_NOARGS, sizeof_doc},
+ DEQUE_APPEND_METHODDEF
+ DEQUE_APPENDLEFT_METHODDEF
+ DEQUE_CLEARMETHOD_METHODDEF
+ DEQUE___COPY___METHODDEF
+ DEQUE_COPY_METHODDEF
+ DEQUE_COUNT_METHODDEF
+ DEQUE_EXTEND_METHODDEF
+ DEQUE_EXTENDLEFT_METHODDEF
+ DEQUE_INDEX_METHODDEF
+ DEQUE_INSERT_METHODDEF
+ DEQUE_POP_METHODDEF
+ DEQUE_POPLEFT_METHODDEF
+ DEQUE___REDUCE___METHODDEF
+ DEQUE_REMOVE_METHODDEF
+ DEQUE___REVERSED___METHODDEF
+ DEQUE_REVERSE_METHODDEF
+ DEQUE_ROTATE_METHODDEF
+ DEQUE___SIZEOF___METHODDEF
{"__class_getitem__", Py_GenericAlias,
METH_O|METH_CLASS, PyDoc_STR("See PEP 585")},
{NULL, NULL} /* sentinel */
@@ -1635,17 +1750,12 @@ static PyMemberDef deque_members[] = {
{NULL},
};
-PyDoc_STRVAR(deque_doc,
-"deque([iterable[, maxlen]]) --> deque object\n\
-\n\
-A list-like sequence optimized for data accesses near its endpoints.");
-
static PyType_Slot deque_slots[] = {
{Py_tp_dealloc, deque_dealloc},
{Py_tp_repr, deque_repr},
{Py_tp_hash, PyObject_HashNotImplemented},
{Py_tp_getattro, PyObject_GenericGetAttr},
- {Py_tp_doc, (void *)deque_doc},
+ {Py_tp_doc, (void *)deque_init__doc__},
{Py_tp_traverse, deque_traverse},
{Py_tp_clear, deque_clear},
{Py_tp_richcompare, deque_richcompare},
@@ -1834,7 +1944,7 @@ static PyType_Spec dequeiter_spec = {
/*********************** Deque Reverse Iterator **************************/
static PyObject *
-deque_reviter(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+deque_reviter(dequeobject *deque)
{
dequeiterobject *it;
collections_state *state = find_module_state_by_def(Py_TYPE(deque));
@@ -1889,7 +1999,7 @@ dequereviter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
return NULL;
assert(type == state->dequereviter_type);
- it = (dequeiterobject*)deque_reviter((dequeobject *)deque, NULL);
+ it = (dequeiterobject *)deque_reviter((dequeobject *)deque);
if (!it)
return NULL;
/* consume items from the queue */
diff --git a/Modules/_csv.c b/Modules/_csv.c
index 929c21584ac2ef..3aa648b8e9cec4 100644
--- a/Modules/_csv.c
+++ b/Modules/_csv.c
@@ -131,7 +131,7 @@ typedef struct {
Py_UCS4 *field; /* temporary buffer */
Py_ssize_t field_size; /* size of allocated buffer */
Py_ssize_t field_len; /* length of current field */
- int numeric_field; /* treat field as numeric */
+ bool unquoted_field; /* true if no quotes around the current field */
unsigned long line_num; /* Source-file line number */
} ReaderObj;
@@ -644,22 +644,33 @@ _call_dialect(_csvstate *module_state, PyObject *dialect_inst, PyObject *kwargs)
static int
parse_save_field(ReaderObj *self)
{
+ int quoting = self->dialect->quoting;
PyObject *field;
- field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
- (void *) self->field, self->field_len);
- if (field == NULL)
- return -1;
- self->field_len = 0;
- if (self->numeric_field) {
- PyObject *tmp;
-
- self->numeric_field = 0;
- tmp = PyNumber_Float(field);
- Py_DECREF(field);
- if (tmp == NULL)
+ if (self->unquoted_field &&
+ self->field_len == 0 &&
+ (quoting == QUOTE_NOTNULL || quoting == QUOTE_STRINGS))
+ {
+ field = Py_NewRef(Py_None);
+ }
+ else {
+ field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
+ (void *) self->field, self->field_len);
+ if (field == NULL) {
return -1;
- field = tmp;
+ }
+ if (self->unquoted_field &&
+ self->field_len != 0 &&
+ (quoting == QUOTE_NONNUMERIC || quoting == QUOTE_STRINGS))
+ {
+ PyObject *tmp = PyNumber_Float(field);
+ Py_DECREF(field);
+ if (tmp == NULL) {
+ return -1;
+ }
+ field = tmp;
+ }
+ self->field_len = 0;
}
if (PyList_Append(self->fields, field) < 0) {
Py_DECREF(field);
@@ -721,6 +732,7 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c)
/* fallthru */
case START_FIELD:
/* expecting field */
+ self->unquoted_field = true;
if (c == '\n' || c == '\r' || c == EOL) {
/* save empty field - return [fields] */
if (parse_save_field(self) < 0)
@@ -730,10 +742,12 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c)
else if (c == dialect->quotechar &&
dialect->quoting != QUOTE_NONE) {
/* start quoted field */
+ self->unquoted_field = false;
self->state = IN_QUOTED_FIELD;
}
else if (c == dialect->escapechar) {
/* possible escaped character */
+ self->unquoted_field = false;
self->state = ESCAPED_CHAR;
}
else if (c == ' ' && dialect->skipinitialspace)
@@ -746,8 +760,6 @@ parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c)
}
else {
/* begin new unquoted field */
- if (dialect->quoting == QUOTE_NONNUMERIC)
- self->numeric_field = 1;
if (parse_add_char(self, module_state, c) < 0)
return -1;
self->state = IN_FIELD;
@@ -892,7 +904,7 @@ parse_reset(ReaderObj *self)
return -1;
self->field_len = 0;
self->state = START_RECORD;
- self->numeric_field = 0;
+ self->unquoted_field = false;
return 0;
}
diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c
index cb5403e8461ff0..9b8e0a719d9048 100644
--- a/Modules/_datetimemodule.c
+++ b/Modules/_datetimemodule.c
@@ -61,16 +61,6 @@ static datetime_state _datetime_global_state;
#define STATIC_STATE() (&_datetime_global_state)
-/*[clinic input]
-module datetime
-class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType"
-class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType"
-class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType"
-[clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=81bec0fa19837f63]*/
-
-#include "clinic/_datetimemodule.c.h"
-
/* We require that C int be at least 32 bits, and use int virtually
* everywhere. In just a few cases we use a temp long, where a Python
* API returns a C long. In such cases, we have to ensure that the
@@ -161,6 +151,17 @@ static PyTypeObject PyDateTime_TimeZoneType;
static int check_tzinfo_subclass(PyObject *p);
+/*[clinic input]
+module datetime
+class datetime.datetime "PyDateTime_DateTime *" "&PyDateTime_DateTimeType"
+class datetime.date "PyDateTime_Date *" "&PyDateTime_DateType"
+class datetime.time "PyDateTime_Time *" "&PyDateTime_TimeType"
+class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "&PyDateTime_IsoCalendarDateType"
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6f65a48dd22fa40f]*/
+
+#include "clinic/_datetimemodule.c.h"
+
/* ---------------------------------------------------------------------------
* Math utilities.
@@ -3466,24 +3467,22 @@ date_timetuple(PyDateTime_Date *self, PyObject *Py_UNUSED(ignored))
0, 0, 0, -1);
}
+/*[clinic input]
+datetime.date.replace
+
+ year: int(c_default="GET_YEAR(self)") = unchanged
+ month: int(c_default="GET_MONTH(self)") = unchanged
+ day: int(c_default="GET_DAY(self)") = unchanged
+
+Return date with new specified fields.
+[clinic start generated code]*/
+
static PyObject *
-date_replace(PyDateTime_Date *self, PyObject *args, PyObject *kw)
+datetime_date_replace_impl(PyDateTime_Date *self, int year, int month,
+ int day)
+/*[clinic end generated code: output=2a9430d1e6318aeb input=0d1f02685b3e90f6]*/
{
- PyObject *clone;
- PyObject *tuple;
- int year = GET_YEAR(self);
- int month = GET_MONTH(self);
- int day = GET_DAY(self);
-
- if (! PyArg_ParseTupleAndKeywords(args, kw, "|iii:replace", date_kws,
- &year, &month, &day))
- return NULL;
- tuple = Py_BuildValue("iii", year, month, day);
- if (tuple == NULL)
- return NULL;
- clone = date_new(Py_TYPE(self), tuple, NULL);
- Py_DECREF(tuple);
- return clone;
+ return new_date_ex(year, month, day, Py_TYPE(self));
}
static Py_hash_t
@@ -3596,10 +3595,9 @@ static PyMethodDef date_methods[] = {
PyDoc_STR("Return the day of the week represented by the date.\n"
"Monday == 0 ... Sunday == 6")},
- {"replace", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS,
- PyDoc_STR("Return date with new specified fields.")},
+ DATETIME_DATE_REPLACE_METHODDEF
- {"__replace__", _PyCFunction_CAST(date_replace), METH_VARARGS | METH_KEYWORDS},
+ {"__replace__", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL | METH_KEYWORDS},
{"__reduce__", (PyCFunction)date_reduce, METH_NOARGS,
PyDoc_STR("__reduce__() -> (cls, state)")},
@@ -4573,36 +4571,28 @@ time_hash(PyDateTime_Time *self)
return self->hashcode;
}
+/*[clinic input]
+datetime.time.replace
+
+ hour: int(c_default="TIME_GET_HOUR(self)") = unchanged
+ minute: int(c_default="TIME_GET_MINUTE(self)") = unchanged
+ second: int(c_default="TIME_GET_SECOND(self)") = unchanged
+ microsecond: int(c_default="TIME_GET_MICROSECOND(self)") = unchanged
+ tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged
+ *
+ fold: int(c_default="TIME_GET_FOLD(self)") = unchanged
+
+Return time with new specified fields.
+[clinic start generated code]*/
+
static PyObject *
-time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw)
+datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute,
+ int second, int microsecond, PyObject *tzinfo,
+ int fold)
+/*[clinic end generated code: output=0b89a44c299e4f80 input=9b6a35b1e704b0ca]*/
{
- PyObject *clone;
- PyObject *tuple;
- int hh = TIME_GET_HOUR(self);
- int mm = TIME_GET_MINUTE(self);
- int ss = TIME_GET_SECOND(self);
- int us = TIME_GET_MICROSECOND(self);
- PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None;
- int fold = TIME_GET_FOLD(self);
-
- if (! PyArg_ParseTupleAndKeywords(args, kw, "|iiiiO$i:replace",
- time_kws,
- &hh, &mm, &ss, &us, &tzinfo, &fold))
- return NULL;
- if (fold != 0 && fold != 1) {
- PyErr_SetString(PyExc_ValueError,
- "fold must be either 0 or 1");
- return NULL;
- }
- tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo);
- if (tuple == NULL)
- return NULL;
- clone = time_new(Py_TYPE(self), tuple, NULL);
- if (clone != NULL) {
- TIME_SET_FOLD(clone, fold);
- }
- Py_DECREF(tuple);
- return clone;
+ return new_time_ex2(hour, minute, second, microsecond, tzinfo, fold,
+ Py_TYPE(self));
}
static PyObject *
@@ -4732,10 +4722,9 @@ static PyMethodDef time_methods[] = {
{"dst", (PyCFunction)time_dst, METH_NOARGS,
PyDoc_STR("Return self.tzinfo.dst(self).")},
- {"replace", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS,
- PyDoc_STR("Return time with new specified fields.")},
+ DATETIME_TIME_REPLACE_METHODDEF
- {"__replace__", _PyCFunction_CAST(time_replace), METH_VARARGS | METH_KEYWORDS},
+ {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS},
{"fromisoformat", (PyCFunction)time_fromisoformat, METH_O | METH_CLASS,
PyDoc_STR("string -> time from a string in ISO 8601 format")},
@@ -6042,40 +6031,32 @@ datetime_hash(PyDateTime_DateTime *self)
return self->hashcode;
}
+/*[clinic input]
+datetime.datetime.replace
+
+ year: int(c_default="GET_YEAR(self)") = unchanged
+ month: int(c_default="GET_MONTH(self)") = unchanged
+ day: int(c_default="GET_DAY(self)") = unchanged
+ hour: int(c_default="DATE_GET_HOUR(self)") = unchanged
+ minute: int(c_default="DATE_GET_MINUTE(self)") = unchanged
+ second: int(c_default="DATE_GET_SECOND(self)") = unchanged
+ microsecond: int(c_default="DATE_GET_MICROSECOND(self)") = unchanged
+ tzinfo: object(c_default="HASTZINFO(self) ? self->tzinfo : Py_None") = unchanged
+ *
+ fold: int(c_default="DATE_GET_FOLD(self)") = unchanged
+
+Return datetime with new specified fields.
+[clinic start generated code]*/
+
static PyObject *
-datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw)
+datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year,
+ int month, int day, int hour, int minute,
+ int second, int microsecond, PyObject *tzinfo,
+ int fold)
+/*[clinic end generated code: output=00bc96536833fddb input=9b38253d56d9bcad]*/
{
- PyObject *clone;
- PyObject *tuple;
- int y = GET_YEAR(self);
- int m = GET_MONTH(self);
- int d = GET_DAY(self);
- int hh = DATE_GET_HOUR(self);
- int mm = DATE_GET_MINUTE(self);
- int ss = DATE_GET_SECOND(self);
- int us = DATE_GET_MICROSECOND(self);
- PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None;
- int fold = DATE_GET_FOLD(self);
-
- if (! PyArg_ParseTupleAndKeywords(args, kw, "|iiiiiiiO$i:replace",
- datetime_kws,
- &y, &m, &d, &hh, &mm, &ss, &us,
- &tzinfo, &fold))
- return NULL;
- if (fold != 0 && fold != 1) {
- PyErr_SetString(PyExc_ValueError,
- "fold must be either 0 or 1");
- return NULL;
- }
- tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo);
- if (tuple == NULL)
- return NULL;
- clone = datetime_new(Py_TYPE(self), tuple, NULL);
- if (clone != NULL) {
- DATE_SET_FOLD(clone, fold);
- }
- Py_DECREF(tuple);
- return clone;
+ return new_datetime_ex2(year, month, day, hour, minute, second,
+ microsecond, tzinfo, fold, Py_TYPE(self));
}
static PyObject *
@@ -6597,10 +6578,9 @@ static PyMethodDef datetime_methods[] = {
{"dst", (PyCFunction)datetime_dst, METH_NOARGS,
PyDoc_STR("Return self.tzinfo.dst(self).")},
- {"replace", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS,
- PyDoc_STR("Return datetime with new specified fields.")},
+ DATETIME_DATETIME_REPLACE_METHODDEF
- {"__replace__", _PyCFunction_CAST(datetime_replace), METH_VARARGS | METH_KEYWORDS},
+ {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS},
{"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS,
PyDoc_STR("tz -> convert to local time in new timezone tz\n")},
diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c
index 8b93f8e2cbcf0b..127f5f2887d4cd 100644
--- a/Modules/_decimal/_decimal.c
+++ b/Modules/_decimal/_decimal.c
@@ -3446,6 +3446,14 @@ dec_format(PyObject *dec, PyObject *args)
if (fmt == NULL) {
return NULL;
}
+
+ if (size > 0 && fmt[size-1] == 'N') {
+ if (PyErr_WarnEx(PyExc_DeprecationWarning,
+ "Format specifier 'N' is deprecated", 1) < 0) {
+ return NULL;
+ }
+ }
+
/* NOTE: If https://github.com/python/cpython/pull/29438 lands, the
* format string manipulation below can be eliminated by enhancing
* the forked mpd_parse_fmt_str(). */
@@ -3593,12 +3601,6 @@ dec_format(PyObject *dec, PyObject *args)
if (replace_fillchar) {
dec_replace_fillchar(decstring);
}
- if (strchr(fmt, 'N') != NULL) {
- if (PyErr_WarnEx(PyExc_DeprecationWarning,
- "Format specifier 'N' is deprecated", 1) < 0) {
- goto finish;
- }
- }
result = PyUnicode_DecodeUTF8(decstring, size, NULL);
diff --git a/Modules/_testcapi_feature_macros.inc b/Modules/_testcapi_feature_macros.inc
index a076e714980074..f5f3524f2c0177 100644
--- a/Modules/_testcapi_feature_macros.inc
+++ b/Modules/_testcapi_feature_macros.inc
@@ -38,6 +38,15 @@ if (res) {
Py_DECREF(result); return NULL;
}
+#ifdef Py_TRACE_REFS
+ res = PyDict_SetItemString(result, "Py_TRACE_REFS", Py_True);
+#else
+ res = PyDict_SetItemString(result, "Py_TRACE_REFS", Py_False);
+#endif
+if (res) {
+ Py_DECREF(result); return NULL;
+}
+
#ifdef USE_STACKCHECK
res = PyDict_SetItemString(result, "USE_STACKCHECK", Py_True);
#else
diff --git a/Modules/_winapi.c b/Modules/_winapi.c
index 26302b559817b3..5e5eb123c4ccff 100644
--- a/Modules/_winapi.c
+++ b/Modules/_winapi.c
@@ -441,7 +441,7 @@ _winapi_ConnectNamedPipe_impl(PyObject *module, HANDLE handle,
/*[clinic input]
_winapi.CreateFile -> HANDLE
- file_name: LPCTSTR
+ file_name: LPCWSTR
desired_access: DWORD
share_mode: DWORD
security_attributes: LPSECURITY_ATTRIBUTES
@@ -452,12 +452,12 @@ _winapi.CreateFile -> HANDLE
[clinic start generated code]*/
static HANDLE
-_winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name,
+_winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name,
DWORD desired_access, DWORD share_mode,
LPSECURITY_ATTRIBUTES security_attributes,
DWORD creation_disposition,
DWORD flags_and_attributes, HANDLE template_file)
-/*[clinic end generated code: output=417ddcebfc5a3d53 input=6423c3e40372dbd5]*/
+/*[clinic end generated code: output=818c811e5e04d550 input=1fa870ed1c2e3d69]*/
{
HANDLE handle;
@@ -468,14 +468,15 @@ _winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name,
}
Py_BEGIN_ALLOW_THREADS
- handle = CreateFile(file_name, desired_access,
- share_mode, security_attributes,
- creation_disposition,
- flags_and_attributes, template_file);
+ handle = CreateFileW(file_name, desired_access,
+ share_mode, security_attributes,
+ creation_disposition,
+ flags_and_attributes, template_file);
Py_END_ALLOW_THREADS
- if (handle == INVALID_HANDLE_VALUE)
+ if (handle == INVALID_HANDLE_VALUE) {
PyErr_SetFromWindowsErr(0);
+ }
return handle;
}
diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h
index 591ab50c76a8e8..60fb12a2231619 100644
--- a/Modules/clinic/_collectionsmodule.c.h
+++ b/Modules/clinic/_collectionsmodule.c.h
@@ -2,9 +2,425 @@
preserve
[clinic start generated code]*/
+#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+# include "pycore_gc.h" // PyGC_Head
+# include "pycore_runtime.h" // _Py_ID()
+#endif
#include "pycore_abstract.h" // _PyNumber_Index()
#include "pycore_modsupport.h" // _PyArg_CheckPositional()
+PyDoc_STRVAR(deque_pop__doc__,
+"pop($self, /)\n"
+"--\n"
+"\n"
+"Remove and return the rightmost element.");
+
+#define DEQUE_POP_METHODDEF \
+ {"pop", (PyCFunction)deque_pop, METH_NOARGS, deque_pop__doc__},
+
+static PyObject *
+deque_pop_impl(dequeobject *deque);
+
+static PyObject *
+deque_pop(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+{
+ return deque_pop_impl(deque);
+}
+
+PyDoc_STRVAR(deque_popleft__doc__,
+"popleft($self, /)\n"
+"--\n"
+"\n"
+"Remove and return the leftmost element.");
+
+#define DEQUE_POPLEFT_METHODDEF \
+ {"popleft", (PyCFunction)deque_popleft, METH_NOARGS, deque_popleft__doc__},
+
+static PyObject *
+deque_popleft_impl(dequeobject *deque);
+
+static PyObject *
+deque_popleft(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+{
+ return deque_popleft_impl(deque);
+}
+
+PyDoc_STRVAR(deque_append__doc__,
+"append($self, item, /)\n"
+"--\n"
+"\n"
+"Add an element to the right side of the deque.");
+
+#define DEQUE_APPEND_METHODDEF \
+ {"append", (PyCFunction)deque_append, METH_O, deque_append__doc__},
+
+PyDoc_STRVAR(deque_appendleft__doc__,
+"appendleft($self, item, /)\n"
+"--\n"
+"\n"
+"Add an element to the left side of the deque.");
+
+#define DEQUE_APPENDLEFT_METHODDEF \
+ {"appendleft", (PyCFunction)deque_appendleft, METH_O, deque_appendleft__doc__},
+
+PyDoc_STRVAR(deque_extend__doc__,
+"extend($self, iterable, /)\n"
+"--\n"
+"\n"
+"Extend the right side of the deque with elements from the iterable.");
+
+#define DEQUE_EXTEND_METHODDEF \
+ {"extend", (PyCFunction)deque_extend, METH_O, deque_extend__doc__},
+
+PyDoc_STRVAR(deque_extendleft__doc__,
+"extendleft($self, iterable, /)\n"
+"--\n"
+"\n"
+"Extend the left side of the deque with elements from the iterable.");
+
+#define DEQUE_EXTENDLEFT_METHODDEF \
+ {"extendleft", (PyCFunction)deque_extendleft, METH_O, deque_extendleft__doc__},
+
+PyDoc_STRVAR(deque_copy__doc__,
+"copy($self, /)\n"
+"--\n"
+"\n"
+"Return a shallow copy of a deque.");
+
+#define DEQUE_COPY_METHODDEF \
+ {"copy", (PyCFunction)deque_copy, METH_NOARGS, deque_copy__doc__},
+
+static PyObject *
+deque_copy_impl(dequeobject *deque);
+
+static PyObject *
+deque_copy(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+{
+ return deque_copy_impl(deque);
+}
+
+PyDoc_STRVAR(deque___copy____doc__,
+"__copy__($self, /)\n"
+"--\n"
+"\n"
+"Return a shallow copy of a deque.");
+
+#define DEQUE___COPY___METHODDEF \
+ {"__copy__", (PyCFunction)deque___copy__, METH_NOARGS, deque___copy____doc__},
+
+static PyObject *
+deque___copy___impl(dequeobject *deque);
+
+static PyObject *
+deque___copy__(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+{
+ return deque___copy___impl(deque);
+}
+
+PyDoc_STRVAR(deque_clearmethod__doc__,
+"clear($self, /)\n"
+"--\n"
+"\n"
+"Remove all elements from the deque.");
+
+#define DEQUE_CLEARMETHOD_METHODDEF \
+ {"clear", (PyCFunction)deque_clearmethod, METH_NOARGS, deque_clearmethod__doc__},
+
+static PyObject *
+deque_clearmethod_impl(dequeobject *deque);
+
+static PyObject *
+deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+{
+ return deque_clearmethod_impl(deque);
+}
+
+PyDoc_STRVAR(deque_rotate__doc__,
+"rotate($self, n=1, /)\n"
+"--\n"
+"\n"
+"Rotate the deque n steps to the right. If n is negative, rotates left.");
+
+#define DEQUE_ROTATE_METHODDEF \
+ {"rotate", _PyCFunction_CAST(deque_rotate), METH_FASTCALL, deque_rotate__doc__},
+
+static PyObject *
+deque_rotate_impl(dequeobject *deque, Py_ssize_t n);
+
+static PyObject *
+deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs)
+{
+ PyObject *return_value = NULL;
+ Py_ssize_t n = 1;
+
+ if (!_PyArg_CheckPositional("rotate", nargs, 0, 1)) {
+ goto exit;
+ }
+ if (nargs < 1) {
+ goto skip_optional;
+ }
+ {
+ Py_ssize_t ival = -1;
+ PyObject *iobj = _PyNumber_Index(args[0]);
+ if (iobj != NULL) {
+ ival = PyLong_AsSsize_t(iobj);
+ Py_DECREF(iobj);
+ }
+ if (ival == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ n = ival;
+ }
+skip_optional:
+ return_value = deque_rotate_impl(deque, n);
+
+exit:
+ return return_value;
+}
+
+PyDoc_STRVAR(deque_reverse__doc__,
+"reverse($self, /)\n"
+"--\n"
+"\n"
+"Reverse *IN PLACE*.");
+
+#define DEQUE_REVERSE_METHODDEF \
+ {"reverse", (PyCFunction)deque_reverse, METH_NOARGS, deque_reverse__doc__},
+
+static PyObject *
+deque_reverse_impl(dequeobject *deque);
+
+static PyObject *
+deque_reverse(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+{
+ return deque_reverse_impl(deque);
+}
+
+PyDoc_STRVAR(deque_count__doc__,
+"count($self, value, /)\n"
+"--\n"
+"\n"
+"Return number of occurrences of value.");
+
+#define DEQUE_COUNT_METHODDEF \
+ {"count", (PyCFunction)deque_count, METH_O, deque_count__doc__},
+
+PyDoc_STRVAR(deque_index__doc__,
+"index($self, value, [start, [stop]])\n"
+"--\n"
+"\n"
+"Return first index of value.\n"
+"\n"
+"Raises ValueError if the value is not present.");
+
+#define DEQUE_INDEX_METHODDEF \
+ {"index", _PyCFunction_CAST(deque_index), METH_FASTCALL, deque_index__doc__},
+
+static PyObject *
+deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start,
+ Py_ssize_t stop);
+
+static PyObject *
+deque_index(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs)
+{
+ PyObject *return_value = NULL;
+ PyObject *v;
+ Py_ssize_t start = 0;
+ Py_ssize_t stop = Py_SIZE(deque);
+
+ if (!_PyArg_CheckPositional("index", nargs, 1, 3)) {
+ goto exit;
+ }
+ v = args[0];
+ if (nargs < 2) {
+ goto skip_optional;
+ }
+ if (!_PyEval_SliceIndexNotNone(args[1], &start)) {
+ goto exit;
+ }
+ if (nargs < 3) {
+ goto skip_optional;
+ }
+ if (!_PyEval_SliceIndexNotNone(args[2], &stop)) {
+ goto exit;
+ }
+skip_optional:
+ return_value = deque_index_impl(deque, v, start, stop);
+
+exit:
+ return return_value;
+}
+
+PyDoc_STRVAR(deque_insert__doc__,
+"insert($self, index, value, /)\n"
+"--\n"
+"\n"
+"Insert value before index.");
+
+#define DEQUE_INSERT_METHODDEF \
+ {"insert", _PyCFunction_CAST(deque_insert), METH_FASTCALL, deque_insert__doc__},
+
+static PyObject *
+deque_insert_impl(dequeobject *deque, Py_ssize_t index, PyObject *value);
+
+static PyObject *
+deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs)
+{
+ PyObject *return_value = NULL;
+ Py_ssize_t index;
+ PyObject *value;
+
+ if (!_PyArg_CheckPositional("insert", nargs, 2, 2)) {
+ goto exit;
+ }
+ {
+ Py_ssize_t ival = -1;
+ PyObject *iobj = _PyNumber_Index(args[0]);
+ if (iobj != NULL) {
+ ival = PyLong_AsSsize_t(iobj);
+ Py_DECREF(iobj);
+ }
+ if (ival == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ index = ival;
+ }
+ value = args[1];
+ return_value = deque_insert_impl(deque, index, value);
+
+exit:
+ return return_value;
+}
+
+PyDoc_STRVAR(deque_remove__doc__,
+"remove($self, value, /)\n"
+"--\n"
+"\n"
+"Remove first occurrence of value.");
+
+#define DEQUE_REMOVE_METHODDEF \
+ {"remove", (PyCFunction)deque_remove, METH_O, deque_remove__doc__},
+
+PyDoc_STRVAR(deque___reduce____doc__,
+"__reduce__($self, /)\n"
+"--\n"
+"\n"
+"Return state information for pickling.");
+
+#define DEQUE___REDUCE___METHODDEF \
+ {"__reduce__", (PyCFunction)deque___reduce__, METH_NOARGS, deque___reduce____doc__},
+
+static PyObject *
+deque___reduce___impl(dequeobject *deque);
+
+static PyObject *
+deque___reduce__(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+{
+ return deque___reduce___impl(deque);
+}
+
+PyDoc_STRVAR(deque_init__doc__,
+"deque([iterable[, maxlen]])\n"
+"--\n"
+"\n"
+"A list-like sequence optimized for data accesses near its endpoints.");
+
+static int
+deque_init_impl(dequeobject *deque, PyObject *iterable, PyObject *maxlenobj);
+
+static int
+deque_init(PyObject *deque, PyObject *args, PyObject *kwargs)
+{
+ int return_value = -1;
+ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+ #define NUM_KEYWORDS 2
+ static struct {
+ PyGC_Head _this_is_not_used;
+ PyObject_VAR_HEAD
+ PyObject *ob_item[NUM_KEYWORDS];
+ } _kwtuple = {
+ .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+ .ob_item = { &_Py_ID(iterable), &_Py_ID(maxlen), },
+ };
+ #undef NUM_KEYWORDS
+ #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+ #else // !Py_BUILD_CORE
+ # define KWTUPLE NULL
+ #endif // !Py_BUILD_CORE
+
+ static const char * const _keywords[] = {"iterable", "maxlen", NULL};
+ static _PyArg_Parser _parser = {
+ .keywords = _keywords,
+ .fname = "deque",
+ .kwtuple = KWTUPLE,
+ };
+ #undef KWTUPLE
+ PyObject *argsbuf[2];
+ PyObject * const *fastargs;
+ Py_ssize_t nargs = PyTuple_GET_SIZE(args);
+ Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0;
+ PyObject *iterable = NULL;
+ PyObject *maxlenobj = NULL;
+
+ fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 2, 0, argsbuf);
+ if (!fastargs) {
+ goto exit;
+ }
+ if (!noptargs) {
+ goto skip_optional_pos;
+ }
+ if (fastargs[0]) {
+ iterable = fastargs[0];
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ maxlenobj = fastargs[1];
+skip_optional_pos:
+ return_value = deque_init_impl((dequeobject *)deque, iterable, maxlenobj);
+
+exit:
+ return return_value;
+}
+
+PyDoc_STRVAR(deque___sizeof____doc__,
+"__sizeof__($self, /)\n"
+"--\n"
+"\n"
+"Return the size of the deque in memory, in bytes.");
+
+#define DEQUE___SIZEOF___METHODDEF \
+ {"__sizeof__", (PyCFunction)deque___sizeof__, METH_NOARGS, deque___sizeof____doc__},
+
+static PyObject *
+deque___sizeof___impl(dequeobject *deque);
+
+static PyObject *
+deque___sizeof__(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+{
+ return deque___sizeof___impl(deque);
+}
+
+PyDoc_STRVAR(deque___reversed____doc__,
+"__reversed__($self, /)\n"
+"--\n"
+"\n"
+"Return a reverse iterator over the deque.");
+
+#define DEQUE___REVERSED___METHODDEF \
+ {"__reversed__", (PyCFunction)deque___reversed__, METH_NOARGS, deque___reversed____doc__},
+
+static PyObject *
+deque___reversed___impl(dequeobject *deque);
+
+static PyObject *
+deque___reversed__(dequeobject *deque, PyObject *Py_UNUSED(ignored))
+{
+ return deque___reversed___impl(deque);
+}
+
PyDoc_STRVAR(_collections__count_elements__doc__,
"_count_elements($module, mapping, iterable, /)\n"
"--\n"
@@ -72,4 +488,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
exit:
return return_value;
}
-/*[clinic end generated code: output=c896a72f8c45930d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=3633a5cbc23e8440 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h
index 1ee50fc2a13762..48499e0aaf7783 100644
--- a/Modules/clinic/_datetimemodule.c.h
+++ b/Modules/clinic/_datetimemodule.c.h
@@ -82,6 +82,207 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
return return_value;
}
+PyDoc_STRVAR(datetime_date_replace__doc__,
+"replace($self, /, year=unchanged, month=unchanged, day=unchanged)\n"
+"--\n"
+"\n"
+"Return date with new specified fields.");
+
+#define DATETIME_DATE_REPLACE_METHODDEF \
+ {"replace", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL|METH_KEYWORDS, datetime_date_replace__doc__},
+
+static PyObject *
+datetime_date_replace_impl(PyDateTime_Date *self, int year, int month,
+ int day);
+
+static PyObject *
+datetime_date_replace(PyDateTime_Date *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+ PyObject *return_value = NULL;
+ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+ #define NUM_KEYWORDS 3
+ static struct {
+ PyGC_Head _this_is_not_used;
+ PyObject_VAR_HEAD
+ PyObject *ob_item[NUM_KEYWORDS];
+ } _kwtuple = {
+ .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+ .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), },
+ };
+ #undef NUM_KEYWORDS
+ #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+ #else // !Py_BUILD_CORE
+ # define KWTUPLE NULL
+ #endif // !Py_BUILD_CORE
+
+ static const char * const _keywords[] = {"year", "month", "day", NULL};
+ static _PyArg_Parser _parser = {
+ .keywords = _keywords,
+ .fname = "replace",
+ .kwtuple = KWTUPLE,
+ };
+ #undef KWTUPLE
+ PyObject *argsbuf[3];
+ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
+ int year = GET_YEAR(self);
+ int month = GET_MONTH(self);
+ int day = GET_DAY(self);
+
+ args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf);
+ if (!args) {
+ goto exit;
+ }
+ if (!noptargs) {
+ goto skip_optional_pos;
+ }
+ if (args[0]) {
+ year = PyLong_AsInt(args[0]);
+ if (year == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[1]) {
+ month = PyLong_AsInt(args[1]);
+ if (month == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ day = PyLong_AsInt(args[2]);
+ if (day == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+skip_optional_pos:
+ return_value = datetime_date_replace_impl(self, year, month, day);
+
+exit:
+ return return_value;
+}
+
+PyDoc_STRVAR(datetime_time_replace__doc__,
+"replace($self, /, hour=unchanged, minute=unchanged, second=unchanged,\n"
+" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n"
+"--\n"
+"\n"
+"Return time with new specified fields.");
+
+#define DATETIME_TIME_REPLACE_METHODDEF \
+ {"replace", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL|METH_KEYWORDS, datetime_time_replace__doc__},
+
+static PyObject *
+datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute,
+ int second, int microsecond, PyObject *tzinfo,
+ int fold);
+
+static PyObject *
+datetime_time_replace(PyDateTime_Time *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+ PyObject *return_value = NULL;
+ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+ #define NUM_KEYWORDS 6
+ static struct {
+ PyGC_Head _this_is_not_used;
+ PyObject_VAR_HEAD
+ PyObject *ob_item[NUM_KEYWORDS];
+ } _kwtuple = {
+ .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+ .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), },
+ };
+ #undef NUM_KEYWORDS
+ #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+ #else // !Py_BUILD_CORE
+ # define KWTUPLE NULL
+ #endif // !Py_BUILD_CORE
+
+ static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL};
+ static _PyArg_Parser _parser = {
+ .keywords = _keywords,
+ .fname = "replace",
+ .kwtuple = KWTUPLE,
+ };
+ #undef KWTUPLE
+ PyObject *argsbuf[6];
+ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
+ int hour = TIME_GET_HOUR(self);
+ int minute = TIME_GET_MINUTE(self);
+ int second = TIME_GET_SECOND(self);
+ int microsecond = TIME_GET_MICROSECOND(self);
+ PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None;
+ int fold = TIME_GET_FOLD(self);
+
+ args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 5, 0, argsbuf);
+ if (!args) {
+ goto exit;
+ }
+ if (!noptargs) {
+ goto skip_optional_pos;
+ }
+ if (args[0]) {
+ hour = PyLong_AsInt(args[0]);
+ if (hour == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[1]) {
+ minute = PyLong_AsInt(args[1]);
+ if (minute == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[2]) {
+ second = PyLong_AsInt(args[2]);
+ if (second == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[3]) {
+ microsecond = PyLong_AsInt(args[3]);
+ if (microsecond == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[4]) {
+ tzinfo = args[4];
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+skip_optional_pos:
+ if (!noptargs) {
+ goto skip_optional_kwonly;
+ }
+ fold = PyLong_AsInt(args[5]);
+ if (fold == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+skip_optional_kwonly:
+ return_value = datetime_time_replace_impl(self, hour, minute, second, microsecond, tzinfo, fold);
+
+exit:
+ return return_value;
+}
+
PyDoc_STRVAR(datetime_datetime_now__doc__,
"now($type, /, tz=None)\n"
"--\n"
@@ -146,4 +347,153 @@ datetime_datetime_now(PyTypeObject *type, PyObject *const *args, Py_ssize_t narg
exit:
return return_value;
}
-/*[clinic end generated code: output=562813dd3e164794 input=a9049054013a1b77]*/
+
+PyDoc_STRVAR(datetime_datetime_replace__doc__,
+"replace($self, /, year=unchanged, month=unchanged, day=unchanged,\n"
+" hour=unchanged, minute=unchanged, second=unchanged,\n"
+" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n"
+"--\n"
+"\n"
+"Return datetime with new specified fields.");
+
+#define DATETIME_DATETIME_REPLACE_METHODDEF \
+ {"replace", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_replace__doc__},
+
+static PyObject *
+datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year,
+ int month, int day, int hour, int minute,
+ int second, int microsecond, PyObject *tzinfo,
+ int fold);
+
+static PyObject *
+datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+ PyObject *return_value = NULL;
+ #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+ #define NUM_KEYWORDS 9
+ static struct {
+ PyGC_Head _this_is_not_used;
+ PyObject_VAR_HEAD
+ PyObject *ob_item[NUM_KEYWORDS];
+ } _kwtuple = {
+ .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+ .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), },
+ };
+ #undef NUM_KEYWORDS
+ #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+ #else // !Py_BUILD_CORE
+ # define KWTUPLE NULL
+ #endif // !Py_BUILD_CORE
+
+ static const char * const _keywords[] = {"year", "month", "day", "hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL};
+ static _PyArg_Parser _parser = {
+ .keywords = _keywords,
+ .fname = "replace",
+ .kwtuple = KWTUPLE,
+ };
+ #undef KWTUPLE
+ PyObject *argsbuf[9];
+ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
+ int year = GET_YEAR(self);
+ int month = GET_MONTH(self);
+ int day = GET_DAY(self);
+ int hour = DATE_GET_HOUR(self);
+ int minute = DATE_GET_MINUTE(self);
+ int second = DATE_GET_SECOND(self);
+ int microsecond = DATE_GET_MICROSECOND(self);
+ PyObject *tzinfo = HASTZINFO(self) ? self->tzinfo : Py_None;
+ int fold = DATE_GET_FOLD(self);
+
+ args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 8, 0, argsbuf);
+ if (!args) {
+ goto exit;
+ }
+ if (!noptargs) {
+ goto skip_optional_pos;
+ }
+ if (args[0]) {
+ year = PyLong_AsInt(args[0]);
+ if (year == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[1]) {
+ month = PyLong_AsInt(args[1]);
+ if (month == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[2]) {
+ day = PyLong_AsInt(args[2]);
+ if (day == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[3]) {
+ hour = PyLong_AsInt(args[3]);
+ if (hour == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[4]) {
+ minute = PyLong_AsInt(args[4]);
+ if (minute == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[5]) {
+ second = PyLong_AsInt(args[5]);
+ if (second == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[6]) {
+ microsecond = PyLong_AsInt(args[6]);
+ if (microsecond == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+ if (args[7]) {
+ tzinfo = args[7];
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+skip_optional_pos:
+ if (!noptargs) {
+ goto skip_optional_kwonly;
+ }
+ fold = PyLong_AsInt(args[8]);
+ if (fold == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+skip_optional_kwonly:
+ return_value = datetime_datetime_replace_impl(self, year, month, day, hour, minute, second, microsecond, tzinfo, fold);
+
+exit:
+ return return_value;
+}
+/*[clinic end generated code: output=c7a04b865b1e0890 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h
index 3a3231c051ef71..d1052f38919dde 100644
--- a/Modules/clinic/_winapi.c.h
+++ b/Modules/clinic/_winapi.c.h
@@ -162,7 +162,7 @@ PyDoc_STRVAR(_winapi_CreateFile__doc__,
{"CreateFile", _PyCFunction_CAST(_winapi_CreateFile), METH_FASTCALL, _winapi_CreateFile__doc__},
static HANDLE
-_winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name,
+_winapi_CreateFile_impl(PyObject *module, LPCWSTR file_name,
DWORD desired_access, DWORD share_mode,
LPSECURITY_ATTRIBUTES security_attributes,
DWORD creation_disposition,
@@ -172,7 +172,7 @@ static PyObject *
_winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
- LPCTSTR file_name;
+ LPCWSTR file_name = NULL;
DWORD desired_access;
DWORD share_mode;
LPSECURITY_ATTRIBUTES security_attributes;
@@ -181,8 +181,8 @@ _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
HANDLE template_file;
HANDLE _return_value;
- if (!_PyArg_ParseStack(args, nargs, "skk" F_POINTER "kk" F_HANDLE ":CreateFile",
- &file_name, &desired_access, &share_mode, &security_attributes, &creation_disposition, &flags_and_attributes, &template_file)) {
+ if (!_PyArg_ParseStack(args, nargs, "O&kk" F_POINTER "kk" F_HANDLE ":CreateFile",
+ _PyUnicode_WideCharString_Converter, &file_name, &desired_access, &share_mode, &security_attributes, &creation_disposition, &flags_and_attributes, &template_file)) {
goto exit;
}
_return_value = _winapi_CreateFile_impl(module, file_name, desired_access, share_mode, security_attributes, creation_disposition, flags_and_attributes, template_file);
@@ -195,6 +195,9 @@ _winapi_CreateFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
return_value = HANDLE_TO_PYNUM(_return_value);
exit:
+ /* Cleanup for file_name */
+ PyMem_Free((void *)file_name);
+
return return_value;
}
@@ -1479,4 +1482,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO
return return_value;
}
-/*[clinic end generated code: output=e1a9908bb82a6379 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=2350d4f2275d3a6f input=a9049054013a1b77]*/
diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
index ba3e1cfa8dbc21..1373bdef03ba5e 100644
--- a/Modules/clinic/posixmodule.c.h
+++ b/Modules/clinic/posixmodule.c.h
@@ -4465,6 +4465,156 @@ os_sched_getaffinity(PyObject *module, PyObject *arg)
#endif /* defined(HAVE_SCHED_H) && defined(HAVE_SCHED_SETAFFINITY) */
+#if defined(HAVE_POSIX_OPENPT)
+
+PyDoc_STRVAR(os_posix_openpt__doc__,
+"posix_openpt($module, oflag, /)\n"
+"--\n"
+"\n"
+"Open and return a file descriptor for a master pseudo-terminal device.\n"
+"\n"
+"Performs a posix_openpt() C function call. The oflag argument is used to\n"
+"set file status flags and file access modes as specified in the manual page\n"
+"of posix_openpt() of your system.");
+
+#define OS_POSIX_OPENPT_METHODDEF \
+ {"posix_openpt", (PyCFunction)os_posix_openpt, METH_O, os_posix_openpt__doc__},
+
+static int
+os_posix_openpt_impl(PyObject *module, int oflag);
+
+static PyObject *
+os_posix_openpt(PyObject *module, PyObject *arg)
+{
+ PyObject *return_value = NULL;
+ int oflag;
+ int _return_value;
+
+ oflag = PyLong_AsInt(arg);
+ if (oflag == -1 && PyErr_Occurred()) {
+ goto exit;
+ }
+ _return_value = os_posix_openpt_impl(module, oflag);
+ if ((_return_value == -1) && PyErr_Occurred()) {
+ goto exit;
+ }
+ return_value = PyLong_FromLong((long)_return_value);
+
+exit:
+ return return_value;
+}
+
+#endif /* defined(HAVE_POSIX_OPENPT) */
+
+#if defined(HAVE_GRANTPT)
+
+PyDoc_STRVAR(os_grantpt__doc__,
+"grantpt($module, fd, /)\n"
+"--\n"
+"\n"
+"Grant access to the slave pseudo-terminal device.\n"
+"\n"
+" fd\n"
+" File descriptor of a master pseudo-terminal device.\n"
+"\n"
+"Performs a grantpt() C function call.");
+
+#define OS_GRANTPT_METHODDEF \
+ {"grantpt", (PyCFunction)os_grantpt, METH_O, os_grantpt__doc__},
+
+static PyObject *
+os_grantpt_impl(PyObject *module, int fd);
+
+static PyObject *
+os_grantpt(PyObject *module, PyObject *arg)
+{
+ PyObject *return_value = NULL;
+ int fd;
+
+ if (!_PyLong_FileDescriptor_Converter(arg, &fd)) {
+ goto exit;
+ }
+ return_value = os_grantpt_impl(module, fd);
+
+exit:
+ return return_value;
+}
+
+#endif /* defined(HAVE_GRANTPT) */
+
+#if defined(HAVE_UNLOCKPT)
+
+PyDoc_STRVAR(os_unlockpt__doc__,
+"unlockpt($module, fd, /)\n"
+"--\n"
+"\n"
+"Unlock a pseudo-terminal master/slave pair.\n"
+"\n"
+" fd\n"
+" File descriptor of a master pseudo-terminal device.\n"
+"\n"
+"Performs an unlockpt() C function call.");
+
+#define OS_UNLOCKPT_METHODDEF \
+ {"unlockpt", (PyCFunction)os_unlockpt, METH_O, os_unlockpt__doc__},
+
+static PyObject *
+os_unlockpt_impl(PyObject *module, int fd);
+
+static PyObject *
+os_unlockpt(PyObject *module, PyObject *arg)
+{
+ PyObject *return_value = NULL;
+ int fd;
+
+ if (!_PyLong_FileDescriptor_Converter(arg, &fd)) {
+ goto exit;
+ }
+ return_value = os_unlockpt_impl(module, fd);
+
+exit:
+ return return_value;
+}
+
+#endif /* defined(HAVE_UNLOCKPT) */
+
+#if (defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R))
+
+PyDoc_STRVAR(os_ptsname__doc__,
+"ptsname($module, fd, /)\n"
+"--\n"
+"\n"
+"Return the name of the slave pseudo-terminal device.\n"
+"\n"
+" fd\n"
+" File descriptor of a master pseudo-terminal device.\n"
+"\n"
+"If the ptsname_r() C function is available, it is called;\n"
+"otherwise, performs a ptsname() C function call.");
+
+#define OS_PTSNAME_METHODDEF \
+ {"ptsname", (PyCFunction)os_ptsname, METH_O, os_ptsname__doc__},
+
+static PyObject *
+os_ptsname_impl(PyObject *module, int fd);
+
+static PyObject *
+os_ptsname(PyObject *module, PyObject *arg)
+{
+ PyObject *return_value = NULL;
+ int fd;
+
+ if (!_PyLong_FileDescriptor_Converter(arg, &fd)) {
+ goto exit;
+ }
+ return_value = os_ptsname_impl(module, fd);
+
+exit:
+ return return_value;
+}
+
+#endif /* (defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R)) */
+
#if (defined(HAVE_OPENPTY) || defined(HAVE__GETPTY) || defined(HAVE_DEV_PTMX))
PyDoc_STRVAR(os_openpty__doc__,
@@ -11991,6 +12141,22 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored))
#define OS_SCHED_GETAFFINITY_METHODDEF
#endif /* !defined(OS_SCHED_GETAFFINITY_METHODDEF) */
+#ifndef OS_POSIX_OPENPT_METHODDEF
+ #define OS_POSIX_OPENPT_METHODDEF
+#endif /* !defined(OS_POSIX_OPENPT_METHODDEF) */
+
+#ifndef OS_GRANTPT_METHODDEF
+ #define OS_GRANTPT_METHODDEF
+#endif /* !defined(OS_GRANTPT_METHODDEF) */
+
+#ifndef OS_UNLOCKPT_METHODDEF
+ #define OS_UNLOCKPT_METHODDEF
+#endif /* !defined(OS_UNLOCKPT_METHODDEF) */
+
+#ifndef OS_PTSNAME_METHODDEF
+ #define OS_PTSNAME_METHODDEF
+#endif /* !defined(OS_PTSNAME_METHODDEF) */
+
#ifndef OS_OPENPTY_METHODDEF
#define OS_OPENPTY_METHODDEF
#endif /* !defined(OS_OPENPTY_METHODDEF) */
@@ -12422,4 +12588,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored))
#ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF
#define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF
#endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */
-/*[clinic end generated code: output=18c128534c355d84 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=43e4e557c771358a input=a9049054013a1b77]*/
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 007fc1cb116f84..40ff131b119d66 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -8358,6 +8358,149 @@ os_sched_getaffinity_impl(PyObject *module, pid_t pid)
#endif /* HAVE_SCHED_H */
+#ifdef HAVE_POSIX_OPENPT
+/*[clinic input]
+os.posix_openpt -> int
+
+ oflag: int
+ /
+
+Open and return a file descriptor for a master pseudo-terminal device.
+
+Performs a posix_openpt() C function call. The oflag argument is used to
+set file status flags and file access modes as specified in the manual page
+of posix_openpt() of your system.
+[clinic start generated code]*/
+
+static int
+os_posix_openpt_impl(PyObject *module, int oflag)
+/*[clinic end generated code: output=ee0bc2624305fc79 input=0de33d0e29693caa]*/
+{
+ int fd;
+
+#if defined(O_CLOEXEC)
+ oflag |= O_CLOEXEC;
+#endif
+
+ fd = posix_openpt(oflag);
+ if (fd == -1) {
+ posix_error();
+ return -1;
+ }
+
+ // Just in case, likely a no-op given O_CLOEXEC above.
+ if (_Py_set_inheritable(fd, 0, NULL) < 0) {
+ close(fd);
+ return -1;
+ }
+
+ return fd;
+}
+#endif /* HAVE_POSIX_OPENPT */
+
+#ifdef HAVE_GRANTPT
+/*[clinic input]
+os.grantpt
+
+ fd: fildes
+ File descriptor of a master pseudo-terminal device.
+ /
+
+Grant access to the slave pseudo-terminal device.
+
+Performs a grantpt() C function call.
+[clinic start generated code]*/
+
+static PyObject *
+os_grantpt_impl(PyObject *module, int fd)
+/*[clinic end generated code: output=dfd580015cf548ab input=0668e3b96760e849]*/
+{
+ int ret;
+ int saved_errno;
+ PyOS_sighandler_t sig_saved;
+
+ sig_saved = PyOS_setsig(SIGCHLD, SIG_DFL);
+
+ ret = grantpt(fd);
+ if (ret == -1)
+ saved_errno = errno;
+
+ PyOS_setsig(SIGCHLD, sig_saved);
+
+ if (ret == -1) {
+ errno = saved_errno;
+ return posix_error();
+ }
+
+ Py_RETURN_NONE;
+}
+#endif /* HAVE_GRANTPT */
+
+#ifdef HAVE_UNLOCKPT
+/*[clinic input]
+os.unlockpt
+
+ fd: fildes
+ File descriptor of a master pseudo-terminal device.
+ /
+
+Unlock a pseudo-terminal master/slave pair.
+
+Performs an unlockpt() C function call.
+[clinic start generated code]*/
+
+static PyObject *
+os_unlockpt_impl(PyObject *module, int fd)
+/*[clinic end generated code: output=e08d354dec12d30c input=de7ab1f59f69a2b4]*/
+{
+ if (unlockpt(fd) == -1)
+ return posix_error();
+
+ Py_RETURN_NONE;
+}
+#endif /* HAVE_UNLOCKPT */
+
+#if defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R)
+/*[clinic input]
+os.ptsname
+
+ fd: fildes
+ File descriptor of a master pseudo-terminal device.
+ /
+
+Return the name of the slave pseudo-terminal device.
+
+If the ptsname_r() C function is available, it is called;
+otherwise, performs a ptsname() C function call.
+[clinic start generated code]*/
+
+static PyObject *
+os_ptsname_impl(PyObject *module, int fd)
+/*[clinic end generated code: output=ef300fadc5675872 input=1369ccc0546f3130]*/
+{
+#ifdef HAVE_PTSNAME_R
+ int ret;
+ char name[MAXPATHLEN+1];
+
+ ret = ptsname_r(fd, name, sizeof(name));
+ if (ret != 0) {
+ errno = ret;
+ return posix_error();
+ }
+#else
+ char *name;
+
+ name = ptsname(fd);
+ /* POSIX manpage: Upon failure, ptsname() shall return a null pointer and may set errno.
+ *MAY* set errno? Hmm... */
+ if (name == NULL)
+ return posix_error();
+#endif /* HAVE_PTSNAME_R */
+
+ return PyUnicode_DecodeFSDefault(name);
+}
+#endif /* defined(HAVE_PTSNAME) || defined(HAVE_PTSNAME_R) */
+
/* AIX uses /dev/ptc but is otherwise the same as /dev/ptmx */
#if defined(HAVE_DEV_PTC) && !defined(HAVE_DEV_PTMX)
# define DEV_PTY_FILE "/dev/ptc"
@@ -16275,6 +16418,10 @@ static PyMethodDef posix_methods[] = {
OS_SCHED_YIELD_METHODDEF
OS_SCHED_SETAFFINITY_METHODDEF
OS_SCHED_GETAFFINITY_METHODDEF
+ OS_POSIX_OPENPT_METHODDEF
+ OS_GRANTPT_METHODDEF
+ OS_UNLOCKPT_METHODDEF
+ OS_PTSNAME_METHODDEF
OS_OPENPTY_METHODDEF
OS_LOGIN_TTY_METHODDEF
OS_FORKPTY_METHODDEF
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index e608b91679b568..23d7e9b5e38a35 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -262,7 +262,7 @@ _PyDict_ClearFreeList(PyInterpreterState *interp)
PyObject_GC_Del(op);
}
while (state->keys_numfree) {
- PyObject_Free(state->keys_free_list[--state->keys_numfree]);
+ PyMem_Free(state->keys_free_list[--state->keys_numfree]);
}
#endif
}
@@ -332,6 +332,22 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk)
_Py_DecRefTotal(_PyInterpreterState_GET());
#endif
if (--dk->dk_refcnt == 0) {
+ if (DK_IS_UNICODE(dk)) {
+ PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dk);
+ Py_ssize_t i, n;
+ for (i = 0, n = dk->dk_nentries; i < n; i++) {
+ Py_XDECREF(entries[i].me_key);
+ Py_XDECREF(entries[i].me_value);
+ }
+ }
+ else {
+ PyDictKeyEntry *entries = DK_ENTRIES(dk);
+ Py_ssize_t i, n;
+ for (i = 0, n = dk->dk_nentries; i < n; i++) {
+ Py_XDECREF(entries[i].me_key);
+ Py_XDECREF(entries[i].me_value);
+ }
+ }
free_keys_object(interp, dk);
}
}
@@ -640,9 +656,9 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
else
#endif
{
- dk = PyObject_Malloc(sizeof(PyDictKeysObject)
- + ((size_t)1 << log2_bytes)
- + entry_size * usable);
+ dk = PyMem_Malloc(sizeof(PyDictKeysObject)
+ + ((size_t)1 << log2_bytes)
+ + entry_size * usable);
if (dk == NULL) {
PyErr_NoMemory();
return NULL;
@@ -666,23 +682,6 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
static void
free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys)
{
- assert(keys != Py_EMPTY_KEYS);
- if (DK_IS_UNICODE(keys)) {
- PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
- Py_ssize_t i, n;
- for (i = 0, n = keys->dk_nentries; i < n; i++) {
- Py_XDECREF(entries[i].me_key);
- Py_XDECREF(entries[i].me_value);
- }
- }
- else {
- PyDictKeyEntry *entries = DK_ENTRIES(keys);
- Py_ssize_t i, n;
- for (i = 0, n = keys->dk_nentries; i < n; i++) {
- Py_XDECREF(entries[i].me_key);
- Py_XDECREF(entries[i].me_value);
- }
- }
#if PyDict_MAXFREELIST > 0
struct _Py_dict_state *state = get_dict_state(interp);
#ifdef Py_DEBUG
@@ -697,7 +696,7 @@ free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys)
return;
}
#endif
- PyObject_Free(keys);
+ PyMem_Free(keys);
}
static inline PyDictValues*
@@ -798,7 +797,7 @@ clone_combined_dict_keys(PyDictObject *orig)
assert(orig->ma_keys->dk_refcnt == 1);
size_t keys_size = _PyDict_KeysSize(orig->ma_keys);
- PyDictKeysObject *keys = PyObject_Malloc(keys_size);
+ PyDictKeysObject *keys = PyMem_Malloc(keys_size);
if (keys == NULL) {
PyErr_NoMemory();
return NULL;
@@ -875,11 +874,11 @@ lookdict_index(PyDictKeysObject *k, Py_hash_t hash, Py_ssize_t index)
Py_UNREACHABLE();
}
-// Search non-Unicode key from Unicode table
-static Py_ssize_t
-unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
+static inline Py_ALWAYS_INLINE Py_ssize_t
+do_lookup(PyDictObject *mp, PyDictKeysObject *dk, PyObject *key, Py_hash_t hash,
+ Py_ssize_t (*check_lookup)(PyDictObject *, PyDictKeysObject *, void *, Py_ssize_t ix, PyObject *key, Py_hash_t))
{
- PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(dk);
+ void *ep0 = _DK_ENTRIES(dk);
size_t mask = DK_MASK(dk);
size_t perturb = hash;
size_t i = (size_t)hash & mask;
@@ -887,73 +886,26 @@ unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key
for (;;) {
ix = dictkeys_get_index(dk, i);
if (ix >= 0) {
- PyDictUnicodeEntry *ep = &ep0[ix];
- assert(ep->me_key != NULL);
- assert(PyUnicode_CheckExact(ep->me_key));
- if (ep->me_key == key) {
+ Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash);
+ if (cmp < 0) {
+ return cmp;
+ } else if (cmp) {
return ix;
}
- if (unicode_get_hash(ep->me_key) == hash) {
- PyObject *startkey = ep->me_key;
- Py_INCREF(startkey);
- int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ);
- Py_DECREF(startkey);
- if (cmp < 0) {
- return DKIX_ERROR;
- }
- if (dk == mp->ma_keys && ep->me_key == startkey) {
- if (cmp > 0) {
- return ix;
- }
- }
- else {
- /* The dict was mutated, restart */
- return DKIX_KEY_CHANGED;
- }
- }
}
else if (ix == DKIX_EMPTY) {
return DKIX_EMPTY;
}
perturb >>= PERTURB_SHIFT;
i = mask & (i*5 + perturb + 1);
- }
- Py_UNREACHABLE();
-}
-// Search Unicode key from Unicode table.
-static Py_ssize_t _Py_HOT_FUNCTION
-unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
-{
- PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(dk);
- size_t mask = DK_MASK(dk);
- size_t perturb = hash;
- size_t i = (size_t)hash & mask;
- Py_ssize_t ix;
- for (;;) {
- ix = dictkeys_get_index(dk, i);
- if (ix >= 0) {
- PyDictUnicodeEntry *ep = &ep0[ix];
- assert(ep->me_key != NULL);
- assert(PyUnicode_CheckExact(ep->me_key));
- if (ep->me_key == key ||
- (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) {
- return ix;
- }
- }
- else if (ix == DKIX_EMPTY) {
- return DKIX_EMPTY;
- }
- perturb >>= PERTURB_SHIFT;
- i = mask & (i*5 + perturb + 1);
// Manual loop unrolling
ix = dictkeys_get_index(dk, i);
if (ix >= 0) {
- PyDictUnicodeEntry *ep = &ep0[ix];
- assert(ep->me_key != NULL);
- assert(PyUnicode_CheckExact(ep->me_key));
- if (ep->me_key == key ||
- (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) {
+ Py_ssize_t cmp = check_lookup(mp, dk, ep0, ix, key, hash);
+ if (cmp < 0) {
+ return cmp;
+ } else if (cmp) {
return ix;
}
}
@@ -966,49 +918,94 @@ unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
Py_UNREACHABLE();
}
-// Search key from Generic table.
+static inline Py_ALWAYS_INLINE Py_ssize_t
+compare_unicode_generic(PyDictObject *mp, PyDictKeysObject *dk,
+ void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash)
+{
+ PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix];
+ assert(ep->me_key != NULL);
+ assert(PyUnicode_CheckExact(ep->me_key));
+ assert(!PyUnicode_CheckExact(key));
+ // TODO: Thread safety
+
+ if (unicode_get_hash(ep->me_key) == hash) {
+ PyObject *startkey = ep->me_key;
+ Py_INCREF(startkey);
+ int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ);
+ Py_DECREF(startkey);
+ if (cmp < 0) {
+ return DKIX_ERROR;
+ }
+ if (dk == mp->ma_keys && ep->me_key == startkey) {
+ return cmp;
+ }
+ else {
+ /* The dict was mutated, restart */
+ return DKIX_KEY_CHANGED;
+ }
+ }
+ return 0;
+}
+
+// Search non-Unicode key from Unicode table
static Py_ssize_t
-dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
+unicodekeys_lookup_generic(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
{
- PyDictKeyEntry *ep0 = DK_ENTRIES(dk);
- size_t mask = DK_MASK(dk);
- size_t perturb = hash;
- size_t i = (size_t)hash & mask;
- Py_ssize_t ix;
- for (;;) {
- ix = dictkeys_get_index(dk, i);
- if (ix >= 0) {
- PyDictKeyEntry *ep = &ep0[ix];
- assert(ep->me_key != NULL);
- if (ep->me_key == key) {
- return ix;
- }
- if (ep->me_hash == hash) {
- PyObject *startkey = ep->me_key;
- Py_INCREF(startkey);
- int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ);
- Py_DECREF(startkey);
- if (cmp < 0) {
- return DKIX_ERROR;
- }
- if (dk == mp->ma_keys && ep->me_key == startkey) {
- if (cmp > 0) {
- return ix;
- }
- }
- else {
- /* The dict was mutated, restart */
- return DKIX_KEY_CHANGED;
- }
- }
+ return do_lookup(mp, dk, key, hash, compare_unicode_generic);
+}
+
+static inline Py_ALWAYS_INLINE Py_ssize_t
+compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk,
+ void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash)
+{
+ PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix];
+ assert(ep->me_key != NULL);
+ assert(PyUnicode_CheckExact(ep->me_key));
+ if (ep->me_key == key ||
+ (unicode_get_hash(ep->me_key) == hash && unicode_eq(ep->me_key, key))) {
+ return 1;
+ }
+ return 0;
+}
+
+static Py_ssize_t _Py_HOT_FUNCTION
+unicodekeys_lookup_unicode(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
+{
+ return do_lookup(NULL, dk, key, hash, compare_unicode_unicode);
+}
+
+static inline Py_ALWAYS_INLINE Py_ssize_t
+compare_generic(PyDictObject *mp, PyDictKeysObject *dk,
+ void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash)
+{
+ PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix];
+ assert(ep->me_key != NULL);
+ if (ep->me_key == key) {
+ return 1;
+ }
+ if (ep->me_hash == hash) {
+ PyObject *startkey = ep->me_key;
+ Py_INCREF(startkey);
+ int cmp = PyObject_RichCompareBool(startkey, key, Py_EQ);
+ Py_DECREF(startkey);
+ if (cmp < 0) {
+ return DKIX_ERROR;
}
- else if (ix == DKIX_EMPTY) {
- return DKIX_EMPTY;
+ if (dk == mp->ma_keys && ep->me_key == startkey) {
+ return cmp;
+ }
+ else {
+ /* The dict was mutated, restart */
+ return DKIX_KEY_CHANGED;
}
- perturb >>= PERTURB_SHIFT;
- i = mask & (i*5 + perturb + 1);
}
- Py_UNREACHABLE();
+ return 0;
+}
+
+static Py_ssize_t
+dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, Py_hash_t hash)
+{
+ return do_lookup(mp, dk, key, hash, compare_generic);
}
/* Lookup a string in a (all unicode) dict keys.
@@ -1544,32 +1541,13 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp,
}
}
- // We can not use free_keys_object here because key's reference
- // are moved already.
if (oldkeys != Py_EMPTY_KEYS) {
#ifdef Py_REF_DEBUG
_Py_DecRefTotal(_PyInterpreterState_GET());
#endif
assert(oldkeys->dk_kind != DICT_KEYS_SPLIT);
assert(oldkeys->dk_refcnt == 1);
-#if PyDict_MAXFREELIST > 0
- struct _Py_dict_state *state = get_dict_state(interp);
-#ifdef Py_DEBUG
- // dictresize() must not be called after _PyDict_Fini()
- assert(state->keys_numfree != -1);
-#endif
- if (DK_LOG_SIZE(oldkeys) == PyDict_LOG_MINSIZE &&
- DK_IS_UNICODE(oldkeys) &&
- state->keys_numfree < PyDict_MAXFREELIST)
- {
- state->keys_free_list[state->keys_numfree++] = oldkeys;
- OBJECT_STAT_INC(to_freelist);
- }
- else
-#endif
- {
- PyObject_Free(oldkeys);
- }
+ free_keys_object(interp, oldkeys);
}
}
diff --git a/Objects/exceptions.c b/Objects/exceptions.c
index a685ed803cd02d..cff55d05163b6b 100644
--- a/Objects/exceptions.c
+++ b/Objects/exceptions.c
@@ -2566,6 +2566,11 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError,
MiddlingExtendsException(PyExc_IndentationError, TabError, SyntaxError,
"Improper mixture of spaces and tabs.");
+/*
+ * IncompleteInputError extends SyntaxError
+ */
+MiddlingExtendsException(PyExc_SyntaxError, IncompleteInputError, SyntaxError,
+ "incomplete input.");
/*
* LookupError extends Exception
@@ -3635,6 +3640,7 @@ static struct static_exception static_exceptions[] = {
// Level 4: Other subclasses
ITEM(IndentationError), // base: SyntaxError(Exception)
+ ITEM(IncompleteInputError), // base: SyntaxError(Exception)
ITEM(IndexError), // base: LookupError(Exception)
ITEM(KeyError), // base: LookupError(Exception)
ITEM(ModuleNotFoundError), // base: ImportError(Exception)
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index 912c450a5e1055..b7611d5f96ac3b 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -32,7 +32,7 @@ get_float_state(void)
{
_PyFreeListState *state = _PyFreeListState_GET();
assert(state != NULL);
- return &state->float_state;
+ return &state->floats;
}
#endif
@@ -1993,7 +1993,7 @@ void
_PyFloat_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
{
#ifdef WITH_FREELISTS
- struct _Py_float_state *state = &freelist_state->float_state;
+ struct _Py_float_state *state = &freelist_state->floats;
PyFloatObject *f = state->free_list;
while (f != NULL) {
PyFloatObject *next = (PyFloatObject*) Py_TYPE(f);
diff --git a/Objects/genobject.c b/Objects/genobject.c
index e9aeb7ab9a9fa8..f47197330fdd80 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -1633,7 +1633,7 @@ static struct _Py_async_gen_state *
get_async_gen_state(void)
{
_PyFreeListState *state = _PyFreeListState_GET();
- return &state->async_gen_state;
+ return &state->async_gens;
}
#endif
@@ -1659,7 +1659,7 @@ void
_PyAsyncGen_ClearFreeLists(_PyFreeListState *freelist_state, int is_finalization)
{
#ifdef WITH_FREELISTS
- struct _Py_async_gen_state *state = &freelist_state->async_gen_state;
+ struct _Py_async_gen_state *state = &freelist_state->async_gens;
while (state->value_numfree > 0) {
_PyAsyncGenWrappedValue *o;
diff --git a/Objects/listobject.c b/Objects/listobject.c
index 401d1026133f4e..56785e5f37a450 100644
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -26,7 +26,7 @@ get_list_state(void)
{
_PyFreeListState *state = _PyFreeListState_GET();
assert(state != NULL);
- return &state->list_state;
+ return &state->lists;
}
#endif
@@ -124,7 +124,7 @@ void
_PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
{
#ifdef WITH_FREELISTS
- struct _Py_list_state *state = &freelist_state->list_state;
+ struct _Py_list_state *state = &freelist_state->lists;
while (state->numfree > 0) {
PyListObject *op = state->free_list[--state->numfree];
assert(PyList_CheckExact(op));
@@ -383,18 +383,11 @@ list_dealloc(PyObject *self)
}
static PyObject *
-list_repr(PyObject *self)
+list_repr_impl(PyListObject *v)
{
- PyListObject *v = (PyListObject *)self;
- Py_ssize_t i;
PyObject *s;
_PyUnicodeWriter writer;
-
- if (Py_SIZE(v) == 0) {
- return PyUnicode_FromString("[]");
- }
-
- i = Py_ReprEnter((PyObject*)v);
+ Py_ssize_t i = Py_ReprEnter((PyObject*)v);
if (i != 0) {
return i > 0 ? PyUnicode_FromString("[...]") : NULL;
}
@@ -439,10 +432,24 @@ list_repr(PyObject *self)
return NULL;
}
+static PyObject *
+list_repr(PyObject *self)
+{
+ if (PyList_GET_SIZE(self) == 0) {
+ return PyUnicode_FromString("[]");
+ }
+ PyListObject *v = (PyListObject *)self;
+ PyObject *ret = NULL;
+ Py_BEGIN_CRITICAL_SECTION(v);
+ ret = list_repr_impl(v);
+ Py_END_CRITICAL_SECTION();
+ return ret;
+}
+
static Py_ssize_t
list_length(PyObject *a)
{
- return Py_SIZE(a);
+ return PyList_GET_SIZE(a);
}
static int
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index 16d5bcb53e7eb7..bea4ea85332bdd 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -7,6 +7,7 @@
#include "pycore_pyerrors.h" // _Py_FatalErrorFormat()
#include "pycore_pymem.h"
#include "pycore_pystate.h" // _PyInterpreterState_GET
+#include "pycore_obmalloc_init.h"
#include <stdlib.h> // malloc()
#include <stdbool.h>
@@ -1016,6 +1017,13 @@ static int running_on_valgrind = -1;
typedef struct _obmalloc_state OMState;
+/* obmalloc state for main interpreter and shared by all interpreters without
+ * their own obmalloc state. By not explicitly initalizing this structure, it
+ * will be allocated in the BSS which is a small performance win. The radix
+ * tree arrays are fairly large but are sparsely used. */
+static struct _obmalloc_state obmalloc_state_main;
+static bool obmalloc_state_initialized;
+
static inline int
has_own_state(PyInterpreterState *interp)
{
@@ -1028,10 +1036,8 @@ static inline OMState *
get_state(void)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
- if (!has_own_state(interp)) {
- interp = _PyInterpreterState_Main();
- }
- return &interp->obmalloc;
+ assert(interp->obmalloc != NULL); // otherwise not initialized or freed
+ return interp->obmalloc;
}
// These macros all rely on a local "state" variable.
@@ -1094,7 +1100,11 @@ _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp)
"the interpreter doesn't have its own allocator");
}
#endif
- OMState *state = &interp->obmalloc;
+ OMState *state = interp->obmalloc;
+
+ if (state == NULL) {
+ return 0;
+ }
Py_ssize_t n = raw_allocated_blocks;
/* add up allocated blocks for used pools */
@@ -1116,6 +1126,8 @@ _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *interp)
return n;
}
+static void free_obmalloc_arenas(PyInterpreterState *interp);
+
void
_PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp)
{
@@ -1124,10 +1136,20 @@ _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *interp)
return;
}
#endif
- if (has_own_state(interp)) {
+ if (has_own_state(interp) && interp->obmalloc != NULL) {
Py_ssize_t leaked = _PyInterpreterState_GetAllocatedBlocks(interp);
assert(has_own_state(interp) || leaked == 0);
interp->runtime->obmalloc.interpreter_leaks += leaked;
+ if (_PyMem_obmalloc_state_on_heap(interp) && leaked == 0) {
+ // free the obmalloc arenas and radix tree nodes. If leaked > 0
+ // then some of the memory allocated by obmalloc has not been
+ // freed. It might be safe to free the arenas in that case but
+ // it's possible that extension modules are still using that
+ // memory. So, it is safer to not free and to leak. Perhaps there
+ // should be warning when this happens. It should be possible to
+ // use a tool like "-fsanitize=address" to track down these leaks.
+ free_obmalloc_arenas(interp);
+ }
}
}
@@ -2717,9 +2739,96 @@ _PyDebugAllocatorStats(FILE *out,
(void)printone(out, buf2, num_blocks * sizeof_block);
}
+// Return true if the obmalloc state structure is heap allocated,
+// by PyMem_RawCalloc(). For the main interpreter, this structure
+// allocated in the BSS. Allocating that way gives some memory savings
+// and a small performance win (at least on a demand paged OS). On
+// 64-bit platforms, the obmalloc structure is 256 kB. Most of that
+// memory is for the arena_map_top array. Since normally only one entry
+// of that array is used, only one page of resident memory is actually
+// used, rather than the full 256 kB.
+bool _PyMem_obmalloc_state_on_heap(PyInterpreterState *interp)
+{
+#if WITH_PYMALLOC
+ return interp->obmalloc && interp->obmalloc != &obmalloc_state_main;
+#else
+ return false;
+#endif
+}
+
+#ifdef WITH_PYMALLOC
+static void
+init_obmalloc_pools(PyInterpreterState *interp)
+{
+ // initialize the obmalloc->pools structure. This must be done
+ // before the obmalloc alloc/free functions can be called.
+ poolp temp[OBMALLOC_USED_POOLS_SIZE] =
+ _obmalloc_pools_INIT(interp->obmalloc->pools);
+ memcpy(&interp->obmalloc->pools.used, temp, sizeof(temp));
+}
+#endif /* WITH_PYMALLOC */
+
+int _PyMem_init_obmalloc(PyInterpreterState *interp)
+{
+#ifdef WITH_PYMALLOC
+ /* Initialize obmalloc, but only for subinterpreters,
+ since the main interpreter is initialized statically. */
+ if (_Py_IsMainInterpreter(interp)
+ || _PyInterpreterState_HasFeature(interp,
+ Py_RTFLAGS_USE_MAIN_OBMALLOC)) {
+ interp->obmalloc = &obmalloc_state_main;
+ if (!obmalloc_state_initialized) {
+ init_obmalloc_pools(interp);
+ obmalloc_state_initialized = true;
+ }
+ } else {
+ interp->obmalloc = PyMem_RawCalloc(1, sizeof(struct _obmalloc_state));
+ if (interp->obmalloc == NULL) {
+ return -1;
+ }
+ init_obmalloc_pools(interp);
+ }
+#endif /* WITH_PYMALLOC */
+ return 0; // success
+}
+
#ifdef WITH_PYMALLOC
+static void
+free_obmalloc_arenas(PyInterpreterState *interp)
+{
+ OMState *state = interp->obmalloc;
+ for (uint i = 0; i < maxarenas; ++i) {
+ // free each obmalloc memory arena
+ struct arena_object *ao = &allarenas[i];
+ _PyObject_Arena.free(_PyObject_Arena.ctx,
+ (void *)ao->address, ARENA_SIZE);
+ }
+ // free the array containing pointers to all arenas
+ PyMem_RawFree(allarenas);
+#if WITH_PYMALLOC_RADIX_TREE
+#ifdef USE_INTERIOR_NODES
+ // Free the middle and bottom nodes of the radix tree. These are allocated
+ // by arena_map_mark_used() but not freed when arenas are freed.
+ for (int i1 = 0; i1 < MAP_TOP_LENGTH; i1++) {
+ arena_map_mid_t *mid = arena_map_root.ptrs[i1];
+ if (mid == NULL) {
+ continue;
+ }
+ for (int i2 = 0; i2 < MAP_MID_LENGTH; i2++) {
+ arena_map_bot_t *bot = arena_map_root.ptrs[i1]->ptrs[i2];
+ if (bot == NULL) {
+ continue;
+ }
+ PyMem_RawFree(bot);
+ }
+ PyMem_RawFree(mid);
+ }
+#endif
+#endif
+}
+
#ifdef Py_DEBUG
/* Is target in the list? The list is traversed via the nextpool pointers.
* The list may be NULL-terminated, or circular. Return 1 if target is in
diff --git a/Objects/setobject.c b/Objects/setobject.c
index 88d20019bfb4a7..93de8e84f2ddf9 100644
--- a/Objects/setobject.c
+++ b/Objects/setobject.c
@@ -1770,7 +1770,11 @@ set_issubset(PySetObject *so, PyObject *other)
Py_RETURN_TRUE;
}
-PyDoc_STRVAR(issubset_doc, "Report whether another set contains this set.");
+PyDoc_STRVAR(issubset_doc,
+"issubset($self, other, /)\n\
+--\n\
+\n\
+Test whether every element in the set is in other.");
static PyObject *
set_issuperset(PySetObject *so, PyObject *other)
@@ -1802,7 +1806,11 @@ set_issuperset(PySetObject *so, PyObject *other)
Py_RETURN_TRUE;
}
-PyDoc_STRVAR(issuperset_doc, "Report whether this set contains another set.");
+PyDoc_STRVAR(issuperset_doc,
+"issuperset($self, other, /)\n\
+--\n\
+\n\
+Test whether every element in other is in the set.");
static PyObject *
set_richcompare(PySetObject *v, PyObject *w, int op)
diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c
index 440c1da30620c3..8b9d6bbfd858b7 100644
--- a/Objects/sliceobject.c
+++ b/Objects/sliceobject.c
@@ -106,9 +106,9 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type);
void _PySlice_ClearCache(_PyFreeListState *state)
{
#ifdef WITH_FREELISTS
- PySliceObject *obj = state->slice_state.slice_cache;
+ PySliceObject *obj = state->slices.slice_cache;
if (obj != NULL) {
- state->slice_state.slice_cache = NULL;
+ state->slices.slice_cache = NULL;
PyObject_GC_Del(obj);
}
#endif
@@ -132,9 +132,9 @@ _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step)
PySliceObject *obj;
#ifdef WITH_FREELISTS
_PyFreeListState *state = _PyFreeListState_GET();
- if (state->slice_state.slice_cache != NULL) {
- obj = state->slice_state.slice_cache;
- state->slice_state.slice_cache = NULL;
+ if (state->slices.slice_cache != NULL) {
+ obj = state->slices.slice_cache;
+ state->slices.slice_cache = NULL;
_Py_NewReference((PyObject *)obj);
}
else
@@ -370,8 +370,8 @@ slice_dealloc(PySliceObject *r)
Py_DECREF(r->stop);
#ifdef WITH_FREELISTS
_PyFreeListState *state = _PyFreeListState_GET();
- if (state->slice_state.slice_cache == NULL) {
- state->slice_state.slice_cache = r;
+ if (state->slices.slice_cache == NULL) {
+ state->slices.slice_cache = r;
}
else
#endif
diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
index e1b8e4004c6163..b9bf6cd48f6129 100644
--- a/Objects/tupleobject.c
+++ b/Objects/tupleobject.c
@@ -1125,7 +1125,7 @@ tuple_iter(PyObject *seq)
* freelists *
*************/
-#define STATE (state->tuple_state)
+#define STATE (state->tuples)
#define FREELIST_FINALIZED (STATE.numfree[0] < 0)
static inline PyTupleObject *
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 114cf21f95e744..a850473cad813d 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -3828,6 +3828,17 @@ type_new_impl(type_new_ctx *ctx)
// Put the proper slots in place
fixup_slot_dispatchers(type);
+ if (!_PyDict_HasOnlyStringKeys(type->tp_dict)) {
+ if (PyErr_WarnFormat(
+ PyExc_RuntimeWarning,
+ 1,
+ "non-string key in the __dict__ of class %.200s",
+ type->tp_name) == -1)
+ {
+ goto error;
+ }
+ }
+
if (type_new_set_names(type) < 0) {
goto error;
}
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 4b03cc3f4da5fa..b236ddba9cdc69 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -996,7 +996,7 @@ resize_compact(PyObject *unicode, Py_ssize_t length)
new_size = (struct_size + (length + 1) * char_size);
if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) {
- PyObject_Free(_PyUnicode_UTF8(unicode));
+ PyMem_Free(_PyUnicode_UTF8(unicode));
_PyUnicode_UTF8(unicode) = NULL;
_PyUnicode_UTF8_LENGTH(unicode) = 0;
}
@@ -1049,7 +1049,7 @@ resize_inplace(PyObject *unicode, Py_ssize_t length)
if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode))
{
- PyObject_Free(_PyUnicode_UTF8(unicode));
+ PyMem_Free(_PyUnicode_UTF8(unicode));
_PyUnicode_UTF8(unicode) = NULL;
_PyUnicode_UTF8_LENGTH(unicode) = 0;
}
@@ -1590,10 +1590,10 @@ unicode_dealloc(PyObject *unicode)
return;
}
if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) {
- PyObject_Free(_PyUnicode_UTF8(unicode));
+ PyMem_Free(_PyUnicode_UTF8(unicode));
}
if (!PyUnicode_IS_COMPACT(unicode) && _PyUnicode_DATA_ANY(unicode)) {
- PyObject_Free(_PyUnicode_DATA_ANY(unicode));
+ PyMem_Free(_PyUnicode_DATA_ANY(unicode));
}
Py_TYPE(unicode)->tp_free(unicode);
@@ -5203,7 +5203,7 @@ unicode_fill_utf8(PyObject *unicode)
PyBytes_AS_STRING(writer.buffer);
Py_ssize_t len = end - start;
- char *cache = PyObject_Malloc(len + 1);
+ char *cache = PyMem_Malloc(len + 1);
if (cache == NULL) {
_PyBytesWriter_Dealloc(&writer);
PyErr_NoMemory();
@@ -14674,7 +14674,7 @@ unicode_subtype_new(PyTypeObject *type, PyObject *unicode)
PyErr_NoMemory();
goto onError;
}
- data = PyObject_Malloc((length + 1) * char_size);
+ data = PyMem_Malloc((length + 1) * char_size);
if (data == NULL) {
PyErr_NoMemory();
goto onError;
diff --git a/PC/python3dll.c b/PC/python3dll.c
index 07aa84c91f9fc7..09ecf98fe56ea6 100755
--- a/PC/python3dll.c
+++ b/PC/python3dll.c
@@ -830,6 +830,7 @@ EXPORT_DATA(PyExc_FutureWarning)
EXPORT_DATA(PyExc_GeneratorExit)
EXPORT_DATA(PyExc_ImportError)
EXPORT_DATA(PyExc_ImportWarning)
+EXPORT_DATA(PyExc_IncompleteInputError)
EXPORT_DATA(PyExc_IndentationError)
EXPORT_DATA(PyExc_IndexError)
EXPORT_DATA(PyExc_InterruptedError)
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index dde801fc0fd525..35788ec4503e8f 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -224,6 +224,7 @@
<ClCompile Include="..\Python\initconfig.c" />
<ClCompile Include="..\Python\intrinsics.c" />
<ClCompile Include="..\Python\instrumentation.c" />
+ <ClCompile Include="..\Python\jit.c" />
<ClCompile Include="..\Python\legacy_tracing.c" />
<ClCompile Include="..\Python\lock.c" />
<ClCompile Include="..\Python\marshal.c" />
diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters
index 90ccb954b424bc..7a44179e356105 100644
--- a/PCbuild/_freeze_module.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -250,6 +250,9 @@
<ClCompile Include="..\Objects\iterobject.c">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="..\Python\jit.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
<ClCompile Include="..\Objects\listobject.c">
<Filter>Source Files</Filter>
</ClCompile>
diff --git a/PCbuild/build.bat b/PCbuild/build.bat
index e61267b5852a8f..83b50db4467033 100644
--- a/PCbuild/build.bat
+++ b/PCbuild/build.bat
@@ -36,6 +36,7 @@ echo. overrides -c and -d
echo. --disable-gil Enable experimental support for running without the GIL.
echo. --test-marker Enable the test marker within the build.
echo. --regen Regenerate all opcodes, grammar and tokens.
+echo. --experimental-jit Enable the experimental just-in-time compiler.
echo.
echo.Available flags to avoid building certain modules.
echo.These flags have no effect if '-e' is not given:
@@ -85,6 +86,7 @@ if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts
if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts
if "%~1"=="-V" shift & goto Version
if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts
+if "%~1"=="--experimental-jit" (set UseJIT=true) & shift & goto CheckOpts
rem These use the actual property names used by MSBuild. We could just let
rem them in through the environment, but we specify them on the command line
rem anyway for visibility so set defaults after this
@@ -176,6 +178,7 @@ echo on
/p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^
/p:DisableGil=%UseDisableGil%^
/p:UseTestMarker=%UseTestMarker% %GITProperty%^
+ /p:UseJIT=%UseJIT%^
%1 %2 %3 %4 %5 %6 %7 %8 %9
@echo off
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index e0b9fc137457a0..e1ff97659659ee 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -104,6 +104,7 @@
<AdditionalIncludeDirectories Condition="$(IncludeExternals)">$(zlibDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions Condition="$(IncludeExternals)">_Py_HAVE_ZLIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions Condition="'$(UseJIT)' == 'true'">_Py_JIT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<AdditionalDependencies>version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies)</AdditionalDependencies>
@@ -247,6 +248,7 @@
<ClInclude Include="..\Include\internal\pycore_initconfig.h" />
<ClInclude Include="..\Include\internal\pycore_interp.h" />
<ClInclude Include="..\Include\internal\pycore_intrinsics.h" />
+ <ClInclude Include="..\Include\internal\pycore_jit.h" />
<ClInclude Include="..\Include\internal\pycore_list.h" />
<ClInclude Include="..\Include\internal\pycore_llist.h" />
<ClInclude Include="..\Include\internal\pycore_lock.h" />
@@ -585,6 +587,7 @@
<ClCompile Include="..\Python\initconfig.c" />
<ClCompile Include="..\Python\intrinsics.c" />
<ClCompile Include="..\Python\instrumentation.c" />
+ <ClCompile Include="..\Python\jit.c" />
<ClCompile Include="..\Python\legacy_tracing.c" />
<ClCompile Include="..\Python\lock.c" />
<ClCompile Include="..\Python\marshal.c" />
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index fd79436f5add97..4c55f23006b2f0 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -669,6 +669,9 @@
<ClInclude Include="..\Include\internal\pycore_intrinsics.h">
<Filter>Include\cpython</Filter>
</ClInclude>
+ <ClInclude Include="..\Include\internal\pycore_jit.h">
+ <Filter>Include\internal</Filter>
+ </ClInclude>
<ClInclude Include="..\Include\internal\pycore_list.h">
<Filter>Include\internal</Filter>
</ClInclude>
@@ -1337,6 +1340,9 @@
<ClCompile Include="..\Python\instrumentation.c">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="..\Python\jit.c">
+ <Filter>Python</Filter>
+ </ClCompile>
<ClCompile Include="..\Python\legacy_tracing.c">
<Filter>Source Files</Filter>
</ClCompile>
diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets
index cc9469c7ddd726..a90620d6ca8b7d 100644
--- a/PCbuild/regen.targets
+++ b/PCbuild/regen.targets
@@ -28,6 +28,9 @@
</_TokenOutputs>
<_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" />
<_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" />
+ <!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: -->
+ <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/>
+ <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/>
</ItemGroup>
<Target Name="_TouchRegenSources" Condition="$(ForceRegen) == 'true'">
@@ -76,10 +79,28 @@
<Exec Command="$(PythonForBuild) Tools\build\generate_global_objects.py"
WorkingDirectory="$(PySourcePath)" />
</Target>
+
+ <Target Name="_RegenJIT"
+ Condition="'$(UseJIT)' == 'true'"
+ DependsOnTargets="_UpdatePyconfig;FindPythonForBuild"
+ Inputs="@(_JITSources)"
+ Outputs="@(_JITOutputs)">
+ <PropertyGroup>
+ <JITArgs Condition="$(Platform) == 'ARM64'">aarch64-pc-windows-msvc</JITArgs>
+ <JITArgs Condition="$(Platform) == 'Win32'">i686-pc-windows-msvc</JITArgs>
+ <JITArgs Condition="$(Platform) == 'x64'">x86_64-pc-windows-msvc</JITArgs>
+ <JITArgs Condition="$(Configuration) == 'Debug'">$(JITArgs) --debug</JITArgs>
+ </PropertyGroup>
+ <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs)'
+ WorkingDirectory="$(GeneratedPyConfigDir)"/>
+ </Target>
- <Target Name="Regen"
+ <Target Name="_RegenNoPGUpdate"
Condition="$(Configuration) != 'PGUpdate'"
DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects">
+ </Target>
+
+ <Target Name="Regen" DependsOnTargets="_RegenNoPGUpdate;_RegenJIT">
<Message Text="Generated sources are up to date" Importance="high" />
</Target>
diff --git a/Parser/pegen.c b/Parser/pegen.c
index 7766253a76066f..3d3e64559403b1 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -844,7 +844,7 @@ _PyPegen_run_parser(Parser *p)
if (res == NULL) {
if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) {
PyErr_Clear();
- return RAISE_SYNTAX_ERROR("incomplete input");
+ return _PyPegen_raise_error(p, PyExc_IncompleteInputError, 0, "incomplete input");
}
if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) {
return NULL;
diff --git a/Python/ceval.c b/Python/ceval.c
index 49388cd20377c0..4f208009086191 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -11,6 +11,7 @@
#include "pycore_function.h"
#include "pycore_instruments.h"
#include "pycore_intrinsics.h"
+#include "pycore_jit.h"
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_moduleobject.h" // PyModuleObject
#include "pycore_object.h" // _PyObject_GC_TRACK()
@@ -955,9 +956,24 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
-// The Tier 2 interpreter is also here!
+// Tier 2 is also here!
enter_tier_two:
+#ifdef _Py_JIT
+
+ ; // ;)
+ jit_func jitted = current_executor->jit_code;
+ next_instr = jitted(frame, stack_pointer, tstate);
+ frame = tstate->current_frame;
+ Py_DECREF(current_executor);
+ if (next_instr == NULL) {
+ goto resume_with_error;
+ }
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ DISPATCH();
+
+#else
+
#undef LOAD_IP
#define LOAD_IP(UNUSED) (void)0
@@ -1073,6 +1089,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
Py_DECREF(current_executor);
DISPATCH();
+#endif // _Py_JIT
+
}
#if defined(__GNUC__)
# pragma GCC diagnostic pop
diff --git a/Python/compile.c b/Python/compile.c
index 7cf05dd0683119..4c1d3bb2d2b475 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -160,7 +160,7 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc,
if (idx >= new_alloc) {
new_alloc = idx + default_alloc;
}
- arr = PyObject_Calloc(new_alloc, item_size);
+ arr = PyMem_Calloc(new_alloc, item_size);
if (arr == NULL) {
PyErr_NoMemory();
return ERROR;
@@ -181,7 +181,7 @@ _PyCompile_EnsureArrayLargeEnough(int idx, void **array, int *alloc,
}
assert(newsize > 0);
- void *tmp = PyObject_Realloc(arr, newsize);
+ void *tmp = PyMem_Realloc(arr, newsize);
if (tmp == NULL) {
PyErr_NoMemory();
return ERROR;
@@ -282,10 +282,10 @@ instr_sequence_insert_instruction(instr_sequence *seq, int pos,
static void
instr_sequence_fini(instr_sequence *seq) {
- PyObject_Free(seq->s_labelmap);
+ PyMem_Free(seq->s_labelmap);
seq->s_labelmap = NULL;
- PyObject_Free(seq->s_instrs);
+ PyMem_Free(seq->s_instrs);
seq->s_instrs = NULL;
}
@@ -690,7 +690,7 @@ compiler_unit_free(struct compiler_unit *u)
Py_CLEAR(u->u_metadata.u_cellvars);
Py_CLEAR(u->u_metadata.u_fasthidden);
Py_CLEAR(u->u_private);
- PyObject_Free(u);
+ PyMem_Free(u);
}
static int
@@ -1262,8 +1262,7 @@ compiler_enter_scope(struct compiler *c, identifier name,
struct compiler_unit *u;
- u = (struct compiler_unit *)PyObject_Calloc(1, sizeof(
- struct compiler_unit));
+ u = (struct compiler_unit *)PyMem_Calloc(1, sizeof(struct compiler_unit));
if (!u) {
PyErr_NoMemory();
return ERROR;
@@ -6657,7 +6656,7 @@ ensure_fail_pop(struct compiler *c, pattern_context *pc, Py_ssize_t n)
return SUCCESS;
}
Py_ssize_t needed = sizeof(jump_target_label) * size;
- jump_target_label *resized = PyObject_Realloc(pc->fail_pop, needed);
+ jump_target_label *resized = PyMem_Realloc(pc->fail_pop, needed);
if (resized == NULL) {
PyErr_NoMemory();
return ERROR;
@@ -6696,13 +6695,13 @@ emit_and_reset_fail_pop(struct compiler *c, location loc,
USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]);
if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, loc) < 0) {
pc->fail_pop_size = 0;
- PyObject_Free(pc->fail_pop);
+ PyMem_Free(pc->fail_pop);
pc->fail_pop = NULL;
return ERROR;
}
}
USE_LABEL(c, pc->fail_pop[0]);
- PyObject_Free(pc->fail_pop);
+ PyMem_Free(pc->fail_pop);
pc->fail_pop = NULL;
return SUCCESS;
}
@@ -7206,7 +7205,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc)
Py_DECREF(pc->stores);
*pc = old_pc;
Py_INCREF(pc->stores);
- // Need to NULL this for the PyObject_Free call in the error block.
+ // Need to NULL this for the PyMem_Free call in the error block.
old_pc.fail_pop = NULL;
// No match. Pop the remaining copy of the subject and fail:
if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, LOC(p)) < 0 ||
@@ -7252,7 +7251,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc)
diff:
compiler_error(c, LOC(p), "alternative patterns bind different names");
error:
- PyObject_Free(old_pc.fail_pop);
+ PyMem_Free(old_pc.fail_pop);
Py_DECREF(old_pc.stores);
Py_XDECREF(control);
return ERROR;
@@ -7453,7 +7452,7 @@ compiler_match(struct compiler *c, stmt_ty s)
pattern_context pc;
pc.fail_pop = NULL;
int result = compiler_match_inner(c, s, &pc);
- PyObject_Free(pc.fail_pop);
+ PyMem_Free(pc.fail_pop);
return result;
}
diff --git a/Python/context.c b/Python/context.c
index 1e90811c374ec6..294485e5b407df 100644
--- a/Python/context.c
+++ b/Python/context.c
@@ -69,7 +69,7 @@ static struct _Py_context_state *
get_context_state(void)
{
_PyFreeListState *state = _PyFreeListState_GET();
- return &state->context_state;
+ return &state->contexts;
}
#endif
@@ -1270,7 +1270,7 @@ void
_PyContext_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
{
#ifdef WITH_FREELISTS
- struct _Py_context_state *state = &freelist_state->context_state;
+ struct _Py_context_state *state = &freelist_state->contexts;
for (; state->numfree > 0; state->numfree--) {
PyContext *ctx = state->freelist;
state->freelist = (PyContext *)ctx->ctx_weakreflist;
diff --git a/Python/flowgraph.c b/Python/flowgraph.c
index 96610b3cb11a43..bfc23a298ff492 100644
--- a/Python/flowgraph.c
+++ b/Python/flowgraph.c
@@ -162,7 +162,7 @@ basicblock_last_instr(const basicblock *b) {
static basicblock *
cfg_builder_new_block(cfg_builder *g)
{
- basicblock *b = (basicblock *)PyObject_Calloc(1, sizeof(basicblock));
+ basicblock *b = (basicblock *)PyMem_Calloc(1, sizeof(basicblock));
if (b == NULL) {
PyErr_NoMemory();
return NULL;
@@ -437,10 +437,10 @@ _PyCfgBuilder_Free(cfg_builder *g)
basicblock *b = g->g_block_list;
while (b != NULL) {
if (b->b_instr) {
- PyObject_Free((void *)b->b_instr);
+ PyMem_Free((void *)b->b_instr);
}
basicblock *next = b->b_list;
- PyObject_Free((void *)b);
+ PyMem_Free((void *)b);
b = next;
}
PyMem_Free(g);
diff --git a/Python/import.c b/Python/import.c
index 2dd95d8364a0be..2fd0c08a6bb5ae 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -3544,7 +3544,7 @@ _imp_get_frozen_object_impl(PyObject *module, PyObject *name,
struct frozen_info info = {0};
Py_buffer buf = {0};
if (PyObject_CheckBuffer(dataobj)) {
- if (PyObject_GetBuffer(dataobj, &buf, PyBUF_READ) != 0) {
+ if (PyObject_GetBuffer(dataobj, &buf, PyBUF_SIMPLE) != 0) {
return NULL;
}
info.data = (const char *)buf.buf;
diff --git a/Python/jit.c b/Python/jit.c
new file mode 100644
index 00000000000000..22949c082da05a
--- /dev/null
+++ b/Python/jit.c
@@ -0,0 +1,369 @@
+#ifdef _Py_JIT
+
+#include "Python.h"
+
+#include "pycore_abstract.h"
+#include "pycore_call.h"
+#include "pycore_ceval.h"
+#include "pycore_dict.h"
+#include "pycore_intrinsics.h"
+#include "pycore_long.h"
+#include "pycore_opcode_metadata.h"
+#include "pycore_opcode_utils.h"
+#include "pycore_optimizer.h"
+#include "pycore_pyerrors.h"
+#include "pycore_setobject.h"
+#include "pycore_sliceobject.h"
+#include "pycore_jit.h"
+
+#include "jit_stencils.h"
+
+// Memory management stuff: ////////////////////////////////////////////////////
+
+#ifndef MS_WINDOWS
+ #include <sys/mman.h>
+#endif
+
+static size_t
+get_page_size(void)
+{
+#ifdef MS_WINDOWS
+ SYSTEM_INFO si;
+ GetSystemInfo(&si);
+ return si.dwPageSize;
+#else
+ return sysconf(_SC_PAGESIZE);
+#endif
+}
+
+static void
+jit_error(const char *message)
+{
+#ifdef MS_WINDOWS
+ int hint = GetLastError();
+#else
+ int hint = errno;
+#endif
+ PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint);
+}
+
+static char *
+jit_alloc(size_t size)
+{
+ assert(size);
+ assert(size % get_page_size() == 0);
+#ifdef MS_WINDOWS
+ int flags = MEM_COMMIT | MEM_RESERVE;
+ char *memory = VirtualAlloc(NULL, size, flags, PAGE_READWRITE);
+ int failed = memory == NULL;
+#else
+ int flags = MAP_ANONYMOUS | MAP_PRIVATE;
+ char *memory = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, -1, 0);
+ int failed = memory == MAP_FAILED;
+#endif
+ if (failed) {
+ jit_error("unable to allocate memory");
+ return NULL;
+ }
+ return memory;
+}
+
+static int
+jit_free(char *memory, size_t size)
+{
+ assert(size);
+ assert(size % get_page_size() == 0);
+#ifdef MS_WINDOWS
+ int failed = !VirtualFree(memory, 0, MEM_RELEASE);
+#else
+ int failed = munmap(memory, size);
+#endif
+ if (failed) {
+ jit_error("unable to free memory");
+ return -1;
+ }
+ return 0;
+}
+
+static int
+mark_executable(char *memory, size_t size)
+{
+ if (size == 0) {
+ return 0;
+ }
+ assert(size % get_page_size() == 0);
+ // Do NOT ever leave the memory writable! Also, don't forget to flush the
+ // i-cache (I cannot begin to tell you how horrible that is to debug):
+#ifdef MS_WINDOWS
+ if (!FlushInstructionCache(GetCurrentProcess(), memory, size)) {
+ jit_error("unable to flush instruction cache");
+ return -1;
+ }
+ int old;
+ int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old);
+#else
+ __builtin___clear_cache((char *)memory, (char *)memory + size);
+ int failed = mprotect(memory, size, PROT_EXEC | PROT_READ);
+#endif
+ if (failed) {
+ jit_error("unable to protect executable memory");
+ return -1;
+ }
+ return 0;
+}
+
+static int
+mark_readable(char *memory, size_t size)
+{
+ if (size == 0) {
+ return 0;
+ }
+ assert(size % get_page_size() == 0);
+#ifdef MS_WINDOWS
+ DWORD old;
+ int failed = !VirtualProtect(memory, size, PAGE_READONLY, &old);
+#else
+ int failed = mprotect(memory, size, PROT_READ);
+#endif
+ if (failed) {
+ jit_error("unable to protect readable memory");
+ return -1;
+ }
+ return 0;
+}
+
+// JIT compiler stuff: /////////////////////////////////////////////////////////
+
+// Warning! AArch64 requires you to get your hands dirty. These are your gloves:
+
+// value[value_start : value_start + len]
+static uint32_t
+get_bits(uint64_t value, uint8_t value_start, uint8_t width)
+{
+ assert(width <= 32);
+ return (value >> value_start) & ((1ULL << width) - 1);
+}
+
+// *loc[loc_start : loc_start + width] = value[value_start : value_start + width]
+static void
+set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start,
+ uint8_t width)
+{
+ assert(loc_start + width <= 32);
+ // Clear the bits we're about to patch:
+ *loc &= ~(((1ULL << width) - 1) << loc_start);
+ assert(get_bits(*loc, loc_start, width) == 0);
+ // Patch the bits:
+ *loc |= get_bits(value, value_start, width) << loc_start;
+ assert(get_bits(*loc, loc_start, width) == get_bits(value, value_start, width));
+}
+
+// See https://developer.arm.com/documentation/ddi0602/2023-09/Base-Instructions
+// for instruction encodings:
+#define IS_AARCH64_ADD_OR_SUB(I) (((I) & 0x11C00000) == 0x11000000)
+#define IS_AARCH64_ADRP(I) (((I) & 0x9F000000) == 0x90000000)
+#define IS_AARCH64_BRANCH(I) (((I) & 0x7C000000) == 0x14000000)
+#define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000)
+#define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000)
+
+// Fill all of stencil's holes in the memory pointed to by base, using the
+// values in patches.
+static void
+patch(char *base, const Stencil *stencil, uint64_t *patches)
+{
+ for (uint64_t i = 0; i < stencil->holes_size; i++) {
+ const Hole *hole = &stencil->holes[i];
+ void *location = base + hole->offset;
+ uint64_t value = patches[hole->value] + (uint64_t)hole->symbol + hole->addend;
+ uint32_t *loc32 = (uint32_t *)location;
+ uint64_t *loc64 = (uint64_t *)location;
+ // LLD is a great reference for performing relocations... just keep in
+ // mind that Tools/jit/build.py does filtering and preprocessing for us!
+ // Here's a good place to start for each platform:
+ // - aarch64-apple-darwin:
+ // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp
+ // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h
+ // - aarch64-unknown-linux-gnu:
+ // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp
+ // - i686-pc-windows-msvc:
+ // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp
+ // - x86_64-apple-darwin:
+ // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp
+ // - x86_64-pc-windows-msvc:
+ // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp
+ // - x86_64-unknown-linux-gnu:
+ // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp
+ switch (hole->kind) {
+ case HoleKind_IMAGE_REL_I386_DIR32:
+ // 32-bit absolute address.
+ // Check that we're not out of range of 32 unsigned bits:
+ assert(value < (1ULL << 32));
+ *loc32 = (uint32_t)value;
+ continue;
+ case HoleKind_ARM64_RELOC_UNSIGNED:
+ case HoleKind_IMAGE_REL_AMD64_ADDR64:
+ case HoleKind_R_AARCH64_ABS64:
+ case HoleKind_X86_64_RELOC_UNSIGNED:
+ case HoleKind_R_X86_64_64:
+ // 64-bit absolute address.
+ *loc64 = value;
+ continue;
+ case HoleKind_R_AARCH64_CALL26:
+ case HoleKind_R_AARCH64_JUMP26:
+ // 28-bit relative branch.
+ assert(IS_AARCH64_BRANCH(*loc32));
+ value -= (uint64_t)location;
+ // Check that we're not out of range of 28 signed bits:
+ assert((int64_t)value >= -(1 << 27));
+ assert((int64_t)value < (1 << 27));
+ // Since instructions are 4-byte aligned, only use 26 bits:
+ assert(get_bits(value, 0, 2) == 0);
+ set_bits(loc32, 0, value, 2, 26);
+ continue;
+ case HoleKind_R_AARCH64_MOVW_UABS_G0_NC:
+ // 16-bit low part of an absolute address.
+ assert(IS_AARCH64_MOV(*loc32));
+ // Check the implicit shift (this is "part 0 of 3"):
+ assert(get_bits(*loc32, 21, 2) == 0);
+ set_bits(loc32, 5, value, 0, 16);
+ continue;
+ case HoleKind_R_AARCH64_MOVW_UABS_G1_NC:
+ // 16-bit middle-low part of an absolute address.
+ assert(IS_AARCH64_MOV(*loc32));
+ // Check the implicit shift (this is "part 1 of 3"):
+ assert(get_bits(*loc32, 21, 2) == 1);
+ set_bits(loc32, 5, value, 16, 16);
+ continue;
+ case HoleKind_R_AARCH64_MOVW_UABS_G2_NC:
+ // 16-bit middle-high part of an absolute address.
+ assert(IS_AARCH64_MOV(*loc32));
+ // Check the implicit shift (this is "part 2 of 3"):
+ assert(get_bits(*loc32, 21, 2) == 2);
+ set_bits(loc32, 5, value, 32, 16);
+ continue;
+ case HoleKind_R_AARCH64_MOVW_UABS_G3:
+ // 16-bit high part of an absolute address.
+ assert(IS_AARCH64_MOV(*loc32));
+ // Check the implicit shift (this is "part 3 of 3"):
+ assert(get_bits(*loc32, 21, 2) == 3);
+ set_bits(loc32, 5, value, 48, 16);
+ continue;
+ case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21:
+ // 21-bit count of pages between this page and an absolute address's
+ // page... I know, I know, it's weird. Pairs nicely with
+ // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below).
+ assert(IS_AARCH64_ADRP(*loc32));
+ // Number of pages between this page and the value's page:
+ value = (value >> 12) - ((uint64_t)location >> 12);
+ // Check that we're not out of range of 21 signed bits:
+ assert((int64_t)value >= -(1 << 20));
+ assert((int64_t)value < (1 << 20));
+ // value[0:2] goes in loc[29:31]:
+ set_bits(loc32, 29, value, 0, 2);
+ // value[2:21] goes in loc[5:26]:
+ set_bits(loc32, 5, value, 2, 19);
+ continue;
+ case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12:
+ // 12-bit low part of an absolute address. Pairs nicely with
+ // ARM64_RELOC_GOT_LOAD_PAGE21 (above).
+ assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32));
+ // There might be an implicit shift encoded in the instruction:
+ uint8_t shift = 0;
+ if (IS_AARCH64_LDR_OR_STR(*loc32)) {
+ shift = (uint8_t)get_bits(*loc32, 30, 2);
+ // If both of these are set, the shift is supposed to be 4.
+ // That's pretty weird, and it's never actually been observed...
+ assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0);
+ }
+ value = get_bits(value, 0, 12);
+ assert(get_bits(value, 0, shift) == 0);
+ set_bits(loc32, 10, value, shift, 12);
+ continue;
+ }
+ Py_UNREACHABLE();
+ }
+}
+
+static void
+copy_and_patch(char *base, const Stencil *stencil, uint64_t *patches)
+{
+ memcpy(base, stencil->body, stencil->body_size);
+ patch(base, stencil, patches);
+}
+
+static void
+emit(const StencilGroup *group, uint64_t patches[])
+{
+ copy_and_patch((char *)patches[HoleValue_CODE], &group->code, patches);
+ copy_and_patch((char *)patches[HoleValue_DATA], &group->data, patches);
+}
+
+// Compiles executor in-place. Don't forget to call _PyJIT_Free later!
+int
+_PyJIT_Compile(_PyExecutorObject *executor, _PyUOpInstruction *trace, size_t length)
+{
+ // Loop once to find the total compiled size:
+ size_t code_size = 0;
+ size_t data_size = 0;
+ for (size_t i = 0; i < length; i++) {
+ _PyUOpInstruction *instruction = &trace[i];
+ const StencilGroup *group = &stencil_groups[instruction->opcode];
+ code_size += group->code.body_size;
+ data_size += group->data.body_size;
+ }
+ // Round up to the nearest page (code and data need separate pages):
+ size_t page_size = get_page_size();
+ assert((page_size & (page_size - 1)) == 0);
+ code_size += page_size - (code_size & (page_size - 1));
+ data_size += page_size - (data_size & (page_size - 1));
+ char *memory = jit_alloc(code_size + data_size);
+ if (memory == NULL) {
+ return -1;
+ }
+ // Loop again to emit the code:
+ char *code = memory;
+ char *data = memory + code_size;
+ for (size_t i = 0; i < length; i++) {
+ _PyUOpInstruction *instruction = &trace[i];
+ const StencilGroup *group = &stencil_groups[instruction->opcode];
+ // Think of patches as a dictionary mapping HoleValue to uint64_t:
+ uint64_t patches[] = GET_PATCHES();
+ patches[HoleValue_CODE] = (uint64_t)code;
+ patches[HoleValue_CONTINUE] = (uint64_t)code + group->code.body_size;
+ patches[HoleValue_DATA] = (uint64_t)data;
+ patches[HoleValue_EXECUTOR] = (uint64_t)executor;
+ patches[HoleValue_OPARG] = instruction->oparg;
+ patches[HoleValue_OPERAND] = instruction->operand;
+ patches[HoleValue_TARGET] = instruction->target;
+ patches[HoleValue_TOP] = (uint64_t)memory;
+ patches[HoleValue_ZERO] = 0;
+ emit(group, patches);
+ code += group->code.body_size;
+ data += group->data.body_size;
+ }
+ if (mark_executable(memory, code_size) ||
+ mark_readable(memory + code_size, data_size))
+ {
+ jit_free(memory, code_size + data_size);
+ return -1;
+ }
+ executor->jit_code = memory;
+ executor->jit_size = code_size + data_size;
+ return 0;
+}
+
+void
+_PyJIT_Free(_PyExecutorObject *executor)
+{
+ char *memory = (char *)executor->jit_code;
+ size_t size = executor->jit_size;
+ if (memory) {
+ executor->jit_code = NULL;
+ executor->jit_size = 0;
+ if (jit_free(memory, size)) {
+ PyErr_WriteUnraisable(NULL);
+ }
+ }
+}
+
+#endif // _Py_JIT
diff --git a/Python/object_stack.c b/Python/object_stack.c
index 66b37bcbb44475..8544892eb71dcb 100644
--- a/Python/object_stack.c
+++ b/Python/object_stack.c
@@ -12,7 +12,7 @@ static struct _Py_object_stack_state *
get_state(void)
{
_PyFreeListState *state = _PyFreeListState_GET();
- return &state->object_stack_state;
+ return &state->object_stacks;
}
_PyObjectStackChunk *
@@ -76,7 +76,7 @@ _PyObjectStackChunk_ClearFreeList(_PyFreeListState *free_lists, int is_finalizat
return;
}
- struct _Py_object_stack_state *state = &free_lists->object_stack_state;
+ struct _Py_object_stack_state *state = &free_lists->object_stacks;
while (state->numfree > 0) {
_PyObjectStackChunk *buf = state->free_list;
state->free_list = buf->prev;
diff --git a/Python/optimizer.c b/Python/optimizer.c
index db615068ff517f..0d04b09fef1e84 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -7,6 +7,7 @@
#include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_uop_ids.h"
+#include "pycore_jit.h"
#include "cpython/optimizer.h"
#include <stdbool.h>
#include <stdint.h>
@@ -227,6 +228,9 @@ static PyMethodDef executor_methods[] = {
static void
uop_dealloc(_PyExecutorObject *self) {
_Py_ExecutorClear(self);
+#ifdef _Py_JIT
+ _PyJIT_Free(self);
+#endif
PyObject_Free(self);
}
@@ -789,6 +793,14 @@ make_executor_from_uops(_PyUOpInstruction *buffer, _PyBloomFilter *dependencies)
executor->trace[i].operand);
}
}
+#endif
+#ifdef _Py_JIT
+ executor->jit_code = NULL;
+ executor->jit_size = 0;
+ if (_PyJIT_Compile(executor, executor->trace, Py_SIZE(executor))) {
+ Py_DECREF(executor);
+ return NULL;
+ }
#endif
return executor;
}
diff --git a/Python/parking_lot.c b/Python/parking_lot.c
index d44c1b4b93b4d2..c83d7443e289c5 100644
--- a/Python/parking_lot.c
+++ b/Python/parking_lot.c
@@ -158,11 +158,15 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, _PyTime_t timeout)
if (sema->counter == 0) {
if (timeout >= 0) {
struct timespec ts;
-
+#if defined(HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP)
+ _PyTime_AsTimespec_clamp(timeout, &ts);
+ err = pthread_cond_timedwait_relative_np(&sema->cond, &sema->mutex, &ts);
+#else
_PyTime_t deadline = _PyTime_Add(_PyTime_GetSystemClock(), timeout);
_PyTime_AsTimespec_clamp(deadline, &ts);
err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts);
+#endif // HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP
}
else {
err = pthread_cond_wait(&sema->cond, &sema->mutex);
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 261622adc4cc77..372f60602375b6 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -32,6 +32,7 @@
#include "pycore_typevarobject.h" // _Py_clear_generic_types()
#include "pycore_unicodeobject.h" // _PyUnicode_InitTypes()
#include "pycore_weakref.h" // _PyWeakref_GET_REF()
+#include "pycore_obmalloc.h" // _PyMem_init_obmalloc()
#include "opcode.h"
@@ -645,6 +646,13 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
return status;
}
+ // initialize the interp->obmalloc state. This must be done after
+ // the settings are loaded (so that feature_flags are set) but before
+ // any calls are made to obmalloc functions.
+ if (_PyMem_init_obmalloc(interp) < 0) {
+ return _PyStatus_NO_MEMORY();
+ }
+
PyThreadState *tstate = _PyThreadState_New(interp,
_PyThreadState_WHENCE_INTERP);
if (tstate == NULL) {
@@ -1232,12 +1240,19 @@ init_interp_main(PyThreadState *tstate)
// Turn on experimental tier 2 (uops-based) optimizer
if (is_main_interp) {
+#ifndef _Py_JIT
+ // No JIT, maybe use the tier two interpreter:
char *envvar = Py_GETENV("PYTHON_UOPS");
int enabled = envvar != NULL && *envvar > '0';
if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) {
enabled = 1;
}
if (enabled) {
+#else
+ // Always enable tier two for JIT builds (ignoring the environment
+ // variable and command-line option above):
+ if (true) {
+#endif
PyObject *opt = PyUnstable_Optimizer_NewUOpOptimizer();
if (opt == NULL) {
return _PyStatus_ERR("can't initialize optimizer");
@@ -2144,6 +2159,14 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config)
goto error;
}
+ // initialize the interp->obmalloc state. This must be done after
+ // the settings are loaded (so that feature_flags are set) but before
+ // any calls are made to obmalloc functions.
+ if (_PyMem_init_obmalloc(interp) < 0) {
+ status = _PyStatus_NO_MEMORY();
+ goto error;
+ }
+
tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INTERP);
if (tstate == NULL) {
status = _PyStatus_NO_MEMORY();
diff --git a/Python/pystate.c b/Python/pystate.c
index 5ad5b6f3fcc634..430121a6a35d7f 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -18,6 +18,7 @@
#include "pycore_pystate.h"
#include "pycore_runtime_init.h" // _PyRuntimeState_INIT
#include "pycore_sysmodule.h" // _PySys_Audit()
+#include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap()
/* --------------------------------------------------------------------------
CAUTION
@@ -66,7 +67,7 @@ _Py_thread_local PyThreadState *_Py_tss_tstate = NULL;
#endif
static inline PyThreadState *
-current_fast_get(_PyRuntimeState *Py_UNUSED(runtime))
+current_fast_get(void)
{
#ifdef HAVE_THREAD_LOCAL
return _Py_tss_tstate;
@@ -100,14 +101,14 @@ current_fast_clear(_PyRuntimeState *Py_UNUSED(runtime))
}
#define tstate_verify_not_active(tstate) \
- if (tstate == current_fast_get((tstate)->interp->runtime)) { \
+ if (tstate == current_fast_get()) { \
_Py_FatalErrorFormat(__func__, "tstate %p is still current", tstate); \
}
PyThreadState *
_PyThreadState_GetCurrent(void)
{
- return current_fast_get(&_PyRuntime);
+ return current_fast_get();
}
@@ -359,10 +360,9 @@ holds_gil(PyThreadState *tstate)
// XXX Fall back to tstate->interp->runtime->ceval.gil.last_holder
// (and tstate->interp->runtime->ceval.gil.locked).
assert(tstate != NULL);
- _PyRuntimeState *runtime = tstate->interp->runtime;
/* Must be the tstate for this thread */
- assert(tstate == gilstate_tss_get(runtime));
- return tstate == current_fast_get(runtime);
+ assert(tstate == gilstate_tss_get(tstate->interp->runtime));
+ return tstate == current_fast_get();
}
@@ -553,6 +553,11 @@ free_interpreter(PyInterpreterState *interp)
// The main interpreter is statically allocated so
// should not be freed.
if (interp != &_PyRuntime._main_interpreter) {
+ if (_PyMem_obmalloc_state_on_heap(interp)) {
+ // interpreter has its own obmalloc state, free it
+ PyMem_RawFree(interp->obmalloc);
+ interp->obmalloc = NULL;
+ }
PyMem_RawFree(interp);
}
}
@@ -595,14 +600,6 @@ init_interpreter(PyInterpreterState *interp,
assert(next != NULL || (interp == runtime->interpreters.main));
interp->next = next;
- /* Initialize obmalloc, but only for subinterpreters,
- since the main interpreter is initialized statically. */
- if (interp != &runtime->_main_interpreter) {
- poolp temp[OBMALLOC_USED_POOLS_SIZE] = \
- _obmalloc_pools_INIT(interp->obmalloc.pools);
- memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp));
- }
-
PyStatus status = _PyObject_InitState(interp);
if (_PyStatus_EXCEPTION(status)) {
return status;
@@ -725,7 +722,7 @@ PyInterpreterState *
PyInterpreterState_New(void)
{
// tstate can be NULL
- PyThreadState *tstate = current_fast_get(&_PyRuntime);
+ PyThreadState *tstate = current_fast_get();
PyInterpreterState *interp;
PyStatus status = _PyInterpreterState_New(tstate, &interp);
@@ -884,7 +881,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp)
// Use the current Python thread state to call audit hooks and to collect
// garbage. It can be different than the current Python thread state
// of 'interp'.
- PyThreadState *current_tstate = current_fast_get(interp->runtime);
+ PyThreadState *current_tstate = current_fast_get();
_PyImport_ClearCore(interp);
interpreter_clear(interp, current_tstate);
}
@@ -910,7 +907,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp)
// XXX Clearing the "current" thread state should happen before
// we start finalizing the interpreter (or the current thread state).
- PyThreadState *tcur = current_fast_get(runtime);
+ PyThreadState *tcur = current_fast_get();
if (tcur != NULL && interp == tcur->interp) {
/* Unset current thread. After this, many C API calls become crashy. */
_PyThreadState_Detach(tcur);
@@ -1012,7 +1009,7 @@ _PyInterpreterState_SetRunningMain(PyInterpreterState *interp)
if (_PyInterpreterState_FailIfRunningMain(interp) < 0) {
return -1;
}
- PyThreadState *tstate = current_fast_get(&_PyRuntime);
+ PyThreadState *tstate = current_fast_get();
_Py_EnsureTstateNotNULL(tstate);
if (tstate->interp != interp) {
PyErr_SetString(PyExc_RuntimeError,
@@ -1027,7 +1024,7 @@ void
_PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp)
{
PyThreadState *tstate = interp->threads.main;
- assert(tstate == current_fast_get(&_PyRuntime));
+ assert(tstate == current_fast_get());
if (tstate->on_delete != NULL) {
// The threading module was imported for the first time in this
@@ -1180,7 +1177,7 @@ PyInterpreterState_GetDict(PyInterpreterState *interp)
PyInterpreterState*
PyInterpreterState_Get(void)
{
- PyThreadState *tstate = current_fast_get(&_PyRuntime);
+ PyThreadState *tstate = current_fast_get();
_Py_EnsureTstateNotNULL(tstate);
PyInterpreterState *interp = tstate->interp;
if (interp == NULL) {
@@ -1476,7 +1473,7 @@ void
PyThreadState_Clear(PyThreadState *tstate)
{
assert(tstate->_status.initialized && !tstate->_status.cleared);
- assert(current_fast_get(&_PyRuntime)->interp == tstate->interp);
+ assert(current_fast_get()->interp == tstate->interp);
// XXX assert(!tstate->_status.bound || tstate->_status.unbound);
tstate->_status.finalizing = 1; // just in case
@@ -1658,7 +1655,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate)
void
PyThreadState_DeleteCurrent(void)
{
- PyThreadState *tstate = current_fast_get(&_PyRuntime);
+ PyThreadState *tstate = current_fast_get();
_PyThreadState_DeleteCurrent(tstate);
}
@@ -1734,7 +1731,7 @@ _PyThreadState_GetDict(PyThreadState *tstate)
PyObject *
PyThreadState_GetDict(void)
{
- PyThreadState *tstate = current_fast_get(&_PyRuntime);
+ PyThreadState *tstate = current_fast_get();
if (tstate == NULL) {
return NULL;
}
@@ -1855,7 +1852,7 @@ _PyThreadState_Attach(PyThreadState *tstate)
#endif
_Py_EnsureTstateNotNULL(tstate);
- if (current_fast_get(&_PyRuntime) != NULL) {
+ if (current_fast_get() != NULL) {
Py_FatalError("non-NULL old thread state");
}
@@ -1885,7 +1882,7 @@ detach_thread(PyThreadState *tstate, int detached_state)
{
// XXX assert(tstate_is_alive(tstate) && tstate_is_bound(tstate));
assert(tstate->state == _Py_THREAD_ATTACHED);
- assert(tstate == current_fast_get(&_PyRuntime));
+ assert(tstate == current_fast_get());
if (tstate->critical_section != 0) {
_PyCriticalSection_SuspendAll(tstate);
}
@@ -2170,14 +2167,14 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc)
PyThreadState *
PyThreadState_GetUnchecked(void)
{
- return current_fast_get(&_PyRuntime);
+ return current_fast_get();
}
PyThreadState *
PyThreadState_Get(void)
{
- PyThreadState *tstate = current_fast_get(&_PyRuntime);
+ PyThreadState *tstate = current_fast_get();
_Py_EnsureTstateNotNULL(tstate);
return tstate;
}
@@ -2185,7 +2182,7 @@ PyThreadState_Get(void)
PyThreadState *
_PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts)
{
- PyThreadState *oldts = current_fast_get(runtime);
+ PyThreadState *oldts = current_fast_get();
if (oldts != NULL) {
_PyThreadState_Detach(oldts);
}
@@ -2280,7 +2277,7 @@ PyObject *
_PyThread_CurrentFrames(void)
{
_PyRuntimeState *runtime = &_PyRuntime;
- PyThreadState *tstate = current_fast_get(runtime);
+ PyThreadState *tstate = current_fast_get();
if (_PySys_Audit(tstate, "sys._current_frames", NULL) < 0) {
return NULL;
}
@@ -2341,7 +2338,7 @@ PyObject *
_PyThread_CurrentExceptions(void)
{
_PyRuntimeState *runtime = &_PyRuntime;
- PyThreadState *tstate = current_fast_get(runtime);
+ PyThreadState *tstate = current_fast_get();
_Py_EnsureTstateNotNULL(tstate);
@@ -2483,7 +2480,7 @@ PyGILState_Check(void)
return 1;
}
- PyThreadState *tstate = current_fast_get(runtime);
+ PyThreadState *tstate = current_fast_get();
if (tstate == NULL) {
return 0;
}
@@ -2581,7 +2578,7 @@ PyGILState_Release(PyGILState_STATE oldstate)
* races; see bugs 225673 and 1061968 (that nasty bug has a
* habit of coming back).
*/
- assert(current_fast_get(runtime) == tstate);
+ assert(current_fast_get() == tstate);
_PyThreadState_DeleteCurrent(tstate);
}
/* Release the lock if necessary */
@@ -2647,9 +2644,8 @@ _PyInterpreterState_GetConfigCopy(PyConfig *config)
const PyConfig*
_Py_GetConfig(void)
{
- _PyRuntimeState *runtime = &_PyRuntime;
assert(PyGILState_Check());
- PyThreadState *tstate = current_fast_get(runtime);
+ PyThreadState *tstate = current_fast_get();
_Py_EnsureTstateNotNULL(tstate);
return _PyInterpreterState_GetConfig(tstate->interp);
}
diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py
index 85c437d521a15a..83146622c74f94 100644
--- a/Tools/build/stable_abi.py
+++ b/Tools/build/stable_abi.py
@@ -278,6 +278,13 @@ def gen_ctypes_test(manifest, args, outfile):
from _testcapi import get_feature_macros
feature_macros = get_feature_macros()
+
+ # Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject
+ # layout differences.
+ # See https://github.com/python/cpython/issues/88299#issuecomment-1113366226
+ if feature_macros['Py_TRACE_REFS']:
+ raise unittest.SkipTest("incompatible with Py_TRACE_REFS.")
+
ctypes_test = import_module('ctypes')
class TestStableABIAvailability(unittest.TestCase):
@@ -308,16 +315,11 @@ def test_windows_feature_macros(self):
{'function', 'data'},
include_abi_only=True,
)
- optional_items = {}
+ feature_macros = list(manifest.select({'feature_macro'}))
+ optional_items = {m.name: [] for m in feature_macros}
for item in items:
- if item.name in (
- # Some symbols aren't exported on all platforms.
- # This is a bug: https://bugs.python.org/issue44133
- 'PyModule_Create2', 'PyModule_FromDefAndSpec2',
- ):
- continue
if item.ifdef:
- optional_items.setdefault(item.ifdef, []).append(item.name)
+ optional_items[item.ifdef].append(item.name)
else:
write(f' "{item.name}",')
write(")")
@@ -328,7 +330,6 @@ def test_windows_feature_macros(self):
write(f" {name!r},")
write(" )")
write("")
- feature_macros = list(manifest.select({'feature_macro'}))
feature_names = sorted(m.name for m in feature_macros)
write(f"EXPECTED_FEATURE_MACROS = set({pprint.pformat(feature_names)})")
diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py
index 29897425da6c03..a37da2f70757e5 100755
--- a/Tools/build/verify_ensurepip_wheels.py
+++ b/Tools/build/verify_ensurepip_wheels.py
@@ -14,7 +14,6 @@
from pathlib import Path
from urllib.request import urlopen
-PACKAGE_NAMES = ("pip",)
ENSURE_PIP_ROOT = Path(__file__).parent.parent.parent / "Lib/ensurepip"
WHEEL_DIR = ENSURE_PIP_ROOT / "_bundled"
ENSURE_PIP_INIT_PY_TEXT = (ENSURE_PIP_ROOT / "__init__.py").read_text(encoding="utf-8")
@@ -97,8 +96,5 @@ def verify_wheel(package_name: str) -> bool:
if __name__ == "__main__":
- exit_status = 0
- for package_name in PACKAGE_NAMES:
- if not verify_wheel(package_name):
- exit_status = 1
+ exit_status = int(not verify_wheel("pip"))
raise SystemExit(exit_status)
diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv
index e3a1b5d532bda2..0b02ad01d39983 100644
--- a/Tools/c-analyzer/cpython/globals-to-fix.tsv
+++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv
@@ -197,6 +197,7 @@ Objects/exceptions.c - _PyExc_AttributeError -
Objects/exceptions.c - _PyExc_SyntaxError -
Objects/exceptions.c - _PyExc_IndentationError -
Objects/exceptions.c - _PyExc_TabError -
+Objects/exceptions.c - _PyExc_IncompleteInputError -
Objects/exceptions.c - _PyExc_LookupError -
Objects/exceptions.c - _PyExc_IndexError -
Objects/exceptions.c - _PyExc_KeyError -
@@ -261,6 +262,7 @@ Objects/exceptions.c - PyExc_AttributeError -
Objects/exceptions.c - PyExc_SyntaxError -
Objects/exceptions.c - PyExc_IndentationError -
Objects/exceptions.c - PyExc_TabError -
+Objects/exceptions.c - PyExc_IncompleteInputError -
Objects/exceptions.c - PyExc_LookupError -
Objects/exceptions.c - PyExc_IndexError -
Objects/exceptions.c - PyExc_KeyError -
diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv
index 2f9e80d6ab6737..c75aff8c1723c1 100644
--- a/Tools/c-analyzer/cpython/ignored.tsv
+++ b/Tools/c-analyzer/cpython/ignored.tsv
@@ -325,7 +325,8 @@ Objects/obmalloc.c - _PyMem_Debug -
Objects/obmalloc.c - _PyMem_Raw -
Objects/obmalloc.c - _PyObject -
Objects/obmalloc.c - last_final_leaks -
-Objects/obmalloc.c - usedpools -
+Objects/obmalloc.c - obmalloc_state_main -
+Objects/obmalloc.c - obmalloc_state_initialized -
Objects/typeobject.c - name_op -
Objects/typeobject.c - slotdefs -
Objects/unicodeobject.c - stripfuncnames -
diff --git a/Tools/jit/README.md b/Tools/jit/README.md
new file mode 100644
index 00000000000000..04a6c0780bf972
--- /dev/null
+++ b/Tools/jit/README.md
@@ -0,0 +1,46 @@
+The JIT Compiler
+================
+
+This version of CPython can be built with an experimental just-in-time compiler. While most everything you already know about building and using CPython is unchanged, you will probably need to install a compatible version of LLVM first.
+
+## Installing LLVM
+
+The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon).
+
+LLVM version 16 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-16`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code.
+
+It's easy to install all of the required tools:
+
+### Linux
+
+Install LLVM 16 on Ubuntu/Debian:
+
+```sh
+wget https://apt.llvm.org/llvm.sh
+chmod +x llvm.sh
+sudo ./llvm.sh 16
+```
+
+### macOS
+
+Install LLVM 16 with [Homebrew](https://brew.sh):
+
+```sh
+brew install llvm@16
+```
+
+Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them.
+
+### Windows
+
+Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
+
+## Building
+
+For `PCbuild`-based builds, pass the new `--experimental-jit` option to `build.bat`.
+
+For all other builds, pass the new `--enable-experimental-jit` option to `configure`.
+
+Otherwise, just configure and build as you normally would. Cross-compiling "just works", since the JIT is built for the host platform.
+
+[^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time.
diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py
new file mode 100644
index 00000000000000..603bbef59ba2e6
--- /dev/null
+++ b/Tools/jit/_llvm.py
@@ -0,0 +1,99 @@
+"""Utilities for invoking LLVM tools."""
+import asyncio
+import functools
+import os
+import re
+import shlex
+import subprocess
+import typing
+
+_LLVM_VERSION = 16
+_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+")
+
+_P = typing.ParamSpec("_P")
+_R = typing.TypeVar("_R")
+_C = typing.Callable[_P, typing.Awaitable[_R]]
+
+
+def _async_cache(f: _C[_P, _R]) -> _C[_P, _R]:
+ cache = {}
+ lock = asyncio.Lock()
+
+ @functools.wraps(f)
+ async def wrapper(
+ *args: _P.args, **kwargs: _P.kwargs # pylint: disable = no-member
+ ) -> _R:
+ async with lock:
+ if args not in cache:
+ cache[args] = await f(*args, **kwargs)
+ return cache[args]
+
+ return wrapper
+
+
+_CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1)
+
+
+async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None:
+ command = [tool, *args]
+ async with _CORES:
+ if echo:
+ print(shlex.join(command))
+ try:
+ process = await asyncio.create_subprocess_exec(
+ *command, stdout=subprocess.PIPE
+ )
+ except FileNotFoundError:
+ return None
+ out, _ = await process.communicate()
+ if process.returncode:
+ raise RuntimeError(f"{tool} exited with return code {process.returncode}")
+ return out.decode()
+
+
+@_async_cache
+async def _check_tool_version(name: str, *, echo: bool = False) -> bool:
+ output = await _run(name, ["--version"], echo=echo)
+ return bool(output and _LLVM_VERSION_PATTERN.search(output))
+
+
+@_async_cache
+async def _get_brew_llvm_prefix(*, echo: bool = False) -> str | None:
+ output = await _run("brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo)
+ return output and output.removesuffix("\n")
+
+
+@_async_cache
+async def _find_tool(tool: str, *, echo: bool = False) -> str | None:
+ # Unversioned executables:
+ path = tool
+ if await _check_tool_version(path, echo=echo):
+ return path
+ # Versioned executables:
+ path = f"{tool}-{_LLVM_VERSION}"
+ if await _check_tool_version(path, echo=echo):
+ return path
+ # Homebrew-installed executables:
+ prefix = await _get_brew_llvm_prefix(echo=echo)
+ if prefix is not None:
+ path = os.path.join(prefix, "bin", tool)
+ if await _check_tool_version(path, echo=echo):
+ return path
+ # Nothing found:
+ return None
+
+
+async def maybe_run(
+ tool: str, args: typing.Iterable[str], echo: bool = False
+) -> str | None:
+ """Run an LLVM tool if it can be found. Otherwise, return None."""
+ path = await _find_tool(tool, echo=echo)
+ return path and await _run(path, args, echo=echo)
+
+
+async def run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str:
+ """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError."""
+ output = await maybe_run(tool, args, echo=echo)
+ if output is None:
+ raise RuntimeError(f"Can't find {tool}-{_LLVM_VERSION}!")
+ return output
diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py
new file mode 100644
index 00000000000000..8eeb78e6cd69ee
--- /dev/null
+++ b/Tools/jit/_schema.py
@@ -0,0 +1,99 @@
+"""Schema for the JSON produced by llvm-readobj --elf-output-style=JSON."""
+import typing
+
+HoleKind: typing.TypeAlias = typing.Literal[
+ "ARM64_RELOC_GOT_LOAD_PAGE21",
+ "ARM64_RELOC_GOT_LOAD_PAGEOFF12",
+ "ARM64_RELOC_UNSIGNED",
+ "IMAGE_REL_AMD64_ADDR64",
+ "IMAGE_REL_I386_DIR32",
+ "R_AARCH64_ABS64",
+ "R_AARCH64_CALL26",
+ "R_AARCH64_JUMP26",
+ "R_AARCH64_MOVW_UABS_G0_NC",
+ "R_AARCH64_MOVW_UABS_G1_NC",
+ "R_AARCH64_MOVW_UABS_G2_NC",
+ "R_AARCH64_MOVW_UABS_G3",
+ "R_X86_64_64",
+ "X86_64_RELOC_UNSIGNED",
+]
+
+
+class COFFRelocation(typing.TypedDict):
+ """A COFF object file relocation record."""
+
+ Type: dict[typing.Literal["Value"], HoleKind]
+ Symbol: str
+ Offset: int
+
+
+class ELFRelocation(typing.TypedDict):
+ """An ELF object file relocation record."""
+
+ Addend: int
+ Offset: int
+ Symbol: dict[typing.Literal["Value"], str]
+ Type: dict[typing.Literal["Value"], HoleKind]
+
+
+class MachORelocation(typing.TypedDict):
+ """A Mach-O object file relocation record."""
+
+ Offset: int
+ Section: typing.NotRequired[dict[typing.Literal["Value"], str]]
+ Symbol: typing.NotRequired[dict[typing.Literal["Value"], str]]
+ Type: dict[typing.Literal["Value"], HoleKind]
+
+
+class _COFFSymbol(typing.TypedDict):
+ Name: str
+ Value: int
+
+
+class _ELFSymbol(typing.TypedDict):
+ Name: dict[typing.Literal["Value"], str]
+ Value: int
+
+
+class _MachOSymbol(typing.TypedDict):
+ Name: dict[typing.Literal["Value"], str]
+ Value: int
+
+
+class COFFSection(typing.TypedDict):
+ """A COFF object file section."""
+
+ Characteristics: dict[
+ typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]
+ ]
+ Number: int
+ RawDataSize: int
+ Relocations: list[dict[typing.Literal["Relocation"], COFFRelocation]]
+ SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]]
+ Symbols: list[dict[typing.Literal["Symbol"], _COFFSymbol]]
+
+
+class ELFSection(typing.TypedDict):
+ """An ELF object file section."""
+
+ Flags: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]]
+ Index: int
+ Info: int
+ Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]]
+ SectionData: dict[typing.Literal["Bytes"], list[int]]
+ Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]]
+ Type: dict[typing.Literal["Value"], str]
+
+
+class MachOSection(typing.TypedDict):
+ """A Mach-O object file section."""
+
+ Address: int
+ Attributes: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]]
+ Index: int
+ Name: dict[typing.Literal["Value"], str]
+ Relocations: typing.NotRequired[
+ list[dict[typing.Literal["Relocation"], MachORelocation]]
+ ]
+ SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]]
+ Symbols: typing.NotRequired[list[dict[typing.Literal["Symbol"], _MachOSymbol]]]
diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py
new file mode 100644
index 00000000000000..71c678e04fbfd5
--- /dev/null
+++ b/Tools/jit/_stencils.py
@@ -0,0 +1,220 @@
+"""Core data structures for compiled code templates."""
+import dataclasses
+import enum
+import sys
+
+import _schema
+
+
+@enum.unique
+class HoleValue(enum.Enum):
+ """
+ Different "base" values that can be patched into holes (usually combined with the
+ address of a symbol and/or an addend).
+ """
+
+ # The base address of the machine code for the current uop (exposed as _JIT_ENTRY):
+ CODE = enum.auto()
+ # The base address of the machine code for the next uop (exposed as _JIT_CONTINUE):
+ CONTINUE = enum.auto()
+ # The base address of the read-only data for this uop:
+ DATA = enum.auto()
+ # The address of the current executor (exposed as _JIT_EXECUTOR):
+ EXECUTOR = enum.auto()
+ # The base address of the "global" offset table located in the read-only data.
+ # Shouldn't be present in the final stencils, since these are all replaced with
+ # equivalent DATA values:
+ GOT = enum.auto()
+ # The current uop's oparg (exposed as _JIT_OPARG):
+ OPARG = enum.auto()
+ # The current uop's operand (exposed as _JIT_OPERAND):
+ OPERAND = enum.auto()
+ # The current uop's target (exposed as _JIT_TARGET):
+ TARGET = enum.auto()
+ # The base address of the machine code for the first uop (exposed as _JIT_TOP):
+ TOP = enum.auto()
+ # A hardcoded value of zero (used for symbol lookups):
+ ZERO = enum.auto()
+
+
+@dataclasses.dataclass
+class Hole:
+ """
+ A "hole" in the stencil to be patched with a computed runtime value.
+
+ Analogous to relocation records in an object file.
+ """
+
+ offset: int
+ kind: _schema.HoleKind
+ # Patch with this base value:
+ value: HoleValue
+ # ...plus the address of this symbol:
+ symbol: str | None
+ # ...plus this addend:
+ addend: int
+ # Convenience method:
+ replace = dataclasses.replace
+
+ def as_c(self) -> str:
+ """Dump this hole as an initialization of a C Hole struct."""
+ parts = [
+ f"{self.offset:#x}",
+ f"HoleKind_{self.kind}",
+ f"HoleValue_{self.value.name}",
+ f"&{self.symbol}" if self.symbol else "NULL",
+ _format_addend(self.addend),
+ ]
+ return f"{{{', '.join(parts)}}}"
+
+
+@dataclasses.dataclass
+class Stencil:
+ """
+ A contiguous block of machine code or data to be copied-and-patched.
+
+ Analogous to a section or segment in an object file.
+ """
+
+ body: bytearray = dataclasses.field(default_factory=bytearray, init=False)
+ holes: list[Hole] = dataclasses.field(default_factory=list, init=False)
+ disassembly: list[str] = dataclasses.field(default_factory=list, init=False)
+
+ def pad(self, alignment: int) -> None:
+ """Pad the stencil to the given alignment."""
+ offset = len(self.body)
+ padding = -offset % alignment
+ self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}")
+ self.body.extend([0] * padding)
+
+ def emit_aarch64_trampoline(self, hole: Hole) -> None:
+ """Even with the large code model, AArch64 Linux insists on 28-bit jumps."""
+ base = len(self.body)
+ where = slice(hole.offset, hole.offset + 4)
+ instruction = int.from_bytes(self.body[where], sys.byteorder)
+ instruction &= 0xFC000000
+ instruction |= ((base - hole.offset) >> 2) & 0x03FFFFFF
+ self.body[where] = instruction.to_bytes(4, sys.byteorder)
+ self.disassembly += [
+ f"{base + 4 * 0: x}: d2800008 mov x8, #0x0",
+ f"{base + 4 * 0:016x}: R_AARCH64_MOVW_UABS_G0_NC {hole.symbol}",
+ f"{base + 4 * 1:x}: f2a00008 movk x8, #0x0, lsl #16",
+ f"{base + 4 * 1:016x}: R_AARCH64_MOVW_UABS_G1_NC {hole.symbol}",
+ f"{base + 4 * 2:x}: f2c00008 movk x8, #0x0, lsl #32",
+ f"{base + 4 * 2:016x}: R_AARCH64_MOVW_UABS_G2_NC {hole.symbol}",
+ f"{base + 4 * 3:x}: f2e00008 movk x8, #0x0, lsl #48",
+ f"{base + 4 * 3:016x}: R_AARCH64_MOVW_UABS_G3 {hole.symbol}",
+ f"{base + 4 * 4:x}: d61f0100 br x8",
+ ]
+ for code in [
+ 0xD2800008.to_bytes(4, sys.byteorder),
+ 0xF2A00008.to_bytes(4, sys.byteorder),
+ 0xF2C00008.to_bytes(4, sys.byteorder),
+ 0xF2E00008.to_bytes(4, sys.byteorder),
+ 0xD61F0100.to_bytes(4, sys.byteorder),
+ ]:
+ self.body.extend(code)
+ for i, kind in enumerate(
+ [
+ "R_AARCH64_MOVW_UABS_G0_NC",
+ "R_AARCH64_MOVW_UABS_G1_NC",
+ "R_AARCH64_MOVW_UABS_G2_NC",
+ "R_AARCH64_MOVW_UABS_G3",
+ ]
+ ):
+ self.holes.append(hole.replace(offset=base + 4 * i, kind=kind))
+
+
+@dataclasses.dataclass
+class StencilGroup:
+ """
+ Code and data corresponding to a given micro-opcode.
+
+ Analogous to an entire object file.
+ """
+
+ code: Stencil = dataclasses.field(default_factory=Stencil, init=False)
+ data: Stencil = dataclasses.field(default_factory=Stencil, init=False)
+ symbols: dict[int | str, tuple[HoleValue, int]] = dataclasses.field(
+ default_factory=dict, init=False
+ )
+ _got: dict[str, int] = dataclasses.field(default_factory=dict, init=False)
+
+ def process_relocations(self, *, alignment: int = 1) -> None:
+ """Fix up all GOT and internal relocations for this stencil group."""
+ self.code.pad(alignment)
+ self.data.pad(8)
+ for stencil in [self.code, self.data]:
+ holes = []
+ for hole in stencil.holes:
+ if hole.value is HoleValue.GOT:
+ assert hole.symbol is not None
+ hole.value = HoleValue.DATA
+ hole.addend += self._global_offset_table_lookup(hole.symbol)
+ hole.symbol = None
+ elif hole.symbol in self.symbols:
+ hole.value, addend = self.symbols[hole.symbol]
+ hole.addend += addend
+ hole.symbol = None
+ elif (
+ hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"}
+ and hole.value is HoleValue.ZERO
+ ):
+ self.code.emit_aarch64_trampoline(hole)
+ continue
+ holes.append(hole)
+ stencil.holes[:] = holes
+ self.code.pad(alignment)
+ self._emit_global_offset_table()
+ self.code.holes.sort(key=lambda hole: hole.offset)
+ self.data.holes.sort(key=lambda hole: hole.offset)
+
+ def _global_offset_table_lookup(self, symbol: str) -> int:
+ return len(self.data.body) + self._got.setdefault(symbol, 8 * len(self._got))
+
+ def _emit_global_offset_table(self) -> None:
+ got = len(self.data.body)
+ for s, offset in self._got.items():
+ if s in self.symbols:
+ value, addend = self.symbols[s]
+ symbol = None
+ else:
+ value, symbol = symbol_to_value(s)
+ addend = 0
+ self.data.holes.append(
+ Hole(got + offset, "R_X86_64_64", value, symbol, addend)
+ )
+ value_part = value.name if value is not HoleValue.ZERO else ""
+ if value_part and not symbol and not addend:
+ addend_part = ""
+ else:
+ addend_part = f"&{symbol}" if symbol else ""
+ addend_part += _format_addend(addend, signed=symbol is not None)
+ if value_part:
+ value_part += "+"
+ self.data.disassembly.append(
+ f"{len(self.data.body):x}: {value_part}{addend_part}"
+ )
+ self.data.body.extend([0] * 8)
+
+
+def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]:
+ """
+ Convert a symbol name to a HoleValue and a symbol name.
+
+ Some symbols (starting with "_JIT_") are special and are converted to their
+ own HoleValues.
+ """
+ if symbol.startswith("_JIT_"):
+ try:
+ return HoleValue[symbol.removeprefix("_JIT_")], None
+ except KeyError:
+ pass
+ return HoleValue.ZERO, symbol
+
+
+def _format_addend(addend: int, signed: bool = False) -> str:
+ addend %= 1 << 64
+ if addend & (1 << 63):
+ addend -= 1 << 64
+ return f"{addend:{'+#x' if signed else '#x'}}"
diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py
new file mode 100644
index 00000000000000..51b091eb246413
--- /dev/null
+++ b/Tools/jit/_targets.py
@@ -0,0 +1,394 @@
+"""Target-specific code generation, parsing, and processing."""
+import asyncio
+import dataclasses
+import hashlib
+import json
+import os
+import pathlib
+import re
+import sys
+import tempfile
+import typing
+
+import _llvm
+import _schema
+import _stencils
+import _writer
+
+if sys.version_info < (3, 11):
+ raise RuntimeError("Building the JIT compiler requires Python 3.11 or newer!")
+
+TOOLS_JIT_BUILD = pathlib.Path(__file__).resolve()
+TOOLS_JIT = TOOLS_JIT_BUILD.parent
+TOOLS = TOOLS_JIT.parent
+CPYTHON = TOOLS.parent
+PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h"
+TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c"
+
+
+_S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection)
+_R = typing.TypeVar(
+ "_R", _schema.COFFRelocation, _schema.ELFRelocation, _schema.MachORelocation
+)
+
+
+@dataclasses.dataclass
+class _Target(typing.Generic[_S, _R]):
+ triple: str
+ _: dataclasses.KW_ONLY
+ alignment: int = 1
+ prefix: str = ""
+ debug: bool = False
+ force: bool = False
+ verbose: bool = False
+
+ def _compute_digest(self, out: pathlib.Path) -> str:
+ hasher = hashlib.sha256()
+ hasher.update(self.triple.encode())
+ hasher.update(self.alignment.to_bytes())
+ hasher.update(self.prefix.encode())
+ # These dependencies are also reflected in _JITSources in regen.targets:
+ hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes())
+ hasher.update((out / "pyconfig.h").read_bytes())
+ for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)):
+ for filename in filenames:
+ hasher.update(pathlib.Path(dirpath, filename).read_bytes())
+ return hasher.hexdigest()
+
+ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup:
+ group = _stencils.StencilGroup()
+ args = ["--disassemble", "--reloc", f"{path}"]
+ output = await _llvm.maybe_run("llvm-objdump", args, echo=self.verbose)
+ if output is not None:
+ group.code.disassembly.extend(
+ line.expandtabs().strip()
+ for line in output.splitlines()
+ if not line.isspace()
+ )
+ args = [
+ "--elf-output-style=JSON",
+ "--expand-relocs",
+ # "--pretty-print",
+ "--section-data",
+ "--section-relocations",
+ "--section-symbols",
+ "--sections",
+ f"{path}",
+ ]
+ output = await _llvm.run("llvm-readobj", args, echo=self.verbose)
+ # --elf-output-style=JSON is only *slightly* broken on Mach-O...
+ output = output.replace("PrivateExtern\n", "\n")
+ output = output.replace("Extern\n", "\n")
+ # ...and also COFF:
+ output = output[output.index("[", 1, None) :]
+ output = output[: output.rindex("]", None, -1) + 1]
+ sections: list[dict[typing.Literal["Section"], _S]] = json.loads(output)
+ for wrapped_section in sections:
+ self._handle_section(wrapped_section["Section"], group)
+ assert group.symbols["_JIT_ENTRY"] == (_stencils.HoleValue.CODE, 0)
+ if group.data.body:
+ line = f"0: {str(bytes(group.data.body)).removeprefix('b')}"
+ group.data.disassembly.append(line)
+ group.process_relocations()
+ return group
+
+ def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None:
+ raise NotImplementedError(type(self))
+
+ def _handle_relocation(
+ self, base: int, relocation: _R, raw: bytes
+ ) -> _stencils.Hole:
+ raise NotImplementedError(type(self))
+
+ async def _compile(
+ self, opname: str, c: pathlib.Path, tempdir: pathlib.Path
+ ) -> _stencils.StencilGroup:
+ o = tempdir / f"{opname}.o"
+ args = [
+ f"--target={self.triple}",
+ "-DPy_BUILD_CORE",
+ "-D_DEBUG" if self.debug else "-DNDEBUG",
+ f"-D_JIT_OPCODE={opname}",
+ "-D_PyJIT_ACTIVE",
+ "-D_Py_JIT",
+ "-I.",
+ f"-I{CPYTHON / 'Include'}",
+ f"-I{CPYTHON / 'Include' / 'internal'}",
+ f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}",
+ f"-I{CPYTHON / 'Python'}",
+ "-O3",
+ "-c",
+ "-fno-asynchronous-unwind-tables",
+ # SET_FUNCTION_ATTRIBUTE on 32-bit Windows debug builds:
+ "-fno-jump-tables",
+ # Position-independent code adds indirection to every load and jump:
+ "-fno-pic",
+ # Don't make calls to weird stack-smashing canaries:
+ "-fno-stack-protector",
+ # We have three options for code model:
+ # - "small": the default, assumes that code and data reside in the
+ # lowest 2GB of memory (128MB on aarch64)
+ # - "medium": assumes that code resides in the lowest 2GB of memory,
+ # and makes no assumptions about data (not available on aarch64)
+ # - "large": makes no assumptions about either code or data
+ "-mcmodel=large",
+ "-o",
+ f"{o}",
+ "-std=c11",
+ f"{c}",
+ ]
+ await _llvm.run("clang", args, echo=self.verbose)
+ return await self._parse(o)
+
+ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]:
+ generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text()
+ opnames = sorted(re.findall(r"\n {8}case (\w+): \{\n", generated_cases))
+ tasks = []
+ with tempfile.TemporaryDirectory() as tempdir:
+ work = pathlib.Path(tempdir).resolve()
+ async with asyncio.TaskGroup() as group:
+ for opname in opnames:
+ coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work)
+ tasks.append(group.create_task(coro, name=opname))
+ return {task.get_name(): task.result() for task in tasks}
+
+ def build(self, out: pathlib.Path, *, comment: str = "") -> None:
+ """Build jit_stencils.h in the given directory."""
+ digest = f"// {self._compute_digest(out)}\n"
+ jit_stencils = out / "jit_stencils.h"
+ if (
+ not self.force
+ and jit_stencils.exists()
+ and jit_stencils.read_text().startswith(digest)
+ ):
+ return
+ stencil_groups = asyncio.run(self._build_stencils())
+ with jit_stencils.open("w") as file:
+ file.write(digest)
+ if comment:
+ file.write(f"// {comment}\n")
+ file.write("")
+ for line in _writer.dump(stencil_groups):
+ file.write(f"{line}\n")
+
+
+class _COFF(
+ _Target[_schema.COFFSection, _schema.COFFRelocation]
+): # pylint: disable = too-few-public-methods
+ def _handle_section(
+ self, section: _schema.COFFSection, group: _stencils.StencilGroup
+ ) -> None:
+ flags = {flag["Name"] for flag in section["Characteristics"]["Flags"]}
+ if "SectionData" in section:
+ section_data_bytes = section["SectionData"]["Bytes"]
+ else:
+ # Zeroed BSS data, seen with printf debugging calls:
+ section_data_bytes = [0] * section["RawDataSize"]
+ if "IMAGE_SCN_MEM_EXECUTE" in flags:
+ value = _stencils.HoleValue.CODE
+ stencil = group.code
+ elif "IMAGE_SCN_MEM_READ" in flags:
+ value = _stencils.HoleValue.DATA
+ stencil = group.data
+ else:
+ return
+ base = len(stencil.body)
+ group.symbols[section["Number"]] = value, base
+ stencil.body.extend(section_data_bytes)
+ for wrapped_symbol in section["Symbols"]:
+ symbol = wrapped_symbol["Symbol"]
+ offset = base + symbol["Value"]
+ name = symbol["Name"]
+ name = name.removeprefix(self.prefix)
+ group.symbols[name] = value, offset
+ for wrapped_relocation in section["Relocations"]:
+ relocation = wrapped_relocation["Relocation"]
+ hole = self._handle_relocation(base, relocation, stencil.body)
+ stencil.holes.append(hole)
+
+ def _handle_relocation(
+ self, base: int, relocation: _schema.COFFRelocation, raw: bytes
+ ) -> _stencils.Hole:
+ match relocation:
+ case {
+ "Offset": offset,
+ "Symbol": s,
+ "Type": {"Value": "IMAGE_REL_AMD64_ADDR64" as kind},
+ }:
+ offset += base
+ s = s.removeprefix(self.prefix)
+ value, symbol = _stencils.symbol_to_value(s)
+ addend = int.from_bytes(raw[offset : offset + 8], "little")
+ case {
+ "Offset": offset,
+ "Symbol": s,
+ "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind},
+ }:
+ offset += base
+ s = s.removeprefix(self.prefix)
+ value, symbol = _stencils.symbol_to_value(s)
+ addend = int.from_bytes(raw[offset : offset + 4], "little")
+ case _:
+ raise NotImplementedError(relocation)
+ return _stencils.Hole(offset, kind, value, symbol, addend)
+
+
+class _ELF(
+ _Target[_schema.ELFSection, _schema.ELFRelocation]
+): # pylint: disable = too-few-public-methods
+ def _handle_section(
+ self, section: _schema.ELFSection, group: _stencils.StencilGroup
+ ) -> None:
+ section_type = section["Type"]["Value"]
+ flags = {flag["Name"] for flag in section["Flags"]["Flags"]}
+ if section_type == "SHT_RELA":
+ assert "SHF_INFO_LINK" in flags, flags
+ assert not section["Symbols"]
+ value, base = group.symbols[section["Info"]]
+ if value is _stencils.HoleValue.CODE:
+ stencil = group.code
+ else:
+ assert value is _stencils.HoleValue.DATA
+ stencil = group.data
+ for wrapped_relocation in section["Relocations"]:
+ relocation = wrapped_relocation["Relocation"]
+ hole = self._handle_relocation(base, relocation, stencil.body)
+ stencil.holes.append(hole)
+ elif section_type == "SHT_PROGBITS":
+ if "SHF_ALLOC" not in flags:
+ return
+ if "SHF_EXECINSTR" in flags:
+ value = _stencils.HoleValue.CODE
+ stencil = group.code
+ else:
+ value = _stencils.HoleValue.DATA
+ stencil = group.data
+ group.symbols[section["Index"]] = value, len(stencil.body)
+ for wrapped_symbol in section["Symbols"]:
+ symbol = wrapped_symbol["Symbol"]
+ offset = len(stencil.body) + symbol["Value"]
+ name = symbol["Name"]["Value"]
+ name = name.removeprefix(self.prefix)
+ group.symbols[name] = value, offset
+ stencil.body.extend(section["SectionData"]["Bytes"])
+ assert not section["Relocations"]
+ else:
+ assert section_type in {
+ "SHT_GROUP",
+ "SHT_LLVM_ADDRSIG",
+ "SHT_NULL",
+ "SHT_STRTAB",
+ "SHT_SYMTAB",
+ }, section_type
+
+ def _handle_relocation(
+ self, base: int, relocation: _schema.ELFRelocation, raw: bytes
+ ) -> _stencils.Hole:
+ match relocation:
+ case {
+ "Addend": addend,
+ "Offset": offset,
+ "Symbol": {"Value": s},
+ "Type": {"Value": kind},
+ }:
+ offset += base
+ s = s.removeprefix(self.prefix)
+ value, symbol = _stencils.symbol_to_value(s)
+ case _:
+ raise NotImplementedError(relocation)
+ return _stencils.Hole(offset, kind, value, symbol, addend)
+
+
+class _MachO(
+ _Target[_schema.MachOSection, _schema.MachORelocation]
+): # pylint: disable = too-few-public-methods
+ def _handle_section(
+ self, section: _schema.MachOSection, group: _stencils.StencilGroup
+ ) -> None:
+ assert section["Address"] >= len(group.code.body)
+ assert "SectionData" in section
+ flags = {flag["Name"] for flag in section["Attributes"]["Flags"]}
+ name = section["Name"]["Value"]
+ name = name.removeprefix(self.prefix)
+ if "SomeInstructions" in flags:
+ value = _stencils.HoleValue.CODE
+ stencil = group.code
+ start_address = 0
+ group.symbols[name] = value, section["Address"] - start_address
+ else:
+ value = _stencils.HoleValue.DATA
+ stencil = group.data
+ start_address = len(group.code.body)
+ group.symbols[name] = value, len(group.code.body)
+ base = section["Address"] - start_address
+ group.symbols[section["Index"]] = value, base
+ stencil.body.extend(
+ [0] * (section["Address"] - len(group.code.body) - len(group.data.body))
+ )
+ stencil.body.extend(section["SectionData"]["Bytes"])
+ assert "Symbols" in section
+ for wrapped_symbol in section["Symbols"]:
+ symbol = wrapped_symbol["Symbol"]
+ offset = symbol["Value"] - start_address
+ name = symbol["Name"]["Value"]
+ name = name.removeprefix(self.prefix)
+ group.symbols[name] = value, offset
+ assert "Relocations" in section
+ for wrapped_relocation in section["Relocations"]:
+ relocation = wrapped_relocation["Relocation"]
+ hole = self._handle_relocation(base, relocation, stencil.body)
+ stencil.holes.append(hole)
+
+ def _handle_relocation(
+ self, base: int, relocation: _schema.MachORelocation, raw: bytes
+ ) -> _stencils.Hole:
+ symbol: str | None
+ match relocation:
+ case {
+ "Offset": offset,
+ "Symbol": {"Value": s},
+ "Type": {
+ "Value": "ARM64_RELOC_GOT_LOAD_PAGE21"
+ | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind
+ },
+ }:
+ offset += base
+ s = s.removeprefix(self.prefix)
+ value, symbol = _stencils.HoleValue.GOT, s
+ addend = 0
+ case {
+ "Offset": offset,
+ "Section": {"Value": s},
+ "Type": {"Value": kind},
+ } | {
+ "Offset": offset,
+ "Symbol": {"Value": s},
+ "Type": {"Value": kind},
+ }:
+ offset += base
+ s = s.removeprefix(self.prefix)
+ value, symbol = _stencils.symbol_to_value(s)
+ addend = 0
+ case _:
+ raise NotImplementedError(relocation)
+ # Turn Clang's weird __bzero calls into normal bzero calls:
+ if symbol == "__bzero":
+ symbol = "bzero"
+ return _stencils.Hole(offset, kind, value, symbol, addend)
+
+
+def get_target(host: str) -> _COFF | _ELF | _MachO:
+ """Build a _Target for the given host "triple" and options."""
+ if re.fullmatch(r"aarch64-apple-darwin.*", host):
+ return _MachO(host, alignment=8, prefix="_")
+ if re.fullmatch(r"aarch64-.*-linux-gnu", host):
+ return _ELF(host, alignment=8)
+ if re.fullmatch(r"i686-pc-windows-msvc", host):
+ return _COFF(host, prefix="_")
+ if re.fullmatch(r"x86_64-apple-darwin.*", host):
+ return _MachO(host, prefix="_")
+ if re.fullmatch(r"x86_64-pc-windows-msvc", host):
+ return _COFF(host)
+ if re.fullmatch(r"x86_64-.*-linux-gnu", host):
+ return _ELF(host)
+ raise ValueError(host)
diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py
new file mode 100644
index 00000000000000..8a2a42e75cfb9b
--- /dev/null
+++ b/Tools/jit/_writer.py
@@ -0,0 +1,95 @@
+"""Utilities for writing StencilGroups out to a C header file."""
+import typing
+
+import _schema
+import _stencils
+
+
+def _dump_header() -> typing.Iterator[str]:
+ yield "typedef enum {"
+ for kind in typing.get_args(_schema.HoleKind):
+ yield f" HoleKind_{kind},"
+ yield "} HoleKind;"
+ yield ""
+ yield "typedef enum {"
+ for value in _stencils.HoleValue:
+ yield f" HoleValue_{value.name},"
+ yield "} HoleValue;"
+ yield ""
+ yield "typedef struct {"
+ yield " const uint64_t offset;"
+ yield " const HoleKind kind;"
+ yield " const HoleValue value;"
+ yield " const void *symbol;"
+ yield " const uint64_t addend;"
+ yield "} Hole;"
+ yield ""
+ yield "typedef struct {"
+ yield " const size_t body_size;"
+ yield " const unsigned char * const body;"
+ yield " const size_t holes_size;"
+ yield " const Hole * const holes;"
+ yield "} Stencil;"
+ yield ""
+ yield "typedef struct {"
+ yield " const Stencil code;"
+ yield " const Stencil data;"
+ yield "} StencilGroup;"
+ yield ""
+
+
+def _dump_footer(opnames: typing.Iterable[str]) -> typing.Iterator[str]:
+ yield "#define INIT_STENCIL(STENCIL) { \\"
+ yield " .body_size = Py_ARRAY_LENGTH(STENCIL##_body) - 1, \\"
+ yield " .body = STENCIL##_body, \\"
+ yield " .holes_size = Py_ARRAY_LENGTH(STENCIL##_holes) - 1, \\"
+ yield " .holes = STENCIL##_holes, \\"
+ yield "}"
+ yield ""
+ yield "#define INIT_STENCIL_GROUP(OP) { \\"
+ yield " .code = INIT_STENCIL(OP##_code), \\"
+ yield " .data = INIT_STENCIL(OP##_data), \\"
+ yield "}"
+ yield ""
+ yield "static const StencilGroup stencil_groups[512] = {"
+ for opname in opnames:
+ yield f" [{opname}] = INIT_STENCIL_GROUP({opname}),"
+ yield "};"
+ yield ""
+ yield "#define GET_PATCHES() { \\"
+ for value in _stencils.HoleValue:
+ yield f" [HoleValue_{value.name}] = (uint64_t)0xBADBADBADBADBADB, \\"
+ yield "}"
+
+
+def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]:
+ yield f"// {opname}"
+ for part, stencil in [("code", group.code), ("data", group.data)]:
+ for line in stencil.disassembly:
+ yield f"// {line}"
+ if stencil.body:
+ size = len(stencil.body) + 1
+ yield f"static const unsigned char {opname}_{part}_body[{size}] = {{"
+ for i in range(0, len(stencil.body), 8):
+ row = " ".join(f"{byte:#04x}," for byte in stencil.body[i : i + 8])
+ yield f" {row}"
+ yield "};"
+ else:
+ yield f"static const unsigned char {opname}_{part}_body[1];"
+ if stencil.holes:
+ size = len(stencil.holes) + 1
+ yield f"static const Hole {opname}_{part}_holes[{size}] = {{"
+ for hole in stencil.holes:
+ yield f" {hole.as_c()},"
+ yield "};"
+ else:
+ yield f"static const Hole {opname}_{part}_holes[1];"
+ yield ""
+
+
+def dump(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]:
+ """Yield a JIT compiler line-by-line as a C header file."""
+ yield from _dump_header()
+ for opname, group in groups.items():
+ yield from _dump_stencil(opname, group)
+ yield from _dump_footer(groups)
diff --git a/Tools/jit/build.py b/Tools/jit/build.py
new file mode 100644
index 00000000000000..4d4ace14ebf26c
--- /dev/null
+++ b/Tools/jit/build.py
@@ -0,0 +1,28 @@
+"""Build an experimental just-in-time compiler for CPython."""
+import argparse
+import pathlib
+import shlex
+import sys
+
+import _targets
+
+if __name__ == "__main__":
+ comment = f"$ {shlex.join([sys.executable] + sys.argv)}"
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "target", type=_targets.get_target, help="a PEP 11 target triple to compile for"
+ )
+ parser.add_argument(
+ "-d", "--debug", action="store_true", help="compile for a debug build of Python"
+ )
+ parser.add_argument(
+ "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt"
+ )
+ parser.add_argument(
+ "-v", "--verbose", action="store_true", help="echo commands as they are run"
+ )
+ args = parser.parse_args()
+ args.target.debug = args.debug
+ args.target.force = args.force
+ args.target.verbose = args.verbose
+ args.target.build(pathlib.Path.cwd(), comment=comment)
diff --git a/Tools/jit/mypy.ini b/Tools/jit/mypy.ini
new file mode 100644
index 00000000000000..768d0028516abd
--- /dev/null
+++ b/Tools/jit/mypy.ini
@@ -0,0 +1,5 @@
+[mypy]
+files = Tools/jit
+pretty = True
+python_version = 3.11
+strict = True
diff --git a/Tools/jit/template.c b/Tools/jit/template.c
new file mode 100644
index 00000000000000..12303a550d8879
--- /dev/null
+++ b/Tools/jit/template.c
@@ -0,0 +1,98 @@
+#include "Python.h"
+
+#include "pycore_call.h"
+#include "pycore_ceval.h"
+#include "pycore_dict.h"
+#include "pycore_emscripten_signal.h"
+#include "pycore_intrinsics.h"
+#include "pycore_jit.h"
+#include "pycore_long.h"
+#include "pycore_opcode_metadata.h"
+#include "pycore_opcode_utils.h"
+#include "pycore_range.h"
+#include "pycore_setobject.h"
+#include "pycore_sliceobject.h"
+
+#include "ceval_macros.h"
+
+#undef CURRENT_OPARG
+#define CURRENT_OPARG() (_oparg)
+
+#undef CURRENT_OPERAND
+#define CURRENT_OPERAND() (_operand)
+
+#undef DEOPT_IF
+#define DEOPT_IF(COND, INSTNAME) \
+ do { \
+ if ((COND)) { \
+ goto deoptimize; \
+ } \
+ } while (0)
+
+#undef ENABLE_SPECIALIZATION
+#define ENABLE_SPECIALIZATION (0)
+
+#undef GOTO_ERROR
+#define GOTO_ERROR(LABEL) \
+ do { \
+ goto LABEL ## _tier_two; \
+ } while (0)
+
+#undef LOAD_IP
+#define LOAD_IP(UNUSED) \
+ do { \
+ } while (0)
+
+#define PATCH_VALUE(TYPE, NAME, ALIAS) \
+ extern void ALIAS; \
+ TYPE NAME = (TYPE)(uint64_t)&ALIAS;
+
+#define PATCH_JUMP(ALIAS) \
+ extern void ALIAS; \
+ __attribute__((musttail)) \
+ return ((jit_func)&ALIAS)(frame, stack_pointer, tstate);
+
+_Py_CODEUNIT *
+_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate)
+{
+ // Locals that the instruction implementations expect to exist:
+ PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR)
+ int oparg;
+ int opcode = _JIT_OPCODE;
+ _PyUOpInstruction *next_uop;
+ // Other stuff we need handy:
+ PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
+ PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND)
+ PATCH_VALUE(uint32_t, _target, _JIT_TARGET)
+ // The actual instruction definitions (only one will be used):
+ if (opcode == _JUMP_TO_TOP) {
+ CHECK_EVAL_BREAKER();
+ PATCH_JUMP(_JIT_TOP);
+ }
+ switch (opcode) {
+#include "executor_cases.c.h"
+ default:
+ Py_UNREACHABLE();
+ }
+ PATCH_JUMP(_JIT_CONTINUE);
+ // Labels that the instruction implementations expect to exist:
+unbound_local_error_tier_two:
+ _PyEval_FormatExcCheckArg(
+ tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG,
+ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg));
+ goto error_tier_two;
+pop_4_error_tier_two:
+ STACK_SHRINK(1);
+pop_3_error_tier_two:
+ STACK_SHRINK(1);
+pop_2_error_tier_two:
+ STACK_SHRINK(1);
+pop_1_error_tier_two:
+ STACK_SHRINK(1);
+error_tier_two:
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ return NULL;
+deoptimize:
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ return _PyCode_CODE(_PyFrame_GetCode(frame)) + _target;
+}
diff --git a/configure b/configure
index b1153df4d7ec52..7b2119ff7f4f78 100755
--- a/configure
+++ b/configure
@@ -920,6 +920,7 @@ LLVM_AR
PROFILE_TASK
DEF_MAKE_RULE
DEF_MAKE_ALL_RULE
+REGEN_JIT_COMMAND
ABIFLAGS
LN
MKDIR_P
@@ -1074,6 +1075,7 @@ with_pydebug
with_trace_refs
enable_pystats
with_assertions
+enable_experimental_jit
enable_optimizations
with_lto
enable_bolt
@@ -1801,6 +1803,9 @@ Optional Features:
--disable-gil enable experimental support for running without the
GIL (default is no)
--enable-pystats enable internal statistics gathering (default is no)
+ --enable-experimental-jit
+ build the experimental just-in-time compiler
+ (default is no)
--enable-optimizations enable expensive, stable optimizations (PGO, etc.)
(default is no)
--enable-bolt enable usage of the llvm-bolt post-link optimizer
@@ -7997,6 +8002,32 @@ else
printf "%s\n" "no" >&6; }
fi
+# Check for --enable-experimental-jit:
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5
+printf %s "checking for --enable-experimental-jit... " >&6; }
+# Check whether --enable-experimental-jit was given.
+if test ${enable_experimental_jit+y}
+then :
+ enableval=$enable_experimental_jit;
+else $as_nop
+ enable_experimental_jit=no
+fi
+
+if test "x$enable_experimental_jit" = xno
+then :
+
+else $as_nop
+ as_fn_append CFLAGS_NODIST " -D_Py_JIT"
+ REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"
+ if test "x$Py_DEBUG" = xtrue
+then :
+ as_fn_append REGEN_JIT_COMMAND " --debug"
+fi
+fi
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5
+printf "%s\n" "$enable_experimental_jit" >&6; }
+
# Enable optimization flags
@@ -17606,6 +17637,12 @@ if test "x$ac_cv_func_getwd" = xyes
then :
printf "%s\n" "#define HAVE_GETWD 1" >>confdefs.h
+fi
+ac_fn_c_check_func "$LINENO" "grantpt" "ac_cv_func_grantpt"
+if test "x$ac_cv_func_grantpt" = xyes
+then :
+ printf "%s\n" "#define HAVE_GRANTPT 1" >>confdefs.h
+
fi
ac_fn_c_check_func "$LINENO" "if_nameindex" "ac_cv_func_if_nameindex"
if test "x$ac_cv_func_if_nameindex" = xyes
@@ -17792,6 +17829,12 @@ if test "x$ac_cv_func_posix_fallocate" = xyes
then :
printf "%s\n" "#define HAVE_POSIX_FALLOCATE 1" >>confdefs.h
+fi
+ac_fn_c_check_func "$LINENO" "posix_openpt" "ac_cv_func_posix_openpt"
+if test "x$ac_cv_func_posix_openpt" = xyes
+then :
+ printf "%s\n" "#define HAVE_POSIX_OPENPT 1" >>confdefs.h
+
fi
ac_fn_c_check_func "$LINENO" "posix_spawn" "ac_cv_func_posix_spawn"
if test "x$ac_cv_func_posix_spawn" = xyes
@@ -17828,6 +17871,12 @@ if test "x$ac_cv_func_preadv2" = xyes
then :
printf "%s\n" "#define HAVE_PREADV2 1" >>confdefs.h
+fi
+ac_fn_c_check_func "$LINENO" "pthread_cond_timedwait_relative_np" "ac_cv_func_pthread_cond_timedwait_relative_np"
+if test "x$ac_cv_func_pthread_cond_timedwait_relative_np" = xyes
+then :
+ printf "%s\n" "#define HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP 1" >>confdefs.h
+
fi
ac_fn_c_check_func "$LINENO" "pthread_condattr_setclock" "ac_cv_func_pthread_condattr_setclock"
if test "x$ac_cv_func_pthread_condattr_setclock" = xyes
@@ -17846,6 +17895,18 @@ if test "x$ac_cv_func_pthread_kill" = xyes
then :
printf "%s\n" "#define HAVE_PTHREAD_KILL 1" >>confdefs.h
+fi
+ac_fn_c_check_func "$LINENO" "ptsname" "ac_cv_func_ptsname"
+if test "x$ac_cv_func_ptsname" = xyes
+then :
+ printf "%s\n" "#define HAVE_PTSNAME 1" >>confdefs.h
+
+fi
+ac_fn_c_check_func "$LINENO" "ptsname_r" "ac_cv_func_ptsname_r"
+if test "x$ac_cv_func_ptsname_r" = xyes
+then :
+ printf "%s\n" "#define HAVE_PTSNAME_R 1" >>confdefs.h
+
fi
ac_fn_c_check_func "$LINENO" "pwrite" "ac_cv_func_pwrite"
if test "x$ac_cv_func_pwrite" = xyes
@@ -18254,6 +18315,12 @@ if test "x$ac_cv_func_unlinkat" = xyes
then :
printf "%s\n" "#define HAVE_UNLINKAT 1" >>confdefs.h
+fi
+ac_fn_c_check_func "$LINENO" "unlockpt" "ac_cv_func_unlockpt"
+if test "x$ac_cv_func_unlockpt" = xyes
+then :
+ printf "%s\n" "#define HAVE_UNLOCKPT 1" >>confdefs.h
+
fi
ac_fn_c_check_func "$LINENO" "utimensat" "ac_cv_func_utimensat"
if test "x$ac_cv_func_utimensat" = xyes
diff --git a/configure.ac b/configure.ac
index 9587e6d63499aa..5bef2351c987e8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1579,6 +1579,26 @@ else
AC_MSG_RESULT([no])
fi
+# Check for --enable-experimental-jit:
+AC_MSG_CHECKING([for --enable-experimental-jit])
+AC_ARG_ENABLE([experimental-jit],
+ [AS_HELP_STRING([--enable-experimental-jit],
+ [build the experimental just-in-time compiler (default is no)])],
+ [],
+ [enable_experimental_jit=no])
+AS_VAR_IF([enable_experimental_jit],
+ [no],
+ [],
+ [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"])
+ AS_VAR_SET([REGEN_JIT_COMMAND],
+ ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"])
+ AS_VAR_IF([Py_DEBUG],
+ [true],
+ [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])],
+ [])])
+AC_SUBST([REGEN_JIT_COMMAND])
+AC_MSG_RESULT([$enable_experimental_jit])
+
# Enable optimization flags
AC_SUBST([DEF_MAKE_ALL_RULE])
AC_SUBST([DEF_MAKE_RULE])
@@ -4771,13 +4791,13 @@ AC_CHECK_FUNCS([ \
getgrnam_r getgrouplist getgroups gethostname getitimer getloadavg getlogin \
getpeername getpgid getpid getppid getpriority _getpty \
getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \
- getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \
+ getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \
lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \
mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \
- pipe2 plock poll posix_fadvise posix_fallocate posix_spawn posix_spawnp \
+ pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \
posix_spawn_file_actions_addclosefrom_np \
- pread preadv preadv2 pthread_condattr_setclock pthread_init pthread_kill \
- pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \
+ pread preadv preadv2 pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \
+ pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \
rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \
sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \
sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \
@@ -4786,7 +4806,7 @@ AC_CHECK_FUNCS([ \
sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \
sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \
sysconf system tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \
- tmpnam tmpnam_r truncate ttyname umask uname unlinkat utimensat utimes vfork \
+ tmpnam tmpnam_r truncate ttyname umask uname unlinkat unlockpt utimensat utimes vfork \
wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \
])
diff --git a/pyconfig.h.in b/pyconfig.h.in
index d8a9f68951afbd..b22740710bcbee 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -601,6 +601,9 @@
bcopy. */
#undef HAVE_GLIBC_MEMMOVE_BUG
+/* Define to 1 if you have the `grantpt' function. */
+#undef HAVE_GRANTPT
+
/* Define to 1 if you have the <grp.h> header file. */
#undef HAVE_GRP_H
@@ -899,6 +902,9 @@
/* Define to 1 if you have the `posix_fallocate' function. */
#undef HAVE_POSIX_FALLOCATE
+/* Define to 1 if you have the `posix_openpt' function. */
+#undef HAVE_POSIX_OPENPT
+
/* Define to 1 if you have the `posix_spawn' function. */
#undef HAVE_POSIX_SPAWN
@@ -930,6 +936,10 @@
/* Define to 1 if you have the `pthread_condattr_setclock' function. */
#undef HAVE_PTHREAD_CONDATTR_SETCLOCK
+/* Define to 1 if you have the `pthread_cond_timedwait_relative_np' function.
+ */
+#undef HAVE_PTHREAD_COND_TIMEDWAIT_RELATIVE_NP
+
/* Defined for Solaris 2.6 bug in pthread header. */
#undef HAVE_PTHREAD_DESTRUCTOR
@@ -951,6 +961,12 @@
/* Define if platform requires stubbed pthreads support */
#undef HAVE_PTHREAD_STUBS
+/* Define to 1 if you have the `ptsname' function. */
+#undef HAVE_PTSNAME
+
+/* Define to 1 if you have the `ptsname_r' function. */
+#undef HAVE_PTSNAME_R
+
/* Define to 1 if you have the <pty.h> header file. */
#undef HAVE_PTY_H
@@ -1459,6 +1475,9 @@
/* Define to 1 if you have the `unlinkat' function. */
#undef HAVE_UNLINKAT
+/* Define to 1 if you have the `unlockpt' function. */
+#undef HAVE_UNLOCKPT
+
/* Define to 1 if you have the `unshare' function. */
#undef HAVE_UNSHARE