Merge branch 'main' into pythongh-102613-remove-selector-classes

.devcontainer/Dockerfile

@@ -2,11 +2,11 @@ FROM docker.io/library/fedora:37
 
 ENV CC=clang
 
-ENV WASI_SDK_VERSION=19
+ENV WASI_SDK_VERSION=20
 ENV WASI_SDK_PATH=/opt/wasi-sdk
 
 ENV WASMTIME_HOME=/opt/wasmtime
-ENV WASMTIME_VERSION=7.0.0
+ENV WASMTIME_VERSION=9.0.1
 ENV WASMTIME_CPU_ARCH=x86_64
 
 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \

.devcontainer/devcontainer.json

@@ -9,7 +9,9 @@
         "-y",
         "which",
         "zsh",
-        "fish"
+        "fish",
+        // For umask fix below.
+        "/usr/bin/setfacl"
     ],
     "updateContentCommand": {
         // Using the shell for `nproc` usage.
@@ -22,6 +24,11 @@
             "html"
         ]
     },
+    "postCreateCommand": {
+        // https://github.com/orgs/community/discussions/26026
+        "umask fix: workspace": ["sudo", "setfacl", "-bnR", "."],
+        "umask fix: /tmp": ["sudo", "setfacl", "-bnR", "/tmp"]
+    },
     "customizations": {
         "vscode": {
             "extensions": [
@@ -31,10 +38,10 @@
                 "maelvalais.autoconf",
                 // C auto-complete.
                 "ms-vscode.cpptools",
-                // To view built docs.
-                "ms-vscode.live-server"
-                // https://github.com/microsoft/vscode-python/issues/18073
-                // "ms-python.python"
+                // To view HTML build of docs.
+                "ms-vscode.live-server",
+                // Python auto-complete.
+                "ms-python.python"
             ],
             "settings": {
                 "C_Cpp.default.compilerPath": "/usr/bin/clang",

.gitattributes

@@ -18,7 +18,6 @@
 *.zip binary
 
 # Specific binary files
-Lib/test/sndhdrdata/sndhdr.* binary
 PC/classicAppCompat.* binary
 
 # Text files that should not be subject to eol conversion

.github/CODEOWNERS

@@ -97,6 +97,8 @@ Include/pytime.h              @pganssle @abalkin
 /Tools/peg_generator/         @pablogsal @lysnikolaou
 /Lib/test/test_peg_generator/ @pablogsal @lysnikolaou
 /Grammar/python.gram          @pablogsal @lysnikolaou
+/Lib/tokenize.py              @pablogsal @lysnikolaou
+/Lib/test/test_tokenize.py    @pablogsal @lysnikolaou
 
 # AST
 Python/ast.c                  @isidentical

.github/workflows/build.yml

@@ -8,6 +8,7 @@ on:
   push:
     branches:
     - 'main'
+    - '3.12'
     - '3.11'
     - '3.10'
     - '3.9'
@@ -16,6 +17,7 @@ on:
   pull_request:
     branches:
     - 'main'
+    - '3.12'
     - '3.11'
     - '3.10'
     - '3.9'
@@ -26,7 +28,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable
   cancel-in-progress: true
 
 jobs:
@@ -35,8 +37,10 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
+      run-docs: ${{ steps.docs-changes.outputs.run-docs || false }}
       run_tests: ${{ steps.check.outputs.run_tests }}
       run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
+      config_hash: ${{ steps.config_hash.outputs.hash }}
     steps:
       - uses: actions/checkout@v3
       - name: Check for source changes
@@ -72,6 +76,32 @@ jobs:
             echo "Run hypothesis tests"
             echo "run_hypothesis=true" >> $GITHUB_OUTPUT
           fi
+      - name: Compute hash for config cache key
+        id: config_hash
+        run: |
+          echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT
+      - name: Get a list of the changed documentation-related files
+        if: github.event_name == 'pull_request'
+        id: changed-docs-files
+        uses: Ana06/get-changed-files@v2.2.0
+        with:
+          filter: |
+            Doc/**
+            Misc/**
+            .github/workflows/reusable-docs.yml
+      - name: Check for docs changes
+        if: >-
+          github.event_name == 'pull_request'
+          && steps.changed-docs-files.outputs.added_modified_renamed != ''
+        id: docs-changes
+        run: |
+          echo "run-docs=true" >> "${GITHUB_OUTPUT}"
+
+  check-docs:
+    name: Docs
+    needs: check_source
+    if: fromJSON(needs.check_source.outputs.run-docs)
+    uses: ./.github/workflows/reusable-docs.yml
 
   check_generated_files:
     name: 'Check if generated files are up to date'
@@ -81,6 +111,11 @@ jobs:
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
       - uses: actions/checkout@v3
+      - name: Restore config.cache
+        uses: actions/cache@v3
+        with:
+          path: config.cache
+          key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
       - uses: actions/setup-python@v3
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
@@ -97,7 +132,7 @@ jobs:
       - name: Configure CPython
         run: |
           # Build Python with the libpython dynamic library
-          ./configure --with-pydebug --enable-shared
+          ./configure --config-cache --with-pydebug --enable-shared
       - name: Regenerate autoconf files with container image
         run: make regen-configure
       - name: Build CPython
@@ -178,6 +213,11 @@ jobs:
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v3
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
     - name: Install Homebrew dependencies
       run: brew install pkg-config openssl@1.1 xz gdbm tcl-tk
     - name: Configure CPython
@@ -186,6 +226,7 @@ jobs:
         LDFLAGS="-L$(brew --prefix gdbm)/lib -I$(brew --prefix xz)/lib" \
         PKG_CONFIG_PATH="$(brew --prefix tcl-tk)/lib/pkgconfig" \
         ./configure \
+          --config-cache \
           --with-pydebug \
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix openssl@1.1)"
@@ -238,9 +279,18 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+      run: |
+        ../cpython-ro-srcdir/configure \
+          --config-cache \
+          --with-pydebug \
+          --with-openssl=$OPENSSL_DIR
     - name: Build CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make -j4
@@ -271,6 +321,11 @@ jobs:
       LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
     steps:
     - uses: actions/checkout@v3
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -295,7 +350,7 @@ jobs:
     - name: Configure ccache action
       uses: hendrikmuhs/ccache-action@v1.2
     - name: Configure CPython
-      run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR
+      run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR
     - name: Build CPython
       run: make -j4
     - name: Display build info
@@ -304,7 +359,7 @@ jobs:
       run: ./python Lib/test/ssltests.py
 
   test_hypothesis:
-    name: "Hypothesis Tests on Ubuntu"
+    name: "Hypothesis tests on Ubuntu"
     runs-on: ubuntu-20.04
     timeout-minutes: 60
     needs: check_source
@@ -345,9 +400,18 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+      run: |
+        ../cpython-ro-srcdir/configure \
+          --config-cache \
+          --with-pydebug \
+          --with-openssl=$OPENSSL_DIR
     - name: Build CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make -j4
@@ -368,6 +432,14 @@ jobs:
         echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
         echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
         ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis
+    - name: 'Restore Hypothesis database'
+      id: cache-hypothesis-database
+      uses: actions/cache@v3
+      with:
+        path: ./hypothesis
+        key: hypothesis-database-${{ github.head_ref || github.run_id }}
+        restore-keys: |
+          - hypothesis-database-
     - name: "Run tests"
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: |
@@ -388,6 +460,11 @@ jobs:
           -x test_subprocess \
           -x test_signal \
           -x test_sysconfig
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: hypothesis-example-db
+        path: .hypothesis/examples/
 
 
   build_asan:
@@ -402,6 +479,11 @@ jobs:
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
     - uses: actions/checkout@v3
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -430,7 +512,7 @@ jobs:
     - name: Configure ccache action
       uses: hendrikmuhs/ccache-action@v1.2
     - name: Configure CPython
-      run: ./configure --with-address-sanitizer --without-pymalloc
+      run: ./configure --config-cache --with-address-sanitizer --without-pymalloc
     - name: Build CPython
       run: make -j4
     - name: Display build info

.github/workflows/build_msi.yml

@@ -8,12 +8,14 @@ on:
     - '3.*'
     paths:
     - 'Tools/msi/**'
+    - '.github/workflows/build_msi.yml'
   pull_request:
     branches:
     - 'main'
     - '3.*'
     paths:
     - 'Tools/msi/**'
+    - '.github/workflows/build_msi.yml'
 
 permissions:
   contents: read
@@ -33,4 +35,4 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Build CPython installer
-      run: .\Tools\msi\build.bat -${{ matrix.type }}
+      run: .\Tools\msi\build.bat --doc -${{ matrix.type }}

.github/workflows/doc.yml → .github/workflows/reusable-docs.yml

@@ -1,29 +1,8 @@
 name: Docs
 
 on:
+  workflow_call:
   workflow_dispatch:
-  #push:
-  #  branches:
-  #  - 'main'
-  #  - '3.11'
-  #  - '3.10'
-  #  - '3.9'
-  #  - '3.8'
-  #  - '3.7'
-  #  paths:
-  #  - 'Doc/**'
-  pull_request:
-    branches:
-    - 'main'
-    - '3.11'
-    - '3.10'
-    - '3.9'
-    - '3.8'
-    - '3.7'
-    paths:
-    - 'Doc/**'
-    - 'Misc/**'
-    - '.github/workflows/doc.yml'
 
 permissions:
   contents: read

.gitignore

@@ -23,6 +23,10 @@
 *.gc??
 *.profclang?
 *.profraw
+# Copies of binaries before BOLT optimizations.
+*.prebolt
+# BOLT profile data.
+*.fdata
 *.dyn
 .gdb_history
 .purify
@@ -57,7 +61,6 @@ Doc/.venv/
 Doc/env/
 Doc/.env/
 Include/pydtrace_probes.h
-Lib/lib2to3/*.pickle
 Lib/site-packages/*
 !Lib/site-packages/README.txt
 Lib/test/data/*
@@ -124,6 +127,7 @@ Tools/unicode/data/
 /platform
 /profile-clean-stamp
 /profile-run-stamp
+/profile-bolt-stamp
 /Python/deepfreeze/*.c
 /pybuilddir.txt
 /pyconfig.h

.readthedocs.yml

@@ -13,6 +13,20 @@ build:
     python: "3"
 
   commands:
+    # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition
+    #
+    # Cancel building pull requests when there aren't changes in the Doc directory.
+    #
+    # If there are no changes (git diff exits with 0) we force the command to return with 183.
+    # This is a special exit code on Read the Docs that will cancel the build immediately.
+    - |
+      if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && [ "$(git diff --quiet origin/main -- Doc/ .readthedocs.yml; echo $?)" -eq 0 ];
+      then
+        echo "No changes to Doc/ - exiting the build.";
+        exit 183;
+      fi
+
     - make -C Doc venv html
     - mkdir _readthedocs
     - mv Doc/build/html _readthedocs/html
+