Merge pull request #30122 from rwgk/pybind11k_merge_sh

git merge smart_holder (pybind/pybind11#5157)

.github/workflows/ci.yml

@@ -197,6 +197,35 @@ jobs:
         pytest tests/extra_setuptools
       if: "!(matrix.runs-on == 'windows-2022')"
 
+  manylinux:
+    name: Manylinux on 🐍 3.13t • GIL
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    container: quay.io/pypa/musllinux_1_2_x86_64:latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Prepare venv
+        run: python3.13t -m venv .venv
+
+      - name: Install Python deps
+        run: .venv/bin/pip install -r tests/requirements.txt
+
+      - name: Configure C++11
+        run: >
+          cmake -S. -Bbuild
+          -DPYBIND11_WERROR=ON
+          -DDOWNLOAD_CATCH=ON
+          -DDOWNLOAD_EIGEN=ON
+          -DPython_ROOT_DIR=.venv
+
+      - name: Build C++11
+        run: cmake --build build -j2
+
+      - name: Python tests C++11
+        run: cmake --build build --target pytest -j2
 
   deadsnakes:
     strategy:

.github/workflows/ci_sh_def.yml

@@ -212,6 +212,35 @@ jobs:
         pytest tests/extra_setuptools
       if: "!(matrix.runs-on == 'windows-2022')"
 
+  manylinux:
+    name: Manylinux on 🐍 3.13t • GIL
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    container: quay.io/pypa/musllinux_1_2_x86_64:latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Prepare venv
+        run: python3.13t -m venv .venv
+
+      - name: Install Python deps
+        run: .venv/bin/pip install -r tests/requirements.txt
+
+      - name: Configure C++11
+        run: >
+          cmake -S. -Bbuild
+          -DPYBIND11_WERROR=ON
+          -DDOWNLOAD_CATCH=ON
+          -DDOWNLOAD_EIGEN=ON
+          -DPython_ROOT_DIR=.venv
+
+      - name: Build C++11
+        run: cmake --build build -j2
+
+      - name: Python tests C++11
+        run: cmake --build build --target pytest -j2
 
   deadsnakes:
     strategy:

.github/workflows/ci_sh_def.yml.patch

@@ -1,5 +1,5 @@
---- ci.yml	2024-05-26 21:14:20.025983967 -0700
-+++ ci_sh_def.yml	2024-05-26 21:14:44.673959335 -0700
+--- ci.yml	2024-06-09 23:55:50.405348417 -0700
++++ ci_sh_def.yml	2024-06-09 23:56:22.061331136 -0700
 @@ -1,4 +1,16 @@
 -name: CI
 +# PLEASE KEEP THIS GROUP OF FILES IN SYNC AT ALL TIMES:
@@ -51,23 +51,23 @@
          -DPYBIND11_INTERNALS_VERSION=10000000
          ${{ matrix.args }}
 
-@@ -264,6 +279,7 @@
+@@ -293,6 +308,7 @@
          -DDOWNLOAD_CATCH=ON
          -DDOWNLOAD_EIGEN=ON
          -DCMAKE_CXX_STANDARD=17
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
 
      - name: Build
        run: cmake --build build -j 2
-@@ -332,6 +348,7 @@
+@@ -361,6 +377,7 @@
          -DPYBIND11_WERROR=ON
          -DDOWNLOAD_CATCH=ON
          -DCMAKE_CXX_STANDARD=${{ matrix.std }}
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
 
      - name: Build
-@@ -361,7 +378,7 @@
+@@ -390,7 +407,7 @@
        run: apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y cmake git python3-dev python3-pytest python3-numpy
 
      - name: Configure
@@ -76,7 +76,7 @@
 
      - name: Build
        run: cmake --build build -j2 --verbose
-@@ -449,7 +466,7 @@
+@@ -478,7 +495,7 @@
          cmake -S . -B build -DDOWNLOAD_CATCH=ON \
                              -DCMAKE_CXX_STANDARD=17 \
                              -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \
@@ -85,127 +85,127 @@
                              -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp"
 
      - name: Build
-@@ -503,6 +520,7 @@
+@@ -532,6 +549,7 @@
          -DPYBIND11_WERROR=ON
          -DDOWNLOAD_CATCH=ON
          -DCMAKE_CXX_STANDARD=${{ matrix.std }}
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
 
      - name: Build
-@@ -525,6 +543,7 @@
+@@ -554,6 +572,7 @@
          -DPYBIND11_WERROR=ON
          -DDOWNLOAD_CATCH=ON
          -DCMAKE_CXX_STANDARD=${{ matrix.std }}
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
          "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp"
 
-@@ -574,6 +593,7 @@
+@@ -603,6 +622,7 @@
          -DDOWNLOAD_CATCH=ON     \
          -DDOWNLOAD_EIGEN=OFF    \
          -DCMAKE_CXX_STANDARD=11             \
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"  \
          -DCMAKE_CXX_COMPILER=$(which icpc)  \
          -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
 
-@@ -606,6 +626,7 @@
+@@ -635,6 +655,7 @@
          -DDOWNLOAD_CATCH=ON     \
          -DDOWNLOAD_EIGEN=OFF    \
          -DCMAKE_CXX_STANDARD=17             \
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"  \
          -DCMAKE_CXX_COMPILER=$(which icpc)  \
          -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
 
-@@ -684,6 +705,7 @@
+@@ -713,6 +734,7 @@
          -DDOWNLOAD_CATCH=ON
          -DDOWNLOAD_EIGEN=ON
          -DCMAKE_CXX_STANDARD=11
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
 
      - name: Build
-@@ -734,6 +756,7 @@
+@@ -763,6 +785,7 @@
          cmake ../pybind11-tests
          -DDOWNLOAD_CATCH=ON
          -DPYBIND11_WERROR=ON
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
        working-directory: /build-tests
 
-@@ -829,6 +852,7 @@
+@@ -858,6 +881,7 @@
          -DPYBIND11_WERROR=ON
          -DDOWNLOAD_CATCH=ON
          -DDOWNLOAD_EIGEN=ON
 +        -DCMAKE_CXX_FLAGS="/GR /EHsc /DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          ${{ matrix.args }}
      - name: Build C++11
        run: cmake --build build -j 2
-@@ -883,6 +907,7 @@
+@@ -912,6 +936,7 @@
          -DPYBIND11_WERROR=ON
          -DDOWNLOAD_CATCH=ON
          -DDOWNLOAD_EIGEN=ON
 +        -DCMAKE_CXX_FLAGS="/GR /EHsc /DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          ${{ matrix.args }}
      - name: Build C++11
        run: cmake --build build --config Debug -j 2
-@@ -925,6 +950,7 @@
+@@ -954,6 +979,7 @@
          -DDOWNLOAD_CATCH=ON
          -DDOWNLOAD_EIGEN=ON
          -DCMAKE_CXX_STANDARD=20
 +        -DCMAKE_CXX_FLAGS="/GR /EHsc /DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
 
      - name: Build C++20
        run: cmake --build build -j 2
-@@ -945,6 +971,7 @@
+@@ -974,6 +1000,7 @@
          -DDOWNLOAD_CATCH=ON
          -DDOWNLOAD_EIGEN=ON
          -DCMAKE_CXX_STANDARD=20
 +        -DCMAKE_CXX_FLAGS="/GR /EHsc /DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp"
 
      - name: Build C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE
-@@ -997,6 +1024,7 @@
+@@ -1026,6 +1053,7 @@
        run: >-
          cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON
          -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)")
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          -S . -B build
 
      - name: Build C++11
-@@ -1018,6 +1046,7 @@
+@@ -1047,6 +1075,7 @@
        run: >-
          cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON
          -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)")
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          -S . -B build2
 
      - name: Build C++14
-@@ -1039,6 +1068,7 @@
+@@ -1068,6 +1097,7 @@
        run: >-
          cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON
          -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)")
 +        -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
          -S . -B build3
 
      - name: Build C++17
-@@ -1106,6 +1136,7 @@
+@@ -1135,6 +1165,7 @@
            -DDOWNLOAD_EIGEN=ON
            -DCMAKE_CXX_COMPILER=clang++
            -DCMAKE_CXX_STANDARD=17
 +          -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
 
        - name: Build
          run: cmake --build . -j 2
-@@ -1171,6 +1202,7 @@
+@@ -1200,6 +1231,7 @@
            -DDOWNLOAD_EIGEN=ON
            -DCMAKE_CXX_COMPILER=clang++
            -DCMAKE_CXX_STANDARD=17
 +          -DCMAKE_CXX_FLAGS="-DPYBIND11_USE_SMART_HOLDER_AS_DEFAULT"
            -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)")
 
        - name: Build
-@@ -1194,6 +1226,7 @@
+@@ -1223,6 +1255,7 @@
            -DDOWNLOAD_EIGEN=ON
            -DCMAKE_CXX_COMPILER=clang++
            -DCMAKE_CXX_STANDARD=17

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
 
 # Ruff, the Python auto-correcting linter/formatter written in Rust
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.5
+  rev: v0.4.7
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]

CMakeLists.txt

@@ -116,6 +116,7 @@ option(PYBIND11_NUMPY_1_ONLY
 set(PYBIND11_INTERNALS_VERSION
     ""
     CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
+option(PYBIND11_USE_CROSSCOMPILING "Respect CMAKE_CROSSCOMPILING" OFF)
 
 if(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
   add_compile_definitions(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
@@ -312,6 +313,7 @@ if(PYBIND11_INSTALL)
           tools/pybind11Common.cmake
           tools/pybind11Tools.cmake
           tools/pybind11NewTools.cmake
+          tools/pybind11GuessPythonExtSuffix.cmake
     DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
 
   if(NOT PYBIND11_EXPORT_NAME)

include/pybind11/detail/class.h

@@ -440,7 +440,11 @@ inline void clear_patients(PyObject *self) {
     auto *instance = reinterpret_cast<detail::instance *>(self);
     auto &internals = get_internals();
     auto pos = internals.patients.find(self);
-    assert(pos != internals.patients.end());
+
+    if (pos == internals.patients.end()) {
+        pybind11_fail("FATAL: Internal consistency check failed: Invalid clear_patients() call.");
+    }
+
     // Clearing the patients can cause more Python code to run, which
     // can invalidate the iterator. Extract the vector of patients
     // from the unordered_map first.

include/pybind11/pybind11.h

@@ -3124,7 +3124,7 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
             PyObject *locals = PyEval_GetFrameLocals();
 #        else
             PyObject *locals = PyEval_GetLocals();
-            Py_INCREF(locals);
+            Py_XINCREF(locals);
 #        endif
             if (locals != nullptr) {
 #        if PY_VERSION_HEX >= 0x030b0000

include/pybind11/pytypes.h

@@ -190,7 +190,15 @@ class object_api : public pyobject_tag {
     str_attr_accessor doc() const;
 
     /// Return the object's current reference count
-    int ref_count() const { return static_cast<int>(Py_REFCNT(derived().ptr())); }
+    ssize_t ref_count() const {
+#ifdef PYPY_VERSION
+        // PyPy uses the top few bits for REFCNT_FROM_PYPY & REFCNT_FROM_PYPY_LIGHT
+        // Following pybind11 2.12.1 and older behavior and removing this part
+        return static_cast<ssize_t>(static_cast<int>(Py_REFCNT(derived().ptr())));
+#else
+        return Py_REFCNT(derived().ptr());
+#endif
+    }
 
     // TODO PYBIND11_DEPRECATED(
     //     "Call py::type::handle_of(h) or py::type::of(h) instead of h.get_type()")
@@ -2195,6 +2203,11 @@ class list : public object {
             throw error_already_set();
         }
     }
+    void clear() /* py-non-const */ {
+        if (PyList_SetSlice(m_ptr, 0, PyList_Size(m_ptr), nullptr) == -1) {
+            throw error_already_set();
+        }
+    }
 };
 
 class args : public tuple {

tests/CMakeLists.txt

@@ -363,7 +363,7 @@ if(Boost_FOUND)
     add_library(Boost::headers IMPORTED INTERFACE)
     if(TARGET Boost::boost)
       # Classic FindBoost
-      set_property(TARGET Boost::boost PROPERTY INTERFACE_LINK_LIBRARIES Boost::boost)
+      set_property(TARGET Boost::headers PROPERTY INTERFACE_LINK_LIBRARIES Boost::boost)
     else()
       # Very old FindBoost, or newer Boost than CMake in older CMakes
       set_property(TARGET Boost::headers PROPERTY INTERFACE_INCLUDE_DIRECTORIES

tests/extra_python_package/test_files.py

@@ -86,6 +86,7 @@
     "share/cmake/pybind11/pybind11Common.cmake",
     "share/cmake/pybind11/pybind11Config.cmake",
     "share/cmake/pybind11/pybind11ConfigVersion.cmake",
+    "share/cmake/pybind11/pybind11GuessPythonExtSuffix.cmake",
     "share/cmake/pybind11/pybind11NewTools.cmake",
     "share/cmake/pybind11/pybind11Targets.cmake",
     "share/cmake/pybind11/pybind11Tools.cmake",

tests/pybind11_tests.cpp

@@ -89,6 +89,8 @@ PYBIND11_MODULE(pybind11_tests, m) {
 #endif
     m.attr("cpp_std") = cpp_std();
     m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID;
+    // Free threaded Python uses UINT32_MAX for immortal objects.
+    m.attr("PYBIND11_REFCNT_IMMORTAL") = UINT32_MAX;
     m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") =
 #if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
         true;

tests/test_class.py

@@ -3,7 +3,7 @@
 import pytest
 
 import env
-from pybind11_tests import ConstructorStats, UserType
+from pybind11_tests import PYBIND11_REFCNT_IMMORTAL, ConstructorStats, UserType
 from pybind11_tests import class_ as m
 
 
@@ -377,7 +377,9 @@ class PyDog(m.Dog):
         refcount_3 = getrefcount(cls)
 
         assert refcount_1 == refcount_3
-        assert refcount_2 > refcount_1
+        assert (refcount_2 > refcount_1) or (
+            refcount_2 == refcount_1 == PYBIND11_REFCNT_IMMORTAL
+        )
 
 
 def test_reentrant_implicit_conversion_failure(msg):

tests/test_embed/CMakeLists.txt

@@ -7,6 +7,13 @@ if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STR
   return()
 endif()
 
+if(TARGET Python::Module AND NOT TARGET Python::Python)
+  message(STATUS "Skipping embed test since no embed libs found")
+  add_custom_target(cpptest) # Dummy target since embedding is not supported.
+  set(_suppress_unused_variable_warning "${DOWNLOAD_CATCH}")
+  return()
+endif()
+
 find_package(Catch 2.13.9)
 
 if(CATCH_FOUND)