Merge branch 'google_pywrapcc_main' into py_nullptr_default_arg_pywrapcc

.github/workflows/ci.yml

@@ -210,7 +210,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Setup Python ${{ matrix.python-version }} (deadsnakes)
-      uses: deadsnakes/action@v3.0.1
+      uses: deadsnakes/action@v3.1.0
       with:
         python-version: ${{ matrix.python-version }}
         debug: ${{ matrix.python-debug }}

.github/workflows/ci_sh_def.yml

@@ -225,7 +225,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Setup Python ${{ matrix.python-version }} (deadsnakes)
-      uses: deadsnakes/action@v3.0.1
+      uses: deadsnakes/action@v3.1.0
       with:
         python-version: ${{ matrix.python-version }}
         debug: ${{ matrix.python-debug }}

.github/workflows/ci_sh_def.yml.patch

@@ -1,5 +1,5 @@
---- ci.yml	2023-10-20 00:00:26.619833296 -0700
-+++ ci_sh_def.yml	2023-10-20 00:00:43.927801556 -0700
+--- ci.yml	2023-12-01 21:59:50.570956290 -0800
++++ ci_sh_def.yml	2023-12-01 22:00:43.098860727 -0800
 @@ -1,4 +1,16 @@
 -name: CI
 +# PLEASE KEEP THIS GROUP OF FILES IN SYNC AT ALL TIMES:

CMakeLists.txt

@@ -301,7 +301,21 @@ if(PYBIND11_INSTALL)
 
   # pkg-config support
   if(NOT prefix_for_pc_file)
-    set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}")
+    if(IS_ABSOLUTE "${CMAKE_INSTALL_DATAROOTDIR}")
+      set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}")
+    else()
+      set(pc_datarootdir "${CMAKE_INSTALL_DATAROOTDIR}")
+      if(CMAKE_VERSION VERSION_LESS 3.20)
+        set(prefix_for_pc_file "\${pcfiledir}/..")
+        while(pc_datarootdir)
+          get_filename_component(pc_datarootdir "${pc_datarootdir}" DIRECTORY)
+          string(APPEND prefix_for_pc_file "/..")
+        endwhile()
+      else()
+        cmake_path(RELATIVE_PATH CMAKE_INSTALL_PREFIX BASE_DIRECTORY CMAKE_INSTALL_DATAROOTDIR
+                   OUTPUT_VARIABLE prefix_for_pc_file)
+      endif()
+    endif()
   endif()
   join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
   configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in"

include/pybind11/stl_bind.h

@@ -525,7 +525,7 @@ class_<Vector, holder_type> bind_vector(handle scope, std::string const &name, A
         [](const Vector &v) -> bool { return !v.empty(); },
         "Check whether the list is nonempty");
 
-    cl.def("__len__", &Vector::size);
+    cl.def("__len__", [](const Vector &vec) { return vec.size(); });
 
 #if 0
     // C++ style functions deprecated, leaving it here as an example
@@ -843,7 +843,8 @@ class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args &&
         m.erase(it);
     });
 
-    cl.def("__len__", &Map::size);
+    // Always use a lambda in case of `using` declaration
+    cl.def("__len__", [](const Map &m) { return m.size(); });
 
     return cl;
 }

noxfile.py

@@ -57,7 +57,7 @@ def tests_packaging(session: nox.Session) -> None:
     Run the packaging tests.
     """
 
-    session.install("-r", "tests/requirements.txt", "--prefer-binary")
+    session.install("-r", "tests/requirements.txt")
     session.run("pytest", "tests/extra_python_package", *session.posargs)
 
 

tests/requirements.txt

@@ -1,10 +1,15 @@
-build==0.8.0
-numpy==1.21.5; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.7"
-numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6"
-numpy==1.21.5; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.10"
-numpy==1.22.2; platform_python_implementation!="PyPy" and python_version>="3.10" and python_version<"3.11"
-pytest==7.0.0; platform_python_implementation!="PyPy" and python_version=="3.6"
-pytest==7.2.0; platform_python_implementation!="PyPy" and python_version>="3.7"
+--only-binary=:all:
+build~=0.9; python_version=="3.6"
+build~=1.0; python_version>="3.7"
+numpy~=1.20.0; python_version=="3.7" and platform_python_implementation=="PyPy"
+numpy~=1.23.0; python_version=="3.8" and platform_python_implementation=="PyPy"
+numpy~=1.25.0; python_version=="3.9" and platform_python_implementation=='PyPy'
+numpy~=1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6"
+numpy~=1.21.5; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.10"
+numpy~=1.22.2; platform_python_implementation!="PyPy" and python_version=="3.10"
+numpy~=1.26.0; platform_python_implementation!="PyPy" and python_version>="3.11"
+pytest~=7.0
 pytest-timeout
-scipy==1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10"
-scipy==1.10.0; platform_python_implementation!="PyPy" and python_version=="3.10"
+scipy~=1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10"
+scipy~=1.8.0; platform_python_implementation!="PyPy" and python_version=="3.10"
+scipy~=1.11.1; platform_python_implementation!="PyPy" and python_version>="3.11"

tests/test_stl_binders.cpp

@@ -15,6 +15,7 @@
 #include <deque>
 #include <map>
 #include <unordered_map>
+#include <vector>
 
 class El {
 public:
@@ -83,6 +84,71 @@ struct RecursiveMap : std::map<int, RecursiveMap> {
     using Parent::Parent;
 };
 
+class UserVectorLike : private std::vector<int> {
+public:
+    // This is only a subset of the member functions, as needed at the time.
+    using Base = std::vector<int>;
+    using typename Base::const_iterator;
+    using typename Base::difference_type;
+    using typename Base::iterator;
+    using typename Base::size_type;
+    using typename Base::value_type;
+
+    using Base::at;
+    using Base::back;
+    using Base::Base;
+    using Base::begin;
+    using Base::cbegin;
+    using Base::cend;
+    using Base::clear;
+    using Base::empty;
+    using Base::end;
+    using Base::erase;
+    using Base::front;
+    using Base::insert;
+    using Base::pop_back;
+    using Base::push_back;
+    using Base::reserve;
+    using Base::shrink_to_fit;
+    using Base::swap;
+    using Base::operator[];
+    using Base::capacity;
+    using Base::size;
+};
+
+bool operator==(UserVectorLike const &, UserVectorLike const &) { return true; }
+bool operator!=(UserVectorLike const &, UserVectorLike const &) { return false; }
+
+class UserMapLike : private std::map<int, int> {
+public:
+    // This is only a subset of the member functions, as needed at the time.
+    using Base = std::map<int, int>;
+    using typename Base::const_iterator;
+    using typename Base::iterator;
+    using typename Base::key_type;
+    using typename Base::mapped_type;
+    using typename Base::size_type;
+    using typename Base::value_type;
+
+    using Base::at;
+    using Base::Base;
+    using Base::begin;
+    using Base::cbegin;
+    using Base::cend;
+    using Base::clear;
+    using Base::emplace;
+    using Base::emplace_hint;
+    using Base::empty;
+    using Base::end;
+    using Base::erase;
+    using Base::find;
+    using Base::insert;
+    using Base::max_size;
+    using Base::swap;
+    using Base::operator[];
+    using Base::size;
+};
+
 /*
  * Pybind11 does not catch more complicated recursion schemes, such as mutual
  * recursion.
@@ -173,6 +239,10 @@ TEST_SUBMODULE(stl_binders, m) {
     py::bind_map<MutuallyRecursiveContainerPairMV>(m, "MutuallyRecursiveContainerPairMV");
     py::bind_vector<MutuallyRecursiveContainerPairVM>(m, "MutuallyRecursiveContainerPairVM");
 
+    // Bind with private inheritance + `using` directives.
+    py::bind_vector<UserVectorLike>(m, "UserVectorLike");
+    py::bind_map<UserMapLike>(m, "UserMapLike");
+
     // The rest depends on numpy:
     try {
         py::module_::import("numpy");

tests/test_stl_binders.py

@@ -353,3 +353,17 @@ def test_recursive_map():
     recursive_map[100][101] = m.RecursiveMap()
     recursive_map[100][102] = m.RecursiveMap()
     assert list(recursive_map[100].keys()) == [101, 102]
+
+
+def test_user_vector_like():
+    vec = m.UserVectorLike()
+    vec.append(2)
+    assert vec[0] == 2
+    assert len(vec) == 1
+
+
+def test_user_like_map():
+    map = m.UserMapLike()
+    map[33] = 44
+    assert map[33] == 44
+    assert len(map) == 1

tools/make_changelog.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+from __future__ import annotations
 
 import re
 
@@ -29,21 +30,46 @@
 )
 issues = (issue for page in issues_pages for issue in page)
 missing = []
+cats_descr = {
+    "feat": "New Features",
+    "fix": "Bug fixes",
+    "fix(types)": "",
+    "fix(cmake)": "",
+    "docs": "Documentation",
+    "tests": "Tests",
+    "ci": "CI",
+    "chore": "Other",
+    "unknown": "Uncategorised",
+}
+cats: dict[str, list[str]] = {c: [] for c in cats_descr}
 
 for issue in issues:
     changelog = ENTRY.findall(issue.body or "")
     if not changelog or not changelog[0]:
         missing.append(issue)
     else:
         (msg,) = changelog
+        if msg.startswith("- "):
+            msg = msg[2:]
         if not msg.startswith("* "):
             msg = "* " + msg
         if not msg.endswith("."):
             msg += "."
 
         msg += f"\n  `#{issue.number} <{issue.html_url}>`_"
+        for cat in cats:
+            if issue.title.lower().startswith(f"{cat}:"):
+                cats[cat].append(msg)
+                break
+        else:
+            cats["unknown"].append(msg)
 
-        print(Syntax(msg, "rst", theme="ansi_light", word_wrap=True))
+for cat, msgs in cats.items():
+    if msgs:
+        desc = cats_descr[cat]
+        print(f"[bold]{desc}:\n" if desc else "")
+        for msg in msgs:
+            print(Syntax(msg, "rst", theme="ansi_light", word_wrap=True))
         print()
 
 if missing:

tools/pybind11Common.cmake

@@ -218,8 +218,15 @@ if(NOT _pybind11_nopython)
 
     execute_process(
       COMMAND
-        ${${_Python}_EXECUTABLE} -c
-        "from pkg_resources import get_distribution; print(get_distribution('${PYPI_NAME}').version)"
+        ${${_Python}_EXECUTABLE} -c "
+try:
+    from importlib.metadata import version
+except ImportError:
+    from pkg_resources import get_distribution
+    def version(s):
+        return get_distribution(s).version
+print(version('${PYPI_NAME}'))
+        "
       RESULT_VARIABLE RESULT_PRESENT
       OUTPUT_VARIABLE PKG_VERSION
       ERROR_QUIET)
@@ -300,21 +307,24 @@ function(_pybind11_generate_lto target prefer_thin_lto)
       set(cxx_append ";-fno-fat-lto-objects")
     endif()
 
-    if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le" OR CMAKE_SYSTEM_PROCESSOR MATCHES "mips64")
-      set(NO_FLTO_ARCH TRUE)
+    if(prefer_thin_lto)
+      set(thin "=thin")
     else()
-      set(NO_FLTO_ARCH FALSE)
+      set(thin "")
     endif()
 
-    if(CMAKE_CXX_COMPILER_ID MATCHES "Clang"
-       AND prefer_thin_lto
-       AND NOT NO_FLTO_ARCH)
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le" OR CMAKE_SYSTEM_PROCESSOR MATCHES "mips64")
+      # Do nothing
+    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES emscripten)
+      # This compile is very costly when cross-compiling, so set this without checking
+      set(PYBIND11_LTO_CXX_FLAGS "-flto${thin}${cxx_append}")
+      set(PYBIND11_LTO_LINKER_FLAGS "-flto${thin}${linker_append}")
+    elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
       _pybind11_return_if_cxx_and_linker_flags_work(
-        HAS_FLTO_THIN "-flto=thin${cxx_append}" "-flto=thin${linker_append}"
+        HAS_FLTO_THIN "-flto${thin}${cxx_append}" "-flto=${thin}${linker_append}"
         PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS)
     endif()
-
-    if(NOT HAS_FLTO_THIN AND NOT NO_FLTO_ARCH)
+    if(NOT HAS_FLTO_THIN)
       _pybind11_return_if_cxx_and_linker_flags_work(
         HAS_FLTO "-flto${cxx_append}" "-flto${linker_append}" PYBIND11_LTO_CXX_FLAGS
         PYBIND11_LTO_LINKER_FLAGS)

tools/pybind11NewTools.cmake

@@ -39,12 +39,23 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
     set(_pybind11_dev_component Development.Module OPTIONAL_COMPONENTS Development.Embed)
   endif()
 
+  # Callers need to be able to access Python_EXECUTABLE
+  set(_pybind11_global_keyword "")
+  if(NOT is_config AND NOT DEFINED Python_ARTIFACTS_INTERACTIVE)
+    set(Python_ARTIFACTS_INTERACTIVE TRUE)
+    if(NOT CMAKE_VERSION VERSION_LESS 3.24)
+      set(_pybind11_global_keyword "GLOBAL")
+    endif()
+  endif()
+
   find_package(Python 3.6 REQUIRED COMPONENTS Interpreter ${_pybind11_dev_component}
-                                              ${_pybind11_quiet})
+                                              ${_pybind11_quiet} ${_pybind11_global_keyword})
 
   # If we are in submodule mode, export the Python targets to global targets.
   # If this behavior is not desired, FindPython _before_ pybind11.
-  if(NOT is_config)
+  if(NOT is_config
+     AND NOT Python_ARTIFACTS_INTERACTIVE
+     AND _pybind11_global_keyword STREQUAL "")
     if(TARGET Python::Python)
       set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE)
     endif()
@@ -53,6 +64,22 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
       set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE)
     endif()
   endif()
+
+  # Explicitly export version for callers (including our own functions)
+  if(NOT is_config AND Python_ARTIFACTS_INTERACTIVE)
+    set(Python_VERSION
+        "${Python_VERSION}"
+        CACHE INTERNAL "")
+    set(Python_VERSION_MAJOR
+        "${Python_VERSION_MAJOR}"
+        CACHE INTERNAL "")
+    set(Python_VERSION_MINOR
+        "${Python_VERSION_MINOR}"
+        CACHE INTERNAL "")
+    set(Python_VERSION_PATCH
+        "${Python_VERSION_PATCH}"
+        CACHE INTERNAL "")
+  endif()
 endif()
 
 if(Python_FOUND)