Add detail/cross_extension_shared_state.h (#30008)

* Transfer PR #4329 from master to smart_holder branch, STEP 1.

The patch .rej below are resolved, but THIS STATE DOES NOT BUILD:

```
clang++ -o pybind11/tests/test_constants_and_functions.os -c -std=c++17 -fPIC -fvisibility=hidden -O0 -g -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -Wunused-result -Werror -isystem /usr/include/python3.10 -isystem /usr/include/eigen3 -DPYBIND11_STRICT_ASSERTS_CLASS_HOLDER_VS_TYPE_CASTER_MIX -DPYBIND11_ENABLE_TYPE_CASTER_ODR_GUARD_IF_AVAILABLE -DPYBIND11_TEST_BOOST -Ipybind11/include -I/usr/local/google/home/rwgk/forked/pybind11/include -I/usr/local/google/home/rwgk/clone/pybind11/include /usr/local/google/home/rwgk/forked/pybind11/tests/test_constants_and_functions.cpp
In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/test_constants_and_functions.cpp:11:
In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/pybind11_tests.h:3:
In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/eval.h:14:
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:1781:9: error: static_assert failed due to requirement '!holder_is_smart_holder == type_caster_type_is_type_caster_base_subtype' "py::class_ holder vs type_caster mismatch: missing PYBIND11_TYPE_CASTER_BASE_HOLDER(T, ...) or collision with custom py::detail::type_caster<T>?"
        static_assert(!holder_is_smart_holder == type_caster_type_is_type_caster_base_subtype,
        ^             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:2460:11: note: in instantiation of function template specialization 'pybind11::class_<MyEnum>::class_<>' requested here
        : class_<Type>(scope, name, extra...), m_base(*this, scope) {
          ^
/usr/local/google/home/rwgk/forked/pybind11/tests/test_constants_and_functions.cpp:106:5: note: in instantiation of function template specialization 'pybind11::enum_<MyEnum>::enum_<>' requested here
    py::enum_<MyEnum>(m, "MyEnum")
    ^
1 error generated.
```

________

```
rwgk.c.googlers.com:~/forked/pybind11 $ patch -p 1 < ~/native_enum_git_diff_master_2022-11-19+131942.patch
patching file .github/workflows/python312.yml
patching file CMakeLists.txt
Hunk #1 FAILED at 111.
Hunk #2 succeeded at 138 (offset 5 lines).
1 out of 2 hunks FAILED -- saving rejects to file CMakeLists.txt.rej
patching file include/pybind11/cast.h
Hunk #1 FAILED at 12.
Hunk #2 succeeded at 78 (offset 30 lines).
Hunk #3 succeeded at 1173 (offset 41 lines).
1 out of 3 hunks FAILED -- saving rejects to file include/pybind11/cast.h.rej
patching file include/pybind11/detail/abi_platform_id.h
patching file include/pybind11/detail/cross_extension_shared_state.h
patching file include/pybind11/detail/internals.h
Hunk #1 FAILED at 16.
Hunk #2 succeeded at 109 (offset 1 line).
Hunk #3 FAILED at 203.
Hunk #4 succeeded at 398 (offset 11 lines).
Hunk #5 succeeded at 457 (offset 11 lines).
2 out of 5 hunks FAILED -- saving rejects to file include/pybind11/detail/internals.h.rej
patching file include/pybind11/detail/native_enum_data.h
patching file include/pybind11/detail/type_map.h
patching file include/pybind11/embed.h
patching file include/pybind11/native_enum.h
patching file include/pybind11/pybind11.h
Hunk #1 FAILED at 12.
Hunk #2 succeeded at 1269 (offset 1 line).
Hunk #3 succeeded at 2457 (offset 255 lines).
1 out of 3 hunks FAILED -- saving rejects to file include/pybind11/pybind11.h.rej
patching file include/pybind11/pytypes.h
patching file tests/CMakeLists.txt
Hunk #1 succeeded at 160 (offset 18 lines).
patching file tests/conftest.py
patching file tests/extra_python_package/test_files.py
Hunk #2 FAILED at 47.
1 out of 2 hunks FAILED -- saving rejects to file tests/extra_python_package/test_files.py.rej
patching file tests/test_embed/test_interpreter.cpp
patching file tests/test_enum.cpp
patching file tests/test_enum.py
patching file tests/test_native_enum.cpp
patching file tests/test_native_enum.py
```

* Make `smart_holder` code compatible with `type_caster_enum_type`

* Fix `if` condition guarding `Unable to cast native enum type to reference`

* WIP native_enum_add_to_parent

* PYBIND11_SILENCE_MSVC_C4127

* Transfer upstream.yml/python312.yml changes from PR #4397

* clang-tidy (clang 15) auto-fix

* Remove python312.yml: this is handled separately under PR #30006

* Fixes for ruff

* Replace `PyEval_GetBuiltins()` in `finalize_interpreter()` with `get_python_state_dict()`

* Bug fix: Ensure `state_dict` is destroyed before `Py_Finalize()`

* Restore tests/test_embed/test_interpreter.cpp from google_pywrapcc_main

* Restore include/pybind11/embed.h from google_pywrapcc_main

* Back out all native_enum changes, leaving only the cross_extension_shared_state changes.

* Undo unrelated one-line change (from `auto` to `internals`).

include/pybind11/detail/abi_platform_id.h

@@ -0,0 +1,103 @@
+// Copyright (c) 2022 The pybind Community.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#pragma once
+
+#include "common.h"
+
+/// On MSVC, debug and release builds are not ABI-compatible!
+#if defined(_MSC_VER) && defined(_DEBUG)
+#    define PYBIND11_BUILD_TYPE "_debug"
+#else
+#    define PYBIND11_BUILD_TYPE ""
+#endif
+
+/// Let's assume that different compilers are ABI-incompatible.
+/// A user can manually set this string if they know their
+/// compiler is compatible.
+#ifndef PYBIND11_COMPILER_TYPE
+#    if defined(_MSC_VER)
+#        define PYBIND11_COMPILER_TYPE "_msvc"
+#    elif defined(__INTEL_COMPILER)
+#        define PYBIND11_COMPILER_TYPE "_icc"
+#    elif defined(__clang__)
+#        define PYBIND11_COMPILER_TYPE "_clang"
+#    elif defined(__PGI)
+#        define PYBIND11_COMPILER_TYPE "_pgi"
+#    elif defined(__MINGW32__)
+#        define PYBIND11_COMPILER_TYPE "_mingw"
+#    elif defined(__CYGWIN__)
+#        define PYBIND11_COMPILER_TYPE "_gcc_cygwin"
+#    elif defined(__GNUC__)
+#        define PYBIND11_COMPILER_TYPE "_gcc"
+#    else
+#        define PYBIND11_COMPILER_TYPE "_unknown"
+#    endif
+#endif
+
+/// Also standard libs
+#ifndef PYBIND11_STDLIB
+#    if defined(_LIBCPP_VERSION)
+#        define PYBIND11_STDLIB "_libcpp"
+#    elif defined(__GLIBCXX__) || defined(__GLIBCPP__)
+#        define PYBIND11_STDLIB "_libstdcpp"
+#    else
+#        define PYBIND11_STDLIB ""
+#    endif
+#endif
+
+/// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility.
+#ifndef PYBIND11_BUILD_ABI
+#    if defined(__GXX_ABI_VERSION)
+#        define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION)
+#    else
+#        define PYBIND11_BUILD_ABI ""
+#    endif
+#endif
+
+#ifndef PYBIND11_INTERNALS_KIND
+#    if defined(WITH_THREAD)
+#        define PYBIND11_INTERNALS_KIND ""
+#    else
+#        define PYBIND11_INTERNALS_KIND "_without_thread"
+#    endif
+#endif
+
+/// See README_smart_holder.rst:
+/// Classic / Conservative / Progressive cross-module compatibility
+#ifndef PYBIND11_INTERNALS_SH_DEF
+#    if defined(PYBIND11_USE_SMART_HOLDER_AS_DEFAULT)
+#        define PYBIND11_INTERNALS_SH_DEF ""
+#    else
+#        define PYBIND11_INTERNALS_SH_DEF "_sh_def"
+#    endif
+#endif
+
+/* NOTE - ATTENTION - WARNING - EXTREME CAUTION
+   Changing this will break compatibility with `PYBIND11_INTERNALS_VERSION 4`
+   See pybind11/detail/type_map.h for more information.
+ */
+#define PYBIND11_PLATFORM_ABI_ID_V4                                                               \
+    PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI             \
+        PYBIND11_BUILD_TYPE PYBIND11_INTERNALS_SH_DEF
+
+/// LEGACY "ABI-breaking" APPROACH, ORIGINAL COMMENT
+/// ------------------------------------------------
+/// Tracks the `internals` and `type_info` ABI version independent of the main library version.
+///
+/// Some portions of the code use an ABI that is conditional depending on this
+/// version number.  That allows ABI-breaking changes to be "pre-implemented".
+/// Once the default version number is incremented, the conditional logic that
+/// no longer applies can be removed.  Additionally, users that need not
+/// maintain ABI compatibility can increase the version number in order to take
+/// advantage of any functionality/efficiency improvements that depend on the
+/// newer ABI.
+///
+/// WARNING: If you choose to manually increase the ABI version, note that
+/// pybind11 may not be tested as thoroughly with a non-default ABI version, and
+/// further ABI-incompatible changes may be made before the ABI is officially
+/// changed to the new version.
+#ifndef PYBIND11_INTERNALS_VERSION
+#    define PYBIND11_INTERNALS_VERSION 4
+#endif

include/pybind11/detail/cross_extension_shared_state.h

@@ -0,0 +1,142 @@
+// Copyright (c) 2022 The pybind Community.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#pragma once
+
+#include "common.h"
+
+#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+#    include "../gil.h"
+#endif
+
+#include "../pytypes.h"
+#include "abi_platform_id.h"
+
+#include <string>
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+inline object get_python_state_dict() {
+    object state_dict;
+#if (PYBIND11_INTERNALS_VERSION <= 4 && PY_VERSION_HEX < 0x030C0000)                              \
+    || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION)
+    state_dict = reinterpret_borrow<object>(PyEval_GetBuiltins());
+#else
+#    if PY_VERSION_HEX < 0x03090000
+    PyInterpreterState *istate = _PyInterpreterState_Get();
+#    else
+    PyInterpreterState *istate = PyInterpreterState_Get();
+#    endif
+    if (istate) {
+        state_dict = reinterpret_borrow<object>(PyInterpreterState_GetDict(istate));
+    }
+#endif
+    if (!state_dict) {
+        raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED");
+    }
+    return state_dict;
+}
+
+#if defined(WITH_THREAD)
+#    if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT)
+using gil_scoped_acquire_simple = gil_scoped_acquire;
+#    else
+// Cannot use py::gil_scoped_acquire here since that constructor calls get_internals.
+struct gil_scoped_acquire_simple {
+    gil_scoped_acquire_simple() : state(PyGILState_Ensure()) {}
+    gil_scoped_acquire_simple(const gil_scoped_acquire_simple &) = delete;
+    gil_scoped_acquire_simple &operator=(const gil_scoped_acquire_simple &) = delete;
+    ~gil_scoped_acquire_simple() { PyGILState_Release(state); }
+    const PyGILState_STATE state;
+};
+#    endif
+#endif
+
+/* NOTE: struct cross_extension_shared_state is in
+             namespace pybind11::detail
+         but all types using this struct are meant to live in
+             namespace pybind11::cross_extension_shared_states
+         to make them easy to discover and reason about.
+ */
+template <typename AdapterType>
+struct cross_extension_shared_state {
+    static constexpr const char *abi_id() { return AdapterType::abi_id(); }
+
+    using payload_type = typename AdapterType::payload_type;
+
+    static payload_type **&payload_pp() {
+        // The reason for the double-indirection is documented here:
+        // https://github.com/pybind/pybind11/pull/1092
+        static payload_type **pp;
+        return pp;
+    }
+
+    static payload_type *get_existing() {
+        if (payload_pp() && *payload_pp()) {
+            return *payload_pp();
+        }
+
+        gil_scoped_acquire_simple gil;
+        error_scope err_scope;
+
+        str abi_id_str(AdapterType::abi_id());
+        dict state_dict = get_python_state_dict();
+        if (!state_dict.contains(abi_id_str)) {
+            return nullptr;
+        }
+
+        void *raw_ptr = PyCapsule_GetPointer(state_dict[abi_id_str].ptr(), AdapterType::abi_id());
+        if (raw_ptr == nullptr) {
+            raise_from(PyExc_SystemError,
+                       ("pybind11::detail::cross_extension_shared_state::get_existing():"
+                        " Retrieve payload_type** from capsule FAILED for ABI ID \""
+                        + std::string(AdapterType::abi_id()) + "\"")
+                           .c_str());
+        }
+        payload_pp() = static_cast<payload_type **>(raw_ptr);
+        return *payload_pp();
+    }
+
+    static payload_type &get() {
+        payload_type *existing = get_existing();
+        if (existing != nullptr) {
+            return *existing;
+        }
+        if (payload_pp() == nullptr) {
+            payload_pp() = new payload_type *();
+        }
+        *payload_pp() = new payload_type();
+        get_python_state_dict()[AdapterType::abi_id()]
+            = capsule(payload_pp(), AdapterType::abi_id());
+        return **payload_pp();
+    }
+
+    struct scoped_clear {
+        // To be called BEFORE Py_Finalize().
+        scoped_clear() {
+            payload_type *existing = get_existing();
+            if (existing != nullptr) {
+                AdapterType::payload_clear(*existing);
+                arm_dtor = true;
+            }
+        }
+
+        // To be called AFTER Py_Finalize().
+        ~scoped_clear() {
+            if (arm_dtor) {
+                delete *payload_pp();
+                *payload_pp() = nullptr;
+            }
+        }
+
+        scoped_clear(const scoped_clear &) = delete;
+        scoped_clear &operator=(const scoped_clear &) = delete;
+
+        bool arm_dtor = false;
+    };
+};
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)