Conversions between Python's native (stdlib) enum types and C++ enums…

…. (#30005)

* Transfer PR pybind#4329 from master to smart_holder branch, STEP 1.

The patch .rej below are resolved, but THIS STATE DOES NOT BUILD:

```
clang++ -o pybind11/tests/test_constants_and_functions.os -c -std=c++17 -fPIC -fvisibility=hidden -O0 -g -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -Wunused-result -Werror -isystem /usr/include/python3.10 -isystem /usr/include/eigen3 -DPYBIND11_STRICT_ASSERTS_CLASS_HOLDER_VS_TYPE_CASTER_MIX -DPYBIND11_ENABLE_TYPE_CASTER_ODR_GUARD_IF_AVAILABLE -DPYBIND11_TEST_BOOST -Ipybind11/include -I/usr/local/google/home/rwgk/forked/pybind11/include -I/usr/local/google/home/rwgk/clone/pybind11/include /usr/local/google/home/rwgk/forked/pybind11/tests/test_constants_and_functions.cpp
In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/test_constants_and_functions.cpp:11:
In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/pybind11_tests.h:3:
In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/eval.h:14:
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:1781:9: error: static_assert failed due to requirement '!holder_is_smart_holder == type_caster_type_is_type_caster_base_subtype' "py::class_ holder vs type_caster mismatch: missing PYBIND11_TYPE_CASTER_BASE_HOLDER(T, ...) or collision with custom py::detail::type_caster<T>?"
        static_assert(!holder_is_smart_holder == type_caster_type_is_type_caster_base_subtype,
        ^             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:2460:11: note: in instantiation of function template specialization 'pybind11::class_<MyEnum>::class_<>' requested here
        : class_<Type>(scope, name, extra...), m_base(*this, scope) {
          ^
/usr/local/google/home/rwgk/forked/pybind11/tests/test_constants_and_functions.cpp:106:5: note: in instantiation of function template specialization 'pybind11::enum_<MyEnum>::enum_<>' requested here
    py::enum_<MyEnum>(m, "MyEnum")
    ^
1 error generated.
```

________

```
rwgk.c.googlers.com:~/forked/pybind11 $ patch -p 1 < ~/native_enum_git_diff_master_2022-11-19+131942.patch
patching file .github/workflows/python312.yml
patching file CMakeLists.txt
Hunk #1 FAILED at 111.
Hunk #2 succeeded at 138 (offset 5 lines).
1 out of 2 hunks FAILED -- saving rejects to file CMakeLists.txt.rej
patching file include/pybind11/cast.h
Hunk #1 FAILED at 12.
Hunk #2 succeeded at 78 (offset 30 lines).
Hunk #3 succeeded at 1173 (offset 41 lines).
1 out of 3 hunks FAILED -- saving rejects to file include/pybind11/cast.h.rej
patching file include/pybind11/detail/abi_platform_id.h
patching file include/pybind11/detail/cross_extension_shared_state.h
patching file include/pybind11/detail/internals.h
Hunk #1 FAILED at 16.
Hunk #2 succeeded at 109 (offset 1 line).
Hunk #3 FAILED at 203.
Hunk #4 succeeded at 398 (offset 11 lines).
Hunk #5 succeeded at 457 (offset 11 lines).
2 out of 5 hunks FAILED -- saving rejects to file include/pybind11/detail/internals.h.rej
patching file include/pybind11/detail/native_enum_data.h
patching file include/pybind11/detail/type_map.h
patching file include/pybind11/embed.h
patching file include/pybind11/native_enum.h
patching file include/pybind11/pybind11.h
Hunk #1 FAILED at 12.
Hunk #2 succeeded at 1269 (offset 1 line).
Hunk #3 succeeded at 2457 (offset 255 lines).
1 out of 3 hunks FAILED -- saving rejects to file include/pybind11/pybind11.h.rej
patching file include/pybind11/pytypes.h
patching file tests/CMakeLists.txt
Hunk #1 succeeded at 160 (offset 18 lines).
patching file tests/conftest.py
patching file tests/extra_python_package/test_files.py
Hunk #2 FAILED at 47.
1 out of 2 hunks FAILED -- saving rejects to file tests/extra_python_package/test_files.py.rej
patching file tests/test_embed/test_interpreter.cpp
patching file tests/test_enum.cpp
patching file tests/test_enum.py
patching file tests/test_native_enum.cpp
patching file tests/test_native_enum.py
```

* Make `smart_holder` code compatible with `type_caster_enum_type`

* Fix `if` condition guarding `Unable to cast native enum type to reference`

* WIP native_enum_add_to_parent

* PYBIND11_SILENCE_MSVC_C4127

* Transfer upstream.yml/python312.yml changes from PR pybind#4397

* clang-tidy (clang 15) auto-fix

* Remove python312.yml: this is handled separately under PR #30006

* Fixes for ruff

* Replace `PyEval_GetBuiltins()` in `finalize_interpreter()` with `get_python_state_dict()`

* Bug fix: Ensure `state_dict` is destroyed before `Py_Finalize()`

* Restore tests/test_embed/test_interpreter.cpp from google_pywrapcc_main

* Restore include/pybind11/embed.h from google_pywrapcc_main

* Undo unrelated one-line change (from `auto` to `internals`).

* Add missing `test_class_with_enum`

* Add note to docs/classes.rst pointing to PR #30005

CMakeLists.txt

@@ -119,6 +119,7 @@ set(PYBIND11_HEADERS
     include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h
     include/pybind11/detail/init.h
     include/pybind11/detail/internals.h
+    include/pybind11/detail/native_enum_data.h
     include/pybind11/detail/smart_holder_poc.h
     include/pybind11/detail/smart_holder_sfinae_hooks_only.h
     include/pybind11/detail/smart_holder_type_casters.h
@@ -141,6 +142,7 @@ set(PYBIND11_HEADERS
     include/pybind11/gil.h
     include/pybind11/iostream.h
     include/pybind11/functional.h
+    include/pybind11/native_enum.h
     include/pybind11/numpy.h
     include/pybind11/operators.h
     include/pybind11/pybind11.h

docs/classes.rst

@@ -539,3 +539,8 @@ The ``name`` property returns the name of the enum value as a unicode string.
            ...
 
     By default, these are omitted to conserve space.
+
+.. note::
+
+    ``py::native_enum`` was added as an alternative to ``py::enum_``
+    with http://github.com/google/pywrapcc/pull/30005

include/pybind11/cast.h

@@ -12,6 +12,7 @@
 
 #include "detail/common.h"
 #include "detail/descr.h"
+#include "detail/native_enum_data.h"
 #include "detail/smart_holder_sfinae_hooks_only.h"
 #include "detail/type_caster_base.h"
 #include "detail/type_caster_odr_guard.h"
@@ -63,12 +64,6 @@ using make_caster_for_intrinsic = type_caster<type>;
 template <typename type>
 using make_caster = make_caster_for_intrinsic<intrinsic_t<type>>;
 
-template <typename T>
-struct type_uses_smart_holder_type_caster {
-    static constexpr bool value
-        = std::is_base_of<smart_holder_type_caster_base_tag, make_caster<T>>::value;
-};
-
 // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T
 template <typename T>
 typename make_caster<T>::template cast_op_type<T> cast_op(make_caster<T> &caster) {
@@ -81,6 +76,126 @@ cast_op(make_caster<T> &&caster) {
         template cast_op_type<typename std::add_rvalue_reference<T>::type>();
 }
 
+template <typename EnumType>
+class type_caster_enum_type {
+private:
+    using Underlying = typename std::underlying_type<EnumType>::type;
+
+public:
+    static constexpr auto name = const_name<EnumType>();
+
+    template <typename SrcType>
+    static handle cast(SrcType &&src, return_value_policy, handle parent) {
+        auto const &natives = cross_extension_shared_states::native_enum_type_map::get();
+        auto found = natives.find(std::type_index(typeid(EnumType)));
+        if (found != natives.end()) {
+            return handle(found->second)(static_cast<Underlying>(src)).release();
+        }
+        return type_caster_for_class_<EnumType>::cast(
+            std::forward<SrcType>(src),
+            // Fixes https://github.com/pybind/pybind11/pull/3643#issuecomment-1022987818:
+            return_value_policy::copy,
+            parent);
+    }
+
+    bool load(handle src, bool convert) {
+        auto const &natives = cross_extension_shared_states::native_enum_type_map::get();
+        auto found = natives.find(std::type_index(typeid(EnumType)));
+        if (found != natives.end()) {
+            if (!isinstance(src, found->second)) {
+                return false;
+            }
+            type_caster<Underlying> underlying_caster;
+            if (!underlying_caster.load(src.attr("value"), convert)) {
+                pybind11_fail("native_enum internal consistency failure.");
+            }
+            value = static_cast<EnumType>(static_cast<Underlying>(underlying_caster));
+            return true;
+        }
+        if (!pybind11_enum_) {
+            pybind11_enum_.reset(new type_caster_for_class_<EnumType>());
+        }
+        return pybind11_enum_->load(src, convert);
+    }
+
+    template <typename T>
+    using cast_op_type = detail::cast_op_type<T>;
+
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator EnumType *() {
+        if (!pybind11_enum_) {
+            return &value;
+        }
+        return pybind11_enum_->operator EnumType *();
+    }
+
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator EnumType &() {
+        if (!pybind11_enum_) {
+            return value;
+        }
+        return pybind11_enum_->operator EnumType &();
+    }
+
+private:
+    std::unique_ptr<type_caster_for_class_<EnumType>> pybind11_enum_;
+    EnumType value;
+};
+
+template <typename EnumType, typename SFINAE = void>
+struct type_caster_enum_type_enabled : std::true_type {};
+
+template <typename T>
+struct type_uses_type_caster_enum_type {
+    static constexpr bool value
+        = std::is_enum<T>::value && type_caster_enum_type_enabled<T>::value;
+};
+
+template <typename EnumType>
+class type_caster<EnumType, detail::enable_if_t<type_uses_type_caster_enum_type<EnumType>::value>>
+    : public type_caster_enum_type<EnumType> {};
+
+template <typename T>
+struct type_uses_smart_holder_type_caster {
+    static constexpr bool value
+        = std::is_base_of<smart_holder_type_caster_base_tag, make_caster<T>>::value
+#ifdef PYBIND11_USE_SMART_HOLDER_AS_DEFAULT
+          || type_uses_type_caster_enum_type<T>::value
+#endif
+        ;
+};
+
+template <typename T, typename SFINAE = void>
+struct type_caster_classh_enum_aware : type_caster<T> {};
+
+#ifdef PYBIND11_USE_SMART_HOLDER_AS_DEFAULT
+template <typename EnumType>
+struct type_caster_classh_enum_aware<
+    EnumType,
+    detail::enable_if_t<type_uses_type_caster_enum_type<EnumType>::value>>
+    : type_caster_for_class_<EnumType> {};
+#endif
+
+template <typename T, detail::enable_if_t<std::is_enum<T>::value, int> = 0>
+bool isinstance_native_enum_impl(handle obj, const std::type_info &tp) {
+    auto const &natives = cross_extension_shared_states::native_enum_type_map::get();
+    auto found = natives.find(tp);
+    if (found == natives.end()) {
+        return false;
+    }
+    return isinstance(obj, found->second);
+}
+
+template <typename T, detail::enable_if_t<!std::is_enum<T>::value, int> = 0>
+bool isinstance_native_enum_impl(handle, const std::type_info &) {
+    return false;
+}
+
+template <typename T>
+bool isinstance_native_enum(handle obj, const std::type_info &tp) {
+    return isinstance_native_enum_impl<intrinsic_t<T>>(obj, tp);
+}
+
 template <typename type>
 class type_caster<std::reference_wrapper<type>> {
 private:
@@ -1024,11 +1139,11 @@ using move_never = none_of<move_always<T>, move_if_unreferenced<T>>;
 // non-reference/pointer `type`s and reference/pointers from a type_caster_generic are safe;
 // everything else returns a reference/pointer to a local variable.
 template <typename type>
-using cast_is_temporary_value_reference
-    = bool_constant<(std::is_reference<type>::value || std::is_pointer<type>::value)
-                    && !std::is_base_of<type_caster_generic, make_caster<type>>::value
-                    && !type_uses_smart_holder_type_caster<intrinsic_t<type>>::value
-                    && !std::is_same<intrinsic_t<type>, void>::value>;
+using cast_is_temporary_value_reference = bool_constant<
+    (std::is_reference<type>::value || std::is_pointer<type>::value)
+    && !std::is_base_of<type_caster_generic, make_caster<type>>::value
+    && !std::is_base_of<smart_holder_type_caster_base_tag, make_caster<type>>::value
+    && !std::is_same<intrinsic_t<type>, void>::value>;
 
 // When a value returned from a C++ function is being cast back to Python, we almost always want to
 // force `policy = move`, regardless of the return value policy the function/method was declared
@@ -1086,8 +1201,18 @@ PYBIND11_NAMESPACE_END(detail)
 template <typename T, detail::enable_if_t<!detail::is_pyobject<T>::value, int> = 0>
 T cast(const handle &handle) {
     using namespace detail;
-    static_assert(!cast_is_temporary_value_reference<T>::value,
+    constexpr bool is_enum_cast = type_uses_type_caster_enum_type<intrinsic_t<T>>::value;
+    static_assert(!cast_is_temporary_value_reference<T>::value || is_enum_cast,
                   "Unable to cast type to reference: value is local to type caster");
+#ifndef NDEBUG
+    if (is_enum_cast && cast_is_temporary_value_reference<T>::value) {
+        if (cross_extension_shared_states::native_enum_type_map::get().count(
+                std::type_index(typeid(intrinsic_t<T>)))
+            != 0) {
+            pybind11_fail("Unable to cast native enum type to reference");
+        }
+    }
+#endif
     return cast_op<T>(load_type<T>(handle));
 }
 

include/pybind11/detail/native_enum_data.h

@@ -0,0 +1,129 @@
+// Copyright (c) 2022 The pybind Community.
+// All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#pragma once
+
+#include "../pytypes.h"
+#include "abi_platform_id.h"
+#include "common.h"
+#include "cross_extension_shared_state.h"
+#include "type_map.h"
+
+#include <string>
+#include <typeindex>
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+class native_enum_data {
+public:
+    native_enum_data(const char *enum_name,
+                     const std::type_index &enum_type_index,
+                     bool use_int_enum)
+        : enum_name_encoded{enum_name}, enum_type_index{enum_type_index},
+          use_int_enum{use_int_enum}, enum_name{enum_name} {}
+
+    native_enum_data(const native_enum_data &) = delete;
+    native_enum_data &operator=(const native_enum_data &) = delete;
+
+    void disarm_correct_use_check() const { correct_use_check = false; }
+    void arm_correct_use_check() const { correct_use_check = true; }
+
+    // This is a separate public function only to enable easy unit testing.
+    std::string was_not_added_error_message() const {
+        return "`native_enum` was not added to any module."
+               " Use e.g. `m += native_enum<...>(\""
+               + enum_name_encoded + "\")` to fix.";
+    }
+
+#if !defined(NDEBUG)
+    // This dtor cannot easily be unit tested because it terminates the process.
+    ~native_enum_data() {
+        if (correct_use_check) {
+            pybind11_fail(was_not_added_error_message());
+        }
+    }
+#endif
+
+private:
+    mutable bool correct_use_check{false};
+
+public:
+    std::string enum_name_encoded;
+    std::type_index enum_type_index;
+    bool use_int_enum;
+    bool export_values_flag{false};
+    str enum_name;
+    list members;
+    list docs;
+};
+
+PYBIND11_NAMESPACE_END(detail)
+
+PYBIND11_NAMESPACE_BEGIN(cross_extension_shared_states)
+
+struct native_enum_type_map_v1_adapter {
+    static constexpr const char *abi_id() {
+        return "__pybind11_native_enum_type_map_v1" PYBIND11_PLATFORM_ABI_ID_V4 "__";
+    }
+
+    using payload_type = detail::type_map<PyObject *>;
+
+    static void payload_clear(payload_type &payload) {
+        for (auto it : payload) {
+            Py_DECREF(it.second);
+        }
+        payload.clear();
+    }
+};
+
+using native_enum_type_map_v1
+    = detail::cross_extension_shared_state<native_enum_type_map_v1_adapter>;
+using native_enum_type_map = native_enum_type_map_v1;
+
+PYBIND11_NAMESPACE_END(cross_extension_shared_states)
+
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+inline void native_enum_add_to_parent(object parent, const detail::native_enum_data &data) {
+    data.disarm_correct_use_check();
+    if (hasattr(parent, data.enum_name)) {
+        pybind11_fail("pybind11::native_enum<...>(\"" + data.enum_name_encoded
+                      + "\"): an object with that name is already defined");
+    }
+    auto enum_module = reinterpret_steal<object>(PyImport_ImportModule("enum"));
+    if (!enum_module) {
+        raise_from(PyExc_SystemError,
+                   "`import enum` FAILED at " __FILE__ ":" PYBIND11_TOSTRING(__LINE__));
+    }
+    auto py_enum_type = enum_module.attr(data.use_int_enum ? "IntEnum" : "Enum");
+    auto py_enum = py_enum_type(data.enum_name, data.members);
+    if (hasattr(parent, "__module__")) {
+        // Enum nested in class:
+        py_enum.attr("__module__") = parent.attr("__module__");
+    } else {
+        py_enum.attr("__module__") = parent;
+    }
+    parent.attr(data.enum_name) = py_enum;
+    if (data.export_values_flag) {
+        for (auto member : data.members) {
+            auto member_name = member[int_(0)];
+            if (hasattr(parent, member_name)) {
+                pybind11_fail("pybind11::native_enum<...>(\"" + data.enum_name_encoded
+                              + "\").value(\"" + member_name.cast<std::string>()
+                              + "\"): an object with that name is already defined");
+            }
+            parent.attr(member_name) = py_enum[member_name];
+        }
+    }
+    for (auto doc : data.docs) {
+        py_enum[doc[int_(0)]].attr("__doc__") = doc[int_(1)];
+    }
+    cross_extension_shared_states::native_enum_type_map::get()[data.enum_type_index]
+        = py_enum.release().ptr();
+}
+
+PYBIND11_NAMESPACE_END(detail)
+
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/detail/smart_holder_sfinae_hooks_only.h

@@ -21,6 +21,13 @@
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
 
+constexpr bool smart_holder_is_default_holder_type =
+#ifdef PYBIND11_USE_SMART_HOLDER_AS_DEFAULT
+    true;
+#else
+    false;
+#endif
+
 template <typename T>
 struct is_smart_holder_type : std::false_type {};
 

include/pybind11/embed.h

@@ -243,6 +243,8 @@ inline void initialize_interpreter(bool init_signal_handlers = true,
 
  \endrst */
 inline void finalize_interpreter() {
+    cross_extension_shared_states::native_enum_type_map::scoped_clear native_enum_type_map_clear;
+
     // Get the internals pointer (without creating it if it doesn't exist).  It's possible for the
     // internals to be created during Py_Finalize() (e.g. if a py::capsule calls `get_internals()`
     // during destruction), so we get the pointer-pointer here and check it after Py_Finalize().