diff --git a/include/pybind11/detail/abi_platform_id.h b/include/pybind11/detail/abi_platform_id.h new file mode 100644 index 00000000..17dcf0de --- /dev/null +++ b/include/pybind11/detail/abi_platform_id.h @@ -0,0 +1,103 @@ +// Copyright (c) 2022 The pybind Community. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" + +/// On MSVC, debug and release builds are not ABI-compatible! +#if defined(_MSC_VER) && defined(_DEBUG) +# define PYBIND11_BUILD_TYPE "_debug" +#else +# define PYBIND11_BUILD_TYPE "" +#endif + +/// Let's assume that different compilers are ABI-incompatible. +/// A user can manually set this string if they know their +/// compiler is compatible. +#ifndef PYBIND11_COMPILER_TYPE +# if defined(_MSC_VER) +# define PYBIND11_COMPILER_TYPE "_msvc" +# elif defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_TYPE "_icc" +# elif defined(__clang__) +# define PYBIND11_COMPILER_TYPE "_clang" +# elif defined(__PGI) +# define PYBIND11_COMPILER_TYPE "_pgi" +# elif defined(__MINGW32__) +# define PYBIND11_COMPILER_TYPE "_mingw" +# elif defined(__CYGWIN__) +# define PYBIND11_COMPILER_TYPE "_gcc_cygwin" +# elif defined(__GNUC__) +# define PYBIND11_COMPILER_TYPE "_gcc" +# else +# define PYBIND11_COMPILER_TYPE "_unknown" +# endif +#endif + +/// Also standard libs +#ifndef PYBIND11_STDLIB +# if defined(_LIBCPP_VERSION) +# define PYBIND11_STDLIB "_libcpp" +# elif defined(__GLIBCXX__) || defined(__GLIBCPP__) +# define PYBIND11_STDLIB "_libstdcpp" +# else +# define PYBIND11_STDLIB "" +# endif +#endif + +/// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility. +#ifndef PYBIND11_BUILD_ABI +# if defined(__GXX_ABI_VERSION) +# define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) +# else +# define PYBIND11_BUILD_ABI "" +# endif +#endif + +#ifndef PYBIND11_INTERNALS_KIND +# if defined(WITH_THREAD) +# define PYBIND11_INTERNALS_KIND "" +# else +# define PYBIND11_INTERNALS_KIND "_without_thread" +# endif +#endif + +/// See README_smart_holder.rst: +/// Classic / Conservative / Progressive cross-module compatibility +#ifndef PYBIND11_INTERNALS_SH_DEF +# if defined(PYBIND11_USE_SMART_HOLDER_AS_DEFAULT) +# define PYBIND11_INTERNALS_SH_DEF "" +# else +# define PYBIND11_INTERNALS_SH_DEF "_sh_def" +# endif +#endif + +/* NOTE - ATTENTION - WARNING - EXTREME CAUTION + Changing this will break compatibility with `PYBIND11_INTERNALS_VERSION 4` + See pybind11/detail/type_map.h for more information. + */ +#define PYBIND11_PLATFORM_ABI_ID_V4 \ + PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \ + PYBIND11_BUILD_TYPE PYBIND11_INTERNALS_SH_DEF + +/// LEGACY "ABI-breaking" APPROACH, ORIGINAL COMMENT +/// ------------------------------------------------ +/// Tracks the `internals` and `type_info` ABI version independent of the main library version. +/// +/// Some portions of the code use an ABI that is conditional depending on this +/// version number. That allows ABI-breaking changes to be "pre-implemented". +/// Once the default version number is incremented, the conditional logic that +/// no longer applies can be removed. Additionally, users that need not +/// maintain ABI compatibility can increase the version number in order to take +/// advantage of any functionality/efficiency improvements that depend on the +/// newer ABI. +/// +/// WARNING: If you choose to manually increase the ABI version, note that +/// pybind11 may not be tested as thoroughly with a non-default ABI version, and +/// further ABI-incompatible changes may be made before the ABI is officially +/// changed to the new version. +#ifndef PYBIND11_INTERNALS_VERSION +# define PYBIND11_INTERNALS_VERSION 4 +#endif diff --git a/include/pybind11/detail/cross_extension_shared_state.h b/include/pybind11/detail/cross_extension_shared_state.h new file mode 100644 index 00000000..150df0b0 --- /dev/null +++ b/include/pybind11/detail/cross_extension_shared_state.h @@ -0,0 +1,142 @@ +// Copyright (c) 2022 The pybind Community. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" + +#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "../gil.h" +#endif + +#include "../pytypes.h" +#include "abi_platform_id.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +inline object get_python_state_dict() { + object state_dict; +#if (PYBIND11_INTERNALS_VERSION <= 4 && PY_VERSION_HEX < 0x030C0000) \ + || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) + state_dict = reinterpret_borrow(PyEval_GetBuiltins()); +#else +# if PY_VERSION_HEX < 0x03090000 + PyInterpreterState *istate = _PyInterpreterState_Get(); +# else + PyInterpreterState *istate = PyInterpreterState_Get(); +# endif + if (istate) { + state_dict = reinterpret_borrow(PyInterpreterState_GetDict(istate)); + } +#endif + if (!state_dict) { + raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); + } + return state_dict; +} + +#if defined(WITH_THREAD) +# if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +using gil_scoped_acquire_simple = gil_scoped_acquire; +# else +// Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. +struct gil_scoped_acquire_simple { + gil_scoped_acquire_simple() : state(PyGILState_Ensure()) {} + gil_scoped_acquire_simple(const gil_scoped_acquire_simple &) = delete; + gil_scoped_acquire_simple &operator=(const gil_scoped_acquire_simple &) = delete; + ~gil_scoped_acquire_simple() { PyGILState_Release(state); } + const PyGILState_STATE state; +}; +# endif +#endif + +/* NOTE: struct cross_extension_shared_state is in + namespace pybind11::detail + but all types using this struct are meant to live in + namespace pybind11::cross_extension_shared_states + to make them easy to discover and reason about. + */ +template +struct cross_extension_shared_state { + static constexpr const char *abi_id() { return AdapterType::abi_id(); } + + using payload_type = typename AdapterType::payload_type; + + static payload_type **&payload_pp() { + // The reason for the double-indirection is documented here: + // https://github.com/pybind/pybind11/pull/1092 + static payload_type **pp; + return pp; + } + + static payload_type *get_existing() { + if (payload_pp() && *payload_pp()) { + return *payload_pp(); + } + + gil_scoped_acquire_simple gil; + error_scope err_scope; + + str abi_id_str(AdapterType::abi_id()); + dict state_dict = get_python_state_dict(); + if (!state_dict.contains(abi_id_str)) { + return nullptr; + } + + void *raw_ptr = PyCapsule_GetPointer(state_dict[abi_id_str].ptr(), AdapterType::abi_id()); + if (raw_ptr == nullptr) { + raise_from(PyExc_SystemError, + ("pybind11::detail::cross_extension_shared_state::get_existing():" + " Retrieve payload_type** from capsule FAILED for ABI ID \"" + + std::string(AdapterType::abi_id()) + "\"") + .c_str()); + } + payload_pp() = static_cast(raw_ptr); + return *payload_pp(); + } + + static payload_type &get() { + payload_type *existing = get_existing(); + if (existing != nullptr) { + return *existing; + } + if (payload_pp() == nullptr) { + payload_pp() = new payload_type *(); + } + *payload_pp() = new payload_type(); + get_python_state_dict()[AdapterType::abi_id()] + = capsule(payload_pp(), AdapterType::abi_id()); + return **payload_pp(); + } + + struct scoped_clear { + // To be called BEFORE Py_Finalize(). + scoped_clear() { + payload_type *existing = get_existing(); + if (existing != nullptr) { + AdapterType::payload_clear(*existing); + arm_dtor = true; + } + } + + // To be called AFTER Py_Finalize(). + ~scoped_clear() { + if (arm_dtor) { + delete *payload_pp(); + *payload_pp() = nullptr; + } + } + + scoped_clear(const scoped_clear &) = delete; + scoped_clear &operator=(const scoped_clear &) = delete; + + bool arm_dtor = false; + }; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 3873aa9d..0c4d6643 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -9,34 +9,24 @@ #pragma once -#include "common.h" - -#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) -# include "../gil.h" -#endif - #include "../pytypes.h" +#include "abi_platform_id.h" +#include "common.h" +#include "cross_extension_shared_state.h" #include "smart_holder_sfinae_hooks_only.h" +#include "type_map.h" #include - -/// Tracks the `internals` and `type_info` ABI version independent of the main library version. -/// -/// Some portions of the code use an ABI that is conditional depending on this -/// version number. That allows ABI-breaking changes to be "pre-implemented". -/// Once the default version number is incremented, the conditional logic that -/// no longer applies can be removed. Additionally, users that need not -/// maintain ABI compatibility can increase the version number in order to take -/// advantage of any functionality/efficiency improvements that depend on the -/// newer ABI. -/// -/// WARNING: If you choose to manually increase the ABI version, note that -/// pybind11 may not be tested as thoroughly with a non-default ABI version, and -/// further ABI-incompatible changes may be made before the ABI is officially -/// changed to the new version. -#ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 4 -#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -109,42 +99,6 @@ inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) { # define PYBIND11_TLS_FREE(key) (void) key #endif -// Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly -// other STLs, this means `typeid(A)` from one module won't equal `typeid(A)` from another module -// even when `A` is the same, non-hidden-visibility type (e.g. from a common include). Under -// libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, -// which works. If not under a known-good stl, provide our own name-based hash and equality -// functions that use the type name. -#if defined(__GLIBCXX__) -inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } -using type_hash = std::hash; -using type_equal_to = std::equal_to; -#else -inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { - return lhs.name() == rhs.name() || std::strcmp(lhs.name(), rhs.name()) == 0; -} - -struct type_hash { - size_t operator()(const std::type_index &t) const { - size_t hash = 5381; - const char *ptr = t.name(); - while (auto c = static_cast(*ptr++)) { - hash = (hash * 33) ^ c; - } - return hash; - } -}; - -struct type_equal_to { - bool operator()(const std::type_index &lhs, const std::type_index &rhs) const { - return lhs.name() == rhs.name() || std::strcmp(lhs.name(), rhs.name()) == 0; - } -}; -#endif - -template -using type_map = std::unordered_map; - struct override_hash { inline size_t operator()(const std::pair &v) const { size_t value = std::hash()(v.first); @@ -240,83 +194,13 @@ struct type_info { bool module_local : 1; }; -/// On MSVC, debug and release builds are not ABI-compatible! -#if defined(_MSC_VER) && defined(_DEBUG) -# define PYBIND11_BUILD_TYPE "_debug" -#else -# define PYBIND11_BUILD_TYPE "" -#endif - -/// Let's assume that different compilers are ABI-incompatible. -/// A user can manually set this string if they know their -/// compiler is compatible. -#ifndef PYBIND11_COMPILER_TYPE -# if defined(_MSC_VER) -# define PYBIND11_COMPILER_TYPE "_msvc" -# elif defined(__INTEL_COMPILER) -# define PYBIND11_COMPILER_TYPE "_icc" -# elif defined(__clang__) -# define PYBIND11_COMPILER_TYPE "_clang" -# elif defined(__PGI) -# define PYBIND11_COMPILER_TYPE "_pgi" -# elif defined(__MINGW32__) -# define PYBIND11_COMPILER_TYPE "_mingw" -# elif defined(__CYGWIN__) -# define PYBIND11_COMPILER_TYPE "_gcc_cygwin" -# elif defined(__GNUC__) -# define PYBIND11_COMPILER_TYPE "_gcc" -# else -# define PYBIND11_COMPILER_TYPE "_unknown" -# endif -#endif - -/// Also standard libs -#ifndef PYBIND11_STDLIB -# if defined(_LIBCPP_VERSION) -# define PYBIND11_STDLIB "_libcpp" -# elif defined(__GLIBCXX__) || defined(__GLIBCPP__) -# define PYBIND11_STDLIB "_libstdcpp" -# else -# define PYBIND11_STDLIB "" -# endif -#endif - -/// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility. -#ifndef PYBIND11_BUILD_ABI -# if defined(__GXX_ABI_VERSION) -# define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) -# else -# define PYBIND11_BUILD_ABI "" -# endif -#endif - -#ifndef PYBIND11_INTERNALS_KIND -# if defined(WITH_THREAD) -# define PYBIND11_INTERNALS_KIND "" -# else -# define PYBIND11_INTERNALS_KIND "_without_thread" -# endif -#endif - -/// See README_smart_holder.rst: -/// Classic / Conservative / Progressive cross-module compatibility -#ifndef PYBIND11_INTERNALS_SH_DEF -# if defined(PYBIND11_USE_SMART_HOLDER_AS_DEFAULT) -# define PYBIND11_INTERNALS_SH_DEF "" -# else -# define PYBIND11_INTERNALS_SH_DEF "_sh_def" -# endif -#endif - #define PYBIND11_INTERNALS_ID \ "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \ - PYBIND11_BUILD_TYPE PYBIND11_INTERNALS_SH_DEF "__" + PYBIND11_PLATFORM_ABI_ID_V4 "__" #define PYBIND11_MODULE_LOCAL_ID \ "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \ - PYBIND11_BUILD_TYPE PYBIND11_INTERNALS_SH_DEF "__" + PYBIND11_PLATFORM_ABI_ID_V4 "__" /// Each module locally stores a pointer to the `internals` data. The data /// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`. @@ -432,27 +316,6 @@ inline void translate_local_exception(std::exception_ptr p) { } #endif -inline object get_python_state_dict() { - object state_dict; -#if (PYBIND11_INTERNALS_VERSION <= 4 && PY_VERSION_HEX < 0x030C0000) \ - || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) - state_dict = reinterpret_borrow(PyEval_GetBuiltins()); -#else -# if PY_VERSION_HEX < 0x03090000 - PyInterpreterState *istate = _PyInterpreterState_Get(); -# else - PyInterpreterState *istate = PyInterpreterState_Get(); -# endif - if (istate) { - state_dict = reinterpret_borrow(PyInterpreterState_GetDict(istate)); - } -#endif - if (!state_dict) { - raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); - } - return state_dict; -} - inline object get_internals_obj_from_state_dict(handle state_dict) { return reinterpret_borrow(dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID)); } @@ -472,21 +335,7 @@ PYBIND11_NOINLINE internals &get_internals() { return **internals_pp; } -#if defined(WITH_THREAD) -# if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) - gil_scoped_acquire gil; -# else - // Ensure that the GIL is held since we will need to make Python calls. - // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. - struct gil_scoped_acquire_local { - gil_scoped_acquire_local() : state(PyGILState_Ensure()) {} - gil_scoped_acquire_local(const gil_scoped_acquire_local &) = delete; - gil_scoped_acquire_local &operator=(const gil_scoped_acquire_local &) = delete; - ~gil_scoped_acquire_local() { PyGILState_Release(state); } - const PyGILState_STATE state; - } gil; -# endif -#endif + gil_scoped_acquire_simple gil; error_scope err_scope; dict state_dict = get_python_state_dict(); diff --git a/include/pybind11/detail/type_map.h b/include/pybind11/detail/type_map.h new file mode 100644 index 00000000..e899f3ea --- /dev/null +++ b/include/pybind11/detail/type_map.h @@ -0,0 +1,72 @@ +// Copyright (c) 2022 The pybind Community. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" + +#include +#include +#include +#include + +/* NOTE - ATTENTION - WARNING - EXTREME CAUTION + + Almost any changes here will break compatibility with `PYBIND11_INTERNALS_VERSION 4` + + Recommendation: + To not break compatibility with many existing PyPI wheels (https://pypi.org/), + DO NOT MAKE CHANGES HERE + until Python 3.11 (at least) has reached EOL. Then remove this file entirely. + + To evolve this code, start with a copy of this file and move the code to + namespace pybind11::cross_extension_shared_states + with new, versioned names, e.g. type_map_v2. + */ + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly +// other STLs, this means `typeid(A)` from one module won't equal `typeid(A)` from another module +// even when `A` is the same, non-hidden-visibility type (e.g. from a common include). Under +// libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, +// which works. If not under a known-good stl, provide our own name-based hash and equality +// functions that use the type name. +#if defined(__GLIBCXX__) + +inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } +using type_hash = std::hash; +using type_equal_to = std::equal_to; + +#else + +inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { + return lhs.name() == rhs.name() || std::strcmp(lhs.name(), rhs.name()) == 0; +} + +struct type_hash { + size_t operator()(const std::type_index &t) const { + size_t hash = 5381; + const char *ptr = t.name(); + while (auto c = static_cast(*ptr++)) { + hash = (hash * 33) ^ c; + } + return hash; + } +}; + +struct type_equal_to { + bool operator()(const std::type_index &lhs, const std::type_index &rhs) const { + return lhs.name() == rhs.name() || std::strcmp(lhs.name(), rhs.name()) == 0; + } +}; + +#endif + +template +using type_map = std::unordered_map; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index cceae2b2..3e9a9a87 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -48,8 +48,10 @@ } detail_headers = { + "include/pybind11/detail/abi_platform_id.h", "include/pybind11/detail/class.h", "include/pybind11/detail/common.h", + "include/pybind11/detail/cross_extension_shared_state.h", "include/pybind11/detail/descr.h", "include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h", "include/pybind11/detail/init.h", @@ -59,6 +61,7 @@ "include/pybind11/detail/smart_holder_type_casters.h", "include/pybind11/detail/type_caster_base.h", "include/pybind11/detail/type_caster_odr_guard.h", + "include/pybind11/detail/type_map.h", "include/pybind11/detail/typeid.h", }