From e76c2eb94a4140cf32264abdfb96e32e3e2f7763 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sun, 27 Dec 2020 17:56:52 +0000 Subject: [PATCH] abi3: add support for dict and weakref from Python 3.9 --- CHANGELOG.md | 3 ++ src/ffi/object.rs | 2 +- src/pycell.rs | 4 +- src/pyclass.rs | 93 +++++++++++++++++++++++++++++------ tests/test_dunder.rs | 11 +++-- tests/test_gc.rs | 4 +- tests/test_unsendable_dict.rs | 4 +- tests/test_various.rs | 2 +- 8 files changed, 96 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac2b3024909..3736561b72f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Add support for `#[pyclass(dict)]` and `#[pyclass(weakref)]` with the `abi3` feature on Python 3.9 and up. [#1342](https://github.com/PyO3/pyo3/pull/1342) + ### Fixed - Stop including `Py_TRACE_REFS` config setting automatically if `Py_DEBUG` is set on Python 3.8 and up. [#1334](https://github.com/PyO3/pyo3/pull/1334) - Remove `#[deny(warnings)]` attribute (and instead refuse warnings only in CI). [#1340](https://github.com/PyO3/pyo3/pull/1340) diff --git a/src/ffi/object.rs b/src/ffi/object.rs index aa7ec8d79a6..b16e1a12f8e 100644 --- a/src/ffi/object.rs +++ b/src/ffi/object.rs @@ -706,7 +706,7 @@ extern "C" { arg2: *mut PyObject, arg3: *mut PyObject, ) -> c_int; - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(all(Py_LIMITED_API, not(Py_3_10))))] pub fn PyObject_GenericGetDict(arg1: *mut PyObject, arg2: *mut c_void) -> *mut PyObject; pub fn PyObject_GenericSetDict( arg1: *mut PyObject, diff --git a/src/pycell.rs b/src/pycell.rs index da6a73f998d..31c998ae8fa 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -170,7 +170,7 @@ pub struct PyCell { impl PyCell { /// Get the offset of the dictionary from the start of the struct in bytes. - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(all(Py_LIMITED_API, not(Py_3_9))))] pub(crate) fn dict_offset() -> Option { if T::Dict::IS_DUMMY { None @@ -184,7 +184,7 @@ impl PyCell { } /// Get the offset of the weakref list from the start of the struct in bytes. - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(all(Py_LIMITED_API, not(Py_3_9))))] pub(crate) fn weakref_offset() -> Option { if T::WeakRef::IS_DUMMY { None diff --git a/src/pyclass.rs b/src/pyclass.rs index 67f5ba6e660..b615345d293 100644 --- a/src/pyclass.rs +++ b/src/pyclass.rs @@ -178,6 +178,14 @@ where slots.maybe_push(ffi::Py_tp_new, new.map(|v| v as _)); slots.maybe_push(ffi::Py_tp_call, call.map(|v| v as _)); + #[cfg(Py_3_9)] + { + let members = py_class_members::(); + if !members.is_empty() { + slots.push(ffi::Py_tp_members, into_raw(members)) + } + } + // normal methods if !methods.is_empty() { slots.push(ffi::Py_tp_methods, into_raw(methods)); @@ -203,7 +211,7 @@ where basicsize: std::mem::size_of::() as c_int, itemsize: 0, flags: py_class_flags::(has_gc_methods), - slots: slots.0.as_mut_slice().as_mut_ptr(), + slots: slots.0.as_mut_ptr(), }; let type_object = unsafe { ffi::PyType_FromSpec(&mut spec) }; @@ -215,7 +223,8 @@ where } } -#[cfg(not(Py_LIMITED_API))] +/// Additional type initializations necessary before Python 3.10 +#[cfg(all(not(Py_LIMITED_API), not(Py_3_10)))] fn tp_init_additional(type_object: *mut ffi::PyTypeObject) { // Just patch the type objects for the things there's no // PyType_FromSpec API for... there's no reason this should work, @@ -247,21 +256,27 @@ fn tp_init_additional(type_object: *mut ffi::PyTypeObject) { (*(*type_object).tp_as_buffer).bf_releasebuffer = buffer.bf_releasebuffer; } } - // __dict__ support - if let Some(dict_offset) = PyCell::::dict_offset() { - unsafe { - (*type_object).tp_dictoffset = dict_offset as ffi::Py_ssize_t; + + // Setting tp_dictoffset and tp_weaklistoffset via slots doesn't work until Python 3.9, so on + // older versions again we must fixup the type object. + #[cfg(not(Py_3_9))] + { + // __dict__ support + if let Some(dict_offset) = PyCell::::dict_offset() { + unsafe { + (*type_object).tp_dictoffset = dict_offset as ffi::Py_ssize_t; + } } - } - // weakref support - if let Some(weakref_offset) = PyCell::::weakref_offset() { - unsafe { - (*type_object).tp_weaklistoffset = weakref_offset as ffi::Py_ssize_t; + // weakref support + if let Some(weakref_offset) = PyCell::::weakref_offset() { + unsafe { + (*type_object).tp_weaklistoffset = weakref_offset as ffi::Py_ssize_t; + } } } } -#[cfg(Py_LIMITED_API)] +#[cfg(any(Py_LIMITED_API, Py_3_10))] fn tp_init_additional(_type_object: *mut ffi::PyTypeObject) {} fn py_class_flags(has_gc_methods: bool) -> c_uint { @@ -333,6 +348,46 @@ fn py_class_method_defs() -> ( (new, call, defs) } +/// Generates the __dictoffset__ and __weaklistoffset__ members, to set tp_dictoffset and +/// tp_weaklistoffset. +/// +/// Only works on Python 3.9 and up. +#[cfg(Py_3_9)] +fn py_class_members() -> Vec { + static DICTOFFSET: &str = "__dictoffset__\0"; + static WEAKLISTOFFSET: &str = "__weaklistoffset__\0"; + + let mut members = Vec::new(); + + // __dict__ support + if let Some(dict_offset) = PyCell::::dict_offset() { + members.push(ffi::structmember::PyMemberDef { + name: DICTOFFSET.as_ptr() as _, + type_code: ffi::structmember::T_PYSSIZET, + offset: dict_offset as _, + flags: ffi::structmember::READONLY, + doc: std::ptr::null_mut(), + }); + } + + // weakref support + if let Some(weakref_offset) = PyCell::::weakref_offset() { + members.push(ffi::structmember::PyMemberDef { + name: WEAKLISTOFFSET.as_ptr() as _, + type_code: ffi::structmember::T_PYSSIZET, + offset: weakref_offset as _, + flags: ffi::structmember::READONLY, + doc: std::ptr::null_mut(), + }); + } + + if !members.is_empty() { + members.push(unsafe { std::mem::zeroed() }); + } + + members +} + fn py_class_properties() -> Vec { let mut defs = std::collections::HashMap::new(); @@ -357,11 +412,21 @@ fn py_class_properties() -> Vec { } let mut props: Vec<_> = defs.values().cloned().collect(); + + // PyPy doesn't automatically adds __dict__ getter / setter. + // PyObject_GenericGetDict not in the limited API until Python 3.10. + #[cfg(not(any(PyPy, all(Py_LIMITED_API, not(Py_3_10)))))] if !T::Dict::IS_DUMMY { - props.push(ffi::PyGetSetDef_DICT); + props.push(ffi::PyGetSetDef { + name: "__dict__\0".as_ptr() as *mut c_char, + get: Some(ffi::PyObject_GenericGetDict), + set: Some(ffi::PyObject_GenericSetDict), + doc: ptr::null_mut(), + closure: ptr::null_mut(), + }); } if !props.is_empty() { - props.push(ffi::PyGetSetDef_INIT); + props.push(unsafe { std::mem::zeroed() }); } props } diff --git a/tests/test_dunder.rs b/tests/test_dunder.rs index e229e448181..5285950f70b 100644 --- a/tests/test_dunder.rs +++ b/tests/test_dunder.rs @@ -457,7 +457,7 @@ fn test_cls_impl() { struct DunderDictSupport {} #[test] -#[cfg_attr(Py_LIMITED_API, ignore)] +#[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)] fn dunder_dict_support() { let gil = Python::acquire_gil(); let py = gil.python(); @@ -472,8 +472,9 @@ fn dunder_dict_support() { ); } +// Accessing inst.__dict__ only supported in limited API from Python 3.10 #[test] -#[cfg_attr(Py_LIMITED_API, ignore)] +#[cfg_attr(all(Py_LIMITED_API, not(Py_3_10)), ignore)] fn access_dunder_dict() { let gil = Python::acquire_gil(); let py = gil.python(); @@ -495,7 +496,7 @@ struct InheritDict { } #[test] -#[cfg_attr(Py_LIMITED_API, ignore)] +#[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)] fn inherited_dict() { let gil = Python::acquire_gil(); let py = gil.python(); @@ -505,7 +506,7 @@ fn inherited_dict() { inst, r#" inst.a = 1 - assert inst.__dict__ == {'a': 1} + assert inst.a == 1 "# ); } @@ -514,7 +515,7 @@ fn inherited_dict() { struct WeakRefDunderDictSupport {} #[test] -#[cfg_attr(Py_LIMITED_API, ignore)] +#[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)] fn weakref_dunder_dict_support() { let gil = Python::acquire_gil(); let py = gil.python(); diff --git a/tests/test_gc.rs b/tests/test_gc.rs index c7e4015a70d..9aa5252e100 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -150,7 +150,7 @@ fn gc_integration2() { struct WeakRefSupport {} #[test] -#[cfg_attr(Py_LIMITED_API, ignore)] +#[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)] fn weakref_support() { let gil = Python::acquire_gil(); let py = gil.python(); @@ -169,7 +169,7 @@ struct InheritWeakRef { } #[test] -#[cfg_attr(Py_LIMITED_API, ignore)] +#[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)] fn inherited_weakref() { let gil = Python::acquire_gil(); let py = gil.python(); diff --git a/tests/test_unsendable_dict.rs b/tests/test_unsendable_dict.rs index 86c974135d8..4222bb2c805 100644 --- a/tests/test_unsendable_dict.rs +++ b/tests/test_unsendable_dict.rs @@ -13,7 +13,7 @@ impl UnsendableDictClass { } #[test] -#[cfg_attr(Py_LIMITED_API, ignore)] +#[cfg_attr(all(Py_LIMITED_API, not(Py_3_10)), ignore)] fn test_unsendable_dict() { let gil = Python::acquire_gil(); let py = gil.python(); @@ -33,7 +33,7 @@ impl UnsendableDictClassWithWeakRef { } #[test] -#[cfg_attr(Py_LIMITED_API, ignore)] +#[cfg_attr(all(Py_LIMITED_API, not(Py_3_10)), ignore)] fn test_unsendable_dict_with_weakref() { let gil = Python::acquire_gil(); let py = gil.python(); diff --git a/tests/test_various.rs b/tests/test_various.rs index 25c5e7bcb97..3c9003729e6 100644 --- a/tests/test_various.rs +++ b/tests/test_various.rs @@ -149,7 +149,7 @@ fn add_module(py: Python, module: &PyModule) -> PyResult<()> { } #[test] -#[cfg_attr(Py_LIMITED_API, ignore)] +#[cfg_attr(all(Py_LIMITED_API, not(Py_3_10)), ignore)] fn test_pickle() { let gil = Python::acquire_gil(); let py = gil.python();