convert &'py PyList -> PyList<'py>

examples/plugin/src/main.rs

@@ -13,7 +13,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     //do useful work
     Python::with_gil(|py| {
         //add the current directory to import path of Python (do not use this in production!)
-        let syspath: &PyList = py.import("sys")?.getattr("path")?.extract()?;
+        let syspath: PyList<'_> = py.import("sys")?.getattr("path")?.extract()?;
         syspath.insert(0, &path)?;
         println!("Import path is: {:?}", syspath);
 

guide/src/conversions/tables.md

@@ -19,7 +19,7 @@ The table below contains the Python type and the corresponding function argument
 | `int`         | Any integer type (`i32`, `u32`, `usize`, etc) | `&PyLong` |
 | `float`       | `f32`, `f64`                    | `&PyFloat`           |
 | `complex`     | `num_complex::Complex`[^1]      | `&PyComplex`         |
-| `list[T]`     | `Vec<T>`                        | `&PyList`            |
+| `list[T]`     | `Vec<T>`                        | `PyList<'py>`        |
 | `dict[K, V]`  | `HashMap<K, V>`, `BTreeMap<K, V>`, `hashbrown::HashMap<K, V>`[^2], `indexmap::IndexMap<K, V>`[^3] | `&PyDict` |
 | `tuple[T, U]` | `(T, U)`, `Vec<T>`              | `&PyTuple`           |
 | `set[T]`      | `HashSet<T>`, `BTreeSet<T>`, `hashbrown::HashSet<T>`[^2] | `&PySet` |

guide/src/python_from_rust.md

@@ -406,7 +406,7 @@ fn main() -> PyResult<()> {
     let path = Path::new("/usr/share/python_app");
     let py_app = fs::read_to_string(path.join("app.py"))?;
     let from_python = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
-        let syspath: &PyList = py.import("sys")?.getattr("path")?.downcast()?;
+        let syspath: PyList<'_> = py.import("sys")?.getattr("path")?.downcast()?;
         syspath.insert(0, &path)?;
         let app: Py<PyAny> = PyModule::from_code(py, &py_app, "", "")?
             .getattr("run")?

src/conversion.rs

@@ -642,7 +642,7 @@ mod tests {
             let list: &PyAny = vec![3, 6, 5, 4, 7].to_object(py).into_ref(py);
             let dict: &PyAny = vec![("reverse", true)].into_py_dict(py).as_ref();
 
-            assert!(<PyList as PyTryFrom<'_>>::try_from(list).is_ok());
+            assert!(<PyList<'_> as PyTryFrom<'_>>::try_from(list).is_ok());
             assert!(<PyDict as PyTryFrom<'_>>::try_from(dict).is_ok());
 
             assert!(<PyAny as PyTryFrom<'_>>::try_from(list).is_ok());
@@ -656,7 +656,7 @@ mod tests {
             let list: &PyAny = vec![3, 6, 5, 4, 7].to_object(py).into_ref(py);
             let dict: &PyAny = vec![("reverse", true)].into_py_dict(py).as_ref();
 
-            assert!(PyList::try_from_exact(list).is_ok());
+            assert!(PyList::<'_>::try_from_exact(list).is_ok());
             assert!(PyDict::try_from_exact(dict).is_ok());
 
             assert!(PyAny::try_from_exact(list).is_err());
@@ -668,7 +668,7 @@ mod tests {
     fn test_try_from_unchecked() {
         Python::with_gil(|py| {
             let list = PyList::new(py, [1, 2, 3]);
-            let val = unsafe { <PyList as PyTryFrom>::try_from_unchecked(list.as_ref()) };
+            let val = unsafe { <PyList<'_> as PyTryFrom>::try_from_unchecked(list.as_ref()) };
             assert!(list.is(val));
         });
     }

src/conversions/std/array.rs

@@ -185,11 +185,11 @@ mod tests {
         Python::with_gil(|py| {
             let array: [f32; 4] = [0.0, -16.0, 16.0, 42.0];
             let pyobject = array.to_object(py);
-            let pylist: &PyList = pyobject.extract(py).unwrap();
-            assert_eq!(pylist[0].extract::<f32>().unwrap(), 0.0);
-            assert_eq!(pylist[1].extract::<f32>().unwrap(), -16.0);
-            assert_eq!(pylist[2].extract::<f32>().unwrap(), 16.0);
-            assert_eq!(pylist[3].extract::<f32>().unwrap(), 42.0);
+            let pylist: PyList<'_> = pyobject.extract(py).unwrap();
+            assert_eq!(pylist.get_item(0).unwrap().extract::<f32>().unwrap(), 0.0);
+            assert_eq!(pylist.get_item(1).unwrap().extract::<f32>().unwrap(), -16.0);
+            assert_eq!(pylist.get_item(2).unwrap().extract::<f32>().unwrap(), 16.0);
+            assert_eq!(pylist.get_item(3).unwrap().extract::<f32>().unwrap(), 42.0);
         });
     }
 
@@ -212,11 +212,11 @@ mod tests {
         Python::with_gil(|py| {
             let array: [f32; 4] = [0.0, -16.0, 16.0, 42.0];
             let pyobject = array.into_py(py);
-            let pylist: &PyList = pyobject.extract(py).unwrap();
-            assert_eq!(pylist[0].extract::<f32>().unwrap(), 0.0);
-            assert_eq!(pylist[1].extract::<f32>().unwrap(), -16.0);
-            assert_eq!(pylist[2].extract::<f32>().unwrap(), 16.0);
-            assert_eq!(pylist[3].extract::<f32>().unwrap(), 42.0);
+            let pylist: PyList<'_> = pyobject.extract(py).unwrap();
+            assert_eq!(pylist.get_item(0).unwrap().extract::<f32>().unwrap(), 0.0);
+            assert_eq!(pylist.get_item(1).unwrap().extract::<f32>().unwrap(), -16.0);
+            assert_eq!(pylist.get_item(2).unwrap().extract::<f32>().unwrap(), 16.0);
+            assert_eq!(pylist.get_item(3).unwrap().extract::<f32>().unwrap(), 42.0);
         });
     }
 
@@ -237,8 +237,8 @@ mod tests {
 
         Python::with_gil(|py| {
             let array: [Foo; 8] = [Foo, Foo, Foo, Foo, Foo, Foo, Foo, Foo];
-            let pyobject = array.into_py(py);
-            let list: &PyList = pyobject.downcast(py).unwrap();
+            let pyobject = array.into_py(py).into_owned(py);
+            let list: &PyList<'_> = pyobject.downcast().unwrap();
             let _cell: &crate::PyCell<Foo> = list.get_item(4).unwrap().extract().unwrap();
         });
     }

src/instance.rs

@@ -3,7 +3,7 @@ use crate::err::{self, PyDowncastError, PyErr, PyResult};
 use crate::gil;
 use crate::pycell::{PyBorrowError, PyBorrowMutError, PyCell};
 use crate::pyclass::boolean_struct::{False, True};
-use crate::types::{PyDict, PyString, PyTuple};
+use crate::types::{PyAnyOwned, PyDict, PyString, PyTuple};
 use crate::{
     ffi, AsPyPointer, FromPyObject, IntoPy, IntoPyPointer, PyAny, PyClass, PyClassInitializer,
     PyRef, PyRefMut, PyTypeInfo, Python, ToPyObject,
@@ -285,7 +285,7 @@ where
     /// #
     /// Python::with_gil(|py| {
     ///     let list: Py<PyList> = PyList::empty(py).into();
-    ///     let list: &PyList = list.as_ref(py);
+    ///     let list: &PyList<'_> = list.as_ref(py);
     ///     assert_eq!(list.len(), 0);
     /// });
     /// ```
@@ -871,13 +871,13 @@ impl<T> Py<T> {
     /// # Safety
     /// `ptr` must point to a Python object of type T.
     #[inline]
-    unsafe fn from_non_null(ptr: NonNull<ffi::PyObject>) -> Self {
+    pub(crate) unsafe fn from_non_null(ptr: NonNull<ffi::PyObject>) -> Self {
         Self(ptr, PhantomData)
     }
 
     /// Returns the inner pointer without decreasing the refcount.
     #[inline]
-    fn into_non_null(self) -> NonNull<ffi::PyObject> {
+    pub(crate) fn into_non_null(self) -> NonNull<ffi::PyObject> {
         let pointer = self.0;
         mem::forget(self);
         pointer
@@ -1106,6 +1106,10 @@ impl PyObject {
     {
         <T as PyTryFrom<'_>>::try_from_unchecked(self.as_ref(py))
     }
+
+    pub(crate) fn into_owned(self, py: Python<'_>) -> PyAnyOwned<'_> {
+        unsafe { PyAnyOwned::from_non_null(py, self.into_non_null()) }
+    }
 }
 
 #[cfg(test)]

src/marker.rs

@@ -1056,7 +1056,7 @@ mod tests {
 
     #[test]
     fn test_allow_threads_pass_stuff_in() {
-        let list: Py<PyList> = Python::with_gil(|py| {
+        let list: Py<PyList<'static>> = Python::with_gil(|py| {
             let list = PyList::new(py, vec!["foo", "bar"]);
             list.into()
         });

src/sync.rs

@@ -70,7 +70,7 @@ unsafe impl<T> Sync for GILProtected<T> where T: Send {}
 ///
 /// static LIST_CELL: GILOnceCell<Py<PyList>> = GILOnceCell::new();
 ///
-/// pub fn get_shared_list(py: Python<'_>) -> &PyList {
+/// pub fn get_shared_list(py: Python<'_>) -> &PyList<'_> {
 ///     LIST_CELL
 ///         .get_or_init(py, || PyList::empty(py).into())
 ///         .as_ref(py)

src/test_hygiene/pymethods.rs

@@ -76,7 +76,7 @@ impl Dummy {
 
     fn __delattr__(&mut self, name: ::std::string::String) {}
 
-    fn __dir__<'py>(&self, py: crate::Python<'py>) -> &'py crate::types::PyList {
+    fn __dir__<'py>(&self, py: crate::Python<'py>) -> crate::types::PyList<'py> {
         crate::types::PyList::new(py, ::std::vec![0_u8])
     }
 

src/types/any.rs

@@ -10,6 +10,7 @@ use crate::{err, ffi, Py, PyNativeType, PyObject, Python};
 use std::cell::UnsafeCell;
 use std::cmp::Ordering;
 use std::os::raw::c_int;
+use std::ptr::NonNull;
 
 /// Represents any Python object.
 ///
@@ -36,6 +37,12 @@ use std::os::raw::c_int;
 #[repr(transparent)]
 pub struct PyAny(UnsafeCell<ffi::PyObject>);
 
+// TODO: this will become the main PyAny type in PyO3 0.20, with normal ownership
+// rather than GIL-bound pool semantics.
+//
+// All other types to be migrated in individual PRs first.
+pub(crate) struct PyAnyOwned<'py>(NonNull<ffi::PyObject>, Python<'py>);
+
 impl crate::AsPyPointer for PyAny {
     #[inline]
     fn as_ptr(&self) -> *mut ffi::PyObject {
@@ -934,8 +941,8 @@ impl PyAny {
     /// Returns the list of attributes of this object.
     ///
     /// This is equivalent to the Python expression `dir(self)`.
-    pub fn dir(&self) -> &PyList {
-        unsafe { self.py().from_owned_ptr(ffi::PyObject_Dir(self.as_ptr())) }
+    pub fn dir(&self) -> PyList<'_> {
+        unsafe { PyList::from_owned_ptr(self.py(), ffi::PyObject_Dir(self.as_ptr())) }
     }
 
     /// Checks whether this object is an instance of type `ty`.
@@ -1007,6 +1014,116 @@ impl PyAny {
     }
 }
 
+impl<'py> PyAnyOwned<'py> {
+    /// Create a `PyAny` by taking ownership of the given FFI pointer.
+    ///
+    /// # Safety
+    /// `ptr` must be a pointer to a Python object of type T.
+    ///
+    /// Callers must own the object referred to by `ptr`, as this function
+    /// implicitly takes ownership of that object.
+    ///
+    /// # Panics
+    /// Panics if `ptr` is null.
+    #[inline]
+    pub unsafe fn from_owned_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self {
+        match NonNull::new(ptr) {
+            Some(nonnull_ptr) => Self(nonnull_ptr, py),
+            None => crate::err::panic_after_error(py),
+        }
+    }
+
+    /// Create a new `PyAny` from a raw pointer.
+    ///
+    /// # Safety
+    ///
+    /// `ptr` must be an owned non-null pointer to a Python object.
+    #[inline]
+    pub unsafe fn from_owned_ptr_or_err(
+        py: Python<'py>,
+        ptr: *mut ffi::PyObject,
+    ) -> PyResult<Self> {
+        match NonNull::new(ptr) {
+            Some(non_null) => Ok(Self(non_null, py)),
+            None => Err(PyErr::fetch(py)),
+        }
+    }
+
+    /// Create a new `PyAny` from a raw pointer.
+    ///
+    /// # Safety
+    ///
+    /// `ptr` must be an owned non-null pointer to a Python object.
+    #[inline]
+    pub unsafe fn from_borrowed_ptr_or_err(
+        py: Python<'py>,
+        ptr: *mut ffi::PyObject,
+    ) -> PyResult<Self> {
+        Self::from_borrowed_ptr_or_opt(py, ptr).ok_or_else(|| PyErr::fetch(py))
+    }
+
+    /// Create a new `PyAny` from a raw pointer.
+    ///
+    /// # Safety
+    ///
+    /// `ptr` must be an owned non-null pointer to a Python object.
+    #[inline]
+    pub(crate) unsafe fn from_borrowed_ptr_or_opt(
+        py: Python<'py>,
+        ptr: *mut ffi::PyObject,
+    ) -> Option<Self> {
+        NonNull::new(ptr).map(|non_null| {
+            ffi::Py_INCREF(ptr);
+            Self(non_null, py)
+        })
+    }
+
+    #[inline]
+    pub fn py(&self) -> Python<'py> {
+        self.1
+    }
+
+    /// For internal conversions.
+    #[inline]
+    pub(crate) unsafe fn into_non_null(self) -> NonNull<ffi::PyObject> {
+        let ptr = self.0;
+        std::mem::forget(self);
+        ptr
+    }
+
+    /// For internal conversions.
+    #[inline]
+    pub(crate) unsafe fn from_non_null(py: Python<'py>, ptr: NonNull<ffi::PyObject>) -> Self {
+        Self(ptr, py)
+    }
+}
+
+impl Clone for PyAnyOwned<'_> {
+    #[inline]
+    fn clone(&self) -> Self {
+        unsafe { ffi::Py_INCREF(self.0.as_ptr()) };
+        Self(self.0.clone(), self.1.clone())
+    }
+}
+
+impl Drop for PyAnyOwned<'_> {
+    #[inline]
+    fn drop(&mut self) {
+        unsafe { ffi::Py_DECREF(self.0.as_ptr()) };
+    }
+}
+
+// This helps with backwards-compatibility between PyAnyOwned -> PyAny,
+// for the purposes of migrating.
+impl std::ops::Deref for PyAnyOwned<'_> {
+    type Target = PyAny;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        unsafe { self.py().from_borrowed_ptr(self.0.as_ptr()) }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use crate::{
@@ -1149,7 +1266,7 @@ class SimpleClass:
             let dir = py
                 .eval("dir(42)", None, None)
                 .unwrap()
-                .downcast::<PyList>()
+                .downcast::<PyList<'_>>()
                 .unwrap();
             let a = obj
                 .dir()
@@ -1175,15 +1292,15 @@ class SimpleClass:
             assert!(x.is_instance_of::<PyLong>());
 
             let l = vec![x, x].to_object(py).into_ref(py);
-            assert!(l.is_instance_of::<PyList>());
+            assert!(l.is_instance_of::<PyList<'_>>());
         });
     }
 
     #[test]
     fn test_any_is_instance() {
         Python::with_gil(|py| {
             let l = vec![1u8, 2].to_object(py).into_ref(py);
-            assert!(l.is_instance(py.get_type::<PyList>()).unwrap());
+            assert!(l.is_instance(py.get_type::<PyList<'_>>()).unwrap());
         });
     }
 
@@ -1199,7 +1316,7 @@ class SimpleClass:
             assert!(t.is_exact_instance_of::<PyBool>());
 
             let l = vec![x, x].to_object(py).into_ref(py);
-            assert!(l.is_exact_instance_of::<PyList>());
+            assert!(l.is_exact_instance_of::<PyList<'_>>());
         });
     }
 

src/types/dict.rs

@@ -213,31 +213,22 @@ impl PyDict {
     /// Returns a list of dict keys.
     ///
     /// This is equivalent to the Python expression `list(dict.keys())`.
-    pub fn keys(&self) -> &PyList {
-        unsafe {
-            self.py()
-                .from_owned_ptr::<PyList>(ffi::PyDict_Keys(self.as_ptr()))
-        }
+    pub fn keys(&self) -> PyList<'_> {
+        unsafe { PyList::from_owned_ptr(self.py(), ffi::PyDict_Keys(self.as_ptr())) }
     }
 
     /// Returns a list of dict values.
     ///
     /// This is equivalent to the Python expression `list(dict.values())`.
-    pub fn values(&self) -> &PyList {
-        unsafe {
-            self.py()
-                .from_owned_ptr::<PyList>(ffi::PyDict_Values(self.as_ptr()))
-        }
+    pub fn values(&self) -> PyList<'_> {
+        unsafe { PyList::from_owned_ptr(self.py(), ffi::PyDict_Values(self.as_ptr())) }
     }
 
     /// Returns a list of dict items.
     ///
     /// This is equivalent to the Python expression `list(dict.items())`.
-    pub fn items(&self) -> &PyList {
-        unsafe {
-            self.py()
-                .from_owned_ptr::<PyList>(ffi::PyDict_Items(self.as_ptr()))
-        }
+    pub fn items(&self) -> PyList<'_> {
+        unsafe { PyList::from_owned_ptr(self.py(), ffi::PyDict_Items(self.as_ptr())) }
     }
 
     /// Returns an iterator of `(key, value)` pairs in this dictionary.