From 0e378a62cd1624d8b3192639c7f0772c838428ce Mon Sep 17 00:00:00 2001
From: Tom Dyas <tom.dyas@gmail.com>
Date: Fri, 4 Oct 2024 20:08:52 -0400
Subject: [PATCH 1/2] PyO3: continue porting to `Bound` smart pointer

---
 src/rust/engine/src/externs/interface.rs      | 32 ++++---
 src/rust/engine/src/externs/mod.rs            | 94 +++++++++----------
 src/rust/engine/src/externs/options.rs        |  2 +-
 src/rust/engine/src/interning.rs              |  8 +-
 .../engine/src/intrinsics/dep_inference.rs    |  4 +-
 src/rust/engine/src/intrinsics/digests.rs     |  2 +-
 .../src/intrinsics/interactive_process.rs     |  8 +-
 src/rust/engine/src/intrinsics/process.rs     |  4 +-
 src/rust/engine/src/nodes/downloaded_file.rs  |  2 +-
 src/rust/engine/src/nodes/execute_process.rs  | 14 ++-
 src/rust/engine/src/nodes/mod.rs              | 30 +++---
 src/rust/engine/src/nodes/snapshot.rs         | 10 +-
 src/rust/engine/src/nodes/task.rs             |  6 +-
 src/rust/engine/src/python.rs                 | 69 +++++---------
 14 files changed, 127 insertions(+), 158 deletions(-)

diff --git a/src/rust/engine/src/externs/interface.rs b/src/rust/engine/src/externs/interface.rs
index 0c625c09054..aa661da0294 100644
--- a/src/rust/engine/src/externs/interface.rs
+++ b/src/rust/engine/src/externs/interface.rs
@@ -599,7 +599,7 @@ fn nailgun_server_create(
             Python::with_gil(|py| {
                 let result = runner.as_ref(py).call1((
                     exe.cmd.command,
-                    PyTuple::new(py, exe.cmd.args),
+                    PyTuple::new_bound(py, exe.cmd.args),
                     exe.cmd.env.into_iter().collect::<HashMap<String, String>>(),
                     exe.cmd.working_dir,
                     PySessionCancellationLatch(exe.cancelled),
@@ -1084,7 +1084,10 @@ fn scheduler_live_items<'py>(
     let (items, sizes) = core
         .executor
         .enter(|| py.allow_threads(|| scheduler.live_items(session)));
-    let py_items = items.into_iter().map(|value| value.to_object(py)).collect();
+    let py_items = items
+        .into_iter()
+        .map(|value| value.bind(py).to_object(py))
+        .collect();
     (py_items, sizes)
 }
 
@@ -1434,15 +1437,15 @@ fn validate_reachability(py_scheduler: &Bound<'_, PyScheduler>) -> PyO3Result<()
 #[pyfunction]
 fn rule_graph_consumed_types<'py>(
     py: Python<'py>,
-    py_scheduler: &PyScheduler,
-    param_types: Vec<&PyType>,
+    py_scheduler: &Bound<'py, PyScheduler>,
+    param_types: Vec<Bound<'py, PyType>>,
     product_type: &Bound<'_, PyType>,
-) -> PyO3Result<Vec<&'py PyType>> {
-    let core = &py_scheduler.0.core;
+) -> PyO3Result<Vec<Bound<'py, PyType>>> {
+    let core = &py_scheduler.borrow().0.core;
     core.executor.enter(|| {
         let param_types = param_types
             .into_iter()
-            .map(|t| TypeId::new(&t.as_borrowed()))
+            .map(|t| TypeId::new(&t))
             .collect::<Vec<_>>();
         let subgraph = core
             .rule_graph
@@ -1452,7 +1455,7 @@ fn rule_graph_consumed_types<'py>(
         Ok(subgraph
             .consumed_types()
             .into_iter()
-            .map(|type_id| type_id.as_py_type(py))
+            .map(|type_id| type_id.as_py_type_bound(py))
             .collect())
     })
 }
@@ -1484,10 +1487,10 @@ fn rule_graph_rule_gets<'py>(
                 let provided_params = dependency_key
                     .provided_params
                     .iter()
-                    .map(|p| p.as_py_type(py))
+                    .map(|p| p.as_py_type_bound(py))
                     .collect::<Vec<_>>();
                 dependencies.push((
-                    dependency_key.product.as_py_type(py),
+                    dependency_key.product.as_py_type_bound(py),
                     provided_params,
                     function.0.value.into_py(py),
                 ));
@@ -1638,10 +1641,11 @@ fn capture_snapshots(
             .into_iter()
             .map(|value| {
                 let root: PathBuf = externs::getattr_bound(&value, "root")?;
-                let path_globs = nodes::Snapshot::lift_prepared_path_globs(externs::getattr_bound(
-                    &value,
-                    "path_globs",
-                )?);
+                let path_globs = {
+                    let path_globs_py_value =
+                        externs::getattr_bound::<Bound<'_, PyAny>>(&value, "path_globs")?;
+                    nodes::Snapshot::lift_prepared_path_globs_bound(&path_globs_py_value)
+                };
                 let digest_hint = {
                     let maybe_digest: Bound<'_, PyAny> =
                         externs::getattr_bound(&value, "digest_hint")?;
diff --git a/src/rust/engine/src/externs/mod.rs b/src/rust/engine/src/externs/mod.rs
index 6b729fbb06d..d21bc96a7d0 100644
--- a/src/rust/engine/src/externs/mod.rs
+++ b/src/rust/engine/src/externs/mod.rs
@@ -70,7 +70,7 @@ pub struct PyFailure(pub Failure);
 impl PyFailure {
     fn get_error(&self, py: Python) -> PyErr {
         match &self.0 {
-            Failure::Throw { val, .. } => PyErr::from_value(val.as_ref().as_ref(py)),
+            Failure::Throw { val, .. } => PyErr::from_value_bound(val.bind(py).to_owned()),
             f @ (Failure::Invalidated | Failure::MissingDigest { .. }) => {
                 EngineError::new_err(format!("{f}"))
             }
@@ -82,12 +82,12 @@ impl PyFailure {
 // additional fields. See https://github.com/PyO3/pyo3/issues/295
 import_exception!(pants.base.exceptions, NativeEngineFailure);
 
-pub fn equals(h1: &PyAny, h2: &PyAny) -> bool {
+pub fn equals(h1: &Bound<'_, PyAny>, h2: &Bound<'_, PyAny>) -> bool {
     // NB: Although it does not precisely align with Python's definition of equality, we ban matches
     // between non-equal types to avoid legacy behavior like `assert True == 1`, which is very
     // surprising in interning, and would likely be surprising anywhere else in the engine where we
     // compare things.
-    if !h1.get_type().is(h2.get_type()) {
+    if !h1.get_type().is(&h2.get_type()) {
         return false;
     }
     h1.eq(h2).unwrap()
@@ -96,7 +96,7 @@ pub fn equals(h1: &PyAny, h2: &PyAny) -> bool {
 /// Return true if the given type is a @union.
 ///
 /// This function is also implemented in Python as `pants.engine.union.is_union`.
-pub fn is_union(py: Python, v: &PyType) -> PyResult<bool> {
+pub fn is_union(py: Python, v: &Bound<'_, PyType>) -> PyResult<bool> {
     let is_union_for_attr = intern!(py, "_is_union_for");
     if !v.hasattr(is_union_for_attr)? {
         return Ok(false);
@@ -109,15 +109,15 @@ pub fn is_union(py: Python, v: &PyType) -> PyResult<bool> {
 /// If the given type is a @union, return its in-scope types.
 ///
 /// This function is also implemented in Python as `pants.engine.union.union_in_scope_types`.
-pub fn union_in_scope_types<'p>(
-    py: Python<'p>,
-    v: &'p PyType,
-) -> PyResult<Option<Vec<&'p PyType>>> {
+pub fn union_in_scope_types<'py>(
+    py: Python<'py>,
+    v: &Bound<'py, PyType>,
+) -> PyResult<Option<Vec<Bound<'py, PyType>>>> {
     if !is_union(py, v)? {
         return Ok(None);
     }
 
-    let union_in_scope_types: Vec<&PyType> =
+    let union_in_scope_types: Vec<Bound<'_, PyType>> =
         v.getattr(intern!(py, "_union_in_scope_types"))?.extract()?;
     Ok(Some(union_in_scope_types))
 }
@@ -127,7 +127,7 @@ pub fn store_tuple(py: Python, values: Vec<Value>) -> Value {
         .into_iter()
         .map(|v| v.consume_into_py_object(py))
         .collect();
-    Value::from(PyTuple::new(py, &arg_handles).to_object(py))
+    Value::from(PyTuple::new_bound(py, &arg_handles).to_object(py))
 }
 
 /// Store a slice containing 2-tuples of (key, value) as a Python dictionary.
@@ -135,7 +135,7 @@ pub fn store_dict(
     py: Python,
     keys_and_values: impl IntoIterator<Item = (Value, Value)>,
 ) -> PyResult<Value> {
-    let dict = PyDict::new(py);
+    let dict = PyDict::new_bound(py);
     for (k, v) in keys_and_values {
         dict.set_item(k.consume_into_py_object(py), v.consume_into_py_object(py))?;
     }
@@ -253,10 +253,6 @@ pub fn val_to_str_bound(obj: &Bound<'_, PyAny>) -> String {
     obj.str().unwrap().extract().unwrap()
 }
 
-pub fn val_to_str(obj: &PyAny) -> String {
-    val_to_str_bound(&obj.as_borrowed())
-}
-
 pub fn val_to_log_level_bound(obj: &Bound<'_, PyAny>) -> Result<log::Level, String> {
     let res: Result<PythonLogLevel, String> = getattr_bound(obj, "_level").and_then(|n: u64| {
         n.try_into()
@@ -306,29 +302,31 @@ pub(crate) fn generator_send(
     let (response_unhandled, maybe_thrown) = match input {
         GeneratorInput::Arg(arg) => {
             let response = generator
-                .getattr(py, intern!(py, "send"))?
-                .call1(py, (&*arg,));
+                .bind(py)
+                .getattr(intern!(py, "send"))?
+                .call1((&arg,));
             (response, None)
         }
         GeneratorInput::Err(err) => {
-            let throw_method = generator.getattr(py, intern!(py, "throw"))?;
+            let throw_method = generator.bind(py).getattr(intern!(py, "throw"))?;
             if err.is_instance_of::<NativeEngineFailure>(py) {
                 let throw = err
                     .value(py)
                     .getattr(intern!(py, "failure"))?
                     .extract::<PyRef<PyFailure>>()?
                     .get_error(py);
-                let response = throw_method.call1(py, (&throw,));
+                let response = throw_method.call1((&throw,));
                 (response, Some((throw, err)))
             } else {
-                let response = throw_method.call1(py, (err,));
+                let response = throw_method.call1((err,));
                 (response, None)
             }
         }
         GeneratorInput::Initial => {
             let response = generator
-                .getattr(py, intern!(py, "send"))?
-                .call1(py, (&py.None(),));
+                .bind(py)
+                .getattr(intern!(py, "send"))?
+                .call1((&py.None(),));
             (response, None)
         }
     };
@@ -354,15 +352,15 @@ pub(crate) fn generator_send(
         Ok(r) => r,
     };
 
-    let result = if let Ok(call) = response.extract::<PyRef<PyGeneratorResponseCall>>(py) {
+    let result = if let Ok(call) = response.extract::<PyRef<PyGeneratorResponseCall>>() {
         Ok(GeneratorResponse::Call(call.take()?))
-    } else if let Ok(get) = response.extract::<PyRef<PyGeneratorResponseGet>>(py) {
+    } else if let Ok(get) = response.extract::<PyRef<PyGeneratorResponseGet>>() {
         // It isn't necessary to differentiate between `Get` and `Effect` here, as the static
         // analysis of `@rule`s has already validated usage.
         Ok(GeneratorResponse::Get(get.take()?))
-    } else if let Ok(call) = response.extract::<PyRef<PyGeneratorResponseNativeCall>>(py) {
+    } else if let Ok(call) = response.extract::<PyRef<PyGeneratorResponseNativeCall>>() {
         Ok(GeneratorResponse::NativeCall(call.take()?))
-    } else if let Ok(get_multi) = response.downcast::<PySequence>(py) {
+    } else if let Ok(get_multi) = response.downcast::<PySequence>() {
         // Was an `All` or `MultiGet`.
         let gogs = get_multi
             .iter()?
@@ -370,8 +368,8 @@ pub(crate) fn generator_send(
                 let gog = gog?;
                 // TODO: Find a better way to check whether something is a coroutine... this seems
                 // unnecessarily awkward.
-                if gog.is_instance(generator_type.as_py_type(py).into())? {
-                    Ok(GetOrGenerator::Generator(Value::new(gog.into())))
+                if gog.is_instance(&generator_type.as_py_type_bound(py))? {
+                    Ok(GetOrGenerator::Generator(Value::new(gog.unbind())))
                 } else if let Ok(get) = gog.extract::<PyRef<PyGeneratorResponseGet>>() {
                     Ok(GetOrGenerator::Get(
                         get.take().map_err(PyException::new_err)?,
@@ -397,8 +395,8 @@ pub(crate) fn generator_send(
 /// NB: Panics on failure. Only recommended for use with built-in types, such as
 /// those configured in types::Types.
 pub fn unsafe_call(py: Python, type_id: TypeId, args: &[Value]) -> Value {
-    let py_type = type_id.as_py_type(py);
-    let args_tuple = PyTuple::new(py, args.iter().map(|v| v.to_object(py)));
+    let py_type = type_id.as_py_type_bound(py);
+    let args_tuple = PyTuple::new_bound(py, args.iter().map(|v| v.bind(py)));
     let res = py_type.call1(args_tuple).unwrap_or_else(|e| {
         panic!(
             "Core type constructor `{}` failed: {:?}",
@@ -418,8 +416,8 @@ lazy_static! {
 #[allow(clippy::type_complexity)]
 fn interpret_get_inputs(
     py: Python,
-    input_arg0: Option<&PyAny>,
-    input_arg1: Option<&PyAny>,
+    input_arg0: Option<Bound<'_, PyAny>>,
+    input_arg1: Option<Bound<'_, PyAny>>,
 ) -> PyResult<(SmallVec<[TypeId; 2]>, SmallVec<[Key; 2]>)> {
     match (input_arg0, input_arg1) {
         (None, None) => Ok((smallvec![], smallvec![])),
@@ -479,7 +477,7 @@ fn interpret_get_inputs(
             }
 
             let actual_type = input_arg1.get_type();
-            if !declared_type.is(actual_type) && !is_union(py, declared_type)? {
+            if !declared_type.is(&actual_type) && !is_union(py, declared_type)? {
                 return Err(PyTypeError::new_err(format!(
           "Invalid Get. The third argument `{input_arg1}` must have the exact same type as the \
           second argument, {declared_type}, but had the type {actual_type}."
@@ -487,7 +485,7 @@ fn interpret_get_inputs(
             }
 
             Ok((
-                smallvec![TypeId::new(&declared_type.as_borrowed())],
+                smallvec![TypeId::new(declared_type)],
                 smallvec![INTERNS.key_insert(py, input_arg1.into())?],
             ))
         }
@@ -526,7 +524,7 @@ impl PyGeneratorResponseNativeCall {
 
     fn send(&self, py: Python, value: PyObject) -> PyResult<()> {
         Err(PyStopIteration::new_err(
-            PyTuple::new(py, [value]).to_object(py),
+            PyTuple::new_bound(py, [value]).to_object(py),
         ))
     }
 }
@@ -551,9 +549,9 @@ impl PyGeneratorResponseCall {
         py: Python,
         rule_id: String,
         output_type: &Bound<'_, PyType>,
-        args: &PyTuple,
-        input_arg0: Option<&PyAny>,
-        input_arg1: Option<&PyAny>,
+        args: &Bound<'_, PyTuple>,
+        input_arg0: Option<Bound<'_, PyAny>>,
+        input_arg1: Option<Bound<'_, PyAny>>,
     ) -> PyResult<Self> {
         let output_type = TypeId::new(output_type);
         let (args, args_arity) = if args.is_empty() {
@@ -579,17 +577,17 @@ impl PyGeneratorResponseCall {
     }
 
     #[getter]
-    fn output_type<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyType> {
-        Ok(self.borrow_inner()?.output_type.as_py_type(py))
+    fn output_type<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyType>> {
+        Ok(self.borrow_inner()?.output_type.as_py_type_bound(py))
     }
 
     #[getter]
-    fn input_types<'p>(&'p self, py: Python<'p>) -> PyResult<Vec<&'p PyType>> {
+    fn input_types<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py, PyType>>> {
         Ok(self
             .borrow_inner()?
             .input_types
             .iter()
-            .map(|t| t.as_py_type(py))
+            .map(|t| t.as_py_type_bound(py))
             .collect())
     }
 
@@ -635,8 +633,8 @@ impl PyGeneratorResponseGet {
     fn __new__(
         py: Python,
         product: &Bound<'_, PyAny>,
-        input_arg0: Option<&PyAny>,
-        input_arg1: Option<&PyAny>,
+        input_arg0: Option<Bound<'_, PyAny>>,
+        input_arg1: Option<Bound<'_, PyAny>>,
     ) -> PyResult<Self> {
         let product = product.downcast::<PyType>().map_err(|_| {
             let actual_type = product.get_type();
@@ -657,7 +655,7 @@ impl PyGeneratorResponseGet {
     }
 
     #[getter]
-    fn output_type<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyType> {
+    fn output_type<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyType>> {
         Ok(self
             .0
             .borrow()
@@ -668,11 +666,11 @@ impl PyGeneratorResponseGet {
                 )
             })?
             .output
-            .as_py_type(py))
+            .as_py_type_bound(py))
     }
 
     #[getter]
-    fn input_types<'p>(&'p self, py: Python<'p>) -> PyResult<Vec<&'p PyType>> {
+    fn input_types<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py, PyType>>> {
         Ok(self
             .0
             .borrow()
@@ -684,7 +682,7 @@ impl PyGeneratorResponseGet {
             })?
             .input_types
             .iter()
-            .map(|t| t.as_py_type(py))
+            .map(|t| t.as_py_type_bound(py))
             .collect())
     }
 
diff --git a/src/rust/engine/src/externs/options.rs b/src/rust/engine/src/externs/options.rs
index 33de3d5d471..c4684cda982 100644
--- a/src/rust/engine/src/externs/options.rs
+++ b/src/rust/engine/src/externs/options.rs
@@ -34,7 +34,7 @@ fn val_to_py_object(py: Python, val: &Val) -> PyResult<PyObject> {
             pylist.into_py(py)
         }
         Val::Dict(dict) => {
-            let pydict = PyDict::new(py);
+            let pydict = PyDict::new_bound(py);
             for (k, v) in dict {
                 pydict.set_item(k.into_py(py), val_to_py_object(py, v)?)?;
             }
diff --git a/src/rust/engine/src/interning.rs b/src/rust/engine/src/interning.rs
index 1b83861b1ae..e8df2e9051c 100644
--- a/src/rust/engine/src/interning.rs
+++ b/src/rust/engine/src/interning.rs
@@ -45,15 +45,15 @@ pub struct Interns {
 impl Interns {
     pub fn new() -> Self {
         Self {
-            keys: Python::with_gil(|py| PyDict::new(py).into()),
+            keys: Python::with_gil(|py| PyDict::new_bound(py).unbind()),
             id_generator: atomic::AtomicU64::default(),
         }
     }
 
     pub fn key_insert(&self, py: Python, v: PyObject) -> PyResult<Key> {
         let (id, type_id): (u64, TypeId) = {
-            let v = v.as_ref(py);
-            let keys = self.keys.as_ref(py);
+            let v = v.bind(py);
+            let keys = self.keys.bind(py);
             let id: u64 = if let Some(key) = keys.get_item(v)? {
                 key.extract()?
             } else {
@@ -61,7 +61,7 @@ impl Interns {
                 keys.set_item(v, id)?;
                 id
             };
-            (id, v.get_type().into())
+            (id, TypeId::new(&v.get_type()))
         };
 
         Ok(Key::new(id, type_id, v.into()))
diff --git a/src/rust/engine/src/intrinsics/dep_inference.rs b/src/rust/engine/src/intrinsics/dep_inference.rs
index 2ed8e259fa9..a14de4830fb 100644
--- a/src/rust/engine/src/intrinsics/dep_inference.rs
+++ b/src/rust/engine/src/intrinsics/dep_inference.rs
@@ -16,7 +16,7 @@ use protos::gen::pants::cache::{
     dependency_inference_request, CacheKey, CacheKeyType, DependencyInferenceRequest,
 };
 use pyo3::prelude::{pyfunction, wrap_pyfunction, PyModule, PyResult, Python, ToPyObject};
-use pyo3::types::PyModuleMethods;
+use pyo3::types::{PyAnyMethods, PyModuleMethods};
 use pyo3::{Bound, IntoPy};
 use store::Store;
 use workunit_store::{in_workunit, Level};
@@ -54,7 +54,7 @@ impl PreparedInferenceRequest {
         let PyNativeDependenciesRequest {
             directory_digest,
             metadata,
-        } = Python::with_gil(|py| deps_request.extract(py))?;
+        } = Python::with_gil(|py| deps_request.bind(py).extract())?;
 
         let (path, digest) = Self::find_one_file(directory_digest, store, backend).await?;
         let str_path = path.display().to_string();
diff --git a/src/rust/engine/src/intrinsics/digests.rs b/src/rust/engine/src/intrinsics/digests.rs
index ef87546fe77..f092355e60b 100644
--- a/src/rust/engine/src/intrinsics/digests.rs
+++ b/src/rust/engine/src/intrinsics/digests.rs
@@ -364,7 +364,7 @@ fn digest_subset_to_digest(digest_subset: Value) -> PyGeneratorResponseNativeCal
 fn path_metadata_request(single_path: Value) -> PyGeneratorResponseNativeCall {
     PyGeneratorResponseNativeCall::new(async move {
         let subject_path = Python::with_gil(|py| -> Result<_, String> {
-            let arg = (*single_path).bind(py);
+            let arg = single_path.bind(py);
             let path = externs::getattr_as_optional_string_bound(arg, "path")
                 .map_err(|e| format!("Failed to get `path` for field: {e}"))?;
             let path = path.ok_or_else(|| "Path must not be `None`.".to_string())?;
diff --git a/src/rust/engine/src/intrinsics/interactive_process.rs b/src/rust/engine/src/intrinsics/interactive_process.rs
index d4dffa4518f..17af501ab58 100644
--- a/src/rust/engine/src/intrinsics/interactive_process.rs
+++ b/src/rust/engine/src/intrinsics/interactive_process.rs
@@ -12,7 +12,7 @@ use process_execution::local::{
 };
 use process_execution::{ManagedChild, ProcessExecutionStrategy};
 use pyo3::prelude::{pyfunction, wrap_pyfunction, PyAny, PyModule, PyResult, Python};
-use pyo3::types::PyModuleMethods;
+use pyo3::types::{PyAnyMethods, PyModuleMethods};
 use pyo3::Bound;
 use stdio::TryCloneAsFile;
 use tokio::process;
@@ -56,9 +56,9 @@ pub async fn interactive_process_inner(
         Value,
         externs::process::PyProcessExecutionEnvironment,
     ) = Python::with_gil(|py| {
-        let py_interactive_process = interactive_process.as_ref().as_ref(py);
-        let py_process: Value = externs::getattr(py_interactive_process, "process").unwrap();
-        let process_config = process_config.as_ref().as_ref(py).extract().unwrap();
+        let py_interactive_process = interactive_process.bind(py);
+        let py_process: Value = externs::getattr_bound(py_interactive_process, "process").unwrap();
+        let process_config = process_config.bind(py).extract().unwrap();
         (
             py_interactive_process.extract().unwrap(),
             py_process,
diff --git a/src/rust/engine/src/intrinsics/process.rs b/src/rust/engine/src/intrinsics/process.rs
index e9ce55f2eb2..f2d8bbd9ba5 100644
--- a/src/rust/engine/src/intrinsics/process.rs
+++ b/src/rust/engine/src/intrinsics/process.rs
@@ -5,7 +5,7 @@ use std::time::Duration;
 
 use futures::future::TryFutureExt;
 use futures::try_join;
-use pyo3::types::{PyModule, PyModuleMethods};
+use pyo3::types::{PyAnyMethods, PyModule, PyModuleMethods};
 use pyo3::{pyfunction, wrap_pyfunction, Bound, IntoPy, PyResult, Python};
 
 use crate::externs::{self, PyGeneratorResponseNativeCall};
@@ -24,7 +24,7 @@ fn execute_process(process: Value, process_config: Value) -> PyGeneratorResponse
         let context = task_get_context();
 
         let process_config: externs::process::PyProcessExecutionEnvironment =
-            Python::with_gil(|py| process_config.extract(py)).map_err(|e| format!("{e}"))?;
+            Python::with_gil(|py| process_config.bind(py).extract()).map_err(|e| format!("{e}"))?;
         let process_request = ExecuteProcess::lift(&context.core.store(), process, process_config)
             .map_err(|e| e.enrich("Error lifting Process"))
             .await?;
diff --git a/src/rust/engine/src/nodes/downloaded_file.rs b/src/rust/engine/src/nodes/downloaded_file.rs
index d0f0c499606..d44bb8e22e8 100644
--- a/src/rust/engine/src/nodes/downloaded_file.rs
+++ b/src/rust/engine/src/nodes/downloaded_file.rs
@@ -98,7 +98,7 @@ impl DownloadedFile {
         let (url_str, expected_digest, auth_headers, retry_delay_duration, max_attempts) =
             Python::with_gil(|py| {
                 let py_download_file_val = self.0.to_value();
-                let py_download_file = (*py_download_file_val).bind(py);
+                let py_download_file = py_download_file_val.bind(py);
                 let url_str: String = externs::getattr_bound(py_download_file, "url")
                     .map_err(|e| format!("Failed to get `url` for field: {e}"))?;
                 let auth_headers =
diff --git a/src/rust/engine/src/nodes/execute_process.rs b/src/rust/engine/src/nodes/execute_process.rs
index 86acc60eb7a..6cb503f9a4e 100644
--- a/src/rust/engine/src/nodes/execute_process.rs
+++ b/src/rust/engine/src/nodes/execute_process.rs
@@ -19,7 +19,7 @@ use workunit_store::{
     Metric, ObservationMetric, RunningWorkunit, UserMetadataItem, WorkunitMetadata,
 };
 
-use super::{lift_directory_digest, lift_directory_digest_bound, NodeKey, NodeOutput, NodeResult};
+use super::{lift_directory_digest_bound, NodeKey, NodeOutput, NodeResult};
 use crate::context::Context;
 use crate::externs;
 use crate::python::{throw, Value};
@@ -37,9 +37,13 @@ impl ExecuteProcess {
         value: &Value,
     ) -> Result<InputDigests, StoreError> {
         let input_digests_fut: Result<_, String> = Python::with_gil(|py| {
-            let value = (**value).bind(py);
-            let input_files = lift_directory_digest(externs::getattr_bound(value, "input_digest")?)
-                .map_err(|err| format!("Error parsing input_digest {err}"))?;
+            let value = value.bind(py);
+            let input_files = {
+                let input_files_py_value: Bound<'_, PyAny> =
+                    externs::getattr_bound(value, "input_digest")?;
+                lift_directory_digest_bound(&input_files_py_value)
+                    .map_err(|err| format!("Error parsing input_digest {err}"))?
+            };
             let immutable_inputs = externs::getattr_from_str_frozendict_bound::<Bound<PyAny>>(
                 value,
                 "immutable_input_digests",
@@ -164,7 +168,7 @@ impl ExecuteProcess {
     ) -> Result<Self, StoreError> {
         let input_digests = Self::lift_process_input_digests(store, &value).await?;
         let process = Python::with_gil(|py| {
-            Self::lift_process_fields((*value).bind(py), input_digests, process_config)
+            Self::lift_process_fields(value.bind(py), input_digests, process_config)
         })?;
         Ok(Self { process })
     }
diff --git a/src/rust/engine/src/nodes/mod.rs b/src/rust/engine/src/nodes/mod.rs
index 66f7540b099..ddd3ab0beb6 100644
--- a/src/rust/engine/src/nodes/mod.rs
+++ b/src/rust/engine/src/nodes/mod.rs
@@ -17,8 +17,8 @@ use graph::{Node, NodeError};
 use internment::Intern;
 use process_execution::{self, ProcessCacheScope};
 use pyo3::prelude::{PyAny, Python};
-use pyo3::types::PyAnyMethods;
-use pyo3::{Bound, PyNativeType};
+use pyo3::types::{PyAnyMethods, PyTypeMethods};
+use pyo3::Bound;
 use rule_graph::{DependencyKey, Query};
 use store::{self, StoreFileByDigest};
 use workunit_store::{in_workunit, Level};
@@ -225,10 +225,6 @@ pub fn lift_directory_digest_bound(digest: &Bound<'_, PyAny>) -> Result<Director
     Ok(py_digest.0)
 }
 
-pub fn lift_directory_digest(digest: &PyAny) -> Result<DirectoryDigest, String> {
-    lift_directory_digest_bound(&digest.as_borrowed())
-}
-
 pub fn lift_file_digest_bound(digest: &Bound<'_, PyAny>) -> Result<hashing::Digest, String> {
     let py_file_digest: externs::fs::PyFileDigest = digest.extract().map_err(|e| format!("{e}"))?;
     Ok(py_file_digest.0)
@@ -402,7 +398,7 @@ impl NodeKey {
 
                 let displayable_param_names: Vec<_> = Python::with_gil(|py| {
                     Self::engine_aware_params(context, py, &task.params)
-                        .filter_map(|k| EngineAwareParameter::debug_hint((*k.value).bind(py)))
+                        .filter_map(|k| EngineAwareParameter::debug_hint(k.value.bind(py)))
                         .collect()
                 });
 
@@ -474,11 +470,15 @@ impl NodeKey {
         py: Python<'a>,
         params: &'a Params,
     ) -> impl Iterator<Item = &'a Key> + 'a {
-        let engine_aware_param_ty = context.core.types.engine_aware_parameter.as_py_type(py);
+        let engine_aware_param_ty = context
+            .core
+            .types
+            .engine_aware_parameter
+            .as_py_type_bound(py);
         params.keys().filter(move |key| {
             key.type_id()
-                .as_py_type(py)
-                .is_subclass(engine_aware_param_ty)
+                .as_py_type_bound(py)
+                .is_subclass(&engine_aware_param_ty)
                 .unwrap_or(false)
         })
     }
@@ -508,7 +508,7 @@ impl Node for NodeKey {
                 if let Some(params) = maybe_params {
                     Python::with_gil(|py| {
                         Self::engine_aware_params(&context, py, params)
-                            .flat_map(|k| EngineAwareParameter::metadata((*k.value).bind(py)))
+                            .flat_map(|k| EngineAwareParameter::metadata(k.value.bind(py)))
                             .collect()
                     })
                 } else {
@@ -607,7 +607,7 @@ impl Node for NodeKey {
             }
             (NodeKey::Task(ref t), NodeOutput::Value(ref v)) if t.task.engine_aware_return_type => {
                 Python::with_gil(|py| {
-                    EngineAwareReturnType::is_cacheable((**v).bind(py)).unwrap_or(true)
+                    EngineAwareReturnType::is_cacheable(v.bind(py)).unwrap_or(true)
                 })
             }
             _ => true,
@@ -656,11 +656,7 @@ impl Display for NodeKey {
                     Python::with_gil(|py| {
                         task.params
                             .keys()
-                            .filter_map(|k| {
-                                EngineAwareParameter::debug_hint(
-                                    k.to_value().clone_ref(py).bind(py),
-                                )
-                            })
+                            .filter_map(|k| EngineAwareParameter::debug_hint(k.to_value().bind(py)))
                             .collect::<Vec<_>>()
                     })
                 };
diff --git a/src/rust/engine/src/nodes/snapshot.rs b/src/rust/engine/src/nodes/snapshot.rs
index 6e5ee5315f3..a04b42ea797 100644
--- a/src/rust/engine/src/nodes/snapshot.rs
+++ b/src/rust/engine/src/nodes/snapshot.rs
@@ -11,7 +11,7 @@ use fs::{
 use futures::TryFutureExt;
 use graph::CompoundNode;
 use pyo3::prelude::{Py, PyAny, Python};
-use pyo3::{Bound, IntoPy, PyNativeType};
+use pyo3::{Bound, IntoPy};
 
 use super::{unmatched_globs_additional_context, NodeKey, NodeOutput, NodeResult};
 use crate::context::Context;
@@ -59,10 +59,6 @@ impl Snapshot {
         Ok(PathGlobs::new(globs, strict_glob_matching, conjunction))
     }
 
-    pub fn lift_path_globs(item: &PyAny) -> Result<PathGlobs, String> {
-        Self::lift_path_globs_bound(&item.as_borrowed())
-    }
-
     pub fn lift_prepared_path_globs_bound(
         item: &Bound<'_, PyAny>,
     ) -> Result<PreparedPathGlobs, String> {
@@ -72,10 +68,6 @@ impl Snapshot {
             .map_err(|e| format!("Failed to parse PathGlobs for globs({item:?}): {e}"))
     }
 
-    pub fn lift_prepared_path_globs(item: &PyAny) -> Result<PreparedPathGlobs, String> {
-        Self::lift_prepared_path_globs_bound(&item.as_borrowed())
-    }
-
     pub fn store_directory_digest(py: Python, item: DirectoryDigest) -> Result<Value, String> {
         let py_digest = Py::new(py, externs::fs::PyDigest(item)).map_err(|e| format!("{e}"))?;
         Ok(Value::new(py_digest.into_py(py)))
diff --git a/src/rust/engine/src/nodes/task.rs b/src/rust/engine/src/nodes/task.rs
index 7610836c7d1..f9b1003a7fe 100644
--- a/src/rust/engine/src/nodes/task.rs
+++ b/src/rust/engine/src/nodes/task.rs
@@ -276,12 +276,12 @@ impl Task {
             &self.side_effected,
             async move {
                 Python::with_gil(|py| {
-                    let func = (*self.task.func.0.value).bind(py);
+                    let func = self.task.func.0.value.bind(py);
 
                     // If there are explicit positional arguments, apply any computed arguments as
                     // keywords. Otherwise, apply computed arguments as positional.
                     let res = if let Some(args) = args {
-                        let args = args.value.extract::<Bound<'_, PyTuple>>(py)?;
+                        let args = args.value.bind(py).extract::<Bound<'_, PyTuple>>()?;
                         let kwargs = PyDict::new_bound(py);
                         for ((name, _), value) in self
                             .task
@@ -332,7 +332,7 @@ impl Task {
 
         if self.task.engine_aware_return_type {
             Python::with_gil(|py| {
-                EngineAwareReturnType::update_workunit(workunit, (*result_val).bind(py))
+                EngineAwareReturnType::update_workunit(workunit, result_val.bind(py))
             })
         };
 
diff --git a/src/rust/engine/src/python.rs b/src/rust/engine/src/python.rs
index 4ffe93b9a78..b971bc68818 100644
--- a/src/rust/engine/src/python.rs
+++ b/src/rust/engine/src/python.rs
@@ -1,8 +1,6 @@
 // Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 // Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-use std::convert::AsRef;
-use std::ops::Deref;
 use std::sync::Arc;
 use std::{fmt, hash};
 
@@ -142,20 +140,13 @@ impl TypeId {
         unsafe { PyType::from_type_ptr(py, self.0).as_borrowed().to_owned() }
     }
 
-    pub fn as_py_type<'py>(&self, py: Python<'py>) -> &'py PyType {
-        // SAFETY: Dereferencing a pointer to a PyTypeObject is safe as long as the module defining the
-        // type is not unloaded. That is true today, but would not be if we implemented support for hot
-        // reloading of plugins.
-        unsafe { PyType::from_type_ptr(py, self.0) }
-    }
-
     pub fn is_union(&self) -> bool {
-        Python::with_gil(|py| externs::is_union(py, self.as_py_type(py)).unwrap())
+        Python::with_gil(|py| externs::is_union(py, &self.as_py_type_bound(py)).unwrap())
     }
 
     pub fn union_in_scope_types(&self) -> Option<Vec<TypeId>> {
         Python::with_gil(|py| {
-            externs::union_in_scope_types(py, self.as_py_type(py))
+            externs::union_in_scope_types(py, &self.as_py_type_bound(py))
                 .unwrap()
                 .map(|types| {
                     types
@@ -167,16 +158,11 @@ impl TypeId {
     }
 }
 
-impl From<&PyType> for TypeId {
-    fn from(py_type: &PyType) -> Self {
-        TypeId(py_type.as_type_ptr())
-    }
-}
-
 impl fmt::Debug for TypeId {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         Python::with_gil(|py| {
-            let name = self.as_py_type(py).name().unwrap();
+            let type_bound = self.as_py_type_bound(py);
+            let name = type_bound.name().unwrap();
             write!(f, "{name}")
         })
     }
@@ -206,12 +192,12 @@ impl Function {
     /// The function represented as `path.to.module:lineno:func_name`.
     pub fn full_name(&self) -> String {
         let (module, name, line_no) = Python::with_gil(|py| {
-            let obj = (*self.0.value).as_ref(py);
-            let module: String = externs::getattr(obj, "__module__").unwrap();
-            let name: String = externs::getattr(obj, "__name__").unwrap();
+            let obj = self.0.value.bind(py);
+            let module: String = externs::getattr_bound(obj, "__module__").unwrap();
+            let name: String = externs::getattr_bound(obj, "__name__").unwrap();
             // NB: this is a custom dunder method that Python code should populate before sending the
             // function (e.g. an `@rule`) through FFI.
-            let line_no: u64 = externs::getattr(obj, "__line_number__").unwrap();
+            let line_no: u64 = externs::getattr_bound(obj, "__line_number__").unwrap();
             (module, name, line_no)
         });
         format!("{module}:{line_no}:{name}")
@@ -334,31 +320,17 @@ impl workunit_store::Value for Value {
 
 impl PartialEq for Value {
     fn eq(&self, other: &Value) -> bool {
-        Python::with_gil(|py| externs::equals((*self.0).as_ref(py), (*other.0).as_ref(py)))
+        Python::with_gil(|py| externs::equals(self.bind(py), other.0.bind(py)))
     }
 }
 
 impl Eq for Value {}
 
-impl Deref for Value {
-    type Target = PyObject;
-
-    fn deref(&self) -> &PyObject {
-        &self.0
-    }
-}
-
-impl AsRef<PyObject> for Value {
-    fn as_ref(&self) -> &PyObject {
-        &self.0
-    }
-}
-
 impl fmt::Debug for Value {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         let repr = Python::with_gil(|py| {
-            let obj = (*self.0).as_ref(py);
-            externs::val_to_str(obj)
+            let obj = self.0.bind(py);
+            externs::val_to_str_bound(obj)
         });
         write!(f, "{repr}")
     }
@@ -486,7 +458,7 @@ impl Failure {
                 }) => {
                     // Preserve tracebacks (both engine and python) from upstream error by using any existing
                     // engine traceback and restoring the original python exception cause.
-                    py_err.set_cause(py, Some(PyErr::from_value((*val.0).as_ref(py))));
+                    py_err.set_cause(py, Some(PyErr::from_value_bound(val.0.bind(py).to_owned())));
                     (
             format!(
               "{python_traceback}\nDuring handling of the above exception, another exception occurred:\n\n"
@@ -505,22 +477,22 @@ impl Failure {
             .map(|traceback| traceback.to_object(py));
         let val = Value::from(py_err.into_py(py));
         let python_traceback = if let Some(tb) = maybe_ptraceback {
-            let locals = PyDict::new(py);
+            let locals = PyDict::new_bound(py);
             locals
                 .set_item("traceback", py.import("traceback").unwrap())
                 .unwrap();
             locals.set_item("tb", tb).unwrap();
             locals.set_item("val", &val).unwrap();
-            py.eval(
+            py.eval_bound(
                 "''.join(traceback.format_exception(None, value=val, tb=tb))",
                 None,
-                Some(locals),
+                Some(&locals),
             )
             .unwrap()
             .extract::<String>()
             .unwrap()
         } else {
-            Self::native_traceback(&externs::val_to_str((*val).as_ref(py)))
+            Self::native_traceback(&externs::val_to_str_bound(val.bind(py)))
         };
         Failure::Throw {
             val,
@@ -536,7 +508,10 @@ impl Failure {
 
 impl Failure {
     fn from_wrapped_failure(py: Python, py_err: &PyErr) -> Option<Failure> {
-        match py_err.value(py).downcast::<externs::NativeEngineFailure>() {
+        match py_err
+            .value_bound(py)
+            .downcast::<externs::NativeEngineFailure>()
+        {
             Ok(n_e_failure) => {
                 let failure = n_e_failure
                     .getattr("failure")
@@ -559,8 +534,8 @@ impl fmt::Display for Failure {
             }
             Failure::Throw { val, .. } => {
                 let repr = Python::with_gil(|py| {
-                    let obj = (*val.0).as_ref(py);
-                    externs::val_to_str(obj)
+                    let obj = val.0.bind(py);
+                    externs::val_to_str_bound(obj)
                 });
                 write!(f, "{repr}")
             }

From c42671ff8ef348f95252f4824f171bdae4309eb5 Mon Sep 17 00:00:00 2001
From: Tom Dyas <tom.dyas@gmail.com>
Date: Fri, 4 Oct 2024 21:38:15 -0400
Subject: [PATCH 2/2] disable `gil-refs` feature on `pyo3` crate

---
 src/rust/engine/Cargo.toml                    |  2 +-
 src/rust/engine/src/externs/address.rs        | 24 +++++++++++-----
 src/rust/engine/src/externs/fs.rs             | 28 ++++++++++---------
 src/rust/engine/src/externs/interface.rs      |  4 +--
 src/rust/engine/src/externs/mod.rs            | 18 ++++++------
 src/rust/engine/src/externs/nailgun.rs        |  4 +--
 src/rust/engine/src/externs/options.rs        |  2 +-
 src/rust/engine/src/externs/scheduler.rs      |  2 +-
 src/rust/engine/src/externs/target.rs         |  4 +--
 .../src/intrinsics/interactive_process.rs     |  5 +++-
 src/rust/engine/src/nodes/execute_process.rs  |  8 ++++--
 src/rust/engine/src/python.rs                 | 12 +++++---
 12 files changed, 69 insertions(+), 44 deletions(-)

diff --git a/src/rust/engine/Cargo.toml b/src/rust/engine/Cargo.toml
index c812641d83c..ee267123014 100644
--- a/src/rust/engine/Cargo.toml
+++ b/src/rust/engine/Cargo.toml
@@ -294,7 +294,7 @@ prodash = { git = "https://github.com/stuhood/prodash", rev = "stuhood/raw-messa
 prost = "0.13"
 prost-build = "0.13"
 prost-types = "0.13"
-pyo3 = { version = "0.21", features = ["gil-refs"] }
+pyo3 = { version = "0.21" }
 pyo3-build-config = "0.21"
 rand = "0.8"
 regex = "1"
diff --git a/src/rust/engine/src/externs/address.rs b/src/rust/engine/src/externs/address.rs
index 5ae1f9afe48..83fe921f585 100644
--- a/src/rust/engine/src/externs/address.rs
+++ b/src/rust/engine/src/externs/address.rs
@@ -30,7 +30,10 @@ pub fn register(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
         "AddressParseException",
         py.get_type_bound::<AddressParseException>(),
     )?;
-    m.add("InvalidAddressError", py.get_type::<InvalidAddressError>())?;
+    m.add(
+        "InvalidAddressError",
+        py.get_type_bound::<InvalidAddressError>(),
+    )?;
     m.add(
         "InvalidSpecPathError",
         py.get_type_bound::<InvalidSpecPathError>(),
@@ -153,12 +156,19 @@ impl AddressInput {
         _cls: &Bound<'_, PyType>,
         spec: &str,
         description_of_origin: &str,
-        relative_to: Option<&str>,
-        subproject_roots: Option<Vec<&str>>,
+        relative_to: Option<String>,
+        subproject_roots: Option<Vec<String>>,
     ) -> PyResult<Self> {
-        let subproject_info = subproject_roots
-            .zip(relative_to)
-            .and_then(|(roots, relative_to)| split_on_longest_dir_prefix(relative_to, &roots));
+        let roots_as_strs = subproject_roots
+            .as_deref()
+            .map(|roots| roots.iter().map(|s| s.as_str()).collect::<Vec<_>>());
+        let relative_to2 = relative_to.as_deref();
+        let subproject_info = match (roots_as_strs, relative_to2) {
+            (Some(roots), Some(relative_to)) => {
+                split_on_longest_dir_prefix(relative_to, &roots[..])
+            }
+            _ => None,
+        };
 
         let parsed_spec =
             address::parse_address_spec(spec).map_err(AddressParseException::new_err)?;
@@ -173,7 +183,7 @@ impl AddressInput {
         let normalized_relative_to = if let Some((_, normalized_relative_to)) = subproject_info {
             Some(normalized_relative_to)
         } else {
-            relative_to
+            relative_to.as_deref()
         };
 
         let mut path_component: Cow<str> = address.path.into();
diff --git a/src/rust/engine/src/externs/fs.rs b/src/rust/engine/src/externs/fs.rs
index c453d41c70d..dc2b5a301f9 100644
--- a/src/rust/engine/src/externs/fs.rs
+++ b/src/rust/engine/src/externs/fs.rs
@@ -11,6 +11,7 @@ use itertools::Itertools;
 use pyo3::basic::CompareOp;
 use pyo3::exceptions::{PyException, PyTypeError, PyValueError};
 use pyo3::prelude::*;
+use pyo3::pybacked::PyBackedStr;
 use pyo3::types::{PyIterator, PyString, PyTuple, PyType};
 
 use fs::{
@@ -269,13 +270,12 @@ pub struct PyMergeDigests(pub Vec<DirectoryDigest>);
 impl PyMergeDigests {
     #[new]
     fn __new__(digests: &Bound<'_, PyAny>, _py: Python) -> PyResult<Self> {
-        let digests: PyResult<Vec<DirectoryDigest>> =
-            PyIterator::from_object(digests.as_gil_ref())?
-                .map(|v| {
-                    let py_digest = v?.extract::<PyDigest>()?;
-                    Ok(py_digest.0)
-                })
-                .collect();
+        let digests: PyResult<Vec<DirectoryDigest>> = PyIterator::from_bound_object(digests)?
+            .map(|v| {
+                let py_digest = v?.extract::<PyDigest>()?;
+                Ok(py_digest.0)
+            })
+            .collect();
         Ok(Self(digests?))
     }
 
@@ -415,16 +415,18 @@ impl<'py> FromPyObject<'py> for PyPathGlobs {
             Some(description_of_origin_field.extract()?)
         };
 
-        let match_behavior_str: &str = obj
+        let match_behavior_str: PyBackedStr = obj
             .getattr("glob_match_error_behavior")?
             .getattr("value")?
             .extract()?;
-        let match_behavior = StrictGlobMatching::create(match_behavior_str, description_of_origin)
-            .map_err(PyValueError::new_err)?;
+        let match_behavior =
+            StrictGlobMatching::create(match_behavior_str.as_ref(), description_of_origin)
+                .map_err(PyValueError::new_err)?;
 
-        let conjunction_str: &str = obj.getattr("conjunction")?.getattr("value")?.extract()?;
-        let conjunction =
-            GlobExpansionConjunction::create(conjunction_str).map_err(PyValueError::new_err)?;
+        let conjunction_str: PyBackedStr =
+            obj.getattr("conjunction")?.getattr("value")?.extract()?;
+        let conjunction = GlobExpansionConjunction::create(conjunction_str.as_ref())
+            .map_err(PyValueError::new_err)?;
 
         Ok(PyPathGlobs(PathGlobs::new(
             globs,
diff --git a/src/rust/engine/src/externs/interface.rs b/src/rust/engine/src/externs/interface.rs
index aa661da0294..1c9ee7b304c 100644
--- a/src/rust/engine/src/externs/interface.rs
+++ b/src/rust/engine/src/externs/interface.rs
@@ -74,7 +74,7 @@ fn native_engine(py: Python, m: &Bound<'_, PyModule>) -> PyO3Result<()> {
     externs::workunits::register(m)?;
     externs::dep_inference::register(m)?;
 
-    m.add("PollTimeout", py.get_type::<PollTimeout>())?;
+    m.add("PollTimeout", py.get_type_bound::<PollTimeout>())?;
 
     m.add_class::<PyExecutionRequest>()?;
     m.add_class::<PyExecutionStrategyOptions>()?;
@@ -597,7 +597,7 @@ fn nailgun_server_create(
         let executor = py_executor.0.clone();
         nailgun::Server::new(executor, port, move |exe: nailgun::RawFdExecution| {
             Python::with_gil(|py| {
-                let result = runner.as_ref(py).call1((
+                let result = runner.bind(py).call1((
                     exe.cmd.command,
                     PyTuple::new_bound(py, exe.cmd.args),
                     exe.cmd.env.into_iter().collect::<HashMap<String, String>>(),
diff --git a/src/rust/engine/src/externs/mod.rs b/src/rust/engine/src/externs/mod.rs
index d21bc96a7d0..d3943e1a396 100644
--- a/src/rust/engine/src/externs/mod.rs
+++ b/src/rust/engine/src/externs/mod.rs
@@ -48,11 +48,11 @@ pub fn register(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyGeneratorResponseCall>()?;
     m.add_class::<PyGeneratorResponseGet>()?;
 
-    m.add("EngineError", py.get_type::<EngineError>())?;
-    m.add("IntrinsicError", py.get_type::<IntrinsicError>())?;
+    m.add("EngineError", py.get_type_bound::<EngineError>())?;
+    m.add("IntrinsicError", py.get_type_bound::<IntrinsicError>())?;
     m.add(
         "IncorrectProductError",
-        py.get_type::<IncorrectProductError>(),
+        py.get_type_bound::<IncorrectProductError>(),
     )?;
 
     Ok(())
@@ -144,7 +144,7 @@ pub fn store_dict(
 
 /// Store an opaque buffer of bytes to pass to Python. This will end up as a Python `bytes`.
 pub fn store_bytes(py: Python, bytes: &[u8]) -> Value {
-    Value::from(PyBytes::new(py, bytes).to_object(py))
+    Value::from(PyBytes::new_bound(py, bytes).to_object(py))
 }
 
 /// Store a buffer of utf8 bytes to pass to Python. This will end up as a Python `str`.
@@ -269,7 +269,7 @@ pub fn val_to_log_level_bound(obj: &Bound<'_, PyAny>) -> Result<log::Level, Stri
 
 /// Link to the Pants docs using the current version of Pants.
 pub fn doc_url(py: Python, slug: &str) -> String {
-    let docutil_module = py.import("pants.util.docutil").unwrap();
+    let docutil_module = py.import_bound("pants.util.docutil").unwrap();
     let doc_url_func = docutil_module.getattr("doc_url").unwrap();
     doc_url_func.call1((slug,)).unwrap().extract().unwrap()
 }
@@ -311,7 +311,7 @@ pub(crate) fn generator_send(
             let throw_method = generator.bind(py).getattr(intern!(py, "throw"))?;
             if err.is_instance_of::<NativeEngineFailure>(py) {
                 let throw = err
-                    .value(py)
+                    .value_bound(py)
                     .getattr(intern!(py, "failure"))?
                     .extract::<PyRef<PyFailure>>()?
                     .get_error(py);
@@ -334,12 +334,14 @@ pub(crate) fn generator_send(
     let response = match response_unhandled {
         Err(e) if e.is_instance_of::<PyStopIteration>(py) => {
             let value = e.into_value(py).getattr(py, intern!(py, "value"))?;
-            let type_id = TypeId::new(&value.as_ref(py).get_type().as_borrowed());
+            let type_id = TypeId::new(&value.bind(py).get_type());
             return Ok(GeneratorResponse::Break(Value::new(value), type_id));
         }
         Err(e) => {
             match (maybe_thrown, e.cause(py)) {
-                (Some((thrown, err)), Some(cause)) if thrown.value(py).is(cause.value(py)) => {
+                (Some((thrown, err)), Some(cause))
+                    if thrown.value_bound(py).is(cause.value_bound(py)) =>
+                {
                     // Preserve the engine traceback by using the wrapped failure error as cause. The cause
                     // will be swapped back again in `Failure::from_py_err_with_gil()` to preserve the python
                     // traceback.
diff --git a/src/rust/engine/src/externs/nailgun.rs b/src/rust/engine/src/externs/nailgun.rs
index 21292c99bc4..59f71301004 100644
--- a/src/rust/engine/src/externs/nailgun.rs
+++ b/src/rust/engine/src/externs/nailgun.rs
@@ -12,11 +12,11 @@ use task_executor::Executor;
 pub fn register(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add(
         "PantsdConnectionException",
-        py.get_type::<PantsdConnectionException>(),
+        py.get_type_bound::<PantsdConnectionException>(),
     )?;
     m.add(
         "PantsdClientException",
-        py.get_type::<PantsdClientException>(),
+        py.get_type_bound::<PantsdClientException>(),
     )?;
     m.add_class::<PyNailgunClient>()?;
     Ok(())
diff --git a/src/rust/engine/src/externs/options.rs b/src/rust/engine/src/externs/options.rs
index c4684cda982..a9ad1a463c0 100644
--- a/src/rust/engine/src/externs/options.rs
+++ b/src/rust/engine/src/externs/options.rs
@@ -27,7 +27,7 @@ fn val_to_py_object(py: Python, val: &Val) -> PyResult<PyObject> {
         Val::Float(f) => f.into_py(py),
         Val::String(s) => s.into_py(py),
         Val::List(list) => {
-            let pylist = PyList::empty(py);
+            let pylist = PyList::empty_bound(py);
             for m in list {
                 pylist.append(val_to_py_object(py, m)?)?;
             }
diff --git a/src/rust/engine/src/externs/scheduler.rs b/src/rust/engine/src/externs/scheduler.rs
index 467d4f0c717..9f921e5f673 100644
--- a/src/rust/engine/src/externs/scheduler.rs
+++ b/src/rust/engine/src/externs/scheduler.rs
@@ -38,7 +38,7 @@ impl PyExecutor {
             let _ =
                 unsafe { ffi::PyThreadState_New(Python::with_gil(|_| PyInterpreterState_Main())) };
             Python::with_gil(|py| {
-                let _ = py.eval("__import__('debugpy').debug_this_thread()", None, None);
+                let _ = py.eval_bound("__import__('debugpy').debug_this_thread()", None, None);
             });
         })
         .map(PyExecutor)
diff --git a/src/rust/engine/src/externs/target.rs b/src/rust/engine/src/externs/target.rs
index eaf998f86e4..1cd6af6c0e1 100644
--- a/src/rust/engine/src/externs/target.rs
+++ b/src/rust/engine/src/externs/target.rs
@@ -141,7 +141,7 @@ impl Field {
     }
 
     fn __hash__(self_: &Bound<'_, Self>, py: Python) -> PyResult<isize> {
-        Ok(self_.get_type().hash()? & self_.borrow().value.as_ref(py).hash()?)
+        Ok(self_.get_type().hash()? & self_.borrow().value.bind(py).hash()?)
     }
 
     fn __repr__(self_: &Bound<'_, Self>) -> PyResult<String> {
@@ -180,7 +180,7 @@ impl Field {
             && self_
                 .borrow()
                 .value
-                .as_ref(py)
+                .bind(py)
                 .eq(&other.extract::<PyRef<Field>>()?.value)?;
         match op {
             CompareOp::Eq => Ok(is_eq.into_py(py)),
diff --git a/src/rust/engine/src/intrinsics/interactive_process.rs b/src/rust/engine/src/intrinsics/interactive_process.rs
index 17af501ab58..278af86f485 100644
--- a/src/rust/engine/src/intrinsics/interactive_process.rs
+++ b/src/rust/engine/src/intrinsics/interactive_process.rs
@@ -12,6 +12,7 @@ use process_execution::local::{
 };
 use process_execution::{ManagedChild, ProcessExecutionStrategy};
 use pyo3::prelude::{pyfunction, wrap_pyfunction, PyAny, PyModule, PyResult, Python};
+use pyo3::pybacked::PyBackedStr;
 use pyo3::types::{PyAnyMethods, PyModuleMethods};
 use pyo3::Bound;
 use stdio::TryCloneAsFile;
@@ -87,7 +88,9 @@ pub async fn interactive_process_inner(
         let keep_sandboxes_value: Bound<'_, PyAny> =
             externs::getattr_bound(py_interactive_process, "keep_sandboxes").unwrap();
         let keep_sandboxes = KeepSandboxes::from_str(
-            externs::getattr_bound(&keep_sandboxes_value, "value").unwrap(),
+            externs::getattr_bound::<PyBackedStr>(&keep_sandboxes_value, "value")
+                .unwrap()
+                .as_ref(),
         )
         .unwrap();
         (run_in_workspace, keep_sandboxes)
diff --git a/src/rust/engine/src/nodes/execute_process.rs b/src/rust/engine/src/nodes/execute_process.rs
index 6cb503f9a4e..a2c55b4785d 100644
--- a/src/rust/engine/src/nodes/execute_process.rs
+++ b/src/rust/engine/src/nodes/execute_process.rs
@@ -13,6 +13,7 @@ use process_execution::{
     ProcessResultSource,
 };
 use pyo3::prelude::{PyAny, Python};
+use pyo3::pybacked::PyBackedStr;
 use pyo3::Bound;
 use store::{self, Store, StoreError};
 use workunit_store::{
@@ -113,9 +114,12 @@ impl ExecuteProcess {
         let level = externs::val_to_log_level_bound(&py_level)?;
 
         let append_only_caches =
-            externs::getattr_from_str_frozendict_bound::<&str>(value, "append_only_caches")
+            externs::getattr_from_str_frozendict_bound::<PyBackedStr>(value, "append_only_caches")
                 .into_iter()
-                .map(|(name, dest)| Ok((CacheName::new(name)?, RelativePath::new(dest)?)))
+                .map(|(name, dest)| {
+                    let path: &str = dest.as_ref();
+                    Ok((CacheName::new(name)?, RelativePath::new(path)?))
+                })
                 .collect::<Result<_, String>>()?;
 
         let jdk_home = externs::getattr_as_optional_string_bound(value, "jdk_home")
diff --git a/src/rust/engine/src/python.rs b/src/rust/engine/src/python.rs
index b971bc68818..a251446ae89 100644
--- a/src/rust/engine/src/python.rs
+++ b/src/rust/engine/src/python.rs
@@ -137,7 +137,11 @@ impl TypeId {
         // SAFETY: Dereferencing a pointer to a PyTypeObject is safe as long as the module defining the
         // type is not unloaded. That is true today, but would not be if we implemented support for hot
         // reloading of plugins.
-        unsafe { PyType::from_type_ptr(py, self.0).as_borrowed().to_owned() }
+        unsafe {
+            PyType::from_borrowed_type_ptr(py, self.0)
+                .as_borrowed()
+                .to_owned()
+        }
     }
 
     pub fn is_union(&self) -> bool {
@@ -378,7 +382,7 @@ impl<'py, T> From<&Bound<'py, T>> for Value {
 
 impl IntoPy<PyObject> for &Value {
     fn into_py(self, py: Python) -> PyObject {
-        (*self.0).as_ref(py).into_py(py)
+        (*self.0).bind(py).into_py(py)
     }
 }
 
@@ -473,13 +477,13 @@ impl Failure {
         };
 
         let maybe_ptraceback = py_err
-            .traceback(py)
+            .traceback_bound(py)
             .map(|traceback| traceback.to_object(py));
         let val = Value::from(py_err.into_py(py));
         let python_traceback = if let Some(tb) = maybe_ptraceback {
             let locals = PyDict::new_bound(py);
             locals
-                .set_item("traceback", py.import("traceback").unwrap())
+                .set_item("traceback", py.import_bound("traceback").unwrap())
                 .unwrap();
             locals.set_item("tb", tb).unwrap();
             locals.set_item("val", &val).unwrap();