Skip to content

Commit

Permalink
gil: move finalization from prepare_freethreaded_python to
Browse files Browse the repository at this point in the history 
with_embedded_python_interpreter
  • Loading branch information
@davidhewitt
davidhewitt committed Jan 7, 2021
1 parent cbfb8c4 commit faae995
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 112 deletions.
4 changes: 2 additions & 2 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Add FFI definitions `PyOS_BeforeFork`, `PyOS_AfterFork_Parent`, `PyOS_AfterFork_Child` for Python 3.7 and up. [#1348](https://github.com/PyO3/pyo3/pull/1348)
- Add `auto-initialize` feature to control whether PyO3 should automatically initialize an embedded Python interpreter. For compatibility this feature is enabled by default in PyO3 0.13.1, but is planned to become opt-in from PyO3 0.14.0. [#1347](https://github.com/PyO3/pyo3/pull/1347)
- Add support for cross-compiling to Windows without needing `PYO3_CROSS_INCLUDE_DIR`. [#1350](https://github.com/PyO3/pyo3/pull/1350)
- Add `prepare_freethreaded_python_without_finalizer` to initalize a Python interpreter while avoiding potential finalization issues in C-extensions like SciPy. [#1355](https://github.com/PyO3/pyo3/pull/1355)
- Add unsafe API `with_embedded_python_interpreter` to initalize a Python interpreter, execute a closure, and finalize the interpreter. [#1355](https://github.com/PyO3/pyo3/pull/1355)

### Changed
- Deprecate FFI definitions `PyEval_CallObjectWithKeywords`, `PyEval_CallObject`, `PyEval_CallFunction`, `PyEval_CallMethod` when building for Python 3.9. [#1338](https://github.com/PyO3/pyo3/pull/1338)
Expand All @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Deprecate FFI definition `PyOS_AfterFork` for Python 3.7 and up. [#1348](https://github.com/PyO3/pyo3/pull/1348)
- Deprecate FFI definition `PyCoro_Check` and `PyAsyncGen_Check` in favor of `PyCoro_CheckExact` and `PyAsyncGen_CheckExact` respectively to mirror Python API. [#1348](https://github.com/PyO3/pyo3/pull/1348)
- Deprecate FFI definition `PyCoroWrapper_Check` which has never been in the Python API. [#1348](https://github.com/PyO3/pyo3/pull/1348)
- Call `Py_Finalize` in the same thread as `Py_InitializeEx` inside `prepare_freethreaded_python`. [#1350](https://github.com/PyO3/pyo3/pull/1350)
- `prepare_freethreaded_python` will no longer register an `atexit` handler to call `Py_Finalize`. [#1355](https://github.com/PyO3/pyo3/pull/1355)

### Removed
- Remove FFI definition `PyFrame_ClearFreeList` when building for Python 3.9. [#1341](https://github.com/PyO3/pyo3/pull/1341)
Expand Down
161 changes: 52 additions & 109 deletions src/gil.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,6 @@ pub(crate) fn gil_is_acquired() -> bool {
/// Python signal handling depends on the notion of a 'main thread', which must be
/// the thread that initializes the Python interpreter.
///
/// Additionally, this function will register an `atexit` callback to finalize the Python
/// interpreter. Usually this is desirable - it flushes Python buffers and ensures that all
/// Python objects are cleaned up appropriately. Some C extensions can have memory issues during
/// finalization, so you may get crashes on program exit if your embedded Python program uses
/// these extensions. If this is the case, you should use [`prepare_freethreaded_python_without_finalizer`]
/// which does not register the `atexit` callback.
///
/// If both the Python interpreter and Python threading are already initialized,
/// this function has no effect.
///
Expand Down Expand Up @@ -90,88 +83,26 @@ pub fn prepare_freethreaded_python() {
// as we can't make the existing Python main thread acquire the GIL.
assert_ne!(ffi::PyEval_ThreadsInitialized(), 0);
} else {
use parking_lot::Condvar;

// Initialize Python.
// We use Py_InitializeEx() with initsigs=0 to disable Python signal handling.
// Signal handling depends on the notion of a 'main thread', which doesn't exist in this case.
// Note that the 'main thread' notion in Python isn't documented properly;
// and running Python without one is not officially supported.
static INITIALIZATION_THREAD_SIGNAL: Condvar = Condvar::new();
static INITIALIZATION_THREAD_MUTEX: Mutex<()> = const_mutex(());

// This thread will be responsible for initialization and finalization of the
// Python interpreter.
//
// This is necessary because Python's `threading` module requires that the same
// thread which started the interpreter also calls finalize. (If this is not the case,
// an AssertionError is raised during the Py_Finalize call.)
unsafe fn initialization_thread() {
let mut guard = INITIALIZATION_THREAD_MUTEX.lock();

ffi::Py_InitializeEx(0);

#[cfg(not(Py_3_7))] // Called by Py_InitializeEx in Python 3.7 and up.
ffi::PyEval_InitThreads();

// Import the threading module - this ensures that it will associate this
// thread as the "main" thread.
{
let pool = GILPool::new();
pool.python().import("threading").unwrap();
}

// Release the GIL, notify the original calling thread that Python is now
// initialized, and wait for notification to begin finalization.
let tstate = ffi::PyEval_SaveThread();

INITIALIZATION_THREAD_SIGNAL.notify_one();
INITIALIZATION_THREAD_SIGNAL.wait(&mut guard);

// Signal to finalize received.
ffi::PyEval_RestoreThread(tstate);
ffi::Py_Finalize();

INITIALIZATION_THREAD_SIGNAL.notify_one();
}

let mut guard = INITIALIZATION_THREAD_MUTEX.lock();
std::thread::spawn(|| initialization_thread());
INITIALIZATION_THREAD_SIGNAL.wait(&mut guard);
ffi::Py_InitializeEx(0);

// Make sure Py_Finalize will be called before exiting.
extern "C" fn finalize_callback() {
unsafe {
if ffi::Py_IsInitialized() != 0 {
// Before blocking on the finalization thread, ensure this thread does not
// hold the GIL - otherwise can result in a deadlock!
ffi::PyGILState_Ensure();
ffi::PyEval_SaveThread();

// Notify initialization_thread to finalize, and wait.
let mut guard = INITIALIZATION_THREAD_MUTEX.lock();
INITIALIZATION_THREAD_SIGNAL.notify_one();
INITIALIZATION_THREAD_SIGNAL.wait(&mut guard);
assert_eq!(ffi::Py_IsInitialized(), 0);
}
}
}
#[cfg(not(Py_3_7))] // Called by Py_InitializeEx in Python 3.7 and up.
ffi::PyEval_InitThreads();

libc::atexit(finalize_callback);
// Release the GIL.
ffi::PyEval_SaveThread();
}
});
}

/// Prepares the use of Python in a free-threaded context.
/// Executes the provided closure with an embedded Python interpreter.
///
/// If the Python interpreter is not already initialized, this function
/// will initialize it with disabled signal handling
/// (Python will not raise the `KeyboardInterrupt` exception).
/// Python signal handling depends on the notion of a 'main thread', which must be
/// the thread that initializes the Python interpreter.
/// This function intializes the Python interpreter, executes the provided closure, and then
/// finalizes the Python interpreter.
///
/// If both the Python interpreter and Python threading are already initialized,
/// this function has no effect.
/// After execution all Python resources are cleaned up, and no further Python APIs can be called.
/// Because many Python modules implemented in C do not support multiple Python interpreters in a
/// single process, it is not safe to call this function more than once. (Many such modules will not
/// initialize correctly on the second run.)
///
/// # Availability
/// This function is only available when linking against Python distributions that contain a
Expand All @@ -180,46 +111,58 @@ pub fn prepare_freethreaded_python() {
/// This function is not available on PyPy.
///
/// # Panics
/// - If the Python interpreter is initialized but Python threading is not,
/// a panic occurs.
/// It is not possible to safely access the Python runtime unless the main
/// thread (the thread which originally initialized Python) also initializes
/// threading.
/// - If the Python interpreter is already initalized before calling this function.
///
/// # Safety
/// - This function should only ever be called once per process (usually as part of the `main`
/// function). It is also not thread-safe.
/// - No Python APIs can be used after this function has finished executing.
/// - The return value of the closure must not contain any Python value, _including_ `PyResult`.
///
/// # Example
/// ```rust
/// use pyo3::prelude::*;
///
/// # #[allow(clippy::needless_doctest_main)]
/// fn main() {
/// pyo3::prepare_freethreaded_python_without_finalizer();
/// Python::with_gil(|py| {
/// py.run("print('Hello World')", None, None)
/// });
/// unsafe {
/// pyo3::with_embedded_python_interpreter(|py| {
/// py.run("print('Hello World')", None, None)
/// });
/// }
/// }
/// ```
#[cfg(all(Py_SHARED, not(PyPy)))]
pub fn prepare_freethreaded_python_without_finalizer() {
// Protect against race conditions when Python is not yet initialized
// and multiple threads concurrently call 'prepare_freethreaded_python()'.
// Note that we do not protect against concurrent initialization of the Python runtime
// by other users of the Python C API.
START.call_once_force(|_| unsafe {
// Use call_once_force because if initialization panics, it's okay to try again.
if ffi::Py_IsInitialized() != 0 {
// If Python is already initialized, we expect Python threading to also be initialized,
// as we can't make the existing Python main thread acquire the GIL.
assert_ne!(ffi::PyEval_ThreadsInitialized(), 0);
} else {
ffi::Py_InitializeEx(0);
pub unsafe fn with_embedded_python_interpreter<F, R>(f: F) -> R
where
F: for<'p> FnOnce(Python<'p>) -> R,
{
assert_eq!(
ffi::Py_IsInitialized(),
0,
"called `with_embedded_python_interpreter` but a Python interpreter is already running."
);

#[cfg(not(Py_3_7))] // Called by Py_InitializeEx in Python 3.7 and up.
ffi::PyEval_InitThreads();
ffi::Py_InitializeEx(0);

// Release the GIL.
ffi::PyEval_SaveThread();
}
});
#[cfg(not(Py_3_7))] // Called by Py_InitializeEx in Python 3.7 and up.
ffi::PyEval_InitThreads();

// Import the threading module - this ensures that it will associate this thread as the "main"
// thread, which is important to avoid an `AssertionError` at finalization.
let pool = GILPool::new();
pool.python().import("threading").unwrap();

// Execute the closure.
let result = f(pool.python());

// Drop the pool before finalizing.
drop(pool);

// Finalize the Python interpreter.
ffi::Py_Finalize();

result
}

/// RAII type that represents the Global Interpreter Lock acquisition. To get hold of a value
Expand Down
2 changes: 1 addition & 1 deletion src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ pub use crate::conversion::{
};
pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult};
#[cfg(all(Py_SHARED, not(PyPy)))]
pub use crate::gil::{prepare_freethreaded_python, prepare_freethreaded_python_without_finalizer};
pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter};
pub use crate::gil::{GILGuard, GILPool};
pub use crate::instance::{Py, PyNativeType, PyObject};
pub use crate::pycell::{PyCell, PyRef, PyRefMut};
Expand Down

0 comments on commit faae995

Please sign in to comment.