diff --git a/CHANGELOG.md b/CHANGELOG.md index 70805f8ae60..03114b27738 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add FFI definitions `PyOS_BeforeFork`, `PyOS_AfterFork_Parent`, `PyOS_AfterFork_Child` for Python 3.7 and up. [#1348](https://github.com/PyO3/pyo3/pull/1348) - Add `auto-initialize` feature to control whether PyO3 should automatically initialize an embedded Python interpreter. For compatibility this feature is enabled by default in PyO3 0.13.1, but is planned to become opt-in from PyO3 0.14.0. [#1347](https://github.com/PyO3/pyo3/pull/1347) - Add support for cross-compiling to Windows without needing `PYO3_CROSS_INCLUDE_DIR`. [#1350](https://github.com/PyO3/pyo3/pull/1350) -- Add `prepare_freethreaded_python_without_finalizer` to initalize a Python interpreter while avoiding potential finalization issues in C-extensions like SciPy. [#1355](https://github.com/PyO3/pyo3/pull/1355) +- Add unsafe API `with_embedded_python_interpreter` to initalize a Python interpreter, execute a closure, and finalize the interpreter. [#1355](https://github.com/PyO3/pyo3/pull/1355) ### Changed - Deprecate FFI definitions `PyEval_CallObjectWithKeywords`, `PyEval_CallObject`, `PyEval_CallFunction`, `PyEval_CallMethod` when building for Python 3.9. [#1338](https://github.com/PyO3/pyo3/pull/1338) @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Deprecate FFI definition `PyOS_AfterFork` for Python 3.7 and up. [#1348](https://github.com/PyO3/pyo3/pull/1348) - Deprecate FFI definition `PyCoro_Check` and `PyAsyncGen_Check` in favor of `PyCoro_CheckExact` and `PyAsyncGen_CheckExact` respectively to mirror Python API. [#1348](https://github.com/PyO3/pyo3/pull/1348) - Deprecate FFI definition `PyCoroWrapper_Check` which has never been in the Python API. [#1348](https://github.com/PyO3/pyo3/pull/1348) -- Call `Py_Finalize` in the same thread as `Py_InitializeEx` inside `prepare_freethreaded_python`. [#1350](https://github.com/PyO3/pyo3/pull/1350) +- `prepare_freethreaded_python` will no longer register an `atexit` handler to call `Py_Finalize`. [#1355](https://github.com/PyO3/pyo3/pull/1355) ### Removed - Remove FFI definition `PyFrame_ClearFreeList` when building for Python 3.9. [#1341](https://github.com/PyO3/pyo3/pull/1341) diff --git a/src/gil.rs b/src/gil.rs index 5683880d857..7ed4d39ac3f 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -42,13 +42,6 @@ pub(crate) fn gil_is_acquired() -> bool { /// Python signal handling depends on the notion of a 'main thread', which must be /// the thread that initializes the Python interpreter. /// -/// Additionally, this function will register an `atexit` callback to finalize the Python -/// interpreter. Usually this is desirable - it flushes Python buffers and ensures that all -/// Python objects are cleaned up appropriately. Some C extensions can have memory issues during -/// finalization, so you may get crashes on program exit if your embedded Python program uses -/// these extensions. If this is the case, you should use [`prepare_freethreaded_python_without_finalizer`] -/// which does not register the `atexit` callback. -/// /// If both the Python interpreter and Python threading are already initialized, /// this function has no effect. /// @@ -90,88 +83,26 @@ pub fn prepare_freethreaded_python() { // as we can't make the existing Python main thread acquire the GIL. assert_ne!(ffi::PyEval_ThreadsInitialized(), 0); } else { - use parking_lot::Condvar; - - // Initialize Python. - // We use Py_InitializeEx() with initsigs=0 to disable Python signal handling. - // Signal handling depends on the notion of a 'main thread', which doesn't exist in this case. - // Note that the 'main thread' notion in Python isn't documented properly; - // and running Python without one is not officially supported. - static INITIALIZATION_THREAD_SIGNAL: Condvar = Condvar::new(); - static INITIALIZATION_THREAD_MUTEX: Mutex<()> = const_mutex(()); - - // This thread will be responsible for initialization and finalization of the - // Python interpreter. - // - // This is necessary because Python's `threading` module requires that the same - // thread which started the interpreter also calls finalize. (If this is not the case, - // an AssertionError is raised during the Py_Finalize call.) - unsafe fn initialization_thread() { - let mut guard = INITIALIZATION_THREAD_MUTEX.lock(); - - ffi::Py_InitializeEx(0); - - #[cfg(not(Py_3_7))] // Called by Py_InitializeEx in Python 3.7 and up. - ffi::PyEval_InitThreads(); - - // Import the threading module - this ensures that it will associate this - // thread as the "main" thread. - { - let pool = GILPool::new(); - pool.python().import("threading").unwrap(); - } - - // Release the GIL, notify the original calling thread that Python is now - // initialized, and wait for notification to begin finalization. - let tstate = ffi::PyEval_SaveThread(); - - INITIALIZATION_THREAD_SIGNAL.notify_one(); - INITIALIZATION_THREAD_SIGNAL.wait(&mut guard); - - // Signal to finalize received. - ffi::PyEval_RestoreThread(tstate); - ffi::Py_Finalize(); - - INITIALIZATION_THREAD_SIGNAL.notify_one(); - } - - let mut guard = INITIALIZATION_THREAD_MUTEX.lock(); - std::thread::spawn(|| initialization_thread()); - INITIALIZATION_THREAD_SIGNAL.wait(&mut guard); + ffi::Py_InitializeEx(0); - // Make sure Py_Finalize will be called before exiting. - extern "C" fn finalize_callback() { - unsafe { - if ffi::Py_IsInitialized() != 0 { - // Before blocking on the finalization thread, ensure this thread does not - // hold the GIL - otherwise can result in a deadlock! - ffi::PyGILState_Ensure(); - ffi::PyEval_SaveThread(); - - // Notify initialization_thread to finalize, and wait. - let mut guard = INITIALIZATION_THREAD_MUTEX.lock(); - INITIALIZATION_THREAD_SIGNAL.notify_one(); - INITIALIZATION_THREAD_SIGNAL.wait(&mut guard); - assert_eq!(ffi::Py_IsInitialized(), 0); - } - } - } + #[cfg(not(Py_3_7))] // Called by Py_InitializeEx in Python 3.7 and up. + ffi::PyEval_InitThreads(); - libc::atexit(finalize_callback); + // Release the GIL. + ffi::PyEval_SaveThread(); } }); } -/// Prepares the use of Python in a free-threaded context. +/// Executes the provided closure with an embedded Python interpreter. /// -/// If the Python interpreter is not already initialized, this function -/// will initialize it with disabled signal handling -/// (Python will not raise the `KeyboardInterrupt` exception). -/// Python signal handling depends on the notion of a 'main thread', which must be -/// the thread that initializes the Python interpreter. +/// This function intializes the Python interpreter, executes the provided closure, and then +/// finalizes the Python interpreter. /// -/// If both the Python interpreter and Python threading are already initialized, -/// this function has no effect. +/// After execution all Python resources are cleaned up, and no further Python APIs can be called. +/// Because many Python modules implemented in C do not support multiple Python interpreters in a +/// single process, it is not safe to call this function more than once. (Many such modules will not +/// initialize correctly on the second run.) /// /// # Availability /// This function is only available when linking against Python distributions that contain a @@ -180,11 +111,13 @@ pub fn prepare_freethreaded_python() { /// This function is not available on PyPy. /// /// # Panics -/// - If the Python interpreter is initialized but Python threading is not, -/// a panic occurs. -/// It is not possible to safely access the Python runtime unless the main -/// thread (the thread which originally initialized Python) also initializes -/// threading. +/// - If the Python interpreter is already initalized before calling this function. +/// +/// # Safety +/// - This function should only ever be called once per process (usually as part of the `main` +/// function). It is also not thread-safe. +/// - No Python APIs can be used after this function has finished executing. +/// - The return value of the closure must not contain any Python value, _including_ `PyResult`. /// /// # Example /// ```rust @@ -192,34 +125,44 @@ pub fn prepare_freethreaded_python() { /// /// # #[allow(clippy::needless_doctest_main)] /// fn main() { -/// pyo3::prepare_freethreaded_python_without_finalizer(); -/// Python::with_gil(|py| { -/// py.run("print('Hello World')", None, None) -/// }); +/// unsafe { +/// pyo3::with_embedded_python_interpreter(|py| { +/// py.run("print('Hello World')", None, None) +/// }); +/// } /// } /// ``` #[cfg(all(Py_SHARED, not(PyPy)))] -pub fn prepare_freethreaded_python_without_finalizer() { - // Protect against race conditions when Python is not yet initialized - // and multiple threads concurrently call 'prepare_freethreaded_python()'. - // Note that we do not protect against concurrent initialization of the Python runtime - // by other users of the Python C API. - START.call_once_force(|_| unsafe { - // Use call_once_force because if initialization panics, it's okay to try again. - if ffi::Py_IsInitialized() != 0 { - // If Python is already initialized, we expect Python threading to also be initialized, - // as we can't make the existing Python main thread acquire the GIL. - assert_ne!(ffi::PyEval_ThreadsInitialized(), 0); - } else { - ffi::Py_InitializeEx(0); +pub unsafe fn with_embedded_python_interpreter(f: F) -> R +where + F: for<'p> FnOnce(Python<'p>) -> R, +{ + assert_eq!( + ffi::Py_IsInitialized(), + 0, + "called `with_embedded_python_interpreter` but a Python interpreter is already running." + ); - #[cfg(not(Py_3_7))] // Called by Py_InitializeEx in Python 3.7 and up. - ffi::PyEval_InitThreads(); + ffi::Py_InitializeEx(0); - // Release the GIL. - ffi::PyEval_SaveThread(); - } - }); + #[cfg(not(Py_3_7))] // Called by Py_InitializeEx in Python 3.7 and up. + ffi::PyEval_InitThreads(); + + // Import the threading module - this ensures that it will associate this thread as the "main" + // thread, which is important to avoid an `AssertionError` at finalization. + let pool = GILPool::new(); + pool.python().import("threading").unwrap(); + + // Execute the closure. + let result = f(pool.python()); + + // Drop the pool before finalizing. + drop(pool); + + // Finalize the Python interpreter. + ffi::Py_Finalize(); + + result } /// RAII type that represents the Global Interpreter Lock acquisition. To get hold of a value diff --git a/src/lib.rs b/src/lib.rs index e02aed53f1a..b461de41ccd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,7 +147,7 @@ pub use crate::conversion::{ }; pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult}; #[cfg(all(Py_SHARED, not(PyPy)))] -pub use crate::gil::{prepare_freethreaded_python, prepare_freethreaded_python_without_finalizer}; +pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter}; pub use crate::gil::{GILGuard, GILPool}; pub use crate::instance::{Py, PyNativeType, PyObject}; pub use crate::pycell::{PyCell, PyRef, PyRefMut};