From 8ee5510b8a1384f99514f1ec91bff5b651bc0073 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Wed, 4 Sep 2024 11:26:49 -0600 Subject: [PATCH] Remove GILProtected on free-threaded build (#4504) * do not define GILProtected if Py_GIL_DISABLED is set * add stub for migration guide on free-threaded support * remove internal uses of GILProtected on gil-enabled builds as well * add newsfragment * flesh out documentation * fixup migration guide examples * simplify migration guide example --- guide/src/migration.md | 74 +++++++++++++++++++++++++++ newsfragments/4504.changed.md | 2 + src/impl_/pyclass/lazy_type_object.rs | 24 +++++---- src/sync.rs | 10 +++- 4 files changed, 98 insertions(+), 12 deletions(-) create mode 100644 newsfragments/4504.changed.md diff --git a/guide/src/migration.md b/guide/src/migration.md index e366c825a3b..fb212743702 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -198,6 +198,80 @@ impl<'a, 'py> IntoPyObject<'py> for &'a MyPyObjectWrapper { ``` +### Free-threaded Python Support +
+Click to expand + +PyO3 0.23 introduces preliminary support for the new free-threaded build of +CPython 3.13. PyO3 features that implicitly assumed the existence of the GIL +are not exposed in the free-threaded build, since they are no longer safe. + +If you make use of these features then you will need to account for the +unavailability of this API in the free-threaded build. One way to handle it is +via conditional compilation -- extensions built for the free-threaded build will +have the `Py_GIL_DISABLED` attribute defined. + +### `GILProtected` + +`GILProtected` allows mutable access to static data by leveraging the GIL to +lock concurrent access from other threads. In free-threaded python there is no +GIL, so you will need to replace this type with some other form of locking. In +many cases, `std::sync::Atomic` or `std::sync::Mutex` will be sufficient. If the +locks do not guard the execution of arbitrary Python code or use of the CPython +C API then conditional compilation is likely unnecessary since `GILProtected` +was not needed in the first place. + +Before: + +```rust +# fn main() { +# #[cfg(not(Py_GIL_DISABLED))] { +# use pyo3::prelude::*; +use pyo3::sync::GILProtected; +use pyo3::types::{PyDict, PyNone}; +use std::cell::RefCell; + +static OBJECTS: GILProtected>>> = + GILProtected::new(RefCell::new(Vec::new())); + +Python::with_gil(|py| { + // stand-in for something that executes arbitrary python code + let d = PyDict::new(py); + d.set_item(PyNone::get(py), PyNone::get(py)).unwrap(); + OBJECTS.get(py).borrow_mut().push(d.unbind()); +}); +# }} +``` + +After: + +```rust +# use pyo3::prelude::*; +# fn main() { +use pyo3::types::{PyDict, PyNone}; +use std::sync::Mutex; + +static OBJECTS: Mutex>> = Mutex::new(Vec::new()); + +Python::with_gil(|py| { + // stand-in for something that executes arbitrary python code + let d = PyDict::new(py); + d.set_item(PyNone::get(py), PyNone::get(py)).unwrap(); + // we're not executing python code while holding the lock, so GILProtected + // was never needed + OBJECTS.lock().unwrap().push(d.unbind()); +}); +# } +``` + +If you are executing arbitrary Python code while holding the lock, then you will +need to use conditional compilation to use `GILProtected` on GIL-enabled python +builds and mutexes otherwise. Python 3.13 introduces `PyMutex`, which releases +the GIL while the lock is held, so that is another option if you only need to +support newer Python versions. + +
+ ## from 0.21.* to 0.22 ### Deprecation of `gil-refs` feature continues diff --git a/newsfragments/4504.changed.md b/newsfragments/4504.changed.md new file mode 100644 index 00000000000..94d056dcef9 --- /dev/null +++ b/newsfragments/4504.changed.md @@ -0,0 +1,2 @@ +* The `GILProtected` struct is not available on the free-threaded build of + Python 3.13. diff --git a/src/impl_/pyclass/lazy_type_object.rs b/src/impl_/pyclass/lazy_type_object.rs index be383a272f3..d3bede7b2f3 100644 --- a/src/impl_/pyclass/lazy_type_object.rs +++ b/src/impl_/pyclass/lazy_type_object.rs @@ -1,5 +1,4 @@ use std::{ - cell::RefCell, ffi::CStr, marker::PhantomData, thread::{self, ThreadId}, @@ -11,11 +10,13 @@ use crate::{ impl_::pyclass::MaybeRuntimePyMethodDef, impl_::pymethods::PyMethodDefType, pyclass::{create_type_object, PyClassTypeObject}, - sync::{GILOnceCell, GILProtected}, + sync::GILOnceCell, types::PyType, Bound, PyClass, PyErr, PyObject, PyResult, Python, }; +use std::sync::Mutex; + use super::PyClassItemsIter; /// Lazy type object for PyClass. @@ -27,7 +28,7 @@ struct LazyTypeObjectInner { value: GILOnceCell, // Threads which have begun initialization of the `tp_dict`. Used for // reentrant initialization detection. - initializing_threads: GILProtected>>, + initializing_threads: Mutex>, tp_dict_filled: GILOnceCell<()>, } @@ -38,7 +39,7 @@ impl LazyTypeObject { LazyTypeObject( LazyTypeObjectInner { value: GILOnceCell::new(), - initializing_threads: GILProtected::new(RefCell::new(Vec::new())), + initializing_threads: Mutex::new(Vec::new()), tp_dict_filled: GILOnceCell::new(), }, PhantomData, @@ -117,7 +118,7 @@ impl LazyTypeObjectInner { let thread_id = thread::current().id(); { - let mut threads = self.initializing_threads.get(py).borrow_mut(); + let mut threads = self.initializing_threads.lock().unwrap(); if threads.contains(&thread_id) { // Reentrant call: just return the type object, even if the // `tp_dict` is not filled yet. @@ -127,20 +128,18 @@ impl LazyTypeObjectInner { } struct InitializationGuard<'a> { - initializing_threads: &'a GILProtected>>, - py: Python<'a>, + initializing_threads: &'a Mutex>, thread_id: ThreadId, } impl Drop for InitializationGuard<'_> { fn drop(&mut self) { - let mut threads = self.initializing_threads.get(self.py).borrow_mut(); + let mut threads = self.initializing_threads.lock().unwrap(); threads.retain(|id| *id != self.thread_id); } } let guard = InitializationGuard { initializing_threads: &self.initializing_threads, - py, thread_id, }; @@ -185,8 +184,11 @@ impl LazyTypeObjectInner { // Initialization successfully complete, can clear the thread list. // (No further calls to get_or_init() will try to init, on any thread.) - std::mem::forget(guard); - self.initializing_threads.get(py).replace(Vec::new()); + let mut threads = { + drop(guard); + self.initializing_threads.lock().unwrap() + }; + threads.clear(); result }); diff --git a/src/sync.rs b/src/sync.rs index 33d247b7856..59f669f2627 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -6,16 +6,21 @@ //! [PEP 703]: https://peps.python.org/pep-703/ use crate::{ types::{any::PyAnyMethods, PyString, PyType}, - Bound, Py, PyResult, PyVisit, Python, + Bound, Py, PyResult, Python, }; use std::cell::UnsafeCell; +#[cfg(not(Py_GIL_DISABLED))] +use crate::PyVisit; + /// Value with concurrent access protected by the GIL. /// /// This is a synchronization primitive based on Python's global interpreter lock (GIL). /// It ensures that only one thread at a time can access the inner value via shared references. /// It can be combined with interior mutability to obtain mutable references. /// +/// This type is not defined for extensions built against the free-threaded CPython ABI. +/// /// # Example /// /// Combining `GILProtected` with `RefCell` enables mutable access to static data: @@ -31,10 +36,12 @@ use std::cell::UnsafeCell; /// NUMBERS.get(py).borrow_mut().push(42); /// }); /// ``` +#[cfg(not(Py_GIL_DISABLED))] pub struct GILProtected { value: T, } +#[cfg(not(Py_GIL_DISABLED))] impl GILProtected { /// Place the given value under the protection of the GIL. pub const fn new(value: T) -> Self { @@ -52,6 +59,7 @@ impl GILProtected { } } +#[cfg(not(Py_GIL_DISABLED))] unsafe impl Sync for GILProtected where T: Send {} /// A write-once cell similar to [`once_cell::OnceCell`](https://docs.rs/once_cell/latest/once_cell/).