From 46fa1b2b80429367302c4b17db56cfb1888a0129 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 19 Dec 2023 16:16:13 +0000 Subject: [PATCH 001/349] add test which is broken with gevent --- pytests/pyproject.toml | 1 + pytests/src/misc.rs | 12 +++++++++++- pytests/tests/test_misc.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index fb1ac3dbdff..4d5b42bf46f 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -20,6 +20,7 @@ classifiers = [ [project.optional-dependencies] dev = [ + "gevent>=23.9.1", "hypothesis>=3.55", "pytest-asyncio>=0.21", "pytest-benchmark>=3.4", diff --git a/pytests/src/misc.rs b/pytests/src/misc.rs index 029e8b16528..548c5cebd77 100644 --- a/pytests/src/misc.rs +++ b/pytests/src/misc.rs @@ -1,4 +1,4 @@ -use pyo3::prelude::*; +use pyo3::{prelude::*, types::PyDict}; use std::borrow::Cow; #[pyfunction] @@ -17,10 +17,20 @@ fn accepts_bool(val: bool) -> bool { val } +#[pyfunction] +fn get_item_and_run_callback(dict: &PyDict, callback: &PyAny) -> PyResult<()> { + let item = dict.get_item("key")?.expect("key not found in dict"); + let string = item.to_string(); + callback.call0()?; + assert_eq!(item.to_string(), string); + Ok(()) +} + #[pymodule] pub fn misc(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(issue_219, m)?)?; m.add_function(wrap_pyfunction!(get_type_full_name, m)?)?; m.add_function(wrap_pyfunction!(accepts_bool, m)?)?; + m.add_function(wrap_pyfunction!(get_item_and_run_callback, m)?)?; Ok(()) } diff --git a/pytests/tests/test_misc.py b/pytests/tests/test_misc.py index 06b2ce73e10..2f6cee6354e 100644 --- a/pytests/tests/test_misc.py +++ b/pytests/tests/test_misc.py @@ -2,6 +2,7 @@ import platform import sys +import gevent import pyo3_pytests.misc import pytest @@ -64,3 +65,39 @@ def test_accepts_numpy_bool(): assert pyo3_pytests.misc.accepts_bool(False) is False assert pyo3_pytests.misc.accepts_bool(numpy.bool_(True)) is True assert pyo3_pytests.misc.accepts_bool(numpy.bool_(False)) is False + + +class ArbitraryClass: + worker_id: int + iteration: int + + def __init__(self, worker_id: int, iteration: int): + self.worker_id = worker_id + self.iteration = iteration + + def __repr__(self): + return f"ArbitraryClass({self.worker_id}, {self.iteration})" + + def __del__(self): + print("del", self.worker_id, self.iteration) + + +def test_gevent(): + def worker(worker_id: int) -> None: + for iteration in range(2): + d = {"key": ArbitraryClass(worker_id, iteration)} + + def arbitrary_python_code(): + # remove the dictionary entry so that the class value can be + # garbage collected + del d["key"] + print("gevent sleep", worker_id, iteration) + gevent.sleep(0) + print("after gevent sleep", worker_id, iteration) + + print("start", worker_id, iteration) + pyo3_pytests.misc.get_item_and_run_callback(d, arbitrary_python_code) + print("end", worker_id, iteration) + + workers = [gevent.spawn(worker, i) for i in range(2)] + gevent.joinall(workers) From 49d7718823b6206d81b56fceac0087705350f009 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 23 Dec 2023 22:10:01 +0000 Subject: [PATCH 002/349] demonstrate switching to `Bound` API resolves `gevent` crash --- pytests/src/misc.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pytests/src/misc.rs b/pytests/src/misc.rs index 548c5cebd77..bd941461e91 100644 --- a/pytests/src/misc.rs +++ b/pytests/src/misc.rs @@ -18,7 +18,11 @@ fn accepts_bool(val: bool) -> bool { } #[pyfunction] -fn get_item_and_run_callback(dict: &PyDict, callback: &PyAny) -> PyResult<()> { +fn get_item_and_run_callback(dict: Bound<'_, PyDict>, callback: Bound<'_, PyAny>) -> PyResult<()> { + // This function gives the opportunity to run a pure-Python callback so that + // gevent can instigate a context switch. This had problematic interactions + // with PyO3's removed "GIL Pool". + // For context, see https://github.com/PyO3/pyo3/issues/3668 let item = dict.get_item("key")?.expect("key not found in dict"); let string = item.to_string(); callback.call0()?; From f5b18468bc6a25980169823bf285a84e860a294b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 24 Dec 2023 15:05:51 +0000 Subject: [PATCH 003/349] partially revert changes to `PyTzInfoAccess` trait --- newsfragments/3679.changed.md | 2 +- pytests/src/datetime.rs | 4 ++-- src/conversions/chrono.rs | 4 ++-- src/types/datetime.rs | 24 +++++++++++++++++------- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/newsfragments/3679.changed.md b/newsfragments/3679.changed.md index 2e8e28a9d61..ab46598ad65 100644 --- a/newsfragments/3679.changed.md +++ b/newsfragments/3679.changed.md @@ -1 +1 @@ -Add lifetime parameter to `PyTzInfoAccess` trait and change the return type of `PyTzInfoAccess::get_tzinfo` to `Option>`. For the deprecated gil-ref API, the trait is now implemented for `&'py PyTime` and `&'py PyDateTime` instead of `PyTime` and `PyDate`. +Add lifetime parameter to `PyTzInfoAccess` trait. For the deprecated gil-ref API, the trait is now implemented for `&'py PyTime` and `&'py PyDateTime` instead of `PyTime` and `PyDate`. diff --git a/pytests/src/datetime.rs b/pytests/src/datetime.rs index 7f0492da75e..f5a15b4f682 100644 --- a/pytests/src/datetime.rs +++ b/pytests/src/datetime.rs @@ -161,12 +161,12 @@ fn datetime_from_timestamp<'p>( #[pyfunction] fn get_datetime_tzinfo(dt: &PyDateTime) -> Option> { - dt.get_tzinfo() + dt.get_tzinfo_bound() } #[pyfunction] fn get_time_tzinfo(dt: &PyTime) -> Option> { - dt.get_tzinfo() + dt.get_tzinfo_bound() } #[pyclass(extends=PyTzInfo)] diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 0ad5b445136..dd541bff135 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -248,7 +248,7 @@ impl FromPyObject<'_> for NaiveDateTime { // we return a hard error. We could silently remove tzinfo, or assume local timezone // and do a conversion, but better leave this decision to the user of the library. #[cfg(not(Py_LIMITED_API))] - let has_tzinfo = dt.get_tzinfo().is_some(); + let has_tzinfo = dt.get_tzinfo_bound().is_some(); #[cfg(Py_LIMITED_API)] let has_tzinfo = !dt.getattr(intern!(dt.py(), "tzinfo"))?.is_none(); if has_tzinfo { @@ -284,7 +284,7 @@ impl FromPyObject<'a>> FromPyObject<'_> for DateTime check_type(dt, &DatetimeTypes::get(dt.py()).datetime, "PyDateTime")?; #[cfg(not(Py_LIMITED_API))] - let tzinfo = dt.get_tzinfo(); + let tzinfo = dt.get_tzinfo_bound(); #[cfg(Py_LIMITED_API)] let tzinfo: Option<&PyAny> = dt.getattr(intern!(dt.py(), "tzinfo"))?.extract()?; diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 94802fe14ef..b2d0f6efa00 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -163,12 +163,21 @@ pub trait PyTimeAccess { /// Trait for accessing the components of a struct containing a tzinfo. pub trait PyTzInfoAccess<'py> { + /// Deprecated form of `get_tzinfo_bound`. + #[deprecated( + since = "0.21.0", + note = "`get_tzinfo` will be replaced by `get_tzinfo_bound` in a future PyO3 version" + )] + fn get_tzinfo(&self) -> Option<&'py PyTzInfo> { + self.get_tzinfo_bound().map(Bound::into_gil_ref) + } + /// Returns the tzinfo (which may be None). /// /// Implementations should conform to the upstream documentation: /// /// - fn get_tzinfo(&self) -> Option>; + fn get_tzinfo_bound(&self) -> Option>; } /// Bindings around `datetime.date` @@ -413,13 +422,13 @@ impl PyTimeAccess for Bound<'_, PyDateTime> { } impl<'py> PyTzInfoAccess<'py> for &'py PyDateTime { - fn get_tzinfo(&self) -> Option> { - Bound::borrowed_from_gil_ref(self).get_tzinfo() + fn get_tzinfo_bound(&self) -> Option> { + Bound::borrowed_from_gil_ref(self).get_tzinfo_bound() } } impl<'py> PyTzInfoAccess<'py> for Bound<'py, PyDateTime> { - fn get_tzinfo(&self) -> Option> { + fn get_tzinfo_bound(&self) -> Option> { let ptr = self.as_ptr() as *mut ffi::PyDateTime_DateTime; unsafe { if (*ptr).hastzinfo != 0 { @@ -543,13 +552,13 @@ impl PyTimeAccess for Bound<'_, PyTime> { } impl<'py> PyTzInfoAccess<'py> for &'py PyTime { - fn get_tzinfo(&self) -> Option> { - Bound::borrowed_from_gil_ref(self).get_tzinfo() + fn get_tzinfo_bound(&self) -> Option> { + Bound::borrowed_from_gil_ref(self).get_tzinfo_bound() } } impl<'py> PyTzInfoAccess<'py> for Bound<'py, PyTime> { - fn get_tzinfo(&self) -> Option> { + fn get_tzinfo_bound(&self) -> Option> { let ptr = self.as_ptr() as *mut ffi::PyDateTime_Time; unsafe { if (*ptr).hastzinfo != 0 { @@ -725,6 +734,7 @@ mod tests { #[test] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons + #[allow(deprecated)] fn test_get_tzinfo() { crate::Python::with_gil(|py| { let utc = timezone_utc(py); From 877e34ac86b46597c30f4ad978868799ba2be791 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 24 Dec 2023 14:49:43 +0000 Subject: [PATCH 004/349] implement `PyErr::write_unraisable_bound` --- src/conversions/chrono.rs | 3 ++- src/err/mod.rs | 16 +++++++++++++--- src/impl_/pyclass.rs | 2 +- src/impl_/trampoline.rs | 6 +++--- src/instance.rs | 7 +++---- src/types/mapping.rs | 5 ++++- src/types/mod.rs | 2 +- src/types/sequence.rs | 5 ++++- tests/test_exceptions.rs | 1 + 9 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 0ad5b445136..3b53e7ce1ed 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -42,6 +42,7 @@ //! } //! ``` use crate::exceptions::{PyTypeError, PyUserWarning, PyValueError}; +use crate::instance::Bound; #[cfg(Py_LIMITED_API)] use crate::sync::GILOnceCell; #[cfg(not(Py_LIMITED_API))] @@ -465,7 +466,7 @@ fn warn_truncated_leap_second(obj: &PyAny) { "ignored leap-second, `datetime` does not support leap-seconds", 0, ) { - e.write_unraisable(py, Some(obj)) + e.write_unraisable_bound(py, Some(Bound::borrowed_from_gil_ref(&obj))) }; } diff --git a/src/err/mod.rs b/src/err/mod.rs index 00ba111f8c5..c2d47a8f17a 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -535,6 +535,16 @@ impl PyErr { .restore(py) } + /// Deprecated form of `PyErr::write_unraisable_bound`. + #[deprecated( + since = "0.21.0", + note = "`PyErr::write_unraisable` will be replaced by `PyErr::write_unraisable_bound` in a future PyO3 version" + )] + #[inline] + pub fn write_unraisable(self, py: Python<'_>, obj: Option<&PyAny>) { + self.write_unraisable_bound(py, obj.as_ref().map(Bound::borrowed_from_gil_ref)) + } + /// Reports the error as unraisable. /// /// This calls `sys.unraisablehook()` using the current exception and obj argument. @@ -557,16 +567,16 @@ impl PyErr { /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { /// match failing_function() { - /// Err(pyerr) => pyerr.write_unraisable(py, None), + /// Err(pyerr) => pyerr.write_unraisable_bound(py, None), /// Ok(..) => { /* do something here */ } /// } /// Ok(()) /// }) /// # } #[inline] - pub fn write_unraisable(self, py: Python<'_>, obj: Option<&PyAny>) { + pub fn write_unraisable_bound(self, py: Python<'_>, obj: Option<&Bound<'_, PyAny>>) { self.restore(py); - unsafe { ffi::PyErr_WriteUnraisable(obj.map_or(std::ptr::null_mut(), |x| x.as_ptr())) } + unsafe { ffi::PyErr_WriteUnraisable(obj.map_or(std::ptr::null_mut(), Bound::as_ptr)) } } /// Issues a warning message. diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 5ee67dc998d..23cebb26705 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1067,7 +1067,7 @@ impl ThreadCheckerImpl { "{} is unsendable, but is being dropped on another thread", type_name )) - .write_unraisable(py, None); + .write_unraisable_bound(py, None); return false; } diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index 5ae75e0fd2e..4b4eac17a15 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -10,8 +10,8 @@ use std::{ }; use crate::{ - callback::PyCallbackOutput, ffi, impl_::panic::PanicTrap, methods::IPowModulo, - panic::PanicException, types::PyModule, GILPool, Py, PyResult, Python, + callback::PyCallbackOutput, ffi, ffi_ptr_ext::FfiPtrExt, impl_::panic::PanicTrap, + methods::IPowModulo, panic::PanicException, types::PyModule, GILPool, Py, PyResult, Python, }; #[inline] @@ -224,7 +224,7 @@ where if let Err(py_err) = panic::catch_unwind(move || body(py)) .unwrap_or_else(|payload| Err(PanicException::from_panic_payload(payload))) { - py_err.write_unraisable(py, py.from_borrowed_ptr_or_opt(ctx)); + py_err.write_unraisable_bound(py, ctx.assume_borrowed_or_opt(py).as_deref()); } trap.disarm(); } diff --git a/src/instance.rs b/src/instance.rs index 8406e86abbe..c3de539abe6 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -3,6 +3,7 @@ use crate::pycell::{PyBorrowError, PyBorrowMutError, PyCell}; use crate::pyclass::boolean_struct::{False, True}; use crate::type_object::HasPyGilRef; use crate::types::any::PyAnyMethods; +use crate::types::string::PyStringMethods; use crate::types::{PyDict, PyString, PyTuple}; use crate::{ ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyClass, PyClassInitializer, PyRef, PyRefMut, @@ -97,10 +98,8 @@ fn python_format( f: &mut std::fmt::Formatter<'_>, ) -> Result<(), std::fmt::Error> { match format_result { - Result::Ok(s) => return f.write_str(&s.as_gil_ref().to_string_lossy()), - Result::Err(err) => { - err.write_unraisable(any.py(), std::option::Option::Some(any.as_gil_ref())) - } + Result::Ok(s) => return f.write_str(&s.to_string_lossy()), + Result::Err(err) => err.write_unraisable_bound(any.py(), Some(any)), } match any.get_type().name() { diff --git a/src/types/mapping.rs b/src/types/mapping.rs index 4eae0cec7c5..239ade9b20c 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -257,7 +257,10 @@ impl PyTypeCheck for PyMapping { || get_mapping_abc(object.py()) .and_then(|abc| object.is_instance(abc)) .unwrap_or_else(|err| { - err.write_unraisable(object.py(), Some(object)); + err.write_unraisable_bound( + object.py(), + Some(Bound::borrowed_from_gil_ref(&object)), + ); false }) } diff --git a/src/types/mod.rs b/src/types/mod.rs index 0ff41f80236..f153c4d92ec 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -107,7 +107,7 @@ macro_rules! pyobject_native_type_base( { match self.str() { ::std::result::Result::Ok(s) => return f.write_str(&s.to_string_lossy()), - ::std::result::Result::Err(err) => err.write_unraisable(self.py(), ::std::option::Option::Some(self)), + ::std::result::Result::Err(err) => err.write_unraisable_bound(self.py(), ::std::option::Option::Some($crate::Bound::borrowed_from_gil_ref(&self))), } match self.get_type().name() { diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 714288bd203..9b497ded4f7 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -541,7 +541,10 @@ impl PyTypeCheck for PySequence { || get_sequence_abc(object.py()) .and_then(|abc| object.is_instance(abc)) .unwrap_or_else(|err| { - err.write_unraisable(object.py(), Some(object)); + err.write_unraisable_bound( + object.py(), + Some(Bound::borrowed_from_gil_ref(&object)), + ); false }) } diff --git a/tests/test_exceptions.rs b/tests/test_exceptions.rs index 700a5c54149..1cf443e6773 100644 --- a/tests/test_exceptions.rs +++ b/tests/test_exceptions.rs @@ -100,6 +100,7 @@ fn test_exception_nosegfault() { #[test] #[cfg(Py_3_8)] +#[allow(deprecated)] fn test_write_unraisable() { use common::UnraisableCapture; use pyo3::{exceptions::PyRuntimeError, ffi}; From 1004ffa7d6b4cbdaab163913de76540f3fd21bcb Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 24 Dec 2023 19:35:50 +0000 Subject: [PATCH 005/349] export `Bound` and `Borrowed` from lib.rs --- src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f70ea01b91f..4802cbc2712 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -303,7 +303,7 @@ pub use crate::err::{ pub use crate::gil::GILPool; #[cfg(not(PyPy))] pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter}; -pub use crate::instance::{Py, PyNativeType, PyObject}; +pub use crate::instance::{Borrowed, Bound, Py, PyNativeType, PyObject}; pub use crate::marker::Python; pub use crate::pycell::{PyCell, PyRef, PyRefMut}; pub use crate::pyclass::PyClass; @@ -312,8 +312,6 @@ pub use crate::type_object::{PyTypeCheck, PyTypeInfo}; pub use crate::types::PyAny; pub use crate::version::PythonVersionInfo; -// Expected to become public API in 0.21 under a different name -pub(crate) use crate::instance::Bound; pub(crate) mod ffi_ptr_ext; pub(crate) mod py_result_ext; From d669a943f73b67574890595d57c3f1fffc71ad51 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 23 Dec 2023 14:26:02 +0000 Subject: [PATCH 006/349] implement iterator for `Bound` iterator --- src/ffi_ptr_ext.rs | 6 ++++++ src/instance.rs | 25 +++++++++++++++++++------ src/types/iterator.rs | 43 +++++++++++++++++++++++++++++++++++++------ 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/src/ffi_ptr_ext.rs b/src/ffi_ptr_ext.rs index 0eca2c36dd5..8a45f5d70c0 100644 --- a/src/ffi_ptr_ext.rs +++ b/src/ffi_ptr_ext.rs @@ -16,6 +16,7 @@ use sealed::Sealed; pub(crate) trait FfiPtrExt: Sealed { unsafe fn assume_owned_or_err(self, py: Python<'_>) -> PyResult>; + unsafe fn assume_owned_or_opt(self, py: Python<'_>) -> Option>; unsafe fn assume_owned(self, py: Python<'_>) -> Bound<'_, PyAny>; /// Assumes this pointer is borrowed from a parent object. @@ -41,6 +42,11 @@ impl FfiPtrExt for *mut ffi::PyObject { Bound::from_owned_ptr_or_err(py, self) } + #[inline] + unsafe fn assume_owned_or_opt(self, py: Python<'_>) -> Option> { + Bound::from_owned_ptr_or_opt(py, self) + } + #[inline] unsafe fn assume_owned(self, py: Python<'_>) -> Bound<'_, PyAny> { Bound::from_owned_ptr(py, self) diff --git a/src/instance.rs b/src/instance.rs index 8406e86abbe..0bd55b895e1 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -49,18 +49,31 @@ pub struct Bound<'py, T>(Python<'py>, ManuallyDrop>); impl<'py> Bound<'py, PyAny> { /// Constructs a new Bound from a pointer. Panics if ptr is null. + /// + /// # Safety + /// + /// `ptr`` must be a valid pointer to a Python object. pub(crate) unsafe fn from_owned_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self(py, ManuallyDrop::new(Py::from_owned_ptr(py, ptr))) } - // /// Constructs a new Bound from a pointer. Returns None if ptr is null. - // /// - // /// Safety: ptr must be a valid pointer to a Python object, or NULL. - // pub unsafe fn from_owned_ptr_or_opt(py: Python<'py>, ptr: *mut ffi::PyObject) -> Option { - // Py::from_owned_ptr_or_opt(py, ptr).map(|obj| Self(py, ManuallyDrop::new(obj))) - // } + /// Constructs a new Bound from a pointer. Returns None if ptr is null. + /// + /// # Safety + /// + /// `ptr`` must be a valid pointer to a Python object, or NULL. + pub(crate) unsafe fn from_owned_ptr_or_opt( + py: Python<'py>, + ptr: *mut ffi::PyObject, + ) -> Option { + Py::from_owned_ptr_or_opt(py, ptr).map(|obj| Self(py, ManuallyDrop::new(obj))) + } /// Constructs a new Bound from a pointer. Returns error if ptr is null. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to a Python object, or NULL. pub(crate) unsafe fn from_owned_ptr_or_err( py: Python<'py>, ptr: *mut ffi::PyObject, diff --git a/src/types/iterator.rs b/src/types/iterator.rs index bf392398eb9..0e033232964 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -1,4 +1,5 @@ use crate::ffi_ptr_ext::FfiPtrExt; +use crate::instance::Borrowed; use crate::py_result_ext::PyResultExt; use crate::{ ffi, AsPyPointer, Bound, PyAny, PyDowncastError, PyErr, PyNativeType, PyResult, PyTypeCheck, @@ -56,21 +57,51 @@ impl<'p> Iterator for &'p PyIterator { /// Further `next()` calls after an exception occurs are likely /// to repeatedly result in the same exception. fn next(&mut self) -> Option { - let py = self.0.py(); + Borrowed::::from_gil_ref(self) + .next() + .map(|result| result.map(Bound::into_gil_ref)) + } - match unsafe { py.from_owned_ptr_or_opt(ffi::PyIter_Next(self.0.as_ptr())) } { - Some(obj) => Some(Ok(obj)), - None => PyErr::take(py).map(Err), - } + #[cfg(not(Py_LIMITED_API))] + fn size_hint(&self) -> (usize, Option) { + Bound::borrowed_from_gil_ref(self).size_hint() + } +} + +impl<'py> Iterator for Bound<'py, PyIterator> { + type Item = PyResult>; + + /// Retrieves the next item from an iterator. + /// + /// Returns `None` when the iterator is exhausted. + /// If an exception occurs, returns `Some(Err(..))`. + /// Further `next()` calls after an exception occurs are likely + /// to repeatedly result in the same exception. + #[inline] + fn next(&mut self) -> Option { + Borrowed::from(&*self).next() } #[cfg(not(Py_LIMITED_API))] fn size_hint(&self) -> (usize, Option) { - let hint = unsafe { ffi::PyObject_LengthHint(self.0.as_ptr(), 0) }; + let hint = unsafe { ffi::PyObject_LengthHint(self.as_ptr(), 0) }; (hint.max(0) as usize, None) } } +impl<'py> Borrowed<'_, 'py, PyIterator> { + // TODO: this method is on Borrowed so that &'py PyIterator can use this; once that + // implementation is deleted this method should be moved to the `Bound<'py, PyIterator> impl + fn next(self) -> Option>> { + let py = self.py(); + + match unsafe { ffi::PyIter_Next(self.as_ptr()).assume_owned_or_opt(py) } { + Some(obj) => Some(Ok(obj)), + None => PyErr::take(py).map(Err), + } + } +} + impl PyTypeCheck for PyIterator { const NAME: &'static str = "Iterator"; From d9cc0c0f243e64e3584208f0801502a70fb779f3 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 24 Dec 2023 13:50:38 +0000 Subject: [PATCH 007/349] introduce `PySuper::new_bound` --- src/types/any.rs | 2 +- src/types/pysuper.rs | 23 ++++++++++++++--------- tests/test_super.rs | 1 + 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/types/any.rs b/src/types/any.rs index 2e8a518fad4..b8e8c03ad06 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -2193,7 +2193,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { #[cfg(not(PyPy))] fn py_super(&self) -> PyResult> { - PySuper::new2(Bound::borrowed_from_gil_ref(&self.get_type()), self) + PySuper::new_bound(Bound::borrowed_from_gil_ref(&self.get_type()), self) } } diff --git a/src/types/pysuper.rs b/src/types/pysuper.rs index 5cf05a36a4d..d14670987b5 100644 --- a/src/types/pysuper.rs +++ b/src/types/pysuper.rs @@ -16,6 +16,19 @@ pyobject_native_type_core!( ); impl PySuper { + /// Deprecated form of `PySuper::new_bound`. + #[deprecated( + since = "0.21.0", + note = "`PySuper::new` will be replaced by `PySuper::new_bound` in a future PyO3 version" + )] + pub fn new<'py>(ty: &'py PyType, obj: &'py PyAny) -> PyResult<&'py PySuper> { + Self::new_bound( + Bound::borrowed_from_gil_ref(&ty), + Bound::borrowed_from_gil_ref(&obj), + ) + .map(Bound::into_gil_ref) + } + /// Constructs a new super object. More read about super object: [docs](https://docs.python.org/3/library/functions.html#super) /// /// # Examples @@ -56,15 +69,7 @@ impl PySuper { /// } /// } /// ``` - pub fn new<'py>(ty: &'py PyType, obj: &'py PyAny) -> PyResult<&'py PySuper> { - Self::new2( - Bound::borrowed_from_gil_ref(&ty), - Bound::borrowed_from_gil_ref(&obj), - ) - .map(Bound::into_gil_ref) - } - - pub(crate) fn new2<'py>( + pub fn new_bound<'py>( ty: &Bound<'py, PyType>, obj: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/tests/test_super.rs b/tests/test_super.rs index dca12457011..b71eae55376 100644 --- a/tests/test_super.rs +++ b/tests/test_super.rs @@ -35,6 +35,7 @@ impl SubClass { } fn method_super_new(self_: &PyCell) -> PyResult<&PyAny> { + #[allow(deprecated)] let super_ = PySuper::new(self_.get_type(), self_)?; super_.call_method("method", (), None) } From 91fdfaab459ee5c6701e40f48fc84c6f1da235fc Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 24 Dec 2023 15:21:01 -0500 Subject: [PATCH 008/349] Use a version of gevent that supports py37 The current version of gevent we require is 3.8+ only --- pytests/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index 4d5b42bf46f..920455ca871 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ [project.optional-dependencies] dev = [ - "gevent>=23.9.1", + "gevent>=22.10.2", "hypothesis>=3.55", "pytest-asyncio>=0.21", "pytest-benchmark>=3.4", From 38abfd2eedf4be7c2e38c2436f449e9368522ff0 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 24 Dec 2023 14:34:08 +0000 Subject: [PATCH 009/349] expose `Bound::as_gil_ref` and `Bound::into_gil_ref` --- src/instance.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index c3de539abe6..68bc0b5d54b 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -183,8 +183,9 @@ impl<'py, T> Bound<'py, T> { unsafe { std::mem::transmute(gil_ref) } } - /// Internal helper to get to pool references for backwards compatibility - #[doc(hidden)] // public and doc(hidden) to use in examples and tests for now + /// Casts this `Bound` as the corresponding "GIL Ref" type. + /// + /// This is a helper to be used for migration from the deprecated "GIL Refs" API. pub fn as_gil_ref(&'py self) -> &'py T::AsRefTarget where T: HasPyGilRef, @@ -192,8 +193,10 @@ impl<'py, T> Bound<'py, T> { unsafe { self.py().from_borrowed_ptr(self.as_ptr()) } } - /// Internal helper to get to pool references for backwards compatibility - #[doc(hidden)] // public but hidden, to use for tests for now + /// Casts this `Bound` as the corresponding "GIL Ref" type, registering the pointer on the + /// [release pool](Python::from_owned_ptr). + /// + /// This is a helper to be used for migration from the deprecated "GIL Refs" API. pub fn into_gil_ref(self) -> &'py T::AsRefTarget where T: HasPyGilRef, From 1b61cb015a498d383c4a9406e2179f3f2849d65e Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 23 Dec 2023 21:48:04 +0000 Subject: [PATCH 010/349] Add `.as_borrowed()` conversion from gil-refs to `Bound` --- newsfragments/3692.added.md | 1 + newsfragments/3692.changed.md | 1 + src/conversions/chrono.rs | 7 ++- src/err/mod.rs | 6 +- src/instance.rs | 42 +++++++------- src/prelude.rs | 1 + src/types/any.rs | 100 +++++++++++++++------------------- src/types/boolobject.rs | 6 +- src/types/bytearray.rs | 16 +++--- src/types/bytes.rs | 4 +- src/types/datetime.rs | 42 +++++++------- src/types/dict.rs | 32 +++++------ src/types/float.rs | 2 +- src/types/iterator.rs | 6 +- src/types/list.rs | 44 ++++++--------- src/types/mapping.rs | 31 ++++------- src/types/mod.rs | 3 +- src/types/pysuper.rs | 11 ++-- src/types/sequence.rs | 51 +++++++---------- src/types/string.rs | 10 ++-- 20 files changed, 185 insertions(+), 231 deletions(-) create mode 100644 newsfragments/3692.added.md create mode 100644 newsfragments/3692.changed.md diff --git a/newsfragments/3692.added.md b/newsfragments/3692.added.md new file mode 100644 index 00000000000..45cdd5aba28 --- /dev/null +++ b/newsfragments/3692.added.md @@ -0,0 +1 @@ +Add `PyNativeType::as_bound` to convert "GIL refs" to the new `Bound` smart pointer. diff --git a/newsfragments/3692.changed.md b/newsfragments/3692.changed.md new file mode 100644 index 00000000000..9535cbb23db --- /dev/null +++ b/newsfragments/3692.changed.md @@ -0,0 +1 @@ +Include `PyNativeType` in `pyo3::prelude`. diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 83100f8023a..cc0eb73a8ef 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -42,7 +42,6 @@ //! } //! ``` use crate::exceptions::{PyTypeError, PyUserWarning, PyValueError}; -use crate::instance::Bound; #[cfg(Py_LIMITED_API)] use crate::sync::GILOnceCell; #[cfg(not(Py_LIMITED_API))] @@ -56,7 +55,9 @@ use crate::types::{ }; #[cfg(Py_LIMITED_API)] use crate::{intern, PyDowncastError}; -use crate::{FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject}; +use crate::{ + FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, ToPyObject, +}; use chrono::offset::{FixedOffset, Utc}; use chrono::{ DateTime, Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Offset, TimeZone, Timelike, @@ -466,7 +467,7 @@ fn warn_truncated_leap_second(obj: &PyAny) { "ignored leap-second, `datetime` does not support leap-seconds", 0, ) { - e.write_unraisable_bound(py, Some(Bound::borrowed_from_gil_ref(&obj))) + e.write_unraisable_bound(py, Some(&obj.as_borrowed())) }; } diff --git a/src/err/mod.rs b/src/err/mod.rs index c2d47a8f17a..ed719103646 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -7,7 +7,7 @@ use crate::{ exceptions::{self, PyBaseException}, ffi, }; -use crate::{IntoPy, Py, PyAny, PyObject, Python, ToPyObject}; +use crate::{IntoPy, Py, PyAny, PyNativeType, PyObject, Python, ToPyObject}; use std::borrow::Cow; use std::cell::UnsafeCell; use std::ffi::CString; @@ -542,7 +542,7 @@ impl PyErr { )] #[inline] pub fn write_unraisable(self, py: Python<'_>, obj: Option<&PyAny>) { - self.write_unraisable_bound(py, obj.as_ref().map(Bound::borrowed_from_gil_ref)) + self.write_unraisable_bound(py, obj.map(PyAny::as_borrowed).as_deref()) } /// Reports the error as unraisable. @@ -821,7 +821,7 @@ impl<'a> std::error::Error for PyDowncastError<'a> {} impl<'a> std::fmt::Display for PyDowncastError<'a> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - display_downcast_error(f, Bound::borrowed_from_gil_ref(&self.from), &self.to) + display_downcast_error(f, &self.from.as_borrowed(), &self.to) } } diff --git a/src/instance.rs b/src/instance.rs index dae71e671d8..90e672c4fa5 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -28,6 +28,19 @@ pub unsafe trait PyNativeType: Sized { /// The form of this which is stored inside a `Py` smart pointer. type AsRefSource: HasPyGilRef; + /// Cast `&self` to a `Borrowed` smart pointer. + /// + /// `Borrowed` implements `Deref>`, so can also be used in locations + /// where `Bound` is expected. + /// + /// This is available as a migration tool to adjust code from the deprecated "GIL Refs" + /// API to the `Bound` smart pointer API. + fn as_borrowed(&self) -> Borrowed<'_, '_, Self::AsRefSource> { + // Safety: &'py Self is expected to be a Python pointer, + // so has the same layout as Borrowed<'py, 'py, T> + unsafe { std::mem::transmute(self) } + } + /// Returns a GIL marker constrained to the lifetime of this type. #[inline] fn py(&self) -> Python<'_> { @@ -184,18 +197,6 @@ impl<'py, T> Bound<'py, T> { self.into_non_null().as_ptr() } - /// Internal helper to convert e.g. &'a &'py PyDict to &'a Bound<'py, PyDict> for - /// backwards-compatibility during migration to removal of pool. - #[doc(hidden)] // public and doc(hidden) to use in examples and tests for now - pub fn borrowed_from_gil_ref<'a, U>(gil_ref: &'a &'py U) -> &'a Self - where - U: PyNativeType, - { - // Safety: &'py T::AsRefTarget is expected to be a Python pointer, - // so &'a &'py T::AsRefTarget has the same layout as &'a Bound<'py, T> - unsafe { std::mem::transmute(gil_ref) } - } - /// Casts this `Bound` as the corresponding "GIL Ref" type. /// /// This is a helper to be used for migration from the deprecated "GIL Refs" API. @@ -311,12 +312,6 @@ impl<'py, T> Borrowed<'py, 'py, T> where T: HasPyGilRef, { - pub(crate) fn from_gil_ref(gil_ref: &'py T::AsRefTarget) -> Self { - // Safety: &'py T::AsRefTarget is expected to be a Python pointer, - // so &'py T::AsRefTarget has the same layout as Self. - unsafe { std::mem::transmute(gil_ref) } - } - // pub(crate) fn into_gil_ref(self) -> &'py T::AsRefTarget { // // Safety: self is a borrow over `'py`. // unsafe { self.py().from_borrowed_ptr(self.0.as_ptr()) } @@ -1366,7 +1361,7 @@ where { /// Extracts `Self` from the source `PyObject`. fn extract(ob: &'a PyAny) -> PyResult { - Bound::borrowed_from_gil_ref(&ob) + ob.as_borrowed() .downcast() .map(Clone::clone) .map_err(Into::into) @@ -1485,7 +1480,7 @@ impl PyObject { mod tests { use super::{Bound, Py, PyObject}; use crate::types::{PyDict, PyString}; - use crate::{PyAny, PyResult, Python, ToPyObject}; + use crate::{PyAny, PyNativeType, PyResult, Python, ToPyObject}; #[test] fn test_call0() { @@ -1612,8 +1607,11 @@ a = A() #[test] fn test_py2_into_py_object() { Python::with_gil(|py| { - let instance: Bound<'_, PyAny> = - Bound::borrowed_from_gil_ref(&py.eval("object()", None, None).unwrap()).clone(); + let instance = py + .eval("object()", None, None) + .unwrap() + .as_borrowed() + .to_owned(); let ptr = instance.as_ptr(); let instance: PyObject = instance.clone().into(); assert_eq!(instance.as_ptr(), ptr); diff --git a/src/prelude.rs b/src/prelude.rs index 866b2226765..b06625dc36a 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -17,6 +17,7 @@ pub use crate::marker::Python; pub use crate::pycell::{PyCell, PyRef, PyRefMut}; pub use crate::pyclass_init::PyClassInitializer; pub use crate::types::{PyAny, PyModule}; +pub use crate::PyNativeType; #[cfg(feature = "macros")] pub use pyo3_macros::{pyclass, pyfunction, pymethods, pymodule, FromPyObject}; diff --git a/src/types/any.rs b/src/types/any.rs index b8e8c03ad06..edc74879cb6 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -73,7 +73,7 @@ impl PyAny { /// This is equivalent to the Python expression `self is other`. #[inline] pub fn is(&self, other: &T) -> bool { - Bound::borrowed_from_gil_ref(&self).is(other) + self.as_borrowed().is(other) } /// Determines whether this object has the given attribute. @@ -102,7 +102,7 @@ impl PyAny { where N: IntoPy>, { - Bound::borrowed_from_gil_ref(&self).hasattr(attr_name) + self.as_borrowed().hasattr(attr_name) } /// Retrieves an attribute value. @@ -131,7 +131,7 @@ impl PyAny { where N: IntoPy>, { - Bound::borrowed_from_gil_ref(&self) + self.as_borrowed() .getattr(attr_name) .map(Bound::into_gil_ref) } @@ -208,7 +208,7 @@ impl PyAny { N: IntoPy>, V: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).setattr(attr_name, value) + self.as_borrowed().setattr(attr_name, value) } /// Deletes an attribute. @@ -221,7 +221,7 @@ impl PyAny { where N: IntoPy>, { - Bound::borrowed_from_gil_ref(&self).delattr(attr_name) + self.as_borrowed().delattr(attr_name) } /// Returns an [`Ordering`] between `self` and `other`. @@ -274,7 +274,7 @@ impl PyAny { where O: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).compare(other) + self.as_borrowed().compare(other) } /// Tests whether two Python objects obey a given [`CompareOp`]. @@ -315,7 +315,7 @@ impl PyAny { where O: ToPyObject, { - Bound::borrowed_from_gil_ref(&self) + self.as_borrowed() .rich_compare(other, compare_op) .map(Bound::into_gil_ref) } @@ -327,7 +327,7 @@ impl PyAny { where O: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).lt(other) + self.as_borrowed().lt(other) } /// Tests whether this object is less than or equal to another. @@ -337,7 +337,7 @@ impl PyAny { where O: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).le(other) + self.as_borrowed().le(other) } /// Tests whether this object is equal to another. @@ -347,7 +347,7 @@ impl PyAny { where O: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).eq(other) + self.as_borrowed().eq(other) } /// Tests whether this object is not equal to another. @@ -357,7 +357,7 @@ impl PyAny { where O: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).ne(other) + self.as_borrowed().ne(other) } /// Tests whether this object is greater than another. @@ -367,7 +367,7 @@ impl PyAny { where O: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).gt(other) + self.as_borrowed().gt(other) } /// Tests whether this object is greater than or equal to another. @@ -377,7 +377,7 @@ impl PyAny { where O: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).ge(other) + self.as_borrowed().ge(other) } /// Determines whether this object appears callable. @@ -408,7 +408,7 @@ impl PyAny { /// /// [1]: https://docs.python.org/3/library/functions.html#callable pub fn is_callable(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).is_callable() + self.as_borrowed().is_callable() } /// Calls the object. @@ -446,7 +446,7 @@ impl PyAny { args: impl IntoPy>, kwargs: Option<&PyDict>, ) -> PyResult<&PyAny> { - Bound::borrowed_from_gil_ref(&self) + self.as_borrowed() .call(args, kwargs) .map(Bound::into_gil_ref) } @@ -472,9 +472,7 @@ impl PyAny { /// /// This is equivalent to the Python expression `help()`. pub fn call0(&self) -> PyResult<&PyAny> { - Bound::borrowed_from_gil_ref(&self) - .call0() - .map(Bound::into_gil_ref) + self.as_borrowed().call0().map(Bound::into_gil_ref) } /// Calls the object with only positional arguments. @@ -505,9 +503,7 @@ impl PyAny { /// # } /// ``` pub fn call1(&self, args: impl IntoPy>) -> PyResult<&PyAny> { - Bound::borrowed_from_gil_ref(&self) - .call1(args) - .map(Bound::into_gil_ref) + self.as_borrowed().call1(args).map(Bound::into_gil_ref) } /// Calls a method on the object. @@ -550,7 +546,7 @@ impl PyAny { N: IntoPy>, A: IntoPy>, { - Bound::borrowed_from_gil_ref(&self) + self.as_borrowed() .call_method(name, args, kwargs) .map(Bound::into_gil_ref) } @@ -590,7 +586,7 @@ impl PyAny { where N: IntoPy>, { - Bound::borrowed_from_gil_ref(&self) + self.as_borrowed() .call_method0(name) .map(Bound::into_gil_ref) } @@ -632,7 +628,7 @@ impl PyAny { N: IntoPy>, A: IntoPy>, { - Bound::borrowed_from_gil_ref(&self) + self.as_borrowed() .call_method1(name, args) .map(Bound::into_gil_ref) } @@ -649,7 +645,7 @@ impl PyAny { /// /// This applies truth value testing equivalent to the Python expression `bool(self)`. pub fn is_truthy(&self) -> PyResult { - Bound::borrowed_from_gil_ref(&self).is_truthy() + self.as_borrowed().is_truthy() } /// Returns whether the object is considered to be None. @@ -657,7 +653,7 @@ impl PyAny { /// This is equivalent to the Python expression `self is None`. #[inline] pub fn is_none(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).is_none() + self.as_borrowed().is_none() } /// Returns whether the object is Ellipsis, e.g. `...`. @@ -665,14 +661,14 @@ impl PyAny { /// This is equivalent to the Python expression `self is ...`. #[deprecated(since = "0.20.0", note = "use `.is(py.Ellipsis())` instead")] pub fn is_ellipsis(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).is_ellipsis() + self.as_borrowed().is_ellipsis() } /// Returns true if the sequence or mapping has a length of 0. /// /// This is equivalent to the Python expression `len(self) == 0`. pub fn is_empty(&self) -> PyResult { - Bound::borrowed_from_gil_ref(&self).is_empty() + self.as_borrowed().is_empty() } /// Gets an item from the collection. @@ -682,9 +678,7 @@ impl PyAny { where K: ToPyObject, { - Bound::borrowed_from_gil_ref(&self) - .get_item(key) - .map(Bound::into_gil_ref) + self.as_borrowed().get_item(key).map(Bound::into_gil_ref) } /// Sets a collection item value. @@ -695,7 +689,7 @@ impl PyAny { K: ToPyObject, V: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).set_item(key, value) + self.as_borrowed().set_item(key, value) } /// Deletes an item from the collection. @@ -705,7 +699,7 @@ impl PyAny { where K: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).del_item(key) + self.as_borrowed().del_item(key) } /// Takes an object and returns an iterator for it. @@ -713,20 +707,18 @@ impl PyAny { /// This is typically a new iterator but if the argument is an iterator, /// this returns itself. pub fn iter(&self) -> PyResult<&PyIterator> { - Bound::borrowed_from_gil_ref(&self) - .iter() - .map(Bound::into_gil_ref) + self.as_borrowed().iter().map(Bound::into_gil_ref) } /// Returns the Python type object for this object's type. pub fn get_type(&self) -> &PyType { - Bound::borrowed_from_gil_ref(&self).get_type() + self.as_borrowed().get_type() } /// Returns the Python type pointer for this object. #[inline] pub fn get_type_ptr(&self) -> *mut ffi::PyTypeObject { - Bound::borrowed_from_gil_ref(&self).get_type_ptr() + self.as_borrowed().get_type_ptr() } /// Downcast this `PyAny` to a concrete Python type or pyclass. @@ -863,46 +855,42 @@ impl PyAny { /// Returns the reference count for the Python object. pub fn get_refcnt(&self) -> isize { - Bound::borrowed_from_gil_ref(&self).get_refcnt() + self.as_borrowed().get_refcnt() } /// Computes the "repr" representation of self. /// /// This is equivalent to the Python expression `repr(self)`. pub fn repr(&self) -> PyResult<&PyString> { - Bound::borrowed_from_gil_ref(&self) - .repr() - .map(Bound::into_gil_ref) + self.as_borrowed().repr().map(Bound::into_gil_ref) } /// Computes the "str" representation of self. /// /// This is equivalent to the Python expression `str(self)`. pub fn str(&self) -> PyResult<&PyString> { - Bound::borrowed_from_gil_ref(&self) - .str() - .map(Bound::into_gil_ref) + self.as_borrowed().str().map(Bound::into_gil_ref) } /// Retrieves the hash code of self. /// /// This is equivalent to the Python expression `hash(self)`. pub fn hash(&self) -> PyResult { - Bound::borrowed_from_gil_ref(&self).hash() + self.as_borrowed().hash() } /// Returns the length of the sequence or mapping. /// /// This is equivalent to the Python expression `len(self)`. pub fn len(&self) -> PyResult { - Bound::borrowed_from_gil_ref(&self).len() + self.as_borrowed().len() } /// Returns the list of attributes of this object. /// /// This is equivalent to the Python expression `dir(self)`. pub fn dir(&self) -> &PyList { - Bound::borrowed_from_gil_ref(&self).dir().into_gil_ref() + self.as_borrowed().dir().into_gil_ref() } /// Checks whether this object is an instance of type `ty`. @@ -910,7 +898,7 @@ impl PyAny { /// This is equivalent to the Python expression `isinstance(self, ty)`. #[inline] pub fn is_instance(&self, ty: &PyAny) -> PyResult { - Bound::borrowed_from_gil_ref(&self).is_instance(Bound::borrowed_from_gil_ref(&ty)) + self.as_borrowed().is_instance(&ty.as_borrowed()) } /// Checks whether this object is an instance of exactly type `ty` (not a subclass). @@ -918,7 +906,7 @@ impl PyAny { /// This is equivalent to the Python expression `type(self) is ty`. #[inline] pub fn is_exact_instance(&self, ty: &PyAny) -> bool { - Bound::borrowed_from_gil_ref(&self).is_exact_instance(Bound::borrowed_from_gil_ref(&ty)) + self.as_borrowed().is_exact_instance(&ty.as_borrowed()) } /// Checks whether this object is an instance of type `T`. @@ -927,7 +915,7 @@ impl PyAny { /// if the type `T` is known at compile time. #[inline] pub fn is_instance_of(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).is_instance_of::() + self.as_borrowed().is_instance_of::() } /// Checks whether this object is an instance of exactly type `T`. @@ -936,7 +924,7 @@ impl PyAny { /// if the type `T` is known at compile time. #[inline] pub fn is_exact_instance_of(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).is_exact_instance_of::() + self.as_borrowed().is_exact_instance_of::() } /// Determines if self contains `value`. @@ -946,7 +934,7 @@ impl PyAny { where V: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).contains(value) + self.as_borrowed().contains(value) } /// Returns a GIL marker constrained to the lifetime of this type. @@ -985,9 +973,7 @@ impl PyAny { /// This is equivalent to the Python expression `super()` #[cfg(not(PyPy))] pub fn py_super(&self) -> PyResult<&PySuper> { - Bound::borrowed_from_gil_ref(&self) - .py_super() - .map(Bound::into_gil_ref) + self.as_borrowed().py_super().map(Bound::into_gil_ref) } } @@ -2193,7 +2179,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { #[cfg(not(PyPy))] fn py_super(&self) -> PyResult> { - PySuper::new_bound(Bound::borrowed_from_gil_ref(&self.get_type()), self) + PySuper::new_bound(&self.get_type().as_borrowed(), self) } } diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 55896f877c5..5c246de380d 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -1,8 +1,8 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - exceptions::PyTypeError, ffi, instance::Bound, FromPyObject, IntoPy, PyAny, PyObject, PyResult, - Python, ToPyObject, + exceptions::PyTypeError, ffi, instance::Bound, FromPyObject, IntoPy, PyAny, PyNativeType, + PyObject, PyResult, Python, ToPyObject, }; /// Represents a Python `bool`. @@ -21,7 +21,7 @@ impl PyBool { /// Gets whether this boolean is `true`. #[inline] pub fn is_true(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).is_true() + self.as_borrowed().is_true() } } diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index a90fec82273..f1b8050c066 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -1,6 +1,6 @@ use crate::err::{PyErr, PyResult}; use crate::instance::{Borrowed, Bound}; -use crate::{ffi, AsPyPointer, Py, PyAny, Python}; +use crate::{ffi, AsPyPointer, Py, PyAny, PyNativeType, Python}; use std::os::raw::c_char; use std::slice; @@ -75,12 +75,12 @@ impl PyByteArray { /// Gets the length of the bytearray. #[inline] pub fn len(&self) -> usize { - Bound::borrowed_from_gil_ref(&self).len() + self.as_borrowed().len() } /// Checks if the bytearray is empty. pub fn is_empty(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).is_empty() + self.as_borrowed().is_empty() } /// Gets the start of the buffer containing the contents of the bytearray. @@ -89,7 +89,7 @@ impl PyByteArray { /// /// See the safety requirements of [`PyByteArray::as_bytes`] and [`PyByteArray::as_bytes_mut`]. pub fn data(&self) -> *mut u8 { - Bound::borrowed_from_gil_ref(&self).data() + self.as_borrowed().data() } /// Extracts a slice of the `ByteArray`'s entire buffer. @@ -188,7 +188,7 @@ impl PyByteArray { /// } /// ``` pub unsafe fn as_bytes(&self) -> &[u8] { - Borrowed::from_gil_ref(self).as_bytes() + self.as_borrowed().as_bytes() } /// Extracts a mutable slice of the `ByteArray`'s entire buffer. @@ -200,7 +200,7 @@ impl PyByteArray { /// apply to this function as well. #[allow(clippy::mut_from_ref)] pub unsafe fn as_bytes_mut(&self) -> &mut [u8] { - Borrowed::from_gil_ref(self).as_bytes_mut() + self.as_borrowed().as_bytes_mut() } /// Copies the contents of the bytearray to a Rust vector. @@ -222,7 +222,7 @@ impl PyByteArray { /// # }); /// ``` pub fn to_vec(&self) -> Vec { - Bound::borrowed_from_gil_ref(&self).to_vec() + self.as_borrowed().to_vec() } /// Resizes the bytearray object to the new length `len`. @@ -230,7 +230,7 @@ impl PyByteArray { /// Note that this will invalidate any pointers obtained by [PyByteArray::data], as well as /// any (unsafe) slices obtained from [PyByteArray::as_bytes] and [PyByteArray::as_bytes_mut]. pub fn resize(&self, len: usize) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).resize(len) + self.as_borrowed().resize(len) } } diff --git a/src/types/bytes.rs b/src/types/bytes.rs index 4db098c6bb6..5a7b174ea0c 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -1,5 +1,5 @@ use crate::instance::{Borrowed, Bound}; -use crate::{ffi, FromPyObject, IntoPy, Py, PyAny, PyResult, Python, ToPyObject}; +use crate::{ffi, FromPyObject, IntoPy, Py, PyAny, PyNativeType, PyResult, Python, ToPyObject}; use std::borrow::Cow; use std::ops::Index; use std::os::raw::c_char; @@ -89,7 +89,7 @@ impl PyBytes { /// Gets the Python string as a byte slice. #[inline] pub fn as_bytes(&self) -> &[u8] { - Borrowed::from_gil_ref(self).as_bytes() + self.as_borrowed().as_bytes() } } diff --git a/src/types/datetime.rs b/src/types/datetime.rs index b2d0f6efa00..b671249c983 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -223,15 +223,15 @@ impl PyDate { impl PyDateAccess for PyDate { fn get_year(&self) -> i32 { - Bound::borrowed_from_gil_ref(&self).get_year() + self.as_borrowed().get_year() } fn get_month(&self) -> u8 { - Bound::borrowed_from_gil_ref(&self).get_month() + self.as_borrowed().get_month() } fn get_day(&self) -> u8 { - Bound::borrowed_from_gil_ref(&self).get_day() + self.as_borrowed().get_day() } } @@ -351,15 +351,15 @@ impl PyDateTime { impl PyDateAccess for PyDateTime { fn get_year(&self) -> i32 { - Bound::borrowed_from_gil_ref(&self).get_year() + self.as_borrowed().get_year() } fn get_month(&self) -> u8 { - Bound::borrowed_from_gil_ref(&self).get_month() + self.as_borrowed().get_month() } fn get_day(&self) -> u8 { - Bound::borrowed_from_gil_ref(&self).get_day() + self.as_borrowed().get_day() } } @@ -379,23 +379,23 @@ impl PyDateAccess for Bound<'_, PyDateTime> { impl PyTimeAccess for PyDateTime { fn get_hour(&self) -> u8 { - Bound::borrowed_from_gil_ref(&self).get_hour() + self.as_borrowed().get_hour() } fn get_minute(&self) -> u8 { - Bound::borrowed_from_gil_ref(&self).get_minute() + self.as_borrowed().get_minute() } fn get_second(&self) -> u8 { - Bound::borrowed_from_gil_ref(&self).get_second() + self.as_borrowed().get_second() } fn get_microsecond(&self) -> u32 { - Bound::borrowed_from_gil_ref(&self).get_microsecond() + self.as_borrowed().get_microsecond() } fn get_fold(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).get_fold() + self.as_borrowed().get_fold() } } @@ -423,7 +423,7 @@ impl PyTimeAccess for Bound<'_, PyDateTime> { impl<'py> PyTzInfoAccess<'py> for &'py PyDateTime { fn get_tzinfo_bound(&self) -> Option> { - Bound::borrowed_from_gil_ref(self).get_tzinfo_bound() + self.as_borrowed().get_tzinfo_bound() } } @@ -509,23 +509,23 @@ impl PyTime { impl PyTimeAccess for PyTime { fn get_hour(&self) -> u8 { - Bound::borrowed_from_gil_ref(&self).get_hour() + self.as_borrowed().get_hour() } fn get_minute(&self) -> u8 { - Bound::borrowed_from_gil_ref(&self).get_minute() + self.as_borrowed().get_minute() } fn get_second(&self) -> u8 { - Bound::borrowed_from_gil_ref(&self).get_second() + self.as_borrowed().get_second() } fn get_microsecond(&self) -> u32 { - Bound::borrowed_from_gil_ref(&self).get_microsecond() + self.as_borrowed().get_microsecond() } fn get_fold(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).get_fold() + self.as_borrowed().get_fold() } } @@ -553,7 +553,7 @@ impl PyTimeAccess for Bound<'_, PyTime> { impl<'py> PyTzInfoAccess<'py> for &'py PyTime { fn get_tzinfo_bound(&self) -> Option> { - Bound::borrowed_from_gil_ref(self).get_tzinfo_bound() + self.as_borrowed().get_tzinfo_bound() } } @@ -644,15 +644,15 @@ impl PyDelta { impl PyDeltaAccess for PyDelta { fn get_days(&self) -> i32 { - Bound::borrowed_from_gil_ref(&self).get_days() + self.as_borrowed().get_days() } fn get_seconds(&self) -> i32 { - Bound::borrowed_from_gil_ref(&self).get_seconds() + self.as_borrowed().get_seconds() } fn get_microseconds(&self) -> i32 { - Bound::borrowed_from_gil_ref(&self).get_microseconds() + self.as_borrowed().get_microseconds() } } diff --git a/src/types/dict.rs b/src/types/dict.rs index 89e0df96e63..80d2798a863 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -6,7 +6,7 @@ use crate::instance::{Borrowed, Bound}; use crate::py_result_ext::PyResultExt; use crate::types::any::PyAnyMethods; use crate::types::{PyAny, PyList}; -use crate::{ffi, Python, ToPyObject}; +use crate::{ffi, PyNativeType, Python, ToPyObject}; /// Represents a Python `dict`. #[repr(transparent)] @@ -82,26 +82,24 @@ impl PyDict { /// /// This is equivalent to the Python expression `self.copy()`. pub fn copy(&self) -> PyResult<&PyDict> { - Bound::borrowed_from_gil_ref(&self) - .copy() - .map(Bound::into_gil_ref) + self.as_borrowed().copy().map(Bound::into_gil_ref) } /// Empties an existing dictionary of all key-value pairs. pub fn clear(&self) { - Bound::borrowed_from_gil_ref(&self).clear() + self.as_borrowed().clear() } /// Return the number of items in the dictionary. /// /// This is equivalent to the Python expression `len(self)`. pub fn len(&self) -> usize { - Bound::borrowed_from_gil_ref(&self).len() + self.as_borrowed().len() } /// Checks if the dict is empty, i.e. `len(self) == 0`. pub fn is_empty(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).is_empty() + self.as_borrowed().is_empty() } /// Determines if the dictionary contains the specified key. @@ -111,7 +109,7 @@ impl PyDict { where K: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).contains(key) + self.as_borrowed().contains(key) } /// Gets an item from the dictionary. @@ -160,7 +158,7 @@ impl PyDict { where K: ToPyObject, { - match Bound::borrowed_from_gil_ref(&self).get_item(key) { + match self.as_borrowed().get_item(key) { Ok(Some(item)) => Ok(Some(item.into_gil_ref())), Ok(None) => Ok(None), Err(e) => Err(e), @@ -188,7 +186,7 @@ impl PyDict { K: ToPyObject, V: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).set_item(key, value) + self.as_borrowed().set_item(key, value) } /// Deletes an item. @@ -198,28 +196,28 @@ impl PyDict { where K: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).del_item(key) + self.as_borrowed().del_item(key) } /// Returns a list of dict keys. /// /// This is equivalent to the Python expression `list(dict.keys())`. pub fn keys(&self) -> &PyList { - Bound::borrowed_from_gil_ref(&self).keys().into_gil_ref() + self.as_borrowed().keys().into_gil_ref() } /// Returns a list of dict values. /// /// This is equivalent to the Python expression `list(dict.values())`. pub fn values(&self) -> &PyList { - Bound::borrowed_from_gil_ref(&self).values().into_gil_ref() + self.as_borrowed().values().into_gil_ref() } /// Returns a list of dict items. /// /// This is equivalent to the Python expression `list(dict.items())`. pub fn items(&self) -> &PyList { - Bound::borrowed_from_gil_ref(&self).items().into_gil_ref() + self.as_borrowed().items().into_gil_ref() } /// Returns an iterator of `(key, value)` pairs in this dictionary. @@ -230,7 +228,7 @@ impl PyDict { /// It is allowed to modify values as you iterate over the dictionary, but only /// so long as the set of keys does not change. pub fn iter(&self) -> PyDictIterator<'_> { - PyDictIterator(Bound::borrowed_from_gil_ref(&self).iter()) + PyDictIterator(self.as_borrowed().iter()) } /// Returns `self` cast as a `PyMapping`. @@ -243,7 +241,7 @@ impl PyDict { /// This is equivalent to the Python expression `self.update(other)`. If `other` is a `PyDict`, you may want /// to use `self.update(other.as_mapping())`, note: `PyDict::as_mapping` is a zero-cost conversion. pub fn update(&self, other: &PyMapping) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).update(Bound::borrowed_from_gil_ref(&other)) + self.as_borrowed().update(&other.as_borrowed()) } /// Add key/value pairs from another dictionary to this one only when they do not exist in this. @@ -255,7 +253,7 @@ impl PyDict { /// This method uses [`PyDict_Merge`](https://docs.python.org/3/c-api/dict.html#c.PyDict_Merge) internally, /// so should have the same performance as `update`. pub fn update_if_missing(&self, other: &PyMapping) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).update_if_missing(Bound::borrowed_from_gil_ref(&other)) + self.as_borrowed().update_if_missing(&other.as_borrowed()) } } diff --git a/src/types/float.rs b/src/types/float.rs index bb55d7ed5f5..b6dabdc48e8 100644 --- a/src/types/float.rs +++ b/src/types/float.rs @@ -29,7 +29,7 @@ impl PyFloat { /// Gets the value of this float. pub fn value(&self) -> c_double { - Bound::borrowed_from_gil_ref(&self).value() + self.as_borrowed().value() } } diff --git a/src/types/iterator.rs b/src/types/iterator.rs index 0e033232964..67866348f8c 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -35,7 +35,7 @@ impl PyIterator { /// /// Equivalent to Python's built-in `iter` function. pub fn from_object(obj: &PyAny) -> PyResult<&PyIterator> { - Self::from_object2(Bound::borrowed_from_gil_ref(&obj)).map(Bound::into_gil_ref) + Self::from_object2(&obj.as_borrowed()).map(Bound::into_gil_ref) } pub(crate) fn from_object2<'py>(obj: &Bound<'py, PyAny>) -> PyResult> { @@ -57,14 +57,14 @@ impl<'p> Iterator for &'p PyIterator { /// Further `next()` calls after an exception occurs are likely /// to repeatedly result in the same exception. fn next(&mut self) -> Option { - Borrowed::::from_gil_ref(self) + self.as_borrowed() .next() .map(|result| result.map(Bound::into_gil_ref)) } #[cfg(not(Py_LIMITED_API))] fn size_hint(&self) -> (usize, Option) { - Bound::borrowed_from_gil_ref(self).size_hint() + self.as_borrowed().size_hint() } } diff --git a/src/types/list.rs b/src/types/list.rs index 0e7fc952b00..cdc39349bfd 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -7,7 +7,7 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::Borrowed; use crate::internal_tricks::get_ssize_index; use crate::types::{PySequence, PyTuple}; -use crate::{Bound, PyAny, PyObject, Python, ToPyObject}; +use crate::{Bound, PyAny, PyNativeType, PyObject, Python, ToPyObject}; use crate::types::any::PyAnyMethods; use crate::types::sequence::PySequenceMethods; @@ -98,12 +98,12 @@ impl PyList { /// Returns the length of the list. pub fn len(&self) -> usize { - Bound::borrowed_from_gil_ref(&self).len() + self.as_borrowed().len() } /// Checks if the list is empty. pub fn is_empty(&self) -> bool { - Bound::borrowed_from_gil_ref(&self).is_empty() + self.as_borrowed().is_empty() } /// Returns `self` cast as a `PySequence`. @@ -122,9 +122,7 @@ impl PyList { /// }); /// ``` pub fn get_item(&self, index: usize) -> PyResult<&PyAny> { - Bound::borrowed_from_gil_ref(&self) - .get_item(index) - .map(Bound::into_gil_ref) + self.as_borrowed().get_item(index).map(Bound::into_gil_ref) } /// Gets the list item at the specified index. Undefined behavior on bad index. Use with caution. @@ -134,9 +132,7 @@ impl PyList { /// Caller must verify that the index is within the bounds of the list. #[cfg(not(Py_LIMITED_API))] pub unsafe fn get_item_unchecked(&self, index: usize) -> &PyAny { - Bound::borrowed_from_gil_ref(&self) - .get_item_unchecked(index) - .into_gil_ref() + self.as_borrowed().get_item_unchecked(index).into_gil_ref() } /// Takes the slice `self[low:high]` and returns it as a new list. @@ -144,9 +140,7 @@ impl PyList { /// Indices must be nonnegative, and out-of-range indices are clipped to /// `self.len()`. pub fn get_slice(&self, low: usize, high: usize) -> &PyList { - Bound::borrowed_from_gil_ref(&self) - .get_slice(low, high) - .into_gil_ref() + self.as_borrowed().get_slice(low, high).into_gil_ref() } /// Sets the item at the specified index. @@ -156,7 +150,7 @@ impl PyList { where I: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).set_item(index, item) + self.as_borrowed().set_item(index, item) } /// Deletes the `index`th element of self. @@ -164,7 +158,7 @@ impl PyList { /// This is equivalent to the Python statement `del self[i]`. #[inline] pub fn del_item(&self, index: usize) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).del_item(index) + self.as_borrowed().del_item(index) } /// Assigns the sequence `seq` to the slice of `self` from `low` to `high`. @@ -172,7 +166,7 @@ impl PyList { /// This is equivalent to the Python statement `self[low:high] = v`. #[inline] pub fn set_slice(&self, low: usize, high: usize, seq: &PyAny) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).set_slice(low, high, Bound::borrowed_from_gil_ref(&seq)) + self.as_borrowed().set_slice(low, high, &seq.as_borrowed()) } /// Deletes the slice from `low` to `high` from `self`. @@ -180,7 +174,7 @@ impl PyList { /// This is equivalent to the Python statement `del self[low:high]`. #[inline] pub fn del_slice(&self, low: usize, high: usize) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).del_slice(low, high) + self.as_borrowed().del_slice(low, high) } /// Appends an item to the list. @@ -188,7 +182,7 @@ impl PyList { where I: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).append(item) + self.as_borrowed().append(item) } /// Inserts an item at the specified index. @@ -198,7 +192,7 @@ impl PyList { where I: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).insert(index, item) + self.as_borrowed().insert(index, item) } /// Determines if self contains `value`. @@ -209,7 +203,7 @@ impl PyList { where V: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).contains(value) + self.as_borrowed().contains(value) } /// Returns the first index `i` for which `self[i] == value`. @@ -220,31 +214,29 @@ impl PyList { where V: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).index(value) + self.as_borrowed().index(value) } /// Returns an iterator over this list's items. pub fn iter(&self) -> PyListIterator<'_> { - PyListIterator(Bound::borrowed_from_gil_ref(&self).iter()) + PyListIterator(self.as_borrowed().iter()) } /// Sorts the list in-place. Equivalent to the Python expression `l.sort()`. pub fn sort(&self) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).sort() + self.as_borrowed().sort() } /// Reverses the list in-place. Equivalent to the Python expression `l.reverse()`. pub fn reverse(&self) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).reverse() + self.as_borrowed().reverse() } /// Return a new tuple containing the contents of the list; equivalent to the Python expression `tuple(list)`. /// /// This method is equivalent to `self.as_sequence().to_tuple()` and faster than `PyTuple::new(py, this_list)`. pub fn to_tuple(&self) -> &PyTuple { - Bound::borrowed_from_gil_ref(&self) - .to_tuple() - .into_gil_ref() + self.as_borrowed().to_tuple().into_gil_ref() } } diff --git a/src/types/mapping.rs b/src/types/mapping.rs index 239ade9b20c..4c9677cea8b 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -20,13 +20,13 @@ impl PyMapping { /// This is equivalent to the Python expression `len(self)`. #[inline] pub fn len(&self) -> PyResult { - Bound::borrowed_from_gil_ref(&self).len() + self.as_borrowed().len() } /// Returns whether the mapping is empty. #[inline] pub fn is_empty(&self) -> PyResult { - Bound::borrowed_from_gil_ref(&self).is_empty() + self.as_borrowed().is_empty() } /// Determines if the mapping contains the specified key. @@ -36,7 +36,7 @@ impl PyMapping { where K: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).contains(key) + self.as_borrowed().contains(key) } /// Gets the item in self with key `key`. @@ -49,9 +49,7 @@ impl PyMapping { where K: ToPyObject, { - Bound::borrowed_from_gil_ref(&self) - .get_item(key) - .map(Bound::into_gil_ref) + self.as_borrowed().get_item(key).map(Bound::into_gil_ref) } /// Sets the item in self with key `key`. @@ -63,7 +61,7 @@ impl PyMapping { K: ToPyObject, V: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).set_item(key, value) + self.as_borrowed().set_item(key, value) } /// Deletes the item with key `key`. @@ -74,31 +72,25 @@ impl PyMapping { where K: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).del_item(key) + self.as_borrowed().del_item(key) } /// Returns a sequence containing all keys in the mapping. #[inline] pub fn keys(&self) -> PyResult<&PySequence> { - Bound::borrowed_from_gil_ref(&self) - .keys() - .map(Bound::into_gil_ref) + self.as_borrowed().keys().map(Bound::into_gil_ref) } /// Returns a sequence containing all values in the mapping. #[inline] pub fn values(&self) -> PyResult<&PySequence> { - Bound::borrowed_from_gil_ref(&self) - .values() - .map(Bound::into_gil_ref) + self.as_borrowed().values().map(Bound::into_gil_ref) } /// Returns a sequence of tuples of all (key, value) pairs in the mapping. #[inline] pub fn items(&self) -> PyResult<&PySequence> { - Bound::borrowed_from_gil_ref(&self) - .items() - .map(Bound::into_gil_ref) + self.as_borrowed().items().map(Bound::into_gil_ref) } /// Register a pyclass as a subclass of `collections.abc.Mapping` (from the Python standard @@ -257,10 +249,7 @@ impl PyTypeCheck for PyMapping { || get_mapping_abc(object.py()) .and_then(|abc| object.is_instance(abc)) .unwrap_or_else(|err| { - err.write_unraisable_bound( - object.py(), - Some(Bound::borrowed_from_gil_ref(&object)), - ); + err.write_unraisable_bound(object.py(), Some(&object.as_borrowed())); false }) } diff --git a/src/types/mod.rs b/src/types/mod.rs index f153c4d92ec..41fb5b0adc8 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -105,9 +105,10 @@ macro_rules! pyobject_native_type_base( fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::result::Result<(), ::std::fmt::Error> { + use $crate::PyNativeType; match self.str() { ::std::result::Result::Ok(s) => return f.write_str(&s.to_string_lossy()), - ::std::result::Result::Err(err) => err.write_unraisable_bound(self.py(), ::std::option::Option::Some($crate::Bound::borrowed_from_gil_ref(&self))), + ::std::result::Result::Err(err) => err.write_unraisable_bound(self.py(), ::std::option::Option::Some(&self.as_borrowed())), } match self.get_type().name() { diff --git a/src/types/pysuper.rs b/src/types/pysuper.rs index d14670987b5..23f445f76b3 100644 --- a/src/types/pysuper.rs +++ b/src/types/pysuper.rs @@ -1,7 +1,7 @@ use crate::instance::Bound; use crate::types::any::PyAnyMethods; use crate::types::PyType; -use crate::{ffi, PyTypeInfo}; +use crate::{ffi, PyNativeType, PyTypeInfo}; use crate::{PyAny, PyResult}; /// Represents a Python `super` object. @@ -22,11 +22,7 @@ impl PySuper { note = "`PySuper::new` will be replaced by `PySuper::new_bound` in a future PyO3 version" )] pub fn new<'py>(ty: &'py PyType, obj: &'py PyAny) -> PyResult<&'py PySuper> { - Self::new_bound( - Bound::borrowed_from_gil_ref(&ty), - Bound::borrowed_from_gil_ref(&obj), - ) - .map(Bound::into_gil_ref) + Self::new_bound(&ty.as_borrowed(), &obj.as_borrowed()).map(Bound::into_gil_ref) } /// Constructs a new super object. More read about super object: [docs](https://docs.python.org/3/library/functions.html#super) @@ -73,7 +69,8 @@ impl PySuper { ty: &Bound<'py, PyType>, obj: &Bound<'py, PyAny>, ) -> PyResult> { - Bound::borrowed_from_gil_ref(&PySuper::type_object(ty.py())) + PySuper::type_object(ty.py()) + .as_borrowed() .call1((ty, obj)) .map(|any| { // Safety: super() always returns instance of super diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 9b497ded4f7..f4eae4d05a9 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -23,13 +23,13 @@ impl PySequence { /// This is equivalent to the Python expression `len(self)`. #[inline] pub fn len(&self) -> PyResult { - Bound::borrowed_from_gil_ref(&self).len() + self.as_borrowed().len() } /// Returns whether the sequence is empty. #[inline] pub fn is_empty(&self) -> PyResult { - Bound::borrowed_from_gil_ref(&self).is_empty() + self.as_borrowed().is_empty() } /// Returns the concatenation of `self` and `other`. @@ -37,8 +37,8 @@ impl PySequence { /// This is equivalent to the Python expression `self + other`. #[inline] pub fn concat(&self, other: &PySequence) -> PyResult<&PySequence> { - Bound::borrowed_from_gil_ref(&self) - .concat(Bound::borrowed_from_gil_ref(&other)) + self.as_borrowed() + .concat(&other.as_borrowed()) .map(Bound::into_gil_ref) } @@ -47,9 +47,7 @@ impl PySequence { /// This is equivalent to the Python expression `self * count`. #[inline] pub fn repeat(&self, count: usize) -> PyResult<&PySequence> { - Bound::borrowed_from_gil_ref(&self) - .repeat(count) - .map(Bound::into_gil_ref) + self.as_borrowed().repeat(count).map(Bound::into_gil_ref) } /// Concatenates `self` and `other`, in place if possible. @@ -61,8 +59,8 @@ impl PySequence { /// possible, but create and return a new object if not. #[inline] pub fn in_place_concat(&self, other: &PySequence) -> PyResult<&PySequence> { - Bound::borrowed_from_gil_ref(&self) - .in_place_concat(Bound::borrowed_from_gil_ref(&other)) + self.as_borrowed() + .in_place_concat(&other.as_borrowed()) .map(Bound::into_gil_ref) } @@ -75,7 +73,7 @@ impl PySequence { /// possible, but create and return a new object if not. #[inline] pub fn in_place_repeat(&self, count: usize) -> PyResult<&PySequence> { - Bound::borrowed_from_gil_ref(&self) + self.as_borrowed() .in_place_repeat(count) .map(Bound::into_gil_ref) } @@ -85,9 +83,7 @@ impl PySequence { /// This is equivalent to the Python expression `self[index]` without support of negative indices. #[inline] pub fn get_item(&self, index: usize) -> PyResult<&PyAny> { - Bound::borrowed_from_gil_ref(&self) - .get_item(index) - .map(|py2| py2.into_gil_ref()) + self.as_borrowed().get_item(index).map(Bound::into_gil_ref) } /// Returns the slice of sequence object between `begin` and `end`. @@ -95,7 +91,7 @@ impl PySequence { /// This is equivalent to the Python expression `self[begin:end]`. #[inline] pub fn get_slice(&self, begin: usize, end: usize) -> PyResult<&PySequence> { - Bound::borrowed_from_gil_ref(&self) + self.as_borrowed() .get_slice(begin, end) .map(Bound::into_gil_ref) } @@ -108,7 +104,7 @@ impl PySequence { where I: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).set_item(i, item) + self.as_borrowed().set_item(i, item) } /// Deletes the `i`th element of self. @@ -116,7 +112,7 @@ impl PySequence { /// This is equivalent to the Python statement `del self[i]`. #[inline] pub fn del_item(&self, i: usize) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).del_item(i) + self.as_borrowed().del_item(i) } /// Assigns the sequence `v` to the slice of `self` from `i1` to `i2`. @@ -124,7 +120,7 @@ impl PySequence { /// This is equivalent to the Python statement `self[i1:i2] = v`. #[inline] pub fn set_slice(&self, i1: usize, i2: usize, v: &PyAny) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).set_slice(i1, i2, Bound::borrowed_from_gil_ref(&v)) + self.as_borrowed().set_slice(i1, i2, &v.as_borrowed()) } /// Deletes the slice from `i1` to `i2` from `self`. @@ -132,7 +128,7 @@ impl PySequence { /// This is equivalent to the Python statement `del self[i1:i2]`. #[inline] pub fn del_slice(&self, i1: usize, i2: usize) -> PyResult<()> { - Bound::borrowed_from_gil_ref(&self).del_slice(i1, i2) + self.as_borrowed().del_slice(i1, i2) } /// Returns the number of occurrences of `value` in self, that is, return the @@ -143,7 +139,7 @@ impl PySequence { where V: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).count(value) + self.as_borrowed().count(value) } /// Determines if self contains `value`. @@ -154,7 +150,7 @@ impl PySequence { where V: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).contains(value) + self.as_borrowed().contains(value) } /// Returns the first index `i` for which `self[i] == value`. @@ -165,23 +161,19 @@ impl PySequence { where V: ToPyObject, { - Bound::borrowed_from_gil_ref(&self).index(value) + self.as_borrowed().index(value) } /// Returns a fresh list based on the Sequence. #[inline] pub fn to_list(&self) -> PyResult<&PyList> { - Bound::borrowed_from_gil_ref(&self) - .to_list() - .map(|py2| py2.into_gil_ref()) + self.as_borrowed().to_list().map(Bound::into_gil_ref) } /// Returns a fresh tuple based on the Sequence. #[inline] pub fn to_tuple(&self) -> PyResult<&PyTuple> { - Bound::borrowed_from_gil_ref(&self) - .to_tuple() - .map(|py2| py2.into_gil_ref()) + self.as_borrowed().to_tuple().map(Bound::into_gil_ref) } /// Register a pyclass as a subclass of `collections.abc.Sequence` (from the Python standard @@ -541,10 +533,7 @@ impl PyTypeCheck for PySequence { || get_sequence_abc(object.py()) .and_then(|abc| object.is_instance(abc)) .unwrap_or_else(|err| { - err.write_unraisable_bound( - object.py(), - Some(Bound::borrowed_from_gil_ref(&object)), - ); + err.write_unraisable_bound(object.py(), Some(&object.as_borrowed())); false }) } diff --git a/src/types/string.rs b/src/types/string.rs index 03d41963f1e..111f3112f26 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -5,7 +5,7 @@ use crate::instance::Borrowed; use crate::types::any::PyAnyMethods; use crate::types::bytes::PyBytesMethods; use crate::types::PyBytes; -use crate::{ffi, Bound, IntoPy, Py, PyAny, PyResult, Python}; +use crate::{ffi, Bound, IntoPy, Py, PyAny, PyNativeType, PyResult, Python}; use std::borrow::Cow; use std::os::raw::c_char; use std::str; @@ -184,7 +184,7 @@ impl PyString { pub fn to_str(&self) -> PyResult<&str> { #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] { - Borrowed::from_gil_ref(self).to_str() + self.as_borrowed().to_str() } #[cfg(not(any(Py_3_10, not(Py_LIMITED_API))))] @@ -202,7 +202,7 @@ impl PyString { /// Returns a `UnicodeEncodeError` if the input is not valid unicode /// (containing unpaired surrogates). pub fn to_cow(&self) -> PyResult> { - Borrowed::from_gil_ref(self).to_cow() + self.as_borrowed().to_cow() } /// Converts the `PyString` into a Rust string. @@ -210,7 +210,7 @@ impl PyString { /// Unpaired surrogates invalid UTF-8 sequences are /// replaced with `U+FFFD REPLACEMENT CHARACTER`. pub fn to_string_lossy(&self) -> Cow<'_, str> { - Borrowed::from_gil_ref(self).to_string_lossy() + self.as_borrowed().to_string_lossy() } /// Obtains the raw data backing the Python string. @@ -229,7 +229,7 @@ impl PyString { /// expected on the targets where you plan to distribute your software. #[cfg(not(Py_LIMITED_API))] pub unsafe fn data(&self) -> PyResult> { - Borrowed::from_gil_ref(self).data() + self.as_borrowed().data() } } From d36ad8f61fdb999b6cc21c9f8fa0e4cfb96db5d6 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 24 Dec 2023 13:48:13 +0000 Subject: [PATCH 011/349] introduce `Bound::as_borrowed` --- src/instance.rs | 15 ++++++++++----- src/types/bytearray.rs | 6 +++--- src/types/bytes.rs | 2 +- src/types/string.rs | 8 ++++---- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index 90e672c4fa5..e857efc9889 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -197,6 +197,15 @@ impl<'py, T> Bound<'py, T> { self.into_non_null().as_ptr() } + /// Casts this `Bound` to a `Borrowed` smart pointer. + pub fn as_borrowed<'a>(&'a self) -> Borrowed<'a, 'py, T> { + Borrowed( + unsafe { NonNull::new_unchecked(self.as_ptr()) }, + PhantomData, + self.py(), + ) + } + /// Casts this `Bound` as the corresponding "GIL Ref" type. /// /// This is a helper to be used for migration from the deprecated "GIL Refs" API. @@ -300,11 +309,7 @@ impl<'a, 'py> Borrowed<'a, 'py, PyAny> { impl<'a, 'py, T> From<&'a Bound<'py, T>> for Borrowed<'a, 'py, T> { /// Create borrow on a Bound fn from(instance: &'a Bound<'py, T>) -> Self { - Self( - unsafe { NonNull::new_unchecked(instance.as_ptr()) }, - PhantomData, - instance.py(), - ) + instance.as_borrowed() } } diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index f1b8050c066..2514e987d4d 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -400,16 +400,16 @@ impl<'py> PyByteArrayMethods<'py> for Bound<'py, PyByteArray> { } fn data(&self) -> *mut u8 { - Borrowed::from(self).data() + self.as_borrowed().data() } unsafe fn as_bytes(&self) -> &[u8] { - Borrowed::from(self).as_bytes() + self.as_borrowed().as_bytes() } #[allow(clippy::mut_from_ref)] unsafe fn as_bytes_mut(&self) -> &mut [u8] { - Borrowed::from(self).as_bytes_mut() + self.as_borrowed().as_bytes_mut() } fn to_vec(&self) -> Vec { diff --git a/src/types/bytes.rs b/src/types/bytes.rs index 5a7b174ea0c..d9d22dbb173 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -107,7 +107,7 @@ pub trait PyBytesMethods<'py> { impl<'py> PyBytesMethods<'py> for Bound<'py, PyBytes> { #[inline] fn as_bytes(&self) -> &[u8] { - Borrowed::from(self).as_bytes() + self.as_borrowed().as_bytes() } } diff --git a/src/types/string.rs b/src/types/string.rs index 111f3112f26..65f5a3922b9 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -280,20 +280,20 @@ pub trait PyStringMethods<'py> { impl<'py> PyStringMethods<'py> for Bound<'py, PyString> { #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] fn to_str(&self) -> PyResult<&str> { - Borrowed::from(self).to_str() + self.as_borrowed().to_str() } fn to_cow(&self) -> PyResult> { - Borrowed::from(self).to_cow() + self.as_borrowed().to_cow() } fn to_string_lossy(&self) -> Cow<'_, str> { - Borrowed::from(self).to_string_lossy() + self.as_borrowed().to_string_lossy() } #[cfg(not(Py_LIMITED_API))] unsafe fn data(&self) -> PyResult> { - Borrowed::from(self).data() + self.as_borrowed().data() } } From 271cbf9edb9e50b716d8eae06367e69e42b0b7bd Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 21 Nov 2023 12:22:49 +0000 Subject: [PATCH 012/349] implement `PySetMethods` --- src/prelude.rs | 1 + src/types/mod.rs | 2 +- src/types/set.rs | 297 ++++++++++++++++++++++++++++++++++++----------- 3 files changed, 229 insertions(+), 71 deletions(-) diff --git a/src/prelude.rs b/src/prelude.rs index b06625dc36a..beaa385ff14 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -34,4 +34,5 @@ pub use crate::types::float::PyFloatMethods; pub use crate::types::list::PyListMethods; pub use crate::types::mapping::PyMappingMethods; pub use crate::types::sequence::PySequenceMethods; +pub use crate::types::set::PySetMethods; pub use crate::types::string::PyStringMethods; diff --git a/src/types/mod.rs b/src/types/mod.rs index 41fb5b0adc8..a8770d146b1 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -79,7 +79,7 @@ pub mod iter { pub use super::dict::{BoundDictIterator, PyDictIterator}; pub use super::frozenset::PyFrozenSetIterator; pub use super::list::{BoundListIterator, PyListIterator}; - pub use super::set::PySetIterator; + pub use super::set::{BoundSetIterator, PySetIterator}; pub use super::tuple::PyTupleIterator; } diff --git a/src/types/set.rs b/src/types/set.rs index 68b89d656f3..3721cb2792d 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -2,7 +2,9 @@ use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, - Py, + ffi_ptr_ext::FfiPtrExt, + instance::Bound, + Py, PyNativeType, }; use crate::{ffi, PyAny, PyObject, Python, ToPyObject}; use std::ptr; @@ -46,9 +48,7 @@ impl PySet { /// Removes all elements from the set. #[inline] pub fn clear(&self) { - unsafe { - ffi::PySet_Clear(self.as_ptr()); - } + self.as_borrowed().clear() } /// Returns the number of items in the set. @@ -56,12 +56,12 @@ impl PySet { /// This is equivalent to the Python expression `len(self)`. #[inline] pub fn len(&self) -> usize { - unsafe { ffi::PySet_Size(self.as_ptr()) as usize } + self.as_borrowed().len() } /// Checks if set is empty. pub fn is_empty(&self) -> bool { - self.len() == 0 + self.as_borrowed().is_empty() } /// Determines if the set contains the specified key. @@ -71,7 +71,110 @@ impl PySet { where K: ToPyObject, { - fn inner(set: &PySet, key: PyObject) -> PyResult { + self.as_borrowed().contains(key) + } + + /// Removes the element from the set if it is present. + /// + /// Returns `true` if the element was present in the set. + pub fn discard(&self, key: K) -> PyResult + where + K: ToPyObject, + { + self.as_borrowed().discard(key) + } + + /// Adds an element to the set. + pub fn add(&self, key: K) -> PyResult<()> + where + K: ToPyObject, + { + self.as_borrowed().add(key) + } + + /// Removes and returns an arbitrary element from the set. + pub fn pop(&self) -> Option { + self.as_borrowed().pop().map(Py::from) + } + + /// Returns an iterator of values in this set. + /// + /// # Panics + /// + /// If PyO3 detects that the set is mutated during iteration, it will panic. + pub fn iter(&self) -> PySetIterator<'_> { + PySetIterator(BoundSetIterator::new(self.as_borrowed().to_owned())) + } +} + +/// Implementation of functionality for [`PySet`]. +/// +/// These methods are defined for the `Bound<'py, PySet>` smart pointer, so to use method call +/// syntax these methods are separated into a trait, because stable Rust does not yet support +/// `arbitrary_self_types`. +#[doc(alias = "PySet")] +pub trait PySetMethods<'py> { + /// Removes all elements from the set. + fn clear(&self); + + /// Returns the number of items in the set. + /// + /// This is equivalent to the Python expression `len(self)`. + fn len(&self) -> usize; + + /// Checks if set is empty. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Determines if the set contains the specified key. + /// + /// This is equivalent to the Python expression `key in self`. + fn contains(&self, key: K) -> PyResult + where + K: ToPyObject; + + /// Removes the element from the set if it is present. + /// + /// Returns `true` if the element was present in the set. + fn discard(&self, key: K) -> PyResult + where + K: ToPyObject; + + /// Adds an element to the set. + fn add(&self, key: K) -> PyResult<()> + where + K: ToPyObject; + + /// Removes and returns an arbitrary element from the set. + fn pop(&self) -> Option>; + + /// Returns an iterator of values in this set. + /// + /// # Panics + /// + /// If PyO3 detects that the set is mutated during iteration, it will panic. + fn iter(&self) -> BoundSetIterator<'py>; +} + +impl<'py> PySetMethods<'py> for Bound<'py, PySet> { + #[inline] + fn clear(&self) { + unsafe { + ffi::PySet_Clear(self.as_ptr()); + } + } + + #[inline] + fn len(&self) -> usize { + unsafe { ffi::PySet_Size(self.as_ptr()) as usize } + } + + fn contains(&self, key: K) -> PyResult + where + K: ToPyObject, + { + fn inner(set: &Bound<'_, PySet>, key: Bound<'_, PyAny>) -> PyResult { match unsafe { ffi::PySet_Contains(set.as_ptr(), key.as_ptr()) } { 1 => Ok(true), 0 => Ok(false), @@ -79,17 +182,15 @@ impl PySet { } } - inner(self, key.to_object(self.py())) + let py = self.py(); + inner(self, key.to_object(py).into_bound(py)) } - /// Removes the element from the set if it is present. - /// - /// Returns `true` if the element was present in the set. - pub fn discard(&self, key: K) -> PyResult + fn discard(&self, key: K) -> PyResult where K: ToPyObject, { - fn inner(set: &PySet, key: PyObject) -> PyResult { + fn inner(set: &Bound<'_, PySet>, key: Bound<'_, PyAny>) -> PyResult { match unsafe { ffi::PySet_Discard(set.as_ptr(), key.as_ptr()) } { 1 => Ok(true), 0 => Ok(false), @@ -97,40 +198,96 @@ impl PySet { } } - inner(self, key.to_object(self.py())) + let py = self.py(); + inner(self, key.to_object(py).into_bound(py)) } - /// Adds an element to the set. - pub fn add(&self, key: K) -> PyResult<()> + fn add(&self, key: K) -> PyResult<()> where K: ToPyObject, { - fn inner(set: &PySet, key: PyObject) -> PyResult<()> { + fn inner(set: &Bound<'_, PySet>, key: Bound<'_, PyAny>) -> PyResult<()> { err::error_on_minusone(set.py(), unsafe { ffi::PySet_Add(set.as_ptr(), key.as_ptr()) }) } - inner(self, key.to_object(self.py())) + let py = self.py(); + inner(self, key.to_object(py).into_bound(py)) } - /// Removes and returns an arbitrary element from the set. - pub fn pop(&self) -> Option { - let element = - unsafe { PyObject::from_owned_ptr_or_err(self.py(), ffi::PySet_Pop(self.as_ptr())) }; + fn pop(&self) -> Option> { + let element = unsafe { ffi::PySet_Pop(self.as_ptr()).assume_owned_or_err(self.py()) }; match element { Ok(e) => Some(e), Err(_) => None, } } + fn iter(&self) -> BoundSetIterator<'py> { + BoundSetIterator::new(self.clone()) + } +} + +/// PyO3 implementation of an iterator for a Python `set` object. +pub struct PySetIterator<'py>(BoundSetIterator<'py>); + +impl<'py> Iterator for PySetIterator<'py> { + type Item = &'py super::PyAny; + + /// Advances the iterator and returns the next value. + /// + /// # Panics + /// + /// If PyO3 detects that the set is mutated during iteration, it will panic. + #[inline] + fn next(&mut self) -> Option { + self.0.next().map(Bound::into_gil_ref) + } + + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } +} + +impl<'py> IntoIterator for &'py PySet { + type Item = &'py PyAny; + type IntoIter = PySetIterator<'py>; /// Returns an iterator of values in this set. /// /// # Panics /// /// If PyO3 detects that the set is mutated during iteration, it will panic. - pub fn iter(&self) -> PySetIterator<'_> { - IntoIterator::into_iter(self) + fn into_iter(self) -> Self::IntoIter { + PySetIterator(BoundSetIterator::new(self.as_borrowed().to_owned())) + } +} + +impl<'py> IntoIterator for Bound<'py, PySet> { + type Item = Bound<'py, PyAny>; + type IntoIter = BoundSetIterator<'py>; + + /// Returns an iterator of values in this set. + /// + /// # Panics + /// + /// If PyO3 detects that the set is mutated during iteration, it will panic. + fn into_iter(self) -> Self::IntoIter { + BoundSetIterator::new(self) + } +} + +impl<'py> IntoIterator for &'_ Bound<'py, PySet> { + type Item = Bound<'py, PyAny>; + type IntoIter = BoundSetIterator<'py>; + + /// Returns an iterator of values in this set. + /// + /// # Panics + /// + /// If PyO3 detects that the set is mutated during iteration, it will panic. + fn into_iter(self) -> Self::IntoIter { + BoundSetIterator::new(self.clone()) } } @@ -138,29 +295,21 @@ impl PySet { mod impl_ { use super::*; - impl<'a> std::iter::IntoIterator for &'a PySet { - type Item = &'a PyAny; - type IntoIter = PySetIterator<'a>; + /// PyO3 implementation of an iterator for a Python `set` object. + pub struct BoundSetIterator<'p> { + it: Bound<'p, PyIterator>, + } - /// Returns an iterator of values in this set. - /// - /// # Panics - /// - /// If PyO3 detects that the set is mutated during iteration, it will panic. - fn into_iter(self) -> Self::IntoIter { - PySetIterator { - it: PyIterator::from_object(self).unwrap(), + impl<'py> BoundSetIterator<'py> { + pub(super) fn new(set: Bound<'py, PySet>) -> Self { + Self { + it: PyIterator::from_object2(&set).unwrap(), } } } - /// PyO3 implementation of an iterator for a Python `set` object. - pub struct PySetIterator<'p> { - it: &'p PyIterator, - } - - impl<'py> Iterator for PySetIterator<'py> { - type Item = &'py super::PyAny; + impl<'py> Iterator for BoundSetIterator<'py> { + type Item = Bound<'py, super::PyAny>; /// Advances the iterator and returns the next value. /// @@ -179,31 +328,21 @@ mod impl_ { use super::*; /// PyO3 implementation of an iterator for a Python `set` object. - pub struct PySetIterator<'py> { - set: &'py super::PySet, + pub struct BoundSetIterator<'py> { + set: Bound<'py, super::PySet>, pos: ffi::Py_ssize_t, used: ffi::Py_ssize_t, } - impl<'a> std::iter::IntoIterator for &'a PySet { - type Item = &'a PyAny; - type IntoIter = PySetIterator<'a>; - /// Returns an iterator of values in this set. - /// - /// # Panics - /// - /// If PyO3 detects that the set is mutated during iteration, it will panic. - fn into_iter(self) -> Self::IntoIter { - PySetIterator { - set: self, - pos: 0, - used: unsafe { ffi::PySet_Size(self.as_ptr()) }, - } + impl<'py> BoundSetIterator<'py> { + pub(super) fn new(set: Bound<'py, PySet>) -> Self { + let used = unsafe { ffi::PySet_Size(set.as_ptr()) }; + BoundSetIterator { set, pos: 0, used } } } - impl<'py> Iterator for PySetIterator<'py> { - type Item = &'py super::PyAny; + impl<'py> Iterator for BoundSetIterator<'py> { + type Item = Bound<'py, super::PyAny>; /// Advances the iterator and returns the next value. /// @@ -221,7 +360,7 @@ mod impl_ { if ffi::_PySet_NextEntry(self.set.as_ptr(), &mut self.pos, &mut key, &mut hash) != 0 { // _PySet_NextEntry returns borrowed object; for safety must make owned (see #890) - Some(self.set.py().from_owned_ptr(ffi::_Py_NewRef(key))) + Some(key.assume_borrowed(self.set.py()).to_owned()) } else { None } @@ -235,11 +374,17 @@ mod impl_ { } } - impl<'py> ExactSizeIterator for PySetIterator<'py> { + impl<'py> ExactSizeIterator for BoundSetIterator<'py> { fn len(&self) -> usize { self.set.len().saturating_sub(self.pos as usize) } } + + impl<'py> ExactSizeIterator for PySetIterator<'py> { + fn len(&self) -> usize { + self.0.len() + } + } } pub use impl_::*; @@ -289,6 +434,7 @@ mod tests { Python::with_gil(|py| { let set = PySet::empty(py).unwrap(); assert_eq!(0, set.len()); + assert!(set.is_empty()); }); } @@ -406,19 +552,30 @@ mod tests { } #[test] + #[cfg(not(Py_LIMITED_API))] fn test_set_iter_size_hint() { Python::with_gil(|py| { let set = PySet::new(py, &[1]).unwrap(); - let mut iter = set.iter(); - if cfg!(Py_LIMITED_API) { - assert_eq!(iter.size_hint(), (0, None)); - } else { - assert_eq!(iter.size_hint(), (1, Some(1))); - iter.next(); - assert_eq!(iter.size_hint(), (0, Some(0))); - } + // Exact size + assert_eq!(iter.len(), 1); + assert_eq!(iter.size_hint(), (1, Some(1))); + iter.next(); + assert_eq!(iter.len(), 0); + assert_eq!(iter.size_hint(), (0, Some(0))); + }); + } + + #[test] + #[cfg(Py_LIMITED_API)] + fn test_set_iter_size_hint() { + Python::with_gil(|py| { + let set = PySet::new(py, &[1]).unwrap(); + let iter = set.iter(); + + // No known bounds + assert_eq!(iter.size_hint(), (0, None)); }); } } From 8a28a69c3d51972d9bdc24141fcfe11eec0ac27b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 27 Nov 2023 13:55:53 +0000 Subject: [PATCH 013/349] implement `PyFrozenSetMethods` --- src/prelude.rs | 1 + src/types/frozenset.rs | 199 +++++++++++++++++++++++++++++++++-------- src/types/mod.rs | 4 +- 3 files changed, 164 insertions(+), 40 deletions(-) diff --git a/src/prelude.rs b/src/prelude.rs index beaa385ff14..9be5dc0dbba 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -31,6 +31,7 @@ pub use crate::types::bytearray::PyByteArrayMethods; pub use crate::types::bytes::PyBytesMethods; pub use crate::types::dict::PyDictMethods; pub use crate::types::float::PyFloatMethods; +pub use crate::types::frozenset::PyFrozenSetMethods; pub use crate::types::list::PyListMethods; pub use crate::types::mapping::PyMappingMethods; pub use crate::types::sequence::PySequenceMethods; diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index bbcd7102fac..777abf6f9c8 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -1,11 +1,11 @@ +#[cfg(not(Py_LIMITED_API))] +use crate::ffi_ptr_ext::FfiPtrExt; #[cfg(Py_LIMITED_API)] use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, - Py, PyObject, + ffi, Bound, Py, PyAny, PyNativeType, PyObject, Python, ToPyObject, }; -use crate::{ffi, PyAny, Python, ToPyObject}; - use std::ptr; /// Allows building a Python `frozenset` one item at a time @@ -83,12 +83,12 @@ impl PyFrozenSet { /// This is equivalent to len(p) on a set. #[inline] pub fn len(&self) -> usize { - unsafe { ffi::PySet_Size(self.as_ptr()) as usize } + self.as_borrowed().len() } /// Check if set is empty. pub fn is_empty(&self) -> bool { - self.len() == 0 + self.as_borrowed().is_empty() } /// Determine if the set contains the specified key. @@ -97,7 +97,54 @@ impl PyFrozenSet { where K: ToPyObject, { - fn inner(frozenset: &PyFrozenSet, key: PyObject) -> PyResult { + self.as_borrowed().contains(key) + } + + /// Returns an iterator of values in this frozen set. + pub fn iter(&self) -> PyFrozenSetIterator<'_> { + PyFrozenSetIterator(BoundFrozenSetIterator::new(self.as_borrowed().to_owned())) + } +} + +/// Implementation of functionality for [`PyFrozenSet`]. +/// +/// These methods are defined for the `Bound<'py, PyFrozenSet>` smart pointer, so to use method call +/// syntax these methods are separated into a trait, because stable Rust does not yet support +/// `arbitrary_self_types`. +#[doc(alias = "PyFrozenSet")] +pub trait PyFrozenSetMethods<'py> { + /// Returns the number of items in the set. + /// + /// This is equivalent to the Python expression `len(self)`. + fn len(&self) -> usize; + + /// Checks if set is empty. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Determines if the set contains the specified key. + /// + /// This is equivalent to the Python expression `key in self`. + fn contains(&self, key: K) -> PyResult + where + K: ToPyObject; + + /// Returns an iterator of values in this set. + fn iter(&self) -> BoundFrozenSetIterator<'py>; +} + +impl<'py> PyFrozenSetMethods<'py> for Bound<'py, PyFrozenSet> { + #[inline] + fn len(&self) -> usize { + unsafe { ffi::PySet_Size(self.as_ptr()) as usize } + } + + fn contains(&self, key: K) -> PyResult + where + K: ToPyObject, + { + fn inner(frozenset: &Bound<'_, PyFrozenSet>, key: Bound<'_, PyAny>) -> PyResult { match unsafe { ffi::PySet_Contains(frozenset.as_ptr(), key.as_ptr()) } { 1 => Ok(true), 0 => Ok(false), @@ -105,12 +152,58 @@ impl PyFrozenSet { } } - inner(self, key.to_object(self.py())) + let py = self.py(); + inner(self, key.to_object(py).into_bound(py)) } - /// Returns an iterator of values in this frozen set. - pub fn iter(&self) -> PyFrozenSetIterator<'_> { - IntoIterator::into_iter(self) + fn iter(&self) -> BoundFrozenSetIterator<'py> { + BoundFrozenSetIterator::new(self.clone()) + } +} + +/// PyO3 implementation of an iterator for a Python `frozenset` object. +pub struct PyFrozenSetIterator<'py>(BoundFrozenSetIterator<'py>); + +impl<'py> Iterator for PyFrozenSetIterator<'py> { + type Item = &'py super::PyAny; + + /// Advances the iterator and returns the next value. + #[inline] + fn next(&mut self) -> Option { + self.0.next().map(Bound::into_gil_ref) + } + + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } +} + +impl<'py> IntoIterator for &'py PyFrozenSet { + type Item = &'py PyAny; + type IntoIter = PyFrozenSetIterator<'py>; + /// Returns an iterator of values in this set. + fn into_iter(self) -> Self::IntoIter { + PyFrozenSetIterator(BoundFrozenSetIterator::new(self.as_borrowed().to_owned())) + } +} + +impl<'py> IntoIterator for Bound<'py, PyFrozenSet> { + type Item = Bound<'py, PyAny>; + type IntoIter = BoundFrozenSetIterator<'py>; + + /// Returns an iterator of values in this set. + fn into_iter(self) -> Self::IntoIter { + BoundFrozenSetIterator::new(self) + } +} + +impl<'py> IntoIterator for &'_ Bound<'py, PyFrozenSet> { + type Item = Bound<'py, PyAny>; + type IntoIter = BoundFrozenSetIterator<'py>; + + /// Returns an iterator of values in this set. + fn into_iter(self) -> Self::IntoIter { + BoundFrozenSetIterator::new(self.clone()) } } @@ -118,25 +211,23 @@ impl PyFrozenSet { mod impl_ { use super::*; - impl<'a> std::iter::IntoIterator for &'a PyFrozenSet { - type Item = &'a PyAny; - type IntoIter = PyFrozenSetIterator<'a>; + /// PyO3 implementation of an iterator for a Python `set` object. + pub struct BoundFrozenSetIterator<'p> { + it: Bound<'p, PyIterator>, + } - fn into_iter(self) -> Self::IntoIter { - PyFrozenSetIterator { - it: PyIterator::from_object(self).unwrap(), + impl<'py> BoundFrozenSetIterator<'py> { + pub(super) fn new(frozenset: Bound<'py, PyFrozenSet>) -> Self { + Self { + it: PyIterator::from_object2(&frozenset).unwrap(), } } } - /// PyO3 implementation of an iterator for a Python `frozenset` object. - pub struct PyFrozenSetIterator<'p> { - it: &'p PyIterator, - } - - impl<'py> Iterator for PyFrozenSetIterator<'py> { - type Item = &'py super::PyAny; + impl<'py> Iterator for BoundFrozenSetIterator<'py> { + type Item = Bound<'py, super::PyAny>; + /// Advances the iterator and returns the next value. #[inline] fn next(&mut self) -> Option { self.it.next().map(Result::unwrap) @@ -148,23 +239,20 @@ mod impl_ { mod impl_ { use super::*; - impl<'a> std::iter::IntoIterator for &'a PyFrozenSet { - type Item = &'a PyAny; - type IntoIter = PyFrozenSetIterator<'a>; - - fn into_iter(self) -> Self::IntoIter { - PyFrozenSetIterator { set: self, pos: 0 } - } - } - /// PyO3 implementation of an iterator for a Python `frozenset` object. - pub struct PyFrozenSetIterator<'py> { - set: &'py PyFrozenSet, + pub struct BoundFrozenSetIterator<'py> { + set: Bound<'py, PyFrozenSet>, pos: ffi::Py_ssize_t, } - impl<'py> Iterator for PyFrozenSetIterator<'py> { - type Item = &'py PyAny; + impl<'py> BoundFrozenSetIterator<'py> { + pub(super) fn new(set: Bound<'py, PyFrozenSet>) -> Self { + Self { set, pos: 0 } + } + } + + impl<'py> Iterator for BoundFrozenSetIterator<'py> { + type Item = Bound<'py, PyAny>; #[inline] fn next(&mut self) -> Option { @@ -174,7 +262,7 @@ mod impl_ { if ffi::_PySet_NextEntry(self.set.as_ptr(), &mut self.pos, &mut key, &mut hash) != 0 { // _PySet_NextEntry returns borrowed object; for safety must make owned (see #890) - Some(self.set.py().from_owned_ptr(ffi::_Py_NewRef(key))) + Some(key.assume_borrowed(self.set.py()).to_owned()) } else { None } @@ -188,11 +276,17 @@ mod impl_ { } } - impl<'py> ExactSizeIterator for PyFrozenSetIterator<'py> { + impl<'py> ExactSizeIterator for BoundFrozenSetIterator<'py> { fn len(&self) -> usize { self.set.len().saturating_sub(self.pos as usize) } } + + impl<'py> ExactSizeIterator for PyFrozenSetIterator<'py> { + fn len(&self) -> usize { + self.0.len() + } + } } pub use impl_::*; @@ -243,6 +337,7 @@ mod tests { Python::with_gil(|py| { let set = PyFrozenSet::empty(py).unwrap(); assert_eq!(0, set.len()); + assert!(set.is_empty()); }); } @@ -271,6 +366,34 @@ mod tests { }); } + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_frozenset_iter_size_hint() { + Python::with_gil(|py| { + let set = PyFrozenSet::new(py, &[1]).unwrap(); + let mut iter = set.iter(); + + // Exact size + assert_eq!(iter.len(), 1); + assert_eq!(iter.size_hint(), (1, Some(1))); + iter.next(); + assert_eq!(iter.len(), 0); + assert_eq!(iter.size_hint(), (0, Some(0))); + }); + } + + #[test] + #[cfg(Py_LIMITED_API)] + fn test_frozenset_iter_size_hint() { + Python::with_gil(|py| { + let set = PyFrozenSet::new(py, &[1]).unwrap(); + let iter = set.iter(); + + // No known bounds + assert_eq!(iter.size_hint(), (0, None)); + }); + } + #[test] fn test_frozenset_builder() { use super::PyFrozenSetBuilder; diff --git a/src/types/mod.rs b/src/types/mod.rs index a8770d146b1..705d366601d 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -77,7 +77,7 @@ pub use self::typeobject::PyType; /// In these cases the iterators are implemented by forwarding to [`PyIterator`]. pub mod iter { pub use super::dict::{BoundDictIterator, PyDictIterator}; - pub use super::frozenset::PyFrozenSetIterator; + pub use super::frozenset::{BoundFrozenSetIterator, PyFrozenSetIterator}; pub use super::list::{BoundListIterator, PyListIterator}; pub use super::set::{BoundSetIterator, PySetIterator}; pub use super::tuple::PyTupleIterator; @@ -288,7 +288,7 @@ mod ellipsis; pub(crate) mod float; #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] mod frame; -mod frozenset; +pub(crate) mod frozenset; mod function; mod iterator; pub(crate) mod list; From e4fd557d2a163a2fc9edd3f6e94459a14e73ed85 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 26 Dec 2023 13:07:10 +0000 Subject: [PATCH 014/349] remove `IntoIterator` for `&Bound` --- src/types/dict.rs | 9 --------- src/types/frozenset.rs | 10 ---------- src/types/list.rs | 9 --------- src/types/set.rs | 14 -------------- 4 files changed, 42 deletions(-) diff --git a/src/types/dict.rs b/src/types/dict.rs index 80d2798a863..0fe48662780 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -626,15 +626,6 @@ impl<'py> BoundDictIterator<'py> { } } -impl<'py> IntoIterator for &'_ Bound<'py, PyDict> { - type Item = (Bound<'py, PyAny>, Bound<'py, PyAny>); - type IntoIter = BoundDictIterator<'py>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - impl<'py> IntoIterator for Bound<'py, PyDict> { type Item = (Bound<'py, PyAny>, Bound<'py, PyAny>); type IntoIter = BoundDictIterator<'py>; diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 777abf6f9c8..1bcdce4cf90 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -197,16 +197,6 @@ impl<'py> IntoIterator for Bound<'py, PyFrozenSet> { } } -impl<'py> IntoIterator for &'_ Bound<'py, PyFrozenSet> { - type Item = Bound<'py, PyAny>; - type IntoIter = BoundFrozenSetIterator<'py>; - - /// Returns an iterator of values in this set. - fn into_iter(self) -> Self::IntoIter { - BoundFrozenSetIterator::new(self.clone()) - } -} - #[cfg(Py_LIMITED_API)] mod impl_ { use super::*; diff --git a/src/types/list.rs b/src/types/list.rs index cdc39349bfd..2ba4b10cd76 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -657,15 +657,6 @@ impl ExactSizeIterator for BoundListIterator<'_> { impl FusedIterator for BoundListIterator<'_> {} -impl<'a, 'py> IntoIterator for &'a Bound<'py, PyList> { - type Item = Bound<'py, PyAny>; - type IntoIter = BoundListIterator<'py>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - impl<'py> IntoIterator for Bound<'py, PyList> { type Item = Bound<'py, PyAny>; type IntoIter = BoundListIterator<'py>; diff --git a/src/types/set.rs b/src/types/set.rs index 3721cb2792d..a1ffaa3e961 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -277,20 +277,6 @@ impl<'py> IntoIterator for Bound<'py, PySet> { } } -impl<'py> IntoIterator for &'_ Bound<'py, PySet> { - type Item = Bound<'py, PyAny>; - type IntoIter = BoundSetIterator<'py>; - - /// Returns an iterator of values in this set. - /// - /// # Panics - /// - /// If PyO3 detects that the set is mutated during iteration, it will panic. - fn into_iter(self) -> Self::IntoIter { - BoundSetIterator::new(self.clone()) - } -} - #[cfg(Py_LIMITED_API)] mod impl_ { use super::*; From 442d13dab3875cda4ce371d4cd144fc968f71f63 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 26 Dec 2023 13:16:41 +0000 Subject: [PATCH 015/349] introduce `Bound::unbind` --- src/instance.rs | 26 +++++++++++++++++--------- src/types/set.rs | 2 +- src/types/string.rs | 4 ++-- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index e857efc9889..e3f3743ebf7 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -206,6 +206,14 @@ impl<'py, T> Bound<'py, T> { ) } + /// Removes the connection for this `Bound` from the GIL, allowing + /// it to cross thread boundaries. + pub fn unbind(self) -> Py { + // Safety: the type T is known to be correct and the ownership of the + // pointer is transferred to the new Py instance. + unsafe { Py::from_non_null(self.into_non_null()) } + } + /// Casts this `Bound` as the corresponding "GIL Ref" type. /// /// This is a helper to be used for migration from the deprecated "GIL Refs" API. @@ -973,7 +981,7 @@ impl Py { where N: IntoPy>, { - self.bind(py).as_any().getattr(attr_name).map(Into::into) + self.bind(py).as_any().getattr(attr_name).map(Bound::unbind) } /// Sets an attribute value. @@ -1017,21 +1025,21 @@ impl Py { args: impl IntoPy>, kwargs: Option<&PyDict>, ) -> PyResult { - self.bind(py).as_any().call(args, kwargs).map(Into::into) + self.bind(py).as_any().call(args, kwargs).map(Bound::unbind) } /// Calls the object with only positional arguments. /// /// This is equivalent to the Python expression `self(*args)`. pub fn call1(&self, py: Python<'_>, args: impl IntoPy>) -> PyResult { - self.bind(py).as_any().call1(args).map(Into::into) + self.bind(py).as_any().call1(args).map(Bound::unbind) } /// Calls the object without arguments. /// /// This is equivalent to the Python expression `self()`. pub fn call0(&self, py: Python<'_>) -> PyResult { - self.bind(py).as_any().call0().map(Into::into) + self.bind(py).as_any().call0().map(Bound::unbind) } /// Calls a method on the object. @@ -1054,7 +1062,7 @@ impl Py { self.bind(py) .as_any() .call_method(name, args, kwargs) - .map(Into::into) + .map(Bound::unbind) } /// Calls a method on the object with only positional arguments. @@ -1071,7 +1079,7 @@ impl Py { self.bind(py) .as_any() .call_method1(name, args) - .map(Into::into) + .map(Bound::unbind) } /// Calls a method on the object with no arguments. @@ -1084,7 +1092,7 @@ impl Py { where N: IntoPy>, { - self.bind(py).as_any().call_method0(name).map(Into::into) + self.bind(py).as_any().call_method0(name).map(Bound::unbind) } /// Create a `Py` instance by taking ownership of the given FFI pointer. @@ -1292,7 +1300,7 @@ where impl std::convert::From> for Py { #[inline] fn from(other: Bound<'_, T>) -> Self { - unsafe { Self::from_non_null(other.into_non_null()) } + other.unbind() } } @@ -1618,7 +1626,7 @@ a = A() .as_borrowed() .to_owned(); let ptr = instance.as_ptr(); - let instance: PyObject = instance.clone().into(); + let instance: PyObject = instance.clone().unbind(); assert_eq!(instance.as_ptr(), ptr); }) } diff --git a/src/types/set.rs b/src/types/set.rs index a1ffaa3e961..be65500b4fe 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -94,7 +94,7 @@ impl PySet { /// Removes and returns an arbitrary element from the set. pub fn pop(&self) -> Option { - self.as_borrowed().pop().map(Py::from) + self.as_borrowed().pop().map(Bound::unbind) } /// Returns an iterator of values in this set. diff --git a/src/types/string.rs b/src/types/string.rs index 65f5a3922b9..0ee64be53f9 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -435,13 +435,13 @@ impl Py { impl IntoPy> for Bound<'_, PyString> { fn into_py(self, _py: Python<'_>) -> Py { - self.into() + self.unbind() } } impl IntoPy> for &Bound<'_, PyString> { fn into_py(self, _py: Python<'_>) -> Py { - self.clone().into() + self.clone().unbind() } } From ac5db1fb4b21acd3a76cca8acb8c4b100900bcdd Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 26 Dec 2023 15:30:28 +0000 Subject: [PATCH 016/349] Add link to YouTube introduction to PyO3 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 62b7f32f543..1b163d75056 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,7 @@ about this topic. ## Articles and other media +- [(Video) Extending Python with Rust using PyO3](https://www.youtube.com/watch?v=T45ZEmSR1-s) - Dec 16, 2023 - [A Week of PyO3 + rust-numpy (How to Speed Up Your Data Pipeline X Times)](https://terencezl.github.io/blog/2023/06/06/a-week-of-pyo3-rust-numpy/) - Jun 6, 2023 - [(Podcast) PyO3 with David Hewitt](https://rustacean-station.org/episode/david-hewitt/) - May 19, 2023 - [Making Python 100x faster with less than 100 lines of Rust](https://ohadravid.github.io/posts/2023-03-rusty-python/) - Mar 28, 2023 From e42d8cf6129b27b4c00c7d2e00ae1b396d3bcd9c Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 29 Dec 2023 13:02:44 +0000 Subject: [PATCH 017/349] ci: updates for Rust 1.75 --- tests/ui/abi3_nativetype_inheritance.stderr | 18 +++++++++++++++++- tests/ui/invalid_argument_attributes.stderr | 6 +++--- tests/ui/pyclass_send.stderr | 21 +++++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/tests/ui/abi3_nativetype_inheritance.stderr b/tests/ui/abi3_nativetype_inheritance.stderr index 08fef60654d..f0672fbf0a7 100644 --- a/tests/ui/abi3_nativetype_inheritance.stderr +++ b/tests/ui/abi3_nativetype_inheritance.stderr @@ -6,6 +6,22 @@ error[E0277]: the trait bound `PyDict: PyClass` is not satisfied | = help: the following other types implement trait `PyClass`: TestClass - Coroutine + pyo3::coroutine::Coroutine = note: required for `PyDict` to implement `PyClassBaseType` = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `PyDict: PyClass` is not satisfied + --> tests/ui/abi3_nativetype_inheritance.rs:5:19 + | +5 | #[pyclass(extends=PyDict)] + | ^^^^^^ the trait `PyClass` is not implemented for `PyDict` + | + = help: the following other types implement trait `PyClass`: + TestClass + pyo3::coroutine::Coroutine + = note: required for `PyDict` to implement `PyClassBaseType` +note: required by a bound in `PyClassImpl::BaseType` + --> src/impl_/pyclass.rs + | + | type BaseType: PyTypeInfo + PyClassBaseType; + | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` diff --git a/tests/ui/invalid_argument_attributes.stderr b/tests/ui/invalid_argument_attributes.stderr index 4903fbc5c3a..ef8b59a8691 100644 --- a/tests/ui/invalid_argument_attributes.stderr +++ b/tests/ui/invalid_argument_attributes.stderr @@ -74,7 +74,7 @@ error[E0277]: the trait bound `CancelHandle: PyClass` is not satisfied 41 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} | ^^^^ the trait `PyClass` is not implemented for `CancelHandle` | - = help: the trait `PyClass` is implemented for `Coroutine` + = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` = note: required for `CancelHandle` to implement `FromPyObject<'_>` = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>` note: required by a bound in `extract_argument` @@ -93,8 +93,8 @@ error[E0277]: the trait bound `CancelHandle: Clone` is not satisfied | ^^^^ the trait `Clone` is not implemented for `CancelHandle` | = help: the following other types implement trait `PyFunctionArgument<'a, 'py>`: - &'a Coroutine - &'a mut Coroutine + &'a pyo3::coroutine::Coroutine + &'a mut pyo3::coroutine::Coroutine = note: required for `CancelHandle` to implement `FromPyObject<'_>` = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>` note: required by a bound in `extract_argument` diff --git a/tests/ui/pyclass_send.stderr b/tests/ui/pyclass_send.stderr index f279acc5f9f..7a80989b63f 100644 --- a/tests/ui/pyclass_send.stderr +++ b/tests/ui/pyclass_send.stderr @@ -16,3 +16,24 @@ note: required by a bound in `SendablePyClass` | pub struct SendablePyClass(PhantomData); | ^^^^ required by this bound in `SendablePyClass` = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: `Rc` cannot be sent between threads safely + --> tests/ui/pyclass_send.rs:4:1 + | +4 | #[pyclass] + | ^^^^^^^^^^ `Rc` cannot be sent between threads safely + | + = help: within `NotThreadSafe`, the trait `Send` is not implemented for `Rc` + = help: the trait `pyo3::impl_::pyclass::PyClassThreadChecker` is implemented for `SendablePyClass` +note: required because it appears within the type `NotThreadSafe` + --> tests/ui/pyclass_send.rs:5:8 + | +5 | struct NotThreadSafe { + | ^^^^^^^^^^^^^ + = note: required for `SendablePyClass` to implement `pyo3::impl_::pyclass::PyClassThreadChecker` +note: required by a bound in `PyClassImpl::ThreadChecker` + --> src/impl_/pyclass.rs + | + | type ThreadChecker: PyClassThreadChecker; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::ThreadChecker` + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) From 46c3190a176e9b571e7a56a58e5c63948cbb636c Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 29 Dec 2023 13:08:49 +0000 Subject: [PATCH 018/349] clean up remnants of deprecated & removed features --- guide/src/function/signature.md | 6 +++--- pyo3-macros-backend/src/pyclass.rs | 18 ++---------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/guide/src/function/signature.md b/guide/src/function/signature.md index 599a2f73f8a..8341f51c38c 100644 --- a/guide/src/function/signature.md +++ b/guide/src/function/signature.md @@ -2,9 +2,9 @@ The `#[pyfunction]` attribute also accepts parameters to control how the generated Python function accepts arguments. Just like in Python, arguments can be positional-only, keyword-only, or accept either. `*args` lists and `**kwargs` dicts can also be accepted. These parameters also work for `#[pymethods]` which will be introduced in the [Python Classes](../class.md) section of the guide. -Like Python, by default PyO3 accepts all arguments as either positional or keyword arguments. Most arguments are required by default, except for trailing `Option<_>` arguments, which are [implicitly given a default of `None`](#trailing-optional-arguments). There are two ways to modify this behaviour: - - The `#[pyo3(signature = (...))]` option which allows writing a signature in Python syntax. - - Extra arguments directly to `#[pyfunction]`. (See deprecated form) +Like Python, by default PyO3 accepts all arguments as either positional or keyword arguments. Most arguments are required by default, except for trailing `Option<_>` arguments, which are [implicitly given a default of `None`](#trailing-optional-arguments). This behaviour can be configured by the `#[pyo3(signature = (...))]` option which allows writing a signature in Python syntax. + +This section of the guide goes into detail about use of the `#[pyo3(signature = (...))]` option and its related option `#[pyo3(text_signature = "...")]` ## Using `#[pyo3(signature = (...))]` diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 239514ae38a..f60dbe5f4d4 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -3,8 +3,7 @@ use std::borrow::Cow; use crate::attributes::kw::frozen; use crate::attributes::{ self, kw, take_pyo3_options, CrateAttribute, ExtendsAttribute, FreelistAttribute, - ModuleAttribute, NameAttribute, NameLitStr, RenameAllAttribute, TextSignatureAttribute, - TextSignatureAttributeValue, + ModuleAttribute, NameAttribute, NameLitStr, RenameAllAttribute, }; use crate::deprecations::Deprecations; use crate::konst::{ConstAttributes, ConstSpec}; @@ -68,7 +67,6 @@ pub struct PyClassPyO3Options { pub sequence: Option, pub set_all: Option, pub subclass: Option, - pub text_signature: Option, pub unsendable: Option, pub weakref: Option, } @@ -886,18 +884,6 @@ impl<'a> PyClassImplsBuilder<'a> { fn impl_pyclassimpl(&self) -> Result { let cls = self.cls; let doc = self.doc.as_ref().map_or(quote! {"\0"}, |doc| quote! {#doc}); - let deprecated_text_signature = match self - .attr - .options - .text_signature - .as_ref() - .map(|attr| &attr.value) - { - Some(TextSignatureAttributeValue::Str(s)) => quote!(::std::option::Option::Some(#s)), - Some(TextSignatureAttributeValue::Disabled(_)) | None => { - quote!(::std::option::Option::None) - } - }; let is_basetype = self.attr.options.subclass.is_some(); let base = self .attr @@ -1040,7 +1026,7 @@ impl<'a> PyClassImplsBuilder<'a> { static DOC: _pyo3::sync::GILOnceCell<::std::borrow::Cow<'static, ::std::ffi::CStr>> = _pyo3::sync::GILOnceCell::new(); DOC.get_or_try_init(py, || { let collector = PyClassImplCollector::::new(); - build_pyclass_doc(<#cls as _pyo3::PyTypeInfo>::NAME, #doc, #deprecated_text_signature.or_else(|| collector.new_text_signature())) + build_pyclass_doc(<#cls as _pyo3::PyTypeInfo>::NAME, #doc, collector.new_text_signature()) }).map(::std::ops::Deref::deref) } From 3da1aac2ddf12996ca44dbe683bd0e092d9bbfaf Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 28 Dec 2023 13:41:38 +0000 Subject: [PATCH 019/349] add `gil-refs` feature to aid migration --- Cargo.toml | 3 +++ guide/src/features.md | 6 ++++++ newsfragments/3707.added.md | 1 + src/err/mod.rs | 9 ++++++--- src/types/datetime.rs | 11 +++++++---- src/types/pysuper.rs | 9 ++++++--- tests/test_exceptions.rs | 2 +- tests/test_super.rs | 2 +- 8 files changed, 31 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3707.added.md diff --git a/Cargo.toml b/Cargo.toml index 69d2608536e..a0400c07301 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,6 +96,9 @@ generate-import-lib = ["pyo3-ffi/generate-import-lib"] # Changes `Python::with_gil` to automatically initialize the Python interpreter if needed. auto-initialize = [] +# Allows use of the deprecated "GIL Refs" APIs. +gil-refs = [] + # Optimizes PyObject to Vec conversion and so on. nightly = [] diff --git a/guide/src/features.md b/guide/src/features.md index 8ed2a2ed0bc..edcf3772b26 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -57,6 +57,12 @@ This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` a This is a first step towards adding first-class support for generating type annotations automatically in PyO3, however work is needed to finish this off. All feedback and offers of help welcome on [issue #2454](https://github.com/PyO3/pyo3/issues/2454). +### `gil-refs` + +This feature is a backwards-compatibility feature to allow continued use of the "GIL Refs" APIs deprecated in PyO3 0.21. These APIs have performance drawbacks and soundness edge cases which the newer `Bound` smart pointer and accompanying APIs resolve. + +This feature and the APIs it enables is expected to be removed in a future PyO3 version. + ### `macros` This feature enables a dependency on the `pyo3-macros` crate, which provides the procedural macros portion of PyO3's API: diff --git a/newsfragments/3707.added.md b/newsfragments/3707.added.md new file mode 100644 index 00000000000..bc92e2c0f95 --- /dev/null +++ b/newsfragments/3707.added.md @@ -0,0 +1 @@ +Add `gil-refs` feature to allow continued use of the deprecated GIL Refs APIs. diff --git a/src/err/mod.rs b/src/err/mod.rs index ed719103646..39975183f07 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -536,9 +536,12 @@ impl PyErr { } /// Deprecated form of `PyErr::write_unraisable_bound`. - #[deprecated( - since = "0.21.0", - note = "`PyErr::write_unraisable` will be replaced by `PyErr::write_unraisable_bound` in a future PyO3 version" + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyErr::write_unraisable` will be replaced by `PyErr::write_unraisable_bound` in a future PyO3 version" + ) )] #[inline] pub fn write_unraisable(self, py: Python<'_>, obj: Option<&PyAny>) { diff --git a/src/types/datetime.rs b/src/types/datetime.rs index b671249c983..3d66cee4d95 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -164,9 +164,12 @@ pub trait PyTimeAccess { /// Trait for accessing the components of a struct containing a tzinfo. pub trait PyTzInfoAccess<'py> { /// Deprecated form of `get_tzinfo_bound`. - #[deprecated( - since = "0.21.0", - note = "`get_tzinfo` will be replaced by `get_tzinfo_bound` in a future PyO3 version" + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`get_tzinfo` will be replaced by `get_tzinfo_bound` in a future PyO3 version" + ) )] fn get_tzinfo(&self) -> Option<&'py PyTzInfo> { self.get_tzinfo_bound().map(Bound::into_gil_ref) @@ -734,7 +737,7 @@ mod tests { #[test] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons - #[allow(deprecated)] + #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] fn test_get_tzinfo() { crate::Python::with_gil(|py| { let utc = timezone_utc(py); diff --git a/src/types/pysuper.rs b/src/types/pysuper.rs index 23f445f76b3..25f5e1ceb1c 100644 --- a/src/types/pysuper.rs +++ b/src/types/pysuper.rs @@ -17,9 +17,12 @@ pyobject_native_type_core!( impl PySuper { /// Deprecated form of `PySuper::new_bound`. - #[deprecated( - since = "0.21.0", - note = "`PySuper::new` will be replaced by `PySuper::new_bound` in a future PyO3 version" + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PySuper::new` will be replaced by `PySuper::new_bound` in a future PyO3 version" + ) )] pub fn new<'py>(ty: &'py PyType, obj: &'py PyAny) -> PyResult<&'py PySuper> { Self::new_bound(&ty.as_borrowed(), &obj.as_borrowed()).map(Bound::into_gil_ref) diff --git a/tests/test_exceptions.rs b/tests/test_exceptions.rs index 1cf443e6773..b783e887dcd 100644 --- a/tests/test_exceptions.rs +++ b/tests/test_exceptions.rs @@ -100,7 +100,7 @@ fn test_exception_nosegfault() { #[test] #[cfg(Py_3_8)] -#[allow(deprecated)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] fn test_write_unraisable() { use common::UnraisableCapture; use pyo3::{exceptions::PyRuntimeError, ffi}; diff --git a/tests/test_super.rs b/tests/test_super.rs index b71eae55376..208290df87b 100644 --- a/tests/test_super.rs +++ b/tests/test_super.rs @@ -35,7 +35,7 @@ impl SubClass { } fn method_super_new(self_: &PyCell) -> PyResult<&PyAny> { - #[allow(deprecated)] + #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] let super_ = PySuper::new(self_.get_type(), self_)?; super_.call_method("method", (), None) } From a9f867c2cb3f5fa32e019dbd40dfeffc966b961c Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 24 Dec 2023 10:13:54 +0000 Subject: [PATCH 020/349] begin drafting `Bound` migration guide --- guide/src/migration.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/guide/src/migration.md b/guide/src/migration.md index ca3a2172576..ec195ad1bc3 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -5,6 +5,42 @@ For a detailed list of all changes, see the [CHANGELOG](changelog.md). ## from 0.20.* to 0.21 +PyO3 0.21 introduces a new `Bound<'py, T>` smart pointer which replaces the existing "GIL Refs" API to interact with Python objects. For example, in PyO3 0.20 the reference `&'py PyAny` would be used to interact with Python objects. In PyO3 0.21 the updated type is `Bound<'py, PyAny>`. Making this change moves Rust ownership semantics out of PyO3's internals and into user code. This change fixes [a known soundness edge case of interaction with gevent](https://github.com/PyO3/pyo3/issues/3668) as well as improves CPU and [memory performance](https://github.com/PyO3/pyo3/issues/1056). For a full history of discussion see https://github.com/PyO3/pyo3/issues/3382. + +The "GIL Ref" `&'py PyAny` and similar types such as `&'py PyDict` continue to be available as a deprecated API. Due to the advantages of the new API it is advised that all users make the effort to upgrade as soon as possible. + +In addition to the major API type overhaul, PyO3 has needed to make a few small breaking adjustments to other APIs to close correctness and soundness gaps. + +The recommended steps to update to PyO3 0.21 is as follows: + 1. Enable the `gil-refs` feature to silence deprecations related to the API change + 2. Fix all other PyO3 0.21 migration steps + 3. Disable the `gil-refs` feature and migrate off the deprecated APIs + +The following sections are laid out in this order. + +### Enable the `gil-refs` feature + +To make the transition for the PyO3 ecosystem away from the GIL Refs API as smooth as possible, in PyO3 0.21 no APIs consuming or producing GIL Refs have been altered. Instead, variants using `Bound` smart pointers have been introduced, for example `PyTuple::new_bound` which returns `Bound` is the replacement form of `PyTuple::new`. The GIL Ref APIs have been deprecated, but to make migration easier it is possible to disable these deprecation warnings by enabling the `gil-refs` feature. + +It is recommended that users do this as a first step of updating to PyO3 0.21 so that the deprecation warnings do not get in the way of resolving the rest of the migration steps. + +Before: + +```toml +# Cargo.toml +[dependencies] +pyo3 = "0.20" +``` + +After: + +```toml +# Cargo.toml +[dependencies] +pyo3 = { version = "0.21", features = ["gil-refs"] } +``` + + ### `PyTypeInfo` and `PyTryFrom` have been adjusted The `PyTryFrom` trait has aged poorly, its [`try_from`] method now conflicts with `try_from` in the 2021 edition prelude. A lot of its functionality was also duplicated with `PyTypeInfo`. @@ -196,6 +232,10 @@ impl PyClassAsyncIter { `PyType::name` has been renamed to `PyType::qualname` to indicate that it does indeed return the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name), matching the `__qualname__` attribute. The newly added `PyType::name` yields the full name including the module name now which corresponds to `__module__.__name__` on the level of attributes. +### Migrating from the GIL-Refs API to `Bound` + +TODO + ## from 0.19.* to 0.20 ### Drop support for older technologies From 9a5668572ba16794fddaedf8d9f61f8ac7576542 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 21 Nov 2023 15:54:15 +0000 Subject: [PATCH 021/349] implement `PyModuleMethods` --- src/prelude.rs | 1 + src/py_result_ext.rs | 8 +- src/types/mod.rs | 2 +- src/types/module.rs | 386 +++++++++++++++++++++++++++++++++++++------ 4 files changed, 340 insertions(+), 57 deletions(-) diff --git a/src/prelude.rs b/src/prelude.rs index 9be5dc0dbba..60ce03063b8 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -34,6 +34,7 @@ pub use crate::types::float::PyFloatMethods; pub use crate::types::frozenset::PyFrozenSetMethods; pub use crate::types::list::PyListMethods; pub use crate::types::mapping::PyMappingMethods; +pub use crate::types::module::PyModuleMethods; pub use crate::types::sequence::PySequenceMethods; pub use crate::types::set::PySetMethods; pub use crate::types::string::PyStringMethods; diff --git a/src/py_result_ext.rs b/src/py_result_ext.rs index da0e5c2a5e0..66309988dc4 100644 --- a/src/py_result_ext.rs +++ b/src/py_result_ext.rs @@ -1,4 +1,4 @@ -use crate::{types::any::PyAnyMethods, Bound, PyAny, PyResult}; +use crate::{types::any::PyAnyMethods, Bound, PyAny, PyResult, PyTypeCheck}; mod sealed { use super::*; @@ -11,10 +11,16 @@ mod sealed { use sealed::Sealed; pub(crate) trait PyResultExt<'py>: Sealed { + fn downcast_into(self) -> PyResult>; unsafe fn downcast_into_unchecked(self) -> PyResult>; } impl<'py> PyResultExt<'py> for PyResult> { + #[inline] + fn downcast_into(self) -> PyResult> where { + self.and_then(|instance| instance.downcast_into().map_err(Into::into)) + } + #[inline] unsafe fn downcast_into_unchecked(self) -> PyResult> { self.map(|instance| instance.downcast_into_unchecked()) diff --git a/src/types/mod.rs b/src/types/mod.rs index 705d366601d..c2885f4630c 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -294,7 +294,7 @@ mod iterator; pub(crate) mod list; pub(crate) mod mapping; mod memoryview; -mod module; +pub(crate) mod module; mod none; mod notimplemented; mod num; diff --git a/src/types/module.rs b/src/types/module.rs index 701a9131307..aa6649a9533 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -1,11 +1,12 @@ use crate::callback::IntoPyCallbackOutput; use crate::err::{PyErr, PyResult}; -use crate::exceptions; -use crate::ffi; +use crate::ffi_ptr_ext::FfiPtrExt; use crate::pyclass::PyClass; -use crate::types::{PyAny, PyCFunction, PyDict, PyList, PyString}; -use crate::{IntoPy, Py, PyObject, Python}; -use std::ffi::{CStr, CString}; +use crate::types::{ + any::PyAnyMethods, list::PyListMethods, PyAny, PyCFunction, PyDict, PyList, PyString, +}; +use crate::{exceptions, ffi, Bound, IntoPy, Py, PyNativeType, PyObject, Python}; +use std::ffi::CString; use std::str; /// Represents a Python [`module`][1] object. @@ -151,11 +152,7 @@ impl PyModule { /// Returns the module's `__dict__` attribute, which contains the module's symbol table. pub fn dict(&self) -> &PyDict { - unsafe { - // PyModule_GetDict returns borrowed ptr; must make owned for safety (see #890). - let ptr = ffi::PyModule_GetDict(self.as_ptr()); - self.py().from_owned_ptr(ffi::_Py_NewRef(ptr)) - } + self.as_borrowed().dict().into_gil_ref() } /// Returns the index (the `__all__` attribute) of the module, @@ -163,34 +160,14 @@ impl PyModule { /// /// `__all__` declares the items that will be imported with `from my_module import *`. pub fn index(&self) -> PyResult<&PyList> { - let __all__ = __all__(self.py()); - match self.getattr(__all__) { - Ok(idx) => idx.downcast().map_err(PyErr::from), - Err(err) => { - if err.is_instance_of::(self.py()) { - let l = PyList::empty(self.py()); - self.setattr(__all__, l).map_err(PyErr::from)?; - Ok(l) - } else { - Err(err) - } - } - } + self.as_borrowed().index().map(Bound::into_gil_ref) } /// Returns the name (the `__name__` attribute) of the module. /// /// May fail if the module does not have a `__name__` attribute. pub fn name(&self) -> PyResult<&str> { - let ptr = unsafe { ffi::PyModule_GetName(self.as_ptr()) }; - if ptr.is_null() { - Err(PyErr::fetch(self.py())) - } else { - let name = unsafe { CStr::from_ptr(ptr) } - .to_str() - .expect("PyModule_GetName expected to return utf8"); - Ok(name) - } + self.as_borrowed().name()?.into_gil_ref().to_str() } /// Returns the filename (the `__file__` attribute) of the module. @@ -198,11 +175,7 @@ impl PyModule { /// May fail if the module does not have a `__file__` attribute. #[cfg(not(PyPy))] pub fn filename(&self) -> PyResult<&str> { - unsafe { - self.py() - .from_owned_ptr_or_err::(ffi::PyModule_GetFilenameObject(self.as_ptr()))? - .to_str() - } + self.as_borrowed().filename()?.into_gil_ref().to_str() } /// Adds an attribute to the module. @@ -239,10 +212,7 @@ impl PyModule { where V: IntoPy, { - self.index()? - .append(name) - .expect("could not append __name__ to __all__"); - self.setattr(name, value.into_py(self.py())) + self.as_borrowed().add(name, value) } /// Adds a new class to the module. @@ -287,8 +257,7 @@ impl PyModule { where T: PyClass, { - let py = self.py(); - self.add(T::NAME, T::lazy_type_object().get_or_try_init(py)?) + self.as_borrowed().add_class::() } /// Adds a function or a (sub)module to a module, using the functions name as name. @@ -298,14 +267,7 @@ impl PyModule { where T: IntoPyCallbackOutput, { - self._add_wrapped(wrapper(self.py()).convert(self.py())?) - } - - fn _add_wrapped(&self, object: PyObject) -> PyResult<()> { - let py = self.py(); - let name = object.getattr(py, __name__(py))?; - let name = name.extract(py)?; - self.add(name, object) + self.as_borrowed().add_wrapped(wrapper) } /// Adds a submodule to a module. @@ -349,8 +311,7 @@ impl PyModule { /// [1]: https://github.com/PyO3/pyo3/issues/759 /// [2]: https://github.com/PyO3/pyo3/issues/1517#issuecomment-808664021 pub fn add_submodule(&self, module: &PyModule) -> PyResult<()> { - let name = module.name()?; - self.add(name, module) + self.as_borrowed().add_submodule(&module.as_borrowed()) } /// Add a function to a module. @@ -388,8 +349,314 @@ impl PyModule { /// [1]: crate::prelude::pyfunction /// [2]: crate::wrap_pyfunction pub fn add_function<'a>(&'a self, fun: &'a PyCFunction) -> PyResult<()> { - let name = fun.getattr(__name__(self.py()))?.extract()?; - self.add(name, fun) + self.as_borrowed().add_function(&fun.as_borrowed()) + } +} + +/// Implementation of functionality for [`PyModule`]. +/// +/// These methods are defined for the `Bound<'py, PyModule>` smart pointer, so to use method call +/// syntax these methods are separated into a trait, because stable Rust does not yet support +/// `arbitrary_self_types`. +#[doc(alias = "PyModule")] +pub trait PyModuleMethods<'py> { + /// Returns the module's `__dict__` attribute, which contains the module's symbol table. + fn dict(&self) -> Bound<'py, PyDict>; + + /// Returns the index (the `__all__` attribute) of the module, + /// creating one if needed. + /// + /// `__all__` declares the items that will be imported with `from my_module import *`. + fn index(&self) -> PyResult>; + + /// Returns the name (the `__name__` attribute) of the module. + /// + /// May fail if the module does not have a `__name__` attribute. + fn name(&self) -> PyResult>; + + /// Returns the filename (the `__file__` attribute) of the module. + /// + /// May fail if the module does not have a `__file__` attribute. + #[cfg(not(PyPy))] + fn filename(&self) -> PyResult>; + + /// Adds an attribute to the module. + /// + /// For adding classes, functions or modules, prefer to use [`PyModule::add_class`], + /// [`PyModule::add_function`] or [`PyModule::add_submodule`] instead, respectively. + /// + /// # Examples + /// + /// ```rust + /// use pyo3::prelude::*; + /// + /// #[pymodule] + /// fn my_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// module.add("c", 299_792_458)?; + /// Ok(()) + /// } + /// ``` + /// + /// Python code can then do the following: + /// + /// ```python + /// from my_module import c + /// + /// print("c is", c) + /// ``` + /// + /// This will result in the following output: + /// + /// ```text + /// c is 299792458 + /// ``` + fn add(&self, name: N, value: V) -> PyResult<()> + where + N: IntoPy>, + V: IntoPy; + + /// Adds a new class to the module. + /// + /// Notice that this method does not take an argument. + /// Instead, this method is *generic*, and requires us to use the + /// "turbofish" syntax to specify the class we want to add. + /// + /// # Examples + /// + /// ```rust + /// use pyo3::prelude::*; + /// + /// #[pyclass] + /// struct Foo { /* fields omitted */ } + /// + /// #[pymodule] + /// fn my_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// module.add_class::()?; + /// Ok(()) + /// } + /// ``` + /// + /// Python code can see this class as such: + /// ```python + /// from my_module import Foo + /// + /// print("Foo is", Foo) + /// ``` + /// + /// This will result in the following output: + /// ```text + /// Foo is + /// ``` + /// + /// Note that as we haven't defined a [constructor][1], Python code can't actually + /// make an *instance* of `Foo` (or *get* one for that matter, as we haven't exported + /// anything that can return instances of `Foo`). + /// + /// [1]: https://pyo3.rs/latest/class.html#constructor + fn add_class(&self) -> PyResult<()> + where + T: PyClass; + + /// Adds a function or a (sub)module to a module, using the functions name as name. + /// + /// Prefer to use [`PyModule::add_function`] and/or [`PyModule::add_submodule`] instead. + fn add_wrapped(&self, wrapper: &impl Fn(Python<'py>) -> T) -> PyResult<()> + where + T: IntoPyCallbackOutput; + + /// Adds a submodule to a module. + /// + /// This is especially useful for creating module hierarchies. + /// + /// Note that this doesn't define a *package*, so this won't allow Python code + /// to directly import submodules by using + /// `from my_module import submodule`. + /// For more information, see [#759][1] and [#1517][2]. + /// + /// # Examples + /// + /// ```rust + /// use pyo3::prelude::*; + /// + /// #[pymodule] + /// fn my_module(py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// let submodule = PyModule::new(py, "submodule")?; + /// submodule.add("super_useful_constant", "important")?; + /// + /// module.add_submodule(submodule)?; + /// Ok(()) + /// } + /// ``` + /// + /// Python code can then do the following: + /// + /// ```python + /// import my_module + /// + /// print("super_useful_constant is", my_module.submodule.super_useful_constant) + /// ``` + /// + /// This will result in the following output: + /// + /// ```text + /// super_useful_constant is important + /// ``` + /// + /// [1]: https://github.com/PyO3/pyo3/issues/759 + /// [2]: https://github.com/PyO3/pyo3/issues/1517#issuecomment-808664021 + fn add_submodule(&self, module: &Bound<'_, PyModule>) -> PyResult<()>; + + /// Add a function to a module. + /// + /// Note that this also requires the [`wrap_pyfunction!`][2] macro + /// to wrap a function annotated with [`#[pyfunction]`][1]. + /// + /// ```rust + /// use pyo3::prelude::*; + /// + /// #[pyfunction] + /// fn say_hello() { + /// println!("Hello world!") + /// } + /// #[pymodule] + /// fn my_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// module.add_function(wrap_pyfunction!(say_hello, module)?) + /// } + /// ``` + /// + /// Python code can then do the following: + /// + /// ```python + /// from my_module import say_hello + /// + /// say_hello() + /// ``` + /// + /// This will result in the following output: + /// + /// ```text + /// Hello world! + /// ``` + /// + /// [1]: crate::prelude::pyfunction + /// [2]: crate::wrap_pyfunction + fn add_function(&self, fun: &Bound<'_, PyCFunction>) -> PyResult<()>; +} + +impl<'py> PyModuleMethods<'py> for Bound<'py, PyModule> { + fn dict(&self) -> Bound<'py, PyDict> { + unsafe { + // PyModule_GetDict returns borrowed ptr; must make owned for safety (see #890). + ffi::PyModule_GetDict(self.as_ptr()) + .assume_borrowed(self.py()) + .to_owned() + .downcast_into_unchecked() + } + } + + fn index(&self) -> PyResult> { + let __all__ = __all__(self.py()); + match self.getattr(__all__) { + Ok(idx) => idx.downcast_into().map_err(PyErr::from), + Err(err) => { + if err.is_instance_of::(self.py()) { + let l = PyList::empty(self.py()).as_borrowed().to_owned(); + self.setattr(__all__, &l).map_err(PyErr::from)?; + Ok(l) + } else { + Err(err) + } + } + } + } + + fn name(&self) -> PyResult> { + #[cfg(not(PyPy))] + { + use crate::py_result_ext::PyResultExt; + + unsafe { + ffi::PyModule_GetNameObject(self.as_ptr()) + .assume_owned_or_err(self.py()) + .downcast_into_unchecked() + } + } + + #[cfg(PyPy)] + { + self.dict() + .get_item("__name__") + .map_err(|_| exceptions::PyAttributeError::new_err("__name__"))? + .downcast_into() + .map_err(PyErr::from) + } + } + + #[cfg(not(PyPy))] + fn filename(&self) -> PyResult> { + use crate::py_result_ext::PyResultExt; + + unsafe { + ffi::PyModule_GetFilenameObject(self.as_ptr()) + .assume_owned_or_err(self.py()) + .downcast_into_unchecked() + } + } + + fn add(&self, name: N, value: V) -> PyResult<()> + where + N: IntoPy>, + V: IntoPy, + { + fn inner( + module: &Bound<'_, PyModule>, + name: Bound<'_, PyString>, + value: Bound<'_, PyAny>, + ) -> PyResult<()> { + module + .index()? + .append(&name) + .expect("could not append __name__ to __all__"); + module.setattr(name, value.into_py(module.py())) + } + + let py = self.py(); + inner( + self, + name.into_py(py).into_bound(py), + value.into_py(py).into_bound(py), + ) + } + + fn add_class(&self) -> PyResult<()> + where + T: PyClass, + { + let py = self.py(); + self.add(T::NAME, T::lazy_type_object().get_or_try_init(py)?) + } + + fn add_wrapped(&self, wrapper: &impl Fn(Python<'py>) -> T) -> PyResult<()> + where + T: IntoPyCallbackOutput, + { + fn inner(module: &Bound<'_, PyModule>, object: Bound<'_, PyAny>) -> PyResult<()> { + let name = object.getattr(__name__(module.py()))?; + module.add(name.downcast_into::()?, object) + } + + let py = self.py(); + inner(self, wrapper(py).convert(py)?.into_bound(py)) + } + + fn add_submodule(&self, module: &Bound<'_, PyModule>) -> PyResult<()> { + let name = module.name()?; + self.add(name, module) + } + + fn add_function(&self, fun: &Bound<'_, PyCFunction>) -> PyResult<()> { + let name = fun.getattr(__name__(self.py()))?; + self.add(name.downcast_into::()?, fun) } } @@ -412,4 +679,13 @@ mod tests { assert_eq!(builtins.name().unwrap(), "builtins"); }) } + + #[test] + #[cfg(not(PyPy))] + fn module_filename() { + Python::with_gil(|py| { + let site = PyModule::import(py, "site").unwrap(); + assert!(site.filename().unwrap().ends_with("site.py")); + }) + } } From e852a4b50256fbd56a9f2cb5b3f7a33a31105153 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 26 Dec 2023 22:54:39 +0000 Subject: [PATCH 022/349] be more lax with ordering in `invalid_result_conversion` ui test --- tests/test_compile_error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 0c278bdfa62..0154f3f12bc 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -33,7 +33,7 @@ fn test_compile_errors() { t.compile_fail("tests/ui/invalid_pymethod_receiver.rs"); t.compile_fail("tests/ui/missing_intopy.rs"); // adding extra error conversion impls changes the output - #[cfg(not(any(windows, feature = "eyre", feature = "anyhow")))] + #[cfg(not(any(windows, feature = "eyre", feature = "anyhow", Py_LIMITED_API)))] t.compile_fail("tests/ui/invalid_result_conversion.rs"); t.compile_fail("tests/ui/not_send.rs"); t.compile_fail("tests/ui/not_send2.rs"); From 339660c117921e7e583cb535cd47e77ea354044c Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 29 Dec 2023 16:09:30 -0500 Subject: [PATCH 023/349] add PyAnyMethods for binary operators also pow fixes #3709 --- newsfragments/3712.added.md | 1 + src/tests/common.rs | 7 +++ src/types/any.rs | 108 ++++++++++++++++++++++++++++++++++++ tests/test_arithmetics.rs | 16 ++++++ 4 files changed, 132 insertions(+) create mode 100644 newsfragments/3712.added.md diff --git a/newsfragments/3712.added.md b/newsfragments/3712.added.md new file mode 100644 index 00000000000..d7390f77c14 --- /dev/null +++ b/newsfragments/3712.added.md @@ -0,0 +1 @@ +Added methods to `PyAnyMethods` for binary operators (`add`, `sub`, etc.) diff --git a/src/tests/common.rs b/src/tests/common.rs index 89b4a83fa1d..f2082437693 100644 --- a/src/tests/common.rs +++ b/src/tests/common.rs @@ -23,6 +23,13 @@ mod inner { }; } + #[macro_export] + macro_rules! assert_py_eq { + ($val:expr, $expected:expr) => { + assert!($val.eq($expected).unwrap()); + }; + } + #[macro_export] macro_rules! py_expect_exception { // Case1: idents & no err_msg diff --git a/src/types/any.rs b/src/types/any.rs index edc74879cb6..102099c5cd2 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1208,6 +1208,58 @@ pub trait PyAnyMethods<'py> { where O: ToPyObject; + /// Computes `self + other`. + fn add(&self, other: O) -> PyResult> + where + O: ToPyObject; + + /// Computes `self - other`. + fn sub(&self, other: O) -> PyResult> + where + O: ToPyObject; + + /// Computes `self * other`. + fn mul(&self, other: O) -> PyResult> + where + O: ToPyObject; + + /// Computes `self / other`. + fn div(&self, other: O) -> PyResult> + where + O: ToPyObject; + + /// Computes `self << other`. + fn lshift(&self, other: O) -> PyResult> + where + O: ToPyObject; + + /// Computes `self >> other`. + fn rshift(&self, other: O) -> PyResult> + where + O: ToPyObject; + + /// Computes `self ** other % modulus` (`pow(self, other, modulus)`). + /// `py.None()` may be passed for the `modulus`. + fn pow(&self, other: O1, modulus: O2) -> PyResult> + where + O1: ToPyObject, + O2: ToPyObject; + + /// Computes `self & other`. + fn bitand(&self, other: O) -> PyResult> + where + O: ToPyObject; + + /// Computes `self | other`. + fn bitor(&self, other: O) -> PyResult> + where + O: ToPyObject; + + /// Computes `self ^ other`. + fn bitxor(&self, other: O) -> PyResult> + where + O: ToPyObject; + /// Determines whether this object appears callable. /// /// This is equivalent to Python's [`callable()`][1] function. @@ -1680,6 +1732,26 @@ pub trait PyAnyMethods<'py> { fn py_super(&self) -> PyResult>; } +macro_rules! implement_binop { + ($name:ident, $c_api:ident, $op:expr) => { + #[doc = concat!("Computes `self ", $op, " other`.")] + fn $name(&self, other: O) -> PyResult> + where + O: ToPyObject, + { + fn inner<'py>( + any: &Bound<'py, PyAny>, + other: Bound<'_, PyAny>, + ) -> PyResult> { + unsafe { ffi::$c_api(any.as_ptr(), other.as_ptr()).assume_owned_or_err(any.py()) } + } + + let py = self.py(); + inner(self, other.to_object(py).into_bound(py)) + } + }; +} + impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { #[inline] fn is(&self, other: &T) -> bool { @@ -1855,6 +1927,42 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { .and_then(|any| any.is_truthy()) } + implement_binop!(add, PyNumber_Add, "+"); + implement_binop!(sub, PyNumber_Subtract, "-"); + implement_binop!(mul, PyNumber_Multiply, "*"); + implement_binop!(div, PyNumber_TrueDivide, "/"); + implement_binop!(lshift, PyNumber_Lshift, "<<"); + implement_binop!(rshift, PyNumber_Rshift, ">>"); + implement_binop!(bitand, PyNumber_And, "&"); + implement_binop!(bitor, PyNumber_Or, "|"); + implement_binop!(bitxor, PyNumber_Xor, "^"); + + /// Computes `self ** other % modulus` (`pow(self, other, modulus)`). + /// `py.None()` may be passed for the `modulus`. + fn pow(&self, other: O1, modulus: O2) -> PyResult> + where + O1: ToPyObject, + O2: ToPyObject, + { + fn inner<'py>( + any: &Bound<'py, PyAny>, + other: Bound<'_, PyAny>, + modulus: Bound<'_, PyAny>, + ) -> PyResult> { + unsafe { + ffi::PyNumber_Power(any.as_ptr(), other.as_ptr(), modulus.as_ptr()) + .assume_owned_or_err(any.py()) + } + } + + let py = self.py(); + inner( + self, + other.to_object(py).into_bound(py), + modulus.to_object(py).into_bound(py), + ) + } + fn is_callable(&self) -> bool { unsafe { ffi::PyCallable_Check(self.as_ptr()) != 0 } } diff --git a/tests/test_arithmetics.rs b/tests/test_arithmetics.rs index 86078080176..456d21a3b62 100644 --- a/tests/test_arithmetics.rs +++ b/tests/test_arithmetics.rs @@ -178,6 +178,10 @@ impl BinaryArithmetic { format!("BA * {:?}", rhs) } + fn __truediv__(&self, rhs: &PyAny) -> String { + format!("BA / {:?}", rhs) + } + fn __lshift__(&self, rhs: &PyAny) -> String { format!("BA << {:?}", rhs) } @@ -233,6 +237,18 @@ fn binary_arithmetic() { py_expect_exception!(py, c, "1 ** c", PyTypeError); py_run!(py, c, "assert pow(c, 1, 100) == 'BA ** 1 (mod: Some(100))'"); + + let c: Bound<'_, PyAny> = c.extract().unwrap(); + assert_py_eq!(c.add(&c).unwrap(), "BA + BA"); + assert_py_eq!(c.sub(&c).unwrap(), "BA - BA"); + assert_py_eq!(c.mul(&c).unwrap(), "BA * BA"); + assert_py_eq!(c.div(&c).unwrap(), "BA / BA"); + assert_py_eq!(c.lshift(&c).unwrap(), "BA << BA"); + assert_py_eq!(c.rshift(&c).unwrap(), "BA >> BA"); + assert_py_eq!(c.bitand(&c).unwrap(), "BA & BA"); + assert_py_eq!(c.bitor(&c).unwrap(), "BA | BA"); + assert_py_eq!(c.bitxor(&c).unwrap(), "BA ^ BA"); + assert_py_eq!(c.pow(&c, py.None()).unwrap(), "BA ** BA (mod: None)"); }); } From 8fa5294d9384ad9010221d3f290de4a4642f0e05 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 29 Dec 2023 22:58:16 +0000 Subject: [PATCH 024/349] release notes for 0.20.1 --- CHANGELOG.md | 21 ++++++++++++++++++- README.md | 4 ++-- examples/decorator/.template/pre-script.rhai | 2 +- .../maturin-starter/.template/pre-script.rhai | 2 +- examples/plugin/.template/pre-script.rhai | 2 +- .../.template/pre-script.rhai | 2 +- examples/word-count/.template/pre-script.rhai | 2 +- newsfragments/3456.added.md | 1 - newsfragments/3507.added.md | 1 - newsfragments/3512.fixed.md | 1 - newsfragments/3556.added.md | 1 - newsfragments/3564.fixed.md | 1 - newsfragments/3587.added.md | 2 -- newsfragments/3616.added.md | 1 - newsfragments/3687.added.md | 1 - 15 files changed, 27 insertions(+), 17 deletions(-) delete mode 100644 newsfragments/3456.added.md delete mode 100644 newsfragments/3507.added.md delete mode 100644 newsfragments/3512.fixed.md delete mode 100644 newsfragments/3556.added.md delete mode 100644 newsfragments/3564.fixed.md delete mode 100644 newsfragments/3587.added.md delete mode 100644 newsfragments/3616.added.md delete mode 100644 newsfragments/3687.added.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 97ce9a00537..6f7708ce05b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,24 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.20.1] - 2023-12-30 + +### Added + +- Add optional `either` feature to add conversions for `either::Either` sum type. [#3456](https://github.com/PyO3/pyo3/pull/3456) +- Add optional `smallvec` feature to add conversions for `smallvec::SmallVec`. [#3507](https://github.com/PyO3/pyo3/pull/3507) +- Add `take` and `into_inner` methods to `GILOnceCell` [#3556](https://github.com/PyO3/pyo3/pull/3556) +- `#[classmethod]` methods can now also receive `Py` as their first argument. [#3587](https://github.com/PyO3/pyo3/pull/3587) +- `#[pyfunction(pass_module)]` can now also receive `Py` as their first argument. [#3587](https://github.com/PyO3/pyo3/pull/3587) +- Add `traverse` method to `GILProtected`. [#3616](https://github.com/PyO3/pyo3/pull/3616) +- Added `abi3-py312` feature [#3687](https://github.com/PyO3/pyo3/pull/3687) + +### Fixed + +- Fix minimum version specification for optional `chrono` dependency. [#3512](https://github.com/PyO3/pyo3/pull/3512) +- Silenced new `clippy::unnecessary_fallible_conversions` warning when using a `Py` `self` receiver. [#3564](https://github.com/PyO3/pyo3/pull/3564) + + ## [0.20.0] - 2023-10-11 ### Packaging @@ -1599,7 +1617,8 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.20.0...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.20.1...HEAD +[0.20.1]: https://github.com/pyo3/pyo3/compare/v0.20.0...v0.20.1 [0.20.0]: https://github.com/pyo3/pyo3/compare/v0.19.2...v0.20.0 [0.19.2]: https://github.com/pyo3/pyo3/compare/v0.19.1...v0.19.2 [0.19.1]: https://github.com/pyo3/pyo3/compare/v0.19.0...v0.19.1 diff --git a/README.md b/README.md index 1b163d75056..6c368ce946c 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.20.0", features = ["extension-module"] } +pyo3 = { version = "0.20.1", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.20.0" +version = "0.20.1" features = ["auto-initialize"] ``` diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 5ba02b12ffd..c7199a5303e 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.0"); +variable::set("PYO3_VERSION", "0.20.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 5ba02b12ffd..c7199a5303e 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.0"); +variable::set("PYO3_VERSION", "0.20.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 94a61826dc2..d325aca0eef 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.0"); +variable::set("PYO3_VERSION", "0.20.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index e4ede9b7aff..91eee121a7b 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.0"); +variable::set("PYO3_VERSION", "0.20.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 5ba02b12ffd..c7199a5303e 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.0"); +variable::set("PYO3_VERSION", "0.20.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/newsfragments/3456.added.md b/newsfragments/3456.added.md deleted file mode 100644 index 6e9376ba65d..00000000000 --- a/newsfragments/3456.added.md +++ /dev/null @@ -1 +0,0 @@ -Add optional conversion support for `either::Either` sum type (under "either" feature). diff --git a/newsfragments/3507.added.md b/newsfragments/3507.added.md deleted file mode 100644 index 2068ab4c3f7..00000000000 --- a/newsfragments/3507.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `smallvec` feature to add `ToPyObject`, `IntoPy` and `FromPyObject` implementations for `smallvec::SmallVec`. diff --git a/newsfragments/3512.fixed.md b/newsfragments/3512.fixed.md deleted file mode 100644 index 39b8087669e..00000000000 --- a/newsfragments/3512.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix minimum version specification for optional `chrono` dependency diff --git a/newsfragments/3556.added.md b/newsfragments/3556.added.md deleted file mode 100644 index 014908a1bf5..00000000000 --- a/newsfragments/3556.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `take` and `into_inner` methods to `GILOnceCell` \ No newline at end of file diff --git a/newsfragments/3564.fixed.md b/newsfragments/3564.fixed.md deleted file mode 100644 index 83e4dba05bb..00000000000 --- a/newsfragments/3564.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Silenced new `clippy::unnecessary_fallible_conversions` warning when using a `Py` `self` receiver diff --git a/newsfragments/3587.added.md b/newsfragments/3587.added.md deleted file mode 100644 index f8ea280dd25..00000000000 --- a/newsfragments/3587.added.md +++ /dev/null @@ -1,2 +0,0 @@ -- Classmethods can now receive `Py` as their first argument -- Function annotated with `pass_module` can now receive `Py` as their first argument \ No newline at end of file diff --git a/newsfragments/3616.added.md b/newsfragments/3616.added.md deleted file mode 100644 index 532dc6e56c1..00000000000 --- a/newsfragments/3616.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `traverse` method to `GILProtected` diff --git a/newsfragments/3687.added.md b/newsfragments/3687.added.md deleted file mode 100644 index a6df28d939f..00000000000 --- a/newsfragments/3687.added.md +++ /dev/null @@ -1 +0,0 @@ -Added `abi3-py312` feature From 375e3d4eeefccc3e89ef76bd7fb298597a5c7a7f Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 20 Dec 2023 15:33:54 +0000 Subject: [PATCH 025/349] implement `PyTupleMethods` --- src/instance.rs | 8 +- src/prelude.rs | 1 + src/types/mod.rs | 4 +- src/types/tuple.rs | 413 ++++++++++++++++++---- tests/ui/invalid_result_conversion.stderr | 2 +- 5 files changed, 361 insertions(+), 67 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index e3f3743ebf7..11936d6c94f 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -325,10 +325,10 @@ impl<'py, T> Borrowed<'py, 'py, T> where T: HasPyGilRef, { - // pub(crate) fn into_gil_ref(self) -> &'py T::AsRefTarget { - // // Safety: self is a borrow over `'py`. - // unsafe { self.py().from_borrowed_ptr(self.0.as_ptr()) } - // } + pub(crate) fn into_gil_ref(self) -> &'py T::AsRefTarget { + // Safety: self is a borrow over `'py`. + unsafe { self.py().from_borrowed_ptr(self.0.as_ptr()) } + } } impl std::fmt::Debug for Borrowed<'_, '_, T> { diff --git a/src/prelude.rs b/src/prelude.rs index 60ce03063b8..6be820ae92a 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -38,3 +38,4 @@ pub use crate::types::module::PyModuleMethods; pub use crate::types::sequence::PySequenceMethods; pub use crate::types::set::PySetMethods; pub use crate::types::string::PyStringMethods; +pub use crate::types::tuple::PyTupleMethods; diff --git a/src/types/mod.rs b/src/types/mod.rs index c2885f4630c..00fe81cf37e 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -80,7 +80,7 @@ pub mod iter { pub use super::frozenset::{BoundFrozenSetIterator, PyFrozenSetIterator}; pub use super::list::{BoundListIterator, PyListIterator}; pub use super::set::{BoundSetIterator, PySetIterator}; - pub use super::tuple::PyTupleIterator; + pub use super::tuple::{BorrowedTupleIterator, BoundTupleIterator, PyTupleIterator}; } // Implementations core to all native types @@ -305,5 +305,5 @@ pub(crate) mod set; mod slice; pub(crate) mod string; mod traceback; -mod tuple; +pub(crate) mod tuple; mod typeobject; diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 0165061a7a1..1b87499f1e0 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -2,21 +2,23 @@ use std::convert::TryInto; use std::iter::FusedIterator; use crate::ffi::{self, Py_ssize_t}; +use crate::ffi_ptr_ext::FfiPtrExt; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; +use crate::instance::Borrowed; use crate::internal_tricks::get_ssize_index; -use crate::types::PyList; -use crate::types::PySequence; +use crate::types::{any::PyAnyMethods, sequence::PySequenceMethods, PyList, PySequence}; use crate::{ - exceptions, FromPyObject, IntoPy, Py, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, + exceptions, Bound, FromPyObject, IntoPy, Py, PyAny, PyErr, PyNativeType, PyObject, PyResult, + Python, ToPyObject, }; #[inline] #[track_caller] -fn new_from_iter( - py: Python<'_>, +fn new_from_iter<'py>( + py: Python<'py>, elements: &mut dyn ExactSizeIterator, -) -> Py { +) -> Bound<'py, PyTuple> { unsafe { // PyTuple_New checks for overflow but has a bad error message, so we check ourselves let len: Py_ssize_t = elements @@ -28,7 +30,7 @@ fn new_from_iter( // - Panics if the ptr is null // - Cleans up the tuple if `convert` or the asserts panic - let tup: Py = Py::from_owned_ptr(py, ptr); + let tup = ptr.assume_owned(py).downcast_into_unchecked(); let mut counter: Py_ssize_t = 0; @@ -92,8 +94,7 @@ impl PyTuple { U: ExactSizeIterator, { let mut elements = elements.into_iter().map(|e| e.to_object(py)); - let tup = new_from_iter(py, &mut elements); - tup.into_ref(py) + new_from_iter(py, &mut elements).into_gil_ref() } /// Constructs an empty tuple (on the Python side, a singleton object). @@ -103,19 +104,12 @@ impl PyTuple { /// Gets the length of the tuple. pub fn len(&self) -> usize { - unsafe { - #[cfg(not(any(Py_LIMITED_API, PyPy)))] - let size = ffi::PyTuple_GET_SIZE(self.as_ptr()); - #[cfg(any(Py_LIMITED_API, PyPy))] - let size = ffi::PyTuple_Size(self.as_ptr()); - // non-negative Py_ssize_t should always fit into Rust uint - size as usize - } + self.as_borrowed().len() } /// Checks if the tuple is empty. pub fn is_empty(&self) -> bool { - self.len() == 0 + self.as_borrowed().is_empty() } /// Returns `self` cast as a `PySequence`. @@ -128,13 +122,7 @@ impl PyTuple { /// Indices must be nonnegative, and out-of-range indices are clipped to /// `self.len()`. pub fn get_slice(&self, low: usize, high: usize) -> &PyTuple { - unsafe { - self.py().from_owned_ptr(ffi::PyTuple_GetSlice( - self.as_ptr(), - get_ssize_index(low), - get_ssize_index(high), - )) - } + self.as_borrowed().get_slice(low, high).into_gil_ref() } /// Gets the tuple item at the specified index. @@ -153,10 +141,9 @@ impl PyTuple { /// # } /// ``` pub fn get_item(&self, index: usize) -> PyResult<&PyAny> { - unsafe { - let item = ffi::PyTuple_GetItem(self.as_ptr(), index as Py_ssize_t); - self.py().from_borrowed_ptr_or_err(item) - } + self.as_borrowed() + .get_borrowed_item(index) + .map(Borrowed::into_gil_ref) } /// Gets the tuple item at the specified index. Undefined behavior on bad index. Use with caution. @@ -166,8 +153,9 @@ impl PyTuple { /// Caller must verify that the index is within the bounds of the tuple. #[cfg(not(any(Py_LIMITED_API, PyPy)))] pub unsafe fn get_item_unchecked(&self, index: usize) -> &PyAny { - let item = ffi::PyTuple_GET_ITEM(self.as_ptr(), index as Py_ssize_t); - self.py().from_borrowed_ptr(item) + self.as_borrowed() + .get_borrowed_item_unchecked(index) + .into_gil_ref() } /// Returns `self` as a slice of objects. @@ -190,7 +178,7 @@ impl PyTuple { where V: ToPyObject, { - self.as_sequence().contains(value) + self.as_borrowed().contains(value) } /// Returns the first index `i` for which `self[i] == value`. @@ -201,54 +189,285 @@ impl PyTuple { where V: ToPyObject, { - self.as_sequence().index(value) + self.as_borrowed().index(value) } /// Returns an iterator over the tuple items. pub fn iter(&self) -> PyTupleIterator<'_> { - PyTupleIterator { - tuple: self, - index: 0, - length: self.len(), - } + PyTupleIterator(BorrowedTupleIterator::new(self.as_borrowed())) } /// Return a new list containing the contents of this tuple; equivalent to the Python expression `list(tuple)`. /// /// This method is equivalent to `self.as_sequence().to_list()` and faster than `PyList::new(py, self)`. pub fn to_list(&self) -> &PyList { + self.as_borrowed().to_list().into_gil_ref() + } +} + +index_impls!(PyTuple, "tuple", PyTuple::len, PyTuple::get_slice); + +/// Implementation of functionality for [`PyTuple`]. +/// +/// These methods are defined for the `Bound<'py, PyTuple>` smart pointer, so to use method call +/// syntax these methods are separated into a trait, because stable Rust does not yet support +/// `arbitrary_self_types`. +#[doc(alias = "PyTuple")] +pub trait PyTupleMethods<'py> { + /// Gets the length of the tuple. + fn len(&self) -> usize; + + /// Checks if the tuple is empty. + fn is_empty(&self) -> bool; + + /// Returns `self` cast as a `PySequence`. + fn as_sequence(&self) -> &Bound<'py, PySequence>; + + /// Takes the slice `self[low:high]` and returns it as a new tuple. + /// + /// Indices must be nonnegative, and out-of-range indices are clipped to + /// `self.len()`. + fn get_slice(&self, low: usize, high: usize) -> Bound<'py, PyTuple>; + + /// Gets the tuple item at the specified index. + /// # Example + /// ``` + /// use pyo3::{prelude::*, types::PyTuple}; + /// + /// # fn main() -> PyResult<()> { + /// Python::with_gil(|py| -> PyResult<()> { + /// let ob = (1, 2, 3).to_object(py); + /// let tuple: &PyTuple = ob.downcast(py).unwrap(); + /// let obj = tuple.get_item(0); + /// assert_eq!(obj.unwrap().extract::().unwrap(), 1); + /// Ok(()) + /// }) + /// # } + /// ``` + fn get_item(&self, index: usize) -> PyResult>; + + /// Like [`get_item`][PyTupleMethods::get_item], but returns a borrowed object, which is a slight performance optimization + /// by avoiding a reference count change. + fn get_borrowed_item<'a>(&'a self, index: usize) -> PyResult>; + + /// Gets the tuple item at the specified index. Undefined behavior on bad index. Use with caution. + /// + /// # Safety + /// + /// Caller must verify that the index is within the bounds of the tuple. + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + unsafe fn get_item_unchecked(&self, index: usize) -> Bound<'py, PyAny>; + + /// Like [`get_item_unchecked`][PyTupleMethods::get_item_unchecked], but returns a borrowed object, + /// which is a slight performance optimization by avoiding a reference count change. + /// + /// # Safety + /// + /// Caller must verify that the index is within the bounds of the tuple. + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + unsafe fn get_borrowed_item_unchecked<'a>(&'a self, index: usize) -> Borrowed<'a, 'py, PyAny>; + + /// Returns `self` as a slice of objects. + #[cfg(not(Py_LIMITED_API))] + fn as_slice(&self) -> &[Bound<'py, PyAny>]; + + /// Determines if self contains `value`. + /// + /// This is equivalent to the Python expression `value in self`. + fn contains(&self, value: V) -> PyResult + where + V: ToPyObject; + + /// Returns the first index `i` for which `self[i] == value`. + /// + /// This is equivalent to the Python expression `self.index(value)`. + fn index(&self, value: V) -> PyResult + where + V: ToPyObject; + + /// Returns an iterator over the tuple items. + fn iter(&self) -> BoundTupleIterator<'py>; + + /// Like [`iter`][PyTupleMethods::iter], but produces an iterator which returns borrowed objects, + /// which is a slight performance optimization by avoiding a reference count change. + fn iter_borrowed<'a>(&'a self) -> BorrowedTupleIterator<'a, 'py>; + + /// Return a new list containing the contents of this tuple; equivalent to the Python expression `list(tuple)`. + /// + /// This method is equivalent to `self.as_sequence().to_list()` and faster than `PyList::new(py, self)`. + fn to_list(&self) -> Bound<'py, PyList>; +} + +impl<'py> PyTupleMethods<'py> for Bound<'py, PyTuple> { + fn len(&self) -> usize { + unsafe { + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + let size = ffi::PyTuple_GET_SIZE(self.as_ptr()); + #[cfg(any(Py_LIMITED_API, PyPy))] + let size = ffi::PyTuple_Size(self.as_ptr()); + // non-negative Py_ssize_t should always fit into Rust uint + size as usize + } + } + + fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn as_sequence(&self) -> &Bound<'py, PySequence> { + unsafe { self.downcast_unchecked() } + } + + fn get_slice(&self, low: usize, high: usize) -> Bound<'py, PyTuple> { + unsafe { + ffi::PyTuple_GetSlice(self.as_ptr(), get_ssize_index(low), get_ssize_index(high)) + .assume_owned(self.py()) + .downcast_into_unchecked() + } + } + + fn get_item(&self, index: usize) -> PyResult> { + self.get_borrowed_item(index).map(Borrowed::to_owned) + } + + fn get_borrowed_item<'a>(&'a self, index: usize) -> PyResult> { + Borrowed::from(self).get_borrowed_item(index) + } + + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + unsafe fn get_item_unchecked(&self, index: usize) -> Bound<'py, PyAny> { + self.get_borrowed_item_unchecked(index).to_owned() + } + + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + unsafe fn get_borrowed_item_unchecked<'a>(&'a self, index: usize) -> Borrowed<'a, 'py, PyAny> { + Borrowed::from(self).get_borrowed_item_unchecked(index) + } + + #[cfg(not(Py_LIMITED_API))] + fn as_slice(&self) -> &[Bound<'py, PyAny>] { + // This is safe because Bound<'py, PyAny> has the same memory layout as *mut ffi::PyObject, + // and because tuples are immutable. + unsafe { + let ptr = self.as_ptr() as *mut ffi::PyTupleObject; + let slice = std::slice::from_raw_parts((*ptr).ob_item.as_ptr(), self.len()); + &*(slice as *const [*mut ffi::PyObject] as *const [Bound<'py, PyAny>]) + } + } + + #[inline] + fn contains(&self, value: V) -> PyResult + where + V: ToPyObject, + { + self.as_sequence().contains(value) + } + + #[inline] + fn index(&self, value: V) -> PyResult + where + V: ToPyObject, + { + self.as_sequence().index(value) + } + + fn iter(&self) -> BoundTupleIterator<'py> { + BoundTupleIterator::new(self.clone()) + } + + fn iter_borrowed<'a>(&'a self) -> BorrowedTupleIterator<'a, 'py> { + BorrowedTupleIterator::new(Borrowed::from(self)) + } + + fn to_list(&self) -> Bound<'py, PyList> { self.as_sequence() .to_list() .expect("failed to convert tuple to list") } } -index_impls!(PyTuple, "tuple", PyTuple::len, PyTuple::get_slice); +impl<'a, 'py> Borrowed<'a, 'py, PyTuple> { + fn get_borrowed_item(self, index: usize) -> PyResult> { + unsafe { + ffi::PyTuple_GetItem(self.as_ptr(), index as Py_ssize_t) + .assume_borrowed_or_err(self.py()) + } + } + + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + unsafe fn get_borrowed_item_unchecked(self, index: usize) -> Borrowed<'a, 'py, PyAny> { + ffi::PyTuple_GET_ITEM(self.as_ptr(), index as Py_ssize_t).assume_borrowed(self.py()) + } +} /// Used by `PyTuple::iter()`. -pub struct PyTupleIterator<'a> { - tuple: &'a PyTuple, +pub struct PyTupleIterator<'a>(BorrowedTupleIterator<'a, 'a>); + +impl<'a> Iterator for PyTupleIterator<'a> { + type Item = &'a PyAny; + + #[inline] + fn next(&mut self) -> Option { + self.0.next().map(Borrowed::into_gil_ref) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.0.size_hint() + } +} + +impl<'a> DoubleEndedIterator for PyTupleIterator<'a> { + #[inline] + fn next_back(&mut self) -> Option { + self.0.next_back().map(Borrowed::into_gil_ref) + } +} + +impl<'a> ExactSizeIterator for PyTupleIterator<'a> { + fn len(&self) -> usize { + self.0.len() + } +} + +impl FusedIterator for PyTupleIterator<'_> {} + +impl<'a> IntoIterator for &'a PyTuple { + type Item = &'a PyAny; + type IntoIter = PyTupleIterator<'a>; + + fn into_iter(self) -> Self::IntoIter { + PyTupleIterator(BorrowedTupleIterator::new(self.as_borrowed())) + } +} + +/// Used by `PyTuple::into_iter()`. +pub struct BoundTupleIterator<'py> { + tuple: Bound<'py, PyTuple>, index: usize, length: usize, } -impl<'a> PyTupleIterator<'a> { - unsafe fn get_item(&self, index: usize) -> &'a PyAny { - #[cfg(any(Py_LIMITED_API, PyPy))] - let item = self.tuple.get_item(index).expect("tuple.get failed"); - #[cfg(not(any(Py_LIMITED_API, PyPy)))] - let item = self.tuple.get_item_unchecked(index); - item +impl<'py> BoundTupleIterator<'py> { + fn new(tuple: Bound<'py, PyTuple>) -> Self { + let length = tuple.len(); + BoundTupleIterator { + tuple, + index: 0, + length, + } } } -impl<'a> Iterator for PyTupleIterator<'a> { - type Item = &'a PyAny; +impl<'py> Iterator for BoundTupleIterator<'py> { + type Item = Bound<'py, PyAny>; #[inline] fn next(&mut self) -> Option { if self.index < self.length { - let item = unsafe { self.get_item(self.index) }; + let item = unsafe { + BorrowedTupleIterator::get_item(Borrowed::from(&self.tuple), self.index).to_owned() + }; self.index += 1; Some(item) } else { @@ -263,11 +482,14 @@ impl<'a> Iterator for PyTupleIterator<'a> { } } -impl<'a> DoubleEndedIterator for PyTupleIterator<'a> { +impl<'py> DoubleEndedIterator for BoundTupleIterator<'py> { #[inline] fn next_back(&mut self) -> Option { if self.index < self.length { - let item = unsafe { self.get_item(self.length - 1) }; + let item = unsafe { + BorrowedTupleIterator::get_item(Borrowed::from(&self.tuple), self.length - 1) + .to_owned() + }; self.length -= 1; Some(item) } else { @@ -276,23 +498,94 @@ impl<'a> DoubleEndedIterator for PyTupleIterator<'a> { } } -impl<'a> ExactSizeIterator for PyTupleIterator<'a> { +impl<'py> ExactSizeIterator for BoundTupleIterator<'py> { fn len(&self) -> usize { self.length.saturating_sub(self.index) } } -impl FusedIterator for PyTupleIterator<'_> {} +impl FusedIterator for BoundTupleIterator<'_> {} -impl<'a> IntoIterator for &'a PyTuple { - type Item = &'a PyAny; - type IntoIter = PyTupleIterator<'a>; +impl<'py> IntoIterator for Bound<'py, PyTuple> { + type Item = Bound<'py, PyAny>; + type IntoIter = BoundTupleIterator<'py>; fn into_iter(self) -> Self::IntoIter { - self.iter() + BoundTupleIterator::new(self) } } +/// Used by `PyTuple::iter_borrowed()`. +pub struct BorrowedTupleIterator<'a, 'py> { + tuple: Borrowed<'a, 'py, PyTuple>, + index: usize, + length: usize, +} + +impl<'a, 'py> BorrowedTupleIterator<'a, 'py> { + fn new(tuple: Borrowed<'a, 'py, PyTuple>) -> Self { + let length = tuple.len(); + BorrowedTupleIterator { + tuple, + index: 0, + length, + } + } + + unsafe fn get_item( + tuple: Borrowed<'a, 'py, PyTuple>, + index: usize, + ) -> Borrowed<'a, 'py, PyAny> { + #[cfg(any(Py_LIMITED_API, PyPy))] + let item = tuple.get_borrowed_item(index).expect("tuple.get failed"); + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + let item = tuple.get_borrowed_item_unchecked(index); + item + } +} + +impl<'a, 'py> Iterator for BorrowedTupleIterator<'a, 'py> { + type Item = Borrowed<'a, 'py, PyAny>; + + #[inline] + fn next(&mut self) -> Option { + if self.index < self.length { + let item = unsafe { Self::get_item(self.tuple, self.index) }; + self.index += 1; + Some(item) + } else { + None + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } +} + +impl<'a, 'py> DoubleEndedIterator for BorrowedTupleIterator<'a, 'py> { + #[inline] + fn next_back(&mut self) -> Option { + if self.index < self.length { + let item = unsafe { Self::get_item(self.tuple, self.length - 1) }; + self.length -= 1; + Some(item) + } else { + None + } + } +} + +impl<'a, 'py> ExactSizeIterator for BorrowedTupleIterator<'a, 'py> { + fn len(&self) -> usize { + self.length.saturating_sub(self.index) + } +} + +impl FusedIterator for BorrowedTupleIterator<'_, '_> {} + #[cold] fn wrong_tuple_length(t: &PyTuple, expected_length: usize) -> PyErr { let msg = format!( @@ -334,7 +627,7 @@ fn type_output() -> TypeInfo { impl<'s, $($T: FromPyObject<'s>),+> FromPyObject<'s> for ($($T,)+) { fn extract(obj: &'s PyAny) -> PyResult { - let t: &PyTuple = obj.downcast()?; + let t = obj.downcast::()?; if t.len() == $length { #[cfg(any(Py_LIMITED_API, PyPy))] return Ok(($(t.get_item($n)?.extract::<$T>()?,)+)); diff --git a/tests/ui/invalid_result_conversion.stderr b/tests/ui/invalid_result_conversion.stderr index f1a429a5abd..2720c71db63 100644 --- a/tests/ui/invalid_result_conversion.stderr +++ b/tests/ui/invalid_result_conversion.stderr @@ -6,8 +6,8 @@ error[E0277]: the trait bound `PyErr: From` is not satisfied | = help: the following other types implement trait `From`: > - > > + > >> >> >> From 53d25f7ff28ed72c197a50a841987f6efe605d0a Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 26 Dec 2023 15:07:32 +0000 Subject: [PATCH 026/349] add new `PyTuple` constructors --- guide/src/conversions/traits.md | 6 ++--- guide/src/python_from_rust.md | 2 +- pytests/src/datetime.rs | 24 +++++++++--------- src/conversion.rs | 2 +- src/impl_/extract_argument.rs | 14 +++++------ src/impl_/pymodule.rs | 2 +- src/types/datetime.rs | 2 +- src/types/list.rs | 2 +- src/types/sequence.rs | 8 ++++-- src/types/tuple.rs | 43 ++++++++++++++++++++++++++++----- tests/test_frompyobject.rs | 16 ++++++------ tests/test_various.rs | 8 +++--- 12 files changed, 82 insertions(+), 47 deletions(-) diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index fd136c83747..68304753b05 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -181,7 +181,7 @@ struct RustyTuple(String, String); # use pyo3::types::PyTuple; # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let tuple = PyTuple::new(py, vec!["test", "test2"]); +# let tuple = PyTuple::new_bound(py, vec!["test", "test2"]); # # let rustytuple: RustyTuple = tuple.extract()?; # assert_eq!(rustytuple.0, "test"); @@ -204,7 +204,7 @@ struct RustyTuple((String,)); # use pyo3::types::PyTuple; # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let tuple = PyTuple::new(py, vec!["test"]); +# let tuple = PyTuple::new_bound(py, vec!["test"]); # # let rustytuple: RustyTuple = tuple.extract()?; # assert_eq!((rustytuple.0).0, "test"); @@ -482,7 +482,7 @@ If the input is neither a string nor an integer, the error message will be: - retrieve the field from a mapping, possibly with the custom key specified as an argument. - can be any literal that implements `ToBorrowedObject` - `pyo3(from_py_with = "...")` - - apply a custom function to convert the field from Python the desired Rust type. + - apply a custom function to convert the field from Python the desired Rust type. - the argument must be the name of the function as a string. - the function signature must be `fn(&PyAny) -> PyResult` where `T` is the Rust type of the argument. diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index ed51b772d2d..6d4034822d7 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -51,7 +51,7 @@ fn main() -> PyResult<()> { fun.call0(py)?; // call object with PyTuple - let args = PyTuple::new(py, &[arg1, arg2, arg3]); + let args = PyTuple::new_bound(py, &[arg1, arg2, arg3]); fun.call1(py, args)?; // pass arguments as rust tuple diff --git a/pytests/src/datetime.rs b/pytests/src/datetime.rs index f5a15b4f682..1407da3fe76 100644 --- a/pytests/src/datetime.rs +++ b/pytests/src/datetime.rs @@ -12,8 +12,8 @@ fn make_date(py: Python<'_>, year: i32, month: u8, day: u8) -> PyResult<&PyDate> } #[pyfunction] -fn get_date_tuple<'p>(py: Python<'p>, d: &PyDate) -> &'p PyTuple { - PyTuple::new(py, [d.get_year(), d.get_month() as i32, d.get_day() as i32]) +fn get_date_tuple<'p>(py: Python<'p>, d: &PyDate) -> Bound<'p, PyTuple> { + PyTuple::new_bound(py, [d.get_year(), d.get_month() as i32, d.get_day() as i32]) } #[pyfunction] @@ -48,8 +48,8 @@ fn time_with_fold<'p>( } #[pyfunction] -fn get_time_tuple<'p>(py: Python<'p>, dt: &PyTime) -> &'p PyTuple { - PyTuple::new( +fn get_time_tuple<'p>(py: Python<'p>, dt: &PyTime) -> Bound<'p, PyTuple> { + PyTuple::new_bound( py, [ dt.get_hour() as u32, @@ -61,8 +61,8 @@ fn get_time_tuple<'p>(py: Python<'p>, dt: &PyTime) -> &'p PyTuple { } #[pyfunction] -fn get_time_tuple_fold<'p>(py: Python<'p>, dt: &PyTime) -> &'p PyTuple { - PyTuple::new( +fn get_time_tuple_fold<'p>(py: Python<'p>, dt: &PyTime) -> Bound<'p, PyTuple> { + PyTuple::new_bound( py, [ dt.get_hour() as u32, @@ -80,8 +80,8 @@ fn make_delta(py: Python<'_>, days: i32, seconds: i32, microseconds: i32) -> PyR } #[pyfunction] -fn get_delta_tuple<'p>(py: Python<'p>, delta: &PyDelta) -> &'p PyTuple { - PyTuple::new( +fn get_delta_tuple<'p>(py: Python<'p>, delta: &PyDelta) -> Bound<'p, PyTuple> { + PyTuple::new_bound( py, [ delta.get_days(), @@ -118,8 +118,8 @@ fn make_datetime<'p>( } #[pyfunction] -fn get_datetime_tuple<'p>(py: Python<'p>, dt: &PyDateTime) -> &'p PyTuple { - PyTuple::new( +fn get_datetime_tuple<'p>(py: Python<'p>, dt: &PyDateTime) -> Bound<'p, PyTuple> { + PyTuple::new_bound( py, [ dt.get_year(), @@ -134,8 +134,8 @@ fn get_datetime_tuple<'p>(py: Python<'p>, dt: &PyDateTime) -> &'p PyTuple { } #[pyfunction] -fn get_datetime_tuple_fold<'p>(py: Python<'p>, dt: &PyDateTime) -> &'p PyTuple { - PyTuple::new( +fn get_datetime_tuple_fold<'p>(py: Python<'p>, dt: &PyDateTime) -> Bound<'p, PyTuple> { + PyTuple::new_bound( py, [ dt.get_year(), diff --git a/src/conversion.rs b/src/conversion.rs index 0a842f9a419..429ca9e8779 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -465,7 +465,7 @@ mod implementations { /// Converts `()` to an empty Python tuple. impl IntoPy> for () { fn into_py(self, py: Python<'_>) -> Py { - PyTuple::empty(py).into() + PyTuple::empty_bound(py).unbind() } } diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index 437ece483ce..a18c8d4bc30 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -615,7 +615,7 @@ impl<'py> VarargsHandler<'py> for TupleVarargs { varargs: &[Option<&PyAny>], _function_description: &FunctionDescription, ) -> PyResult { - Ok(PyTuple::new(py, varargs)) + Ok(PyTuple::new_bound(py, varargs).into_gil_ref()) } #[inline] @@ -697,7 +697,7 @@ fn push_parameter_list(msg: &mut String, parameter_names: &[&str]) { mod tests { use crate::{ types::{IntoPyDict, PyTuple}, - PyAny, Python, ToPyObject, + PyAny, Python, }; use super::{push_parameter_list, FunctionDescription, NoVarargs, NoVarkeywords}; @@ -714,8 +714,8 @@ mod tests { }; Python::with_gil(|py| { - let args = PyTuple::new(py, Vec::<&PyAny>::new()); - let kwargs = [("foo".to_object(py).into_ref(py), 0u8)].into_py_dict(py); + let args = PyTuple::new_bound(py, Vec::<&PyAny>::new()); + let kwargs = [("foo", 0u8)].into_py_dict(py); let err = unsafe { function_description .extract_arguments_tuple_dict::( @@ -745,8 +745,8 @@ mod tests { }; Python::with_gil(|py| { - let args = PyTuple::new(py, Vec::<&PyAny>::new()); - let kwargs = [(1u8.to_object(py).into_ref(py), 1u8)].into_py_dict(py); + let args = PyTuple::new_bound(py, Vec::<&PyAny>::new()); + let kwargs = [(1u8, 1u8)].into_py_dict(py); let err = unsafe { function_description .extract_arguments_tuple_dict::( @@ -776,7 +776,7 @@ mod tests { }; Python::with_gil(|py| { - let args = PyTuple::new(py, Vec::<&PyAny>::new()); + let args = PyTuple::new_bound(py, Vec::<&PyAny>::new()); let mut output = [None, None]; let err = unsafe { function_description.extract_arguments_tuple_dict::( diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index 7c5243fcf1c..0fe5c3846c2 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -72,7 +72,7 @@ impl ModuleDef { .import("sys")? .getattr("implementation")? .getattr("version")?; - if version.lt(crate::types::PyTuple::new(py, PYPY_GOOD_VERSION))? { + if version.lt(crate::types::PyTuple::new_bound(py, PYPY_GOOD_VERSION))? { let warn = py.import("warnings")?.getattr("warn")?; warn.call1(( "PyPy 3.7 versions older than 7.3.8 are known to have binary \ diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 3d66cee4d95..354414b8b4c 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -212,7 +212,7 @@ impl PyDate { /// /// This is equivalent to `datetime.date.fromtimestamp` pub fn from_timestamp(py: Python<'_>, timestamp: i64) -> PyResult<&PyDate> { - let time_tuple = PyTuple::new(py, [timestamp]); + let time_tuple = PyTuple::new_bound(py, [timestamp]); // safety ensure that the API is loaded let _api = ensure_datetime_api(py); diff --git a/src/types/list.rs b/src/types/list.rs index 2ba4b10cd76..f8db0e62ee6 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -1231,7 +1231,7 @@ mod tests { Python::with_gil(|py| { let list = PyList::new(py, vec![1, 2, 3]); let tuple = list.to_tuple(); - let tuple_expected = PyTuple::new(py, vec![1, 2, 3]); + let tuple_expected = PyTuple::new_bound(py, vec![1, 2, 3]); assert!(tuple.eq(tuple_expected).unwrap()); }) } diff --git a/src/types/sequence.rs b/src/types/sequence.rs index f4eae4d05a9..fa15fe06858 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -1009,7 +1009,7 @@ mod tests { assert!(seq .to_tuple() .unwrap() - .eq(PyTuple::new(py, ["foo", "bar"])) + .eq(PyTuple::new_bound(py, ["foo", "bar"])) .unwrap()); }); } @@ -1020,7 +1020,11 @@ mod tests { let v = vec!["foo", "bar"]; let ob = v.to_object(py); let seq = ob.downcast::(py).unwrap(); - assert!(seq.to_tuple().unwrap().eq(PyTuple::new(py, &v)).unwrap()); + assert!(seq + .to_tuple() + .unwrap() + .eq(PyTuple::new_bound(py, &v)) + .unwrap()); }); } diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 1b87499f1e0..114c36c4fd4 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -58,6 +58,23 @@ pub struct PyTuple(PyAny); pyobject_native_type_core!(PyTuple, pyobject_native_static_type_object!(ffi::PyTuple_Type), #checkfunction=ffi::PyTuple_Check); impl PyTuple { + /// Deprecated form of `PyTuple::new_bound`. + #[track_caller] + #[deprecated( + since = "0.21.0", + note = "`PyTuple::new` will be replaced by `PyTuple::new_bound` in a future PyO3 version" + )] + pub fn new( + py: Python<'_>, + elements: impl IntoIterator, + ) -> &PyTuple + where + T: ToPyObject, + U: ExactSizeIterator, + { + Self::new_bound(py, elements).into_gil_ref() + } + /// Constructs a new tuple with the given elements. /// /// If you want to create a [`PyTuple`] with elements of different or unknown types, or from an @@ -73,7 +90,7 @@ impl PyTuple { /// # fn main() { /// Python::with_gil(|py| { /// let elements: Vec = vec![0, 1, 2, 3, 4, 5]; - /// let tuple: &PyTuple = PyTuple::new(py, elements); + /// let tuple = PyTuple::new_bound(py, elements); /// assert_eq!(format!("{:?}", tuple), "(0, 1, 2, 3, 4, 5)"); /// }); /// # } @@ -85,21 +102,34 @@ impl PyTuple { /// All standard library structures implement this trait correctly, if they do, so calling this /// function using [`Vec`]`` or `&[T]` will always succeed. #[track_caller] - pub fn new( + pub fn new_bound( py: Python<'_>, elements: impl IntoIterator, - ) -> &PyTuple + ) -> Bound<'_, PyTuple> where T: ToPyObject, U: ExactSizeIterator, { let mut elements = elements.into_iter().map(|e| e.to_object(py)); - new_from_iter(py, &mut elements).into_gil_ref() + new_from_iter(py, &mut elements) } - /// Constructs an empty tuple (on the Python side, a singleton object). + /// Deprecated form of `PyTuple::empty_bound`. + #[deprecated( + since = "0.21.0", + note = "`PyTuple::empty` will be replaced by `PyTuple::empty_bound` in a future PyO3 version" + )] pub fn empty(py: Python<'_>) -> &PyTuple { - unsafe { py.from_owned_ptr(ffi::PyTuple_New(0)) } + Self::empty_bound(py).into_gil_ref() + } + + /// Constructs an empty tuple (on the Python side, a singleton object). + pub fn empty_bound(py: Python<'_>) -> Bound<'_, PyTuple> { + unsafe { + ffi::PyTuple_New(0) + .assume_owned(py) + .downcast_into_unchecked() + } } /// Gets the length of the tuple. @@ -765,6 +795,7 @@ tuple_conversion!( ); #[cfg(test)] +#[allow(deprecated)] // TODO: remove allow when GIL Pool is removed mod tests { use crate::types::{PyAny, PyList, PyTuple}; use crate::{Python, ToPyObject}; diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index 47e5ec53e92..30edf6f7836 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -162,11 +162,11 @@ pub struct Tuple(String, usize); #[test] fn test_tuple_struct() { Python::with_gil(|py| { - let tup = PyTuple::new(py, &[1.into_py(py), "test".into_py(py)]); - let tup = Tuple::extract(tup.as_ref()); + let tup = PyTuple::new_bound(py, &[1.into_py(py), "test".into_py(py)]); + let tup = Tuple::extract(tup.as_gil_ref()); assert!(tup.is_err()); - let tup = PyTuple::new(py, &["test".into_py(py), 1.into_py(py)]); - let tup = Tuple::extract(tup.as_ref()).expect("Failed to extract Tuple from PyTuple"); + let tup = PyTuple::new_bound(py, &["test".into_py(py), 1.into_py(py)]); + let tup = Tuple::extract(tup.as_gil_ref()).expect("Failed to extract Tuple from PyTuple"); assert_eq!(tup.0, "test"); assert_eq!(tup.1, 1); }); @@ -324,8 +324,8 @@ pub struct PyBool { #[test] fn test_enum() { Python::with_gil(|py| { - let tup = PyTuple::new(py, &[1.into_py(py), "test".into_py(py)]); - let f = Foo::extract(tup.as_ref()).expect("Failed to extract Foo from tuple"); + let tup = PyTuple::new_bound(py, &[1.into_py(py), "test".into_py(py)]); + let f = Foo::extract(tup.as_gil_ref()).expect("Failed to extract Foo from tuple"); match f { Foo::TupleVar(test, test2) => { assert_eq!(test, 1); @@ -401,8 +401,8 @@ TypeError: failed to extract enum Foo ('TupleVar | StructVar | TransparentTuple - variant StructWithGetItemArg (StructWithGetItemArg): KeyError: 'foo'" ); - let tup = PyTuple::empty(py); - let err = Foo::extract(tup.as_ref()).unwrap_err(); + let tup = PyTuple::empty_bound(py); + let err = Foo::extract(tup.as_gil_ref()).unwrap_err(); assert_eq!( err.to_string(), "\ diff --git a/tests/test_various.rs b/tests/test_various.rs index 076d2ba2cb5..6560610f35f 100644 --- a/tests/test_various.rs +++ b/tests/test_various.rs @@ -91,7 +91,7 @@ fn intopytuple_pyclass() { #[test] fn pytuple_primitive_iter() { Python::with_gil(|py| { - let tup = PyTuple::new(py, [1u32, 2, 3].iter()); + let tup = PyTuple::new_bound(py, [1u32, 2, 3].iter()); py_assert!(py, tup, "tup == (1, 2, 3)"); }); } @@ -99,7 +99,7 @@ fn pytuple_primitive_iter() { #[test] fn pytuple_pyclass_iter() { Python::with_gil(|py| { - let tup = PyTuple::new( + let tup = PyTuple::new_bound( py, [ PyCell::new(py, SimplePyClass {}).unwrap(), @@ -126,10 +126,10 @@ impl PickleSupport { pub fn __reduce__<'py>( slf: &'py PyCell, py: Python<'py>, - ) -> PyResult<(PyObject, &'py PyTuple, PyObject)> { + ) -> PyResult<(PyObject, Bound<'py, PyTuple>, PyObject)> { let cls = slf.to_object(py).getattr(py, "__class__")?; let dict = slf.to_object(py).getattr(py, "__dict__")?; - Ok((cls, PyTuple::empty(py), dict)) + Ok((cls, PyTuple::empty_bound(py), dict)) } } From 823b5feed33d865775a11133ce2f6412a45769bb Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 26 Dec 2023 22:51:36 +0000 Subject: [PATCH 027/349] improve tuple methods test coverage --- src/types/tuple.rs | 132 ++++++++++++++++++++-- tests/ui/invalid_result_conversion.stderr | 2 +- 2 files changed, 121 insertions(+), 13 deletions(-) diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 114c36c4fd4..11e9631d164 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -60,9 +60,12 @@ pyobject_native_type_core!(PyTuple, pyobject_native_static_type_object!(ffi::PyT impl PyTuple { /// Deprecated form of `PyTuple::new_bound`. #[track_caller] - #[deprecated( - since = "0.21.0", - note = "`PyTuple::new` will be replaced by `PyTuple::new_bound` in a future PyO3 version" + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyTuple::new` will be replaced by `PyTuple::new_bound` in a future PyO3 version" + ) )] pub fn new( py: Python<'_>, @@ -115,9 +118,12 @@ impl PyTuple { } /// Deprecated form of `PyTuple::empty_bound`. - #[deprecated( - since = "0.21.0", - note = "`PyTuple::empty` will be replaced by `PyTuple::empty_bound` in a future PyO3 version" + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyTuple::empty` will be replaced by `PyTuple::empty_bound` in a future PyO3 version" + ) )] pub fn empty(py: Python<'_>) -> &PyTuple { Self::empty_bound(py).into_gil_ref() @@ -361,7 +367,7 @@ impl<'py> PyTupleMethods<'py> for Bound<'py, PyTuple> { } fn get_borrowed_item<'a>(&'a self, index: usize) -> PyResult> { - Borrowed::from(self).get_borrowed_item(index) + self.as_borrowed().get_borrowed_item(index) } #[cfg(not(any(Py_LIMITED_API, PyPy)))] @@ -371,7 +377,7 @@ impl<'py> PyTupleMethods<'py> for Bound<'py, PyTuple> { #[cfg(not(any(Py_LIMITED_API, PyPy)))] unsafe fn get_borrowed_item_unchecked<'a>(&'a self, index: usize) -> Borrowed<'a, 'py, PyAny> { - Borrowed::from(self).get_borrowed_item_unchecked(index) + self.as_borrowed().get_borrowed_item_unchecked(index) } #[cfg(not(Py_LIMITED_API))] @@ -406,7 +412,7 @@ impl<'py> PyTupleMethods<'py> for Bound<'py, PyTuple> { } fn iter_borrowed<'a>(&'a self) -> BorrowedTupleIterator<'a, 'py> { - BorrowedTupleIterator::new(Borrowed::from(self)) + BorrowedTupleIterator::new(self.as_borrowed()) } fn to_list(&self) -> Bound<'py, PyList> { @@ -496,7 +502,7 @@ impl<'py> Iterator for BoundTupleIterator<'py> { fn next(&mut self) -> Option { if self.index < self.length { let item = unsafe { - BorrowedTupleIterator::get_item(Borrowed::from(&self.tuple), self.index).to_owned() + BorrowedTupleIterator::get_item(self.tuple.as_borrowed(), self.index).to_owned() }; self.index += 1; Some(item) @@ -517,7 +523,7 @@ impl<'py> DoubleEndedIterator for BoundTupleIterator<'py> { fn next_back(&mut self) -> Option { if self.index < self.length { let item = unsafe { - BorrowedTupleIterator::get_item(Borrowed::from(&self.tuple), self.length - 1) + BorrowedTupleIterator::get_item(self.tuple.as_borrowed(), self.length - 1) .to_owned() }; self.length -= 1; @@ -797,7 +803,7 @@ tuple_conversion!( #[cfg(test)] #[allow(deprecated)] // TODO: remove allow when GIL Pool is removed mod tests { - use crate::types::{PyAny, PyList, PyTuple}; + use crate::types::{any::PyAnyMethods, tuple::PyTupleMethods, PyAny, PyList, PyTuple}; use crate::{Python, ToPyObject}; use std::collections::HashSet; @@ -822,11 +828,21 @@ mod tests { let ob = (1, 2, 3).to_object(py); let tuple: &PyTuple = ob.downcast(py).unwrap(); assert_eq!(3, tuple.len()); + assert!(!tuple.is_empty()); let ob: &PyAny = tuple.into(); assert_eq!((1, 2, 3), ob.extract().unwrap()); }); } + #[test] + fn test_empty() { + Python::with_gil(|py| { + let tuple = PyTuple::empty(py); + assert!(tuple.is_empty()); + assert_eq!(0, tuple.len()); + }); + } + #[test] fn test_slice() { Python::with_gil(|py| { @@ -886,6 +902,52 @@ mod tests { }); } + #[test] + fn test_bound_iter() { + Python::with_gil(|py| { + let tuple = PyTuple::new_bound(py, [1, 2, 3]); + assert_eq!(3, tuple.len()); + let mut iter = tuple.iter(); + + assert_eq!(iter.size_hint(), (3, Some(3))); + + assert_eq!(1_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(iter.size_hint(), (2, Some(2))); + + assert_eq!(2_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(iter.size_hint(), (1, Some(1))); + + assert_eq!(3_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(iter.size_hint(), (0, Some(0))); + + assert!(iter.next().is_none()); + assert!(iter.next().is_none()); + }); + } + + #[test] + fn test_bound_iter_rev() { + Python::with_gil(|py| { + let tuple = PyTuple::new_bound(py, [1, 2, 3]); + assert_eq!(3, tuple.len()); + let mut iter = tuple.iter().rev(); + + assert_eq!(iter.size_hint(), (3, Some(3))); + + assert_eq!(3_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(iter.size_hint(), (2, Some(2))); + + assert_eq!(2_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(iter.size_hint(), (1, Some(1))); + + assert_eq!(1_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(iter.size_hint(), (0, Some(0))); + + assert!(iter.next().is_none()); + assert!(iter.next().is_none()); + }); + } + #[test] fn test_into_iter() { Python::with_gil(|py| { @@ -1308,4 +1370,50 @@ mod tests { assert!(list.eq(list_expected).unwrap()); }) } + + #[test] + fn test_tuple_as_sequence() { + Python::with_gil(|py| { + let tuple = PyTuple::new(py, vec![1, 2, 3]); + let sequence = tuple.as_sequence(); + assert!(tuple.get_item(0).unwrap().eq(1).unwrap()); + assert!(sequence.get_item(0).unwrap().eq(1).unwrap()); + + assert_eq!(tuple.len(), 3); + assert_eq!(sequence.len().unwrap(), 3); + }) + } + + #[test] + fn test_bound_tuple_get_item() { + Python::with_gil(|py| { + let tuple = PyTuple::new_bound(py, vec![1, 2, 3, 4]); + + assert_eq!(tuple.len(), 4); + assert_eq!(tuple.get_item(0).unwrap().extract::().unwrap(), 1); + assert_eq!( + tuple + .get_borrowed_item(1) + .unwrap() + .extract::() + .unwrap(), + 2 + ); + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + { + assert_eq!( + unsafe { tuple.get_item_unchecked(2) } + .extract::() + .unwrap(), + 3 + ); + assert_eq!( + unsafe { tuple.get_borrowed_item_unchecked(3) } + .extract::() + .unwrap(), + 4 + ); + } + }) + } } diff --git a/tests/ui/invalid_result_conversion.stderr b/tests/ui/invalid_result_conversion.stderr index 2720c71db63..f1a429a5abd 100644 --- a/tests/ui/invalid_result_conversion.stderr +++ b/tests/ui/invalid_result_conversion.stderr @@ -6,8 +6,8 @@ error[E0277]: the trait bound `PyErr: From` is not satisfied | = help: the following other types implement trait `From`: > - > > + > >> >> >> From 4cf58c8303eeaafc2717ace10ac8f68d7d976e2a Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 26 Dec 2023 15:35:01 +0000 Subject: [PATCH 028/349] implement `IntoPy>` for `Bound` --- src/types/tuple.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 11e9631d164..4860b8de30e 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -622,6 +622,12 @@ impl<'a, 'py> ExactSizeIterator for BorrowedTupleIterator<'a, 'py> { impl FusedIterator for BorrowedTupleIterator<'_, '_> {} +impl IntoPy> for Bound<'_, PyTuple> { + fn into_py(self, _: Python<'_>) -> Py { + self.unbind() + } +} + #[cold] fn wrong_tuple_length(t: &PyTuple, expected_length: usize) -> PyErr { let msg = format!( From 03285835bbd03472a84ef232f8a7c8e94b783ac4 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Tue, 2 Jan 2024 09:32:33 +0100 Subject: [PATCH 029/349] Use a definite version specification when depending on pyo3-build-config. We already do this for other internal pyo3-* dependencies and it seems prudent to apply this to pyo3-build-config as well. --- Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a0400c07301..1242c331ec2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ widestring = "0.5.1" futures = "0.3.28" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "0.21.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.0-dev", features = ["resolve-config"] } [features] default = ["macros"] diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index bc40d86ca19..8021dc72b69 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -38,7 +38,7 @@ abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312"] generate-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "0.21.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0-dev", features = ["resolve-config"] } [lints] workspace = true From 783e98b1a821e5b544fcd0b50baf9051197c238f Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 24 Dec 2023 14:01:00 +0000 Subject: [PATCH 030/349] introduce `PyIterator::from_bound_object` --- src/coroutine.rs | 7 +++++-- src/types/any.rs | 2 +- src/types/frozenset.rs | 2 +- src/types/iterator.rs | 20 +++++++++++++++----- src/types/set.rs | 2 +- 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/coroutine.rs b/src/coroutine.rs index 7dd73cbba10..415bbc88db9 100644 --- a/src/coroutine.rs +++ b/src/coroutine.rs @@ -15,7 +15,7 @@ use crate::{ exceptions::{PyAttributeError, PyRuntimeError, PyStopIteration}, panic::PanicException, types::{PyIterator, PyString}, - IntoPy, Py, PyAny, PyErr, PyObject, PyResult, Python, + IntoPy, Py, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, }; pub(crate) mod cancel; @@ -107,7 +107,10 @@ impl Coroutine { if let Some(future) = self.waker.as_ref().unwrap().initialize_future(py)? { // `asyncio.Future` must be awaited; fortunately, it implements `__iter__ = __await__` // and will yield itself if its result has not been set in polling above - if let Some(future) = PyIterator::from_object(future).unwrap().next() { + if let Some(future) = PyIterator::from_bound_object(&future.as_borrowed()) + .unwrap() + .next() + { // future has not been leaked into Python for now, and Rust code can only call // `set_result(None)` in `Wake` implementation, so it's safe to unwrap return Ok(future.unwrap().into()); diff --git a/src/types/any.rs b/src/types/any.rs index edc74879cb6..ffc98e9b3a6 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -2018,7 +2018,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { } fn iter(&self) -> PyResult> { - PyIterator::from_object2(self) + PyIterator::from_bound_object(self) } fn get_type(&self) -> &'py PyType { diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 1bcdce4cf90..ff0be7b59a2 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -209,7 +209,7 @@ mod impl_ { impl<'py> BoundFrozenSetIterator<'py> { pub(super) fn new(frozenset: Bound<'py, PyFrozenSet>) -> Self { Self { - it: PyIterator::from_object2(&frozenset).unwrap(), + it: PyIterator::from_bound_object(&frozenset).unwrap(), } } } diff --git a/src/types/iterator.rs b/src/types/iterator.rs index 67866348f8c..e27032867bc 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -31,14 +31,23 @@ pyobject_native_type_named!(PyIterator); pyobject_native_type_extract!(PyIterator); impl PyIterator { - /// Constructs a `PyIterator` from a Python iterable object. - /// - /// Equivalent to Python's built-in `iter` function. + /// Deprecated form of `PyIterator::from_bound_object`. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyIterator::from_object` will be replaced by `PyIterator::from_bound_object` in a future PyO3 version" + ) + )] pub fn from_object(obj: &PyAny) -> PyResult<&PyIterator> { - Self::from_object2(&obj.as_borrowed()).map(Bound::into_gil_ref) + Self::from_bound_object(&obj.as_borrowed()).map(Bound::into_gil_ref) } - pub(crate) fn from_object2<'py>(obj: &Bound<'py, PyAny>) -> PyResult> { + /// Builds an iterator for an iterable Python object; the equivalent of calling `iter(obj)` in Python. + /// + /// Usually it is more convenient to write [`obj.iter()`][crate::types::any::PyAnyMethods::iter], + /// which is a more concise way of calling this function. + pub fn from_bound_object<'py>(obj: &Bound<'py, PyAny>) -> PyResult> { unsafe { ffi::PyObject_GetIter(obj.as_ptr()) .assume_owned_or_err(obj.py()) @@ -135,6 +144,7 @@ impl<'v> crate::PyTryFrom<'v> for PyIterator { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::PyIterator; use crate::exceptions::PyTypeError; diff --git a/src/types/set.rs b/src/types/set.rs index be65500b4fe..d59a9e3ffc4 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -289,7 +289,7 @@ mod impl_ { impl<'py> BoundSetIterator<'py> { pub(super) fn new(set: Bound<'py, PySet>) -> Self { Self { - it: PyIterator::from_object2(&set).unwrap(), + it: PyIterator::from_bound_object(&set).unwrap(), } } } From e2c6eb86f94d76bf4b82f5a3823bbfde17a038a7 Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Tue, 2 Jan 2024 09:52:45 +0100 Subject: [PATCH 031/349] Fix missing feature flags in implementation of Either conversion. --- src/conversions/either.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/conversions/either.rs b/src/conversions/either.rs index 4a41d2bd52f..759b282e416 100644 --- a/src/conversions/either.rs +++ b/src/conversions/either.rs @@ -43,9 +43,10 @@ //! //! [either](https://docs.rs/either/ "A library for easy idiomatic error handling and reporting in Rust applications")’s +#[cfg(feature = "experimental-inspect")] +use crate::inspect::types::TypeInfo; use crate::{ - exceptions::PyTypeError, inspect::types::TypeInfo, FromPyObject, IntoPy, PyAny, PyObject, - PyResult, Python, ToPyObject, + exceptions::PyTypeError, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject, }; use either::Either; @@ -97,6 +98,7 @@ where } } + #[cfg(feature = "experimental-inspect")] fn type_input() -> TypeInfo { TypeInfo::union_of(&[L::type_input(), R::type_input()]) } From c1c62f1f3c140e26882529d974f78b029239d22d Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Tue, 2 Jan 2024 10:03:11 +0100 Subject: [PATCH 032/349] Add CI job to test the equivalent of a docs.rs build. --- .github/workflows/ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f97cc37cf0f..a69acff13eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -333,6 +333,22 @@ jobs: RUST_BACKTRACE: 1 TRYBUILD: overwrite + docsrs: + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + needs: [fmt] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + - uses: Swatinem/rust-cache@v2 + with: + key: cargo-careful + continue-on-error: true + - uses: dtolnay/rust-toolchain@nightly + with: + components: rust-src + - run: cargo rustdoc --lib --no-default-features --features "macros num-bigint num-complex hashbrown serde multiple-pymethods indexmap eyre either chrono rust_decimal" -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]" + coverage: needs: [fmt] name: coverage-${{ matrix.os }} @@ -407,6 +423,8 @@ jobs: run: nox -s test-emscripten test-debug: + needs: [fmt] + if: github.ref != 'refs/heads/main' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -458,6 +476,7 @@ jobs: - build-full - valgrind - careful + - docsrs - coverage - emscripten if: always() From 68240e16a7fc63fd3f760306507617edbc5f269e Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Tue, 2 Jan 2024 18:55:13 +0100 Subject: [PATCH 033/349] Include the experimental-inspect feature for the docs.rs build thereby making it equivalent to a full build. --- .github/workflows/ci.yml | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a69acff13eb..d21a3d8f34f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -347,7 +347,7 @@ jobs: - uses: dtolnay/rust-toolchain@nightly with: components: rust-src - - run: cargo rustdoc --lib --no-default-features --features "macros num-bigint num-complex hashbrown serde multiple-pymethods indexmap eyre either chrono rust_decimal" -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]" + - run: cargo rustdoc --lib --no-default-features --features full -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]" coverage: needs: [fmt] diff --git a/Cargo.toml b/Cargo.toml index 1242c331ec2..fd1daeb1451 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -133,7 +133,7 @@ members = [ [package.metadata.docs.rs] no-default-features = true -features = ["macros", "num-bigint", "num-complex", "hashbrown", "serde", "multiple-pymethods", "indexmap", "eyre", "either", "chrono", "rust_decimal"] +features = ["full"] rustdoc-args = ["--cfg", "docsrs"] [workspace.lints.clippy] From 50e33d86c73f51514075ab25d06b8d3464a6ebe4 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 29 Dec 2023 14:43:43 +0000 Subject: [PATCH 034/349] add `call_bound` and `call_method_bound` --- examples/decorator/src/lib.rs | 4 +- guide/src/python_from_rust.md | 22 ++++--- src/instance.rs | 76 +++++++++++++++++++---- src/types/any.rs | 15 ++--- tests/ui/invalid_result_conversion.stderr | 2 +- 5 files changed, 89 insertions(+), 30 deletions(-) diff --git a/examples/decorator/src/lib.rs b/examples/decorator/src/lib.rs index bc369c62b1e..fb2f2932dd2 100644 --- a/examples/decorator/src/lib.rs +++ b/examples/decorator/src/lib.rs @@ -41,7 +41,7 @@ impl PyCounter { &self, py: Python<'_>, args: &PyTuple, - kwargs: Option<&PyDict>, + kwargs: Option>, ) -> PyResult> { let old_count = self.count.get(); let new_count = old_count + 1; @@ -51,7 +51,7 @@ impl PyCounter { println!("{} has been called {} time(s).", name, new_count); // After doing something, we finally forward the call to the wrapped function - let ret = self.wraps.call(py, args, kwargs)?; + let ret = self.wraps.call_bound(py, args, kwargs.as_ref())?; // We could do something with the return value of // the function before returning it diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 6d4034822d7..b81ba919637 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -10,13 +10,13 @@ Any Python-native object reference (such as `&PyAny`, `&PyList`, or `&PyCell`](types.html#pyt-and-pyobject) smart pointer also exposes these same six API methods, but needs a `Python` token as an additional first argument to prove the GIL is held. @@ -95,22 +95,30 @@ fn main() -> PyResult<()> { // call object with PyDict let kwargs = [(key1, val1)].into_py_dict(py); - fun.call(py, (), Some(kwargs))?; + fun.call_bound(py, (), Some(&kwargs.as_borrowed()))?; // pass arguments as Vec let kwargs = vec![(key1, val1), (key2, val2)]; - fun.call(py, (), Some(kwargs.into_py_dict(py)))?; + fun.call_bound(py, (), Some(&kwargs.into_py_dict(py).as_borrowed()))?; // pass arguments as HashMap let mut kwargs = HashMap::<&str, i32>::new(); kwargs.insert(key1, 1); - fun.call(py, (), Some(kwargs.into_py_dict(py)))?; + fun.call_bound(py, (), Some(&kwargs.into_py_dict(py).as_borrowed()))?; Ok(()) }) } ``` +
+ +During PyO3's [migration from "GIL Refs" to the `Bound` smart pointer](./migration.md#migrating-from-the-gil-refs-api-to-boundt), [`Py::call`]({{#PYO3_DOCS_URL}}/pyo3/struct.py#method.call) is temporarily named `call_bound` (and `call_method` is temporarily `call_method_bound`). + +(This temporary naming is only the case for the `Py` smart pointer. The methods on the `&PyAny` GIL Ref such as `call` have not been given replacements, and the methods on the `Bound` smart pointer such as [`Bound::call`]({#PYO3_DOCS_URL}}/pyo3/prelude/trait.pyanymethods#tymethod.call) already use follow the newest API conventions.) + +
+ ## Executing existing Python code If you already have some existing Python code that you need to execute from Rust, the following FAQs can help you select the right PyO3 functionality for your situation: diff --git a/src/instance.rs b/src/instance.rs index 11936d6c94f..5779e9ada6f 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1016,14 +1016,30 @@ impl Py { .setattr(attr_name, value.into_py(py).into_bound(py)) } + /// Deprecated form of [`call_bound`][Py::call_bound]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`call` will be replaced by `call_bound` in a future PyO3 version" + ) + )] + #[inline] + pub fn call(&self, py: Python<'_>, args: A, kwargs: Option<&PyDict>) -> PyResult + where + A: IntoPy>, + { + self.call_bound(py, args, kwargs.map(PyDict::as_borrowed).as_deref()) + } + /// Calls the object. /// /// This is equivalent to the Python expression `self(*args, **kwargs)`. - pub fn call( + pub fn call_bound( &self, py: Python<'_>, args: impl IntoPy>, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult { self.bind(py).as_any().call(args, kwargs).map(Bound::unbind) } @@ -1042,18 +1058,41 @@ impl Py { self.bind(py).as_any().call0().map(Bound::unbind) } + /// Deprecated form of [`call_method_bound`][Py::call_method_bound]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`call_method` will be replaced by `call_method_bound` in a future PyO3 version" + ) + )] + #[inline] + pub fn call_method( + &self, + py: Python<'_>, + name: N, + args: A, + kwargs: Option<&PyDict>, + ) -> PyResult + where + N: IntoPy>, + A: IntoPy>, + { + self.call_method_bound(py, name, args, kwargs.map(PyDict::as_borrowed).as_deref()) + } + /// Calls a method on the object. /// /// This is equivalent to the Python expression `self.name(*args, **kwargs)`. /// /// To avoid repeated temporary allocations of Python strings, the [`intern!`](crate::intern) /// macro can be used to intern `name`. - pub fn call_method( + pub fn call_method_bound( &self, py: Python<'_>, name: N, args: A, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult where N: IntoPy>, @@ -1490,23 +1529,34 @@ impl PyObject { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::{Bound, Py, PyObject}; - use crate::types::{PyDict, PyString}; + use crate::types::{dict::IntoPyDict, PyDict, PyString}; use crate::{PyAny, PyNativeType, PyResult, Python, ToPyObject}; #[test] - fn test_call0() { + fn test_call() { Python::with_gil(|py| { let obj = py.get_type::().to_object(py); - assert_eq!( - obj.call0(py) - .unwrap() - .as_ref(py) - .repr() + + let assert_repr = |obj: &PyAny, expected: &str| { + assert_eq!(obj.repr().unwrap().to_str().unwrap(), expected); + }; + + assert_repr(obj.call0(py).unwrap().as_ref(py), "{}"); + assert_repr(obj.call1(py, ()).unwrap().as_ref(py), "{}"); + assert_repr(obj.call(py, (), None).unwrap().as_ref(py), "{}"); + + assert_repr( + obj.call1(py, ((('x', 1),),)).unwrap().as_ref(py), + "{'x': 1}", + ); + assert_repr( + obj.call(py, (), Some([('x', 1)].into_py_dict(py))) .unwrap() - .to_string_lossy(), - "{}" + .as_ref(py), + "{'x': 1}", ); }) } diff --git a/src/types/any.rs b/src/types/any.rs index 7d5f226a703..9cdf78d6c6b 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -447,7 +447,7 @@ impl PyAny { kwargs: Option<&PyDict>, ) -> PyResult<&PyAny> { self.as_borrowed() - .call(args, kwargs) + .call(args, kwargs.map(PyDict::as_borrowed).as_deref()) .map(Bound::into_gil_ref) } @@ -547,7 +547,7 @@ impl PyAny { A: IntoPy>, { self.as_borrowed() - .call_method(name, args, kwargs) + .call_method(name, args, kwargs.map(PyDict::as_borrowed).as_deref()) .map(Bound::into_gil_ref) } @@ -1322,7 +1322,7 @@ pub trait PyAnyMethods<'py> { fn call( &self, args: impl IntoPy>, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult>; /// Calls the object without arguments. @@ -1415,7 +1415,7 @@ pub trait PyAnyMethods<'py> { &self, name: N, args: A, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> where N: IntoPy>, @@ -1970,12 +1970,12 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { fn call( &self, args: impl IntoPy>, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { fn inner<'py>( any: &Bound<'py, PyAny>, args: Bound<'_, PyTuple>, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { unsafe { ffi::PyObject_Call( @@ -2015,7 +2015,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { &self, name: N, args: A, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> where N: IntoPy>, @@ -2292,6 +2292,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use crate::{ basic::CompareOp, diff --git a/tests/ui/invalid_result_conversion.stderr b/tests/ui/invalid_result_conversion.stderr index f1a429a5abd..2720c71db63 100644 --- a/tests/ui/invalid_result_conversion.stderr +++ b/tests/ui/invalid_result_conversion.stderr @@ -6,8 +6,8 @@ error[E0277]: the trait bound `PyErr: From` is not satisfied | = help: the following other types implement trait `From`: > - > > + > >> >> >> From 026c0daf577561f98a30486e3268462bfae5285e Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 3 Jan 2024 13:20:28 +0000 Subject: [PATCH 035/349] release notes for 0.20.2 --- CHANGELOG.md | 14 +++++++++++++- README.md | 4 ++-- examples/decorator/.template/pre-script.rhai | 2 +- examples/maturin-starter/.template/pre-script.rhai | 2 +- examples/plugin/.template/pre-script.rhai | 2 +- .../.template/pre-script.rhai | 2 +- examples/word-count/.template/pre-script.rhai | 2 +- 7 files changed, 20 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f7708ce05b..9104db2921f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,17 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.20.2] - 2024-01-04 + +### Packaging + +- Pin `pyo3` and `pyo3-ffi` dependencies on `pyo3-build-config` to require the same patch version, i.e. `pyo3` 0.20.2 requires _exactly_ `pyo3-build-config` 0.20.2. [#3721](https://github.com/PyO3/pyo3/pull/3721) + +### Fixed + +- Fix compile failure when building `pyo3` 0.20.0 with latest `pyo3-build-config` 0.20.X. [#3724](https://github.com/PyO3/pyo3/pull/3724) +- Fix docs.rs build. [#3722](https://github.com/PyO3/pyo3/pull/3722) + ## [0.20.1] - 2023-12-30 ### Added @@ -1617,7 +1628,8 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.20.1...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.20.2...HEAD +[0.20.2]: https://github.com/pyo3/pyo3/compare/v0.20.1...v0.20.2 [0.20.1]: https://github.com/pyo3/pyo3/compare/v0.20.0...v0.20.1 [0.20.0]: https://github.com/pyo3/pyo3/compare/v0.19.2...v0.20.0 [0.19.2]: https://github.com/pyo3/pyo3/compare/v0.19.1...v0.19.2 diff --git a/README.md b/README.md index 6c368ce946c..aa95ede4c6c 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.20.1", features = ["extension-module"] } +pyo3 = { version = "0.20.2", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.20.1" +version = "0.20.2" features = ["auto-initialize"] ``` diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index c7199a5303e..12b203c3bb4 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.1"); +variable::set("PYO3_VERSION", "0.20.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index c7199a5303e..12b203c3bb4 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.1"); +variable::set("PYO3_VERSION", "0.20.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index d325aca0eef..72cfe2be91d 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.1"); +variable::set("PYO3_VERSION", "0.20.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 91eee121a7b..78a656558f8 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.1"); +variable::set("PYO3_VERSION", "0.20.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index c7199a5303e..12b203c3bb4 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.1"); +variable::set("PYO3_VERSION", "0.20.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); From 72f0c739259a87d6a509adb3761692b183bc392c Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 8 Jan 2024 15:19:49 +0100 Subject: [PATCH 036/349] Conversion between chrono_tz::Tz and zoneinfo.ZoneInfo --- Cargo.toml | 5 +- guide/src/features.md | 6 ++ newsfragments/3730.added.md | 1 + src/conversions/chrono.rs | 4 +- src/conversions/chrono_tz.rs | 113 +++++++++++++++++++++++++++++++++++ src/conversions/mod.rs | 1 + src/lib.rs | 3 + 7 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3730.added.md create mode 100644 src/conversions/chrono_tz.rs diff --git a/Cargo.toml b/Cargo.toml index fd1daeb1451..31179e59a7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ inventory = { version = "0.3.0", optional = true } # crate integrations that can be added using the eponymous features anyhow = { version = "1.0", optional = true } chrono = { version = "0.4.25", default-features = false, optional = true } +chrono-tz = { version = ">= 0.6, < 0.9", default-features = false, optional = true } either = { version = "1.9", optional = true } eyre = { version = ">= 0.4, < 0.7", optional = true } hashbrown = { version = ">= 0.9, < 0.15", optional = true } @@ -47,7 +48,8 @@ smallvec = { version = "1.0", optional = true } [dev-dependencies] assert_approx_eq = "1.1.0" -chrono = { version = "0.4.25" } +chrono = "0.4.25" +chrono-tz = ">= 0.6, < 0.9" # Required for "and $N others" normalization trybuild = ">=1.0.70" proptest = { version = "1.0", default-features = false, features = ["std"] } @@ -108,6 +110,7 @@ full = [ "macros", # "multiple-pymethods", # TODO re-add this when MSRV is greater than 1.62 "chrono", + "chrono-tz", "num-bigint", "num-complex", "hashbrown", diff --git a/guide/src/features.md b/guide/src/features.md index edcf3772b26..43124e0076e 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -115,6 +115,12 @@ Adds a dependency on [chrono](https://docs.rs/chrono). Enables a conversion from - [NaiveTime](https://docs.rs/chrono/latest/chrono/naive/struct.NaiveTime.html) -> [`PyTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTime.html) - [DateTime](https://docs.rs/chrono/latest/chrono/struct.DateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +### `chrono-tz` + +Adds a dependency on [chrono-tz](https://docs.rs/chrono-tz). +Enables conversion from and to [`Tz`](https://docs.rs/chrono-tz/latest/chrono_tz/enum.Tz.html). +It requires at least Python 3.9. + ### `either` Adds a dependency on [either](https://docs.rs/either). Enables a conversions into [either](https://docs.rs/either)’s [`Either`](https://docs.rs/either/latest/either/struct.Report.html) type. diff --git a/newsfragments/3730.added.md b/newsfragments/3730.added.md new file mode 100644 index 00000000000..7e287245eb1 --- /dev/null +++ b/newsfragments/3730.added.md @@ -0,0 +1 @@ +`chrono-tz` feature allowing conversion between `chrono_tz::Tz` and `zoneinfo.ZoneInfo` \ No newline at end of file diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index cc0eb73a8ef..0e95375b5bd 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -9,8 +9,6 @@ //! //! ```toml //! [dependencies] -//! # change * to the latest versions -//! pyo3 = { version = "*", features = ["chrono"] } //! chrono = "0.4" #![doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"chrono\"] }")] //! ``` @@ -18,7 +16,7 @@ //! Note that you must use compatible versions of chrono and PyO3. //! The required chrono version may vary based on the version of PyO3. //! -//! # Example: Convert a `PyDateTime` to chrono's `DateTime` +//! # Example: Convert a `datetime.datetime` to chrono's `DateTime` //! //! ```rust //! use chrono::{DateTime, Duration, TimeZone, Utc}; diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs new file mode 100644 index 00000000000..8740d0bdd98 --- /dev/null +++ b/src/conversions/chrono_tz.rs @@ -0,0 +1,113 @@ +#![cfg(all(Py_3_9, feature = "chrono-tz"))] + +//! Conversions to and from [chrono-tz](https://docs.rs/chrono-tz/)’s `Tz`. +//! +//! This feature requires at least Python 3.9. +//! +//! # Setup +//! +//! To use this feature, add this to your **`Cargo.toml`**: +//! +//! ```toml +//! [dependencies] +//! chrono-tz = "0.8" +#![doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"chrono-tz\"] }")] +//! ``` +//! +//! Note that you must use compatible versions of chrono, chrono-tz and PyO3. +//! The required chrono version may vary based on the version of PyO3. +//! +//! # Example: Convert a `zoneinfo.ZoneInfo` to chrono-tz's `Tz` +//! +//! ```rust,no_run +//! use chrono_tz::Tz; +//! use pyo3::{Python, ToPyObject}; +//! +//! fn main() { +//! pyo3::prepare_freethreaded_python(); +//! Python::with_gil(|py| { +//! // Convert to Python +//! let py_tzinfo = Tz::Europe__Paris.to_object(py); +//! // Convert back to Rust +//! assert_eq!(py_tzinfo.extract::(py).unwrap(), Tz::Europe__Paris); +//! }); +//! } +//! ``` +use crate::exceptions::PyValueError; +use crate::sync::GILOnceCell; +use crate::types::PyType; +use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; +use chrono_tz::Tz; +use std::str::FromStr; + +impl ToPyObject for Tz { + fn to_object(&self, py: Python<'_>) -> PyObject { + static ZONE_INFO: GILOnceCell> = GILOnceCell::new(); + ZONE_INFO + .get_or_try_init_type_ref(py, "zoneinfo", "ZoneInfo") + .unwrap() + .call1((self.name(),)) + .unwrap() + .into() + } +} + +impl IntoPy for Tz { + fn into_py(self, py: Python<'_>) -> PyObject { + self.to_object(py) + } +} + +impl FromPyObject<'_> for Tz { + fn extract(ob: &PyAny) -> PyResult { + Tz::from_str(ob.getattr(intern!(ob.py(), "key"))?.extract()?) + .map_err(|e| PyValueError::new_err(e.to_string())) + } +} + +#[cfg(all(test, not(windows)))] // Troubles loading timezones on Windows +mod tests { + use super::*; + + #[test] + fn test_frompyobject() { + Python::with_gil(|py| { + assert_eq!( + new_zoneinfo(py, "Europe/Paris").extract::().unwrap(), + Tz::Europe__Paris + ); + assert_eq!(new_zoneinfo(py, "UTC").extract::().unwrap(), Tz::UTC); + assert_eq!( + new_zoneinfo(py, "Etc/GMT-5").extract::().unwrap(), + Tz::Etc__GMTMinus5 + ); + }); + } + + #[test] + fn test_topyobject() { + Python::with_gil(|py| { + let assert_eq = |l: PyObject, r: &PyAny| { + assert!(l.as_ref(py).eq(r).unwrap()); + }; + + assert_eq( + Tz::Europe__Paris.to_object(py), + new_zoneinfo(py, "Europe/Paris"), + ); + assert_eq(Tz::UTC.to_object(py), new_zoneinfo(py, "UTC")); + assert_eq( + Tz::Etc__GMTMinus5.to_object(py), + new_zoneinfo(py, "Etc/GMT-5"), + ); + }); + } + + fn new_zoneinfo<'a>(py: Python<'a>, name: &str) -> &'a PyAny { + zoneinfo_class(py).call1((name,)).unwrap() + } + + fn zoneinfo_class(py: Python<'_>) -> &PyAny { + py.import("zoneinfo").unwrap().getattr("ZoneInfo").unwrap() + } +} diff --git a/src/conversions/mod.rs b/src/conversions/mod.rs index 680ad9be6d5..3d785c02381 100644 --- a/src/conversions/mod.rs +++ b/src/conversions/mod.rs @@ -2,6 +2,7 @@ pub mod anyhow; pub mod chrono; +pub mod chrono_tz; pub mod either; pub mod eyre; pub mod hashbrown; diff --git a/src/lib.rs b/src/lib.rs index 4802cbc2712..3f91eb56913 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -82,6 +82,7 @@ //! The following features enable interactions with other crates in the Rust ecosystem: //! - [`anyhow`]: Enables a conversion from [anyhow]’s [`Error`][anyhow_error] type to [`PyErr`]. //! - [`chrono`]: Enables a conversion from [chrono]'s structures to the equivalent Python ones. +//! - [`chrono-tz`]: Enables a conversion from [chrono-tz]'s `Tz` enum. Requires Python 3.9+. //! - [`either`]: Enables conversions between Python objects and [either]'s [`Either`] type. //! - [`eyre`]: Enables a conversion from [eyre]’s [`Report`] type to [`PyErr`]. //! - [`hashbrown`]: Enables conversions between Python objects and [hashbrown]'s [`HashMap`] and @@ -257,7 +258,9 @@ //! [`Deserialize`]: https://docs.rs/serde/latest/serde/trait.Deserialize.html //! [`Serialize`]: https://docs.rs/serde/latest/serde/trait.Serialize.html //! [chrono]: https://docs.rs/chrono/ "Date and Time for Rust." +//! [chrono-tz]: https://docs.rs/chrono-tz/ "TimeZone implementations for chrono from the IANA database." //! [`chrono`]: ./chrono/index.html "Documentation about the `chrono` feature." +//! [`chrono-tz`]: ./chrono-tz/index.html "Documentation about the `chrono-tz` feature." //! [either]: https://docs.rs/either/ "A type that represents one of two alternatives." //! [`either`]: ./either/index.html "Documentation about the `either` feature." //! [`Either`]: https://docs.rs/either/latest/either/enum.Either.html From 800943ab2df5cf6dc566e9d0eeaab2c3b9a4353d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 20:14:38 +0000 Subject: [PATCH 037/349] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d21a3d8f34f..8fe7af2747a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -339,7 +339,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 - uses: Swatinem/rust-cache@v2 with: key: cargo-careful From 4504a7c96e52c120e07842158784ac86cfbbb977 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 12 Jan 2024 10:05:30 +0000 Subject: [PATCH 038/349] fix some nightly lints 2024-01-12 --- guide/src/class.md | 4 ++++ guide/src/class/object.md | 1 + pyo3-macros-backend/src/pyclass.rs | 11 ++++------- pyo3-macros-backend/src/utils.rs | 7 ++++--- src/tests/hygiene/misc.rs | 2 +- tests/test_frompyobject.rs | 1 + 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index e3f3502780b..25df536c6f7 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -80,6 +80,7 @@ To declare a constructor, you need to define a method and annotate it with the ` attribute. Only Python's `__new__` method can be specified, `__init__` is not available. ```rust +# #![allow(dead_code)] # use pyo3::prelude::*; # #[pyclass] # struct Number(i32); @@ -96,6 +97,7 @@ impl Number { Alternatively, if your `new` method may fail you can return `PyResult`. ```rust +# #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::exceptions::PyValueError; # #[pyclass] @@ -130,6 +132,7 @@ For arguments, see the [`Method arguments`](#method-arguments) section below. The next step is to create the module initializer and add our class to it: ```rust +# #![allow(dead_code)] # use pyo3::prelude::*; # #[pyclass] # struct Number(i32); @@ -663,6 +666,7 @@ Declares a class method callable from Python. To create a constructor which takes a positional class argument, you can combine the `#[classmethod]` and `#[new]` modifiers: ```rust +# #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::types::PyType; # #[pyclass] diff --git a/guide/src/class/object.md b/guide/src/class/object.md index cfaa8bb1ddd..5d4f53a17d4 100644 --- a/guide/src/class/object.md +++ b/guide/src/class/object.md @@ -3,6 +3,7 @@ Recall the `Number` class from the previous chapter: ```rust +# #![allow(dead_code)] use pyo3::prelude::*; #[pyclass] diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index f60dbe5f4d4..66f41504d12 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -885,13 +885,10 @@ impl<'a> PyClassImplsBuilder<'a> { let cls = self.cls; let doc = self.doc.as_ref().map_or(quote! {"\0"}, |doc| quote! {#doc}); let is_basetype = self.attr.options.subclass.is_some(); - let base = self - .attr - .options - .extends - .as_ref() - .map(|extends_attr| extends_attr.value.clone()) - .unwrap_or_else(|| parse_quote! { _pyo3::PyAny }); + let base = match &self.attr.options.extends { + Some(extends_attr) => extends_attr.value.clone(), + None => parse_quote! { _pyo3::PyAny }, + }; let is_subclass = self.attr.options.extends.is_some(); let is_mapping: bool = self.attr.options.mapping.is_some(); let is_sequence: bool = self.attr.options.sequence.is_some(); diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index 360b1ec2341..0fc96bd6a1a 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -146,9 +146,10 @@ pub fn unwrap_ty_group(mut ty: &syn::Type) -> &syn::Type { /// Extract the path to the pyo3 crate, or use the default (`::pyo3`). pub(crate) fn get_pyo3_crate(attr: &Option) -> syn::Path { - attr.as_ref() - .map(|p| p.value.0.clone()) - .unwrap_or_else(|| syn::parse_str("::pyo3").unwrap()) + match attr { + Some(attr) => attr.value.0.clone(), + None => syn::parse_str("::pyo3").unwrap(), + } } pub fn apply_renaming_rule(rule: RenamingRule, name: &str) -> String { diff --git a/src/tests/hygiene/misc.rs b/src/tests/hygiene/misc.rs index 2e7d3e6f9ee..66db7f3a28a 100644 --- a/src/tests/hygiene/misc.rs +++ b/src/tests/hygiene/misc.rs @@ -2,7 +2,7 @@ #[derive(crate::FromPyObject)] #[pyo3(crate = "crate")] -struct Derive1(i32); // newtype case +struct Derive1(#[allow(dead_code)] i32); // newtype case #[derive(crate::FromPyObject)] #[pyo3(crate = "crate")] diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index 30edf6f7836..aa0c1cefc09 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -420,6 +420,7 @@ TypeError: failed to extract enum Foo ('TupleVar | StructVar | TransparentTuple #[derive(Debug, FromPyObject)] enum EnumWithCatchAll<'a> { + #[allow(dead_code)] #[pyo3(transparent)] Foo(Foo<'a>), #[pyo3(transparent)] From ab699a07276e877b4fb7ac7ba8f19ee1de3885c5 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 12 Jan 2024 15:55:21 +0000 Subject: [PATCH 039/349] allow dead_code in `IPowModulo` --- src/impl_/pymethods.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index e403aa23c79..4fac39fbdc4 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -21,7 +21,7 @@ pub struct IPowModulo(*mut ffi::PyObject); /// Python 3.7 and older - __ipow__ does not have modulo argument correctly populated. #[cfg(not(Py_3_8))] #[repr(transparent)] -pub struct IPowModulo(std::mem::MaybeUninit<*mut ffi::PyObject>); +pub struct IPowModulo(#[allow(dead_code)] std::mem::MaybeUninit<*mut ffi::PyObject>); /// Helper to use as pymethod ffi definition #[allow(non_camel_case_types)] From 8fef7a5848cdd9c8998ca21884096d1ee7205464 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 12 Jan 2024 10:14:39 +0000 Subject: [PATCH 040/349] fix size of pypy private fields in Py_buffer definition --- .github/workflows/build.yml | 1 + pyo3-ffi/src/cpython/object.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de82dd20c05..bf98dd485ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,6 +32,7 @@ jobs: with: python-version: ${{ inputs.python-version }} architecture: ${{ inputs.python-architecture }} + check-latest: ${{ startsWith(inputs.python-version, 'pypy') }} # PyPy can have FFI changes within Python versions, which creates pain in CI - name: Install nox run: python -m pip install --upgrade pip && pip install nox diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 423af03a0e2..9f46ce62cfc 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -21,7 +21,7 @@ mod bufferinfo { use std::ptr; #[cfg(PyPy)] - const Py_MAX_NDIMS: usize = 36; + const Py_MAX_NDIMS: usize = if cfg!(Py_3_9) { 64 } else { 36 }; #[repr(C)] #[derive(Copy, Clone)] From 83f0f22efebb43b6aab473bbebfd8b0688a42924 Mon Sep 17 00:00:00 2001 From: jadedpasta Date: Tue, 9 Jan 2024 23:47:34 -0600 Subject: [PATCH 041/349] ffi: Add definition for PyType_GetModuleByDef This API is available starting in 3.11. It is not part of the Stable ABI. PyType_GetModuleByDef searches the MRO of the given type for a module matching the given module spec. It can be useful for users use that `pyo3_ffi` directly and want to support multiple interpreters/per interpreter GIL. It is useful to obtain access to the module state from special methods like `tp_new` that can't use the METH_METHOD calling convention. This API is not supported on PyPy yet, but guess the symbol name for the future. --- newsfragments/3734.added.md | 1 + pyo3-ffi/src/cpython/object.rs | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3734.added.md diff --git a/newsfragments/3734.added.md b/newsfragments/3734.added.md new file mode 100644 index 00000000000..e58c2038e70 --- /dev/null +++ b/newsfragments/3734.added.md @@ -0,0 +1 @@ +Add definition for `PyType_GetModuleByDef` to `pyo3_ffi`. diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 9f46ce62cfc..ce4f56f61d5 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -1,5 +1,7 @@ #[cfg(Py_3_8)] use crate::vectorcallfunc; +#[cfg(Py_3_11)] +use crate::PyModuleDef; use crate::{object, PyGetSetDef, PyMemberDef, PyMethodDef, PyObject, Py_ssize_t}; use std::mem; use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; @@ -339,9 +341,12 @@ pub unsafe fn PyHeapType_GET_MEMBERS(etype: *mut PyHeapTypeObject) -> *mut PyMem // skipped _PyType_CalculateMetaclass // skipped _PyType_GetDocFromInternalDoc // skipped _PyType_GetTextSignatureFromInternalDoc -// skipped _PyType_GetModuleByDef extern "C" { + #[cfg(Py_3_11)] + #[cfg_attr(PyPy, link_name = "PyPyType_GetModuleByDef")] + pub fn PyType_GetModuleByDef(ty: *mut PyTypeObject, def: *mut PyModuleDef) -> *mut PyObject; + #[cfg(Py_3_12)] pub fn PyType_GetDict(o: *mut PyTypeObject) -> *mut PyObject; From 0e876d94d6157510069d2d209995a5daae264f48 Mon Sep 17 00:00:00 2001 From: Samuel Colvin Date: Sun, 14 Jan 2024 15:39:05 +0000 Subject: [PATCH 042/349] improve performance of successful int extract by ~30% add newsfragment formatting skip slow path on 3.8+ formatting cfg if,else formatting again dedicated macro, change int_convert_u64_or_i64 too add float tests force index call for PyLong_AsUnsignedLongLong perform PyLong check for 3.8 too perform PyLong check for <3.10 --- newsfragments/3742.changed.md | 1 + src/conversions/std/num.rs | 87 ++++++++++++++++++++++++----------- 2 files changed, 62 insertions(+), 26 deletions(-) create mode 100644 newsfragments/3742.changed.md diff --git a/newsfragments/3742.changed.md b/newsfragments/3742.changed.md new file mode 100644 index 00000000000..b8805abafda --- /dev/null +++ b/newsfragments/3742.changed.md @@ -0,0 +1 @@ +Improve performance of `extract::` (and other integer types) by avoiding call to `__index__()` converting the value to an integer for 3.10+. Gives performance improvement of around 30% for successful extraction. diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 1f3bf673a48..e5c72b98ef0 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -44,8 +44,39 @@ macro_rules! int_fits_larger_int { }; } +macro_rules! extract_int { + ($obj:ident, $error_val:expr, $pylong_as:expr) => { + extract_int!($obj, $error_val, $pylong_as, false) + }; + + ($obj:ident, $error_val:expr, $pylong_as:expr, $force_index_call: literal) => { + // In python 3.8+ `PyLong_AsLong` and friends takes care of calling `PyNumber_Index`, + // however 3.8 & 3.9 do lossy conversion of floats, hence we only use the + // simplest logic for 3.10+ where that was fixed - python/cpython#82180. + // `PyLong_AsUnsignedLongLong` does not call `PyNumber_Index`, hence the `force_index_call` argument + // See https://github.com/PyO3/pyo3/pull/3742 for detials + if cfg!(Py_3_10) && !$force_index_call { + err_if_invalid_value($obj.py(), $error_val, unsafe { $pylong_as($obj.as_ptr()) }) + } else if let Ok(long) = $obj.downcast::() { + // fast path - checking for subclass of `int` just checks a bit in the type $object + err_if_invalid_value($obj.py(), $error_val, unsafe { $pylong_as(long.as_ptr()) }) + } else { + unsafe { + let num = ffi::PyNumber_Index($obj.as_ptr()); + if num.is_null() { + Err(PyErr::fetch($obj.py())) + } else { + let result = err_if_invalid_value($obj.py(), $error_val, $pylong_as(num)); + ffi::Py_DECREF(num); + result + } + } + } + }; +} + macro_rules! int_convert_u64_or_i64 { - ($rust_type:ty, $pylong_from_ll_or_ull:expr, $pylong_as_ll_or_ull:expr) => { + ($rust_type:ty, $pylong_from_ll_or_ull:expr, $pylong_as_ll_or_ull:expr, $force_index_call:literal) => { impl ToPyObject for $rust_type { #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { @@ -64,18 +95,8 @@ macro_rules! int_convert_u64_or_i64 { } } impl<'source> FromPyObject<'source> for $rust_type { - fn extract(ob: &'source PyAny) -> PyResult<$rust_type> { - let ptr = ob.as_ptr(); - unsafe { - let num = ffi::PyNumber_Index(ptr); - if num.is_null() { - Err(PyErr::fetch(ob.py())) - } else { - let result = err_if_invalid_value(ob.py(), !0, $pylong_as_ll_or_ull(num)); - ffi::Py_DECREF(num); - result - } - } + fn extract(obj: &'source PyAny) -> PyResult<$rust_type> { + extract_int!(obj, !0, $pylong_as_ll_or_ull, $force_index_call) } #[cfg(feature = "experimental-inspect")] @@ -106,17 +127,7 @@ macro_rules! int_fits_c_long { impl<'source> FromPyObject<'source> for $rust_type { fn extract(obj: &'source PyAny) -> PyResult { - let ptr = obj.as_ptr(); - let val = unsafe { - let num = ffi::PyNumber_Index(ptr); - if num.is_null() { - Err(PyErr::fetch(obj.py())) - } else { - let val = err_if_invalid_value(obj.py(), -1, ffi::PyLong_AsLong(num)); - ffi::Py_DECREF(num); - val - } - }?; + let val: c_long = extract_int!(obj, -1, ffi::PyLong_AsLong)?; <$rust_type>::try_from(val) .map_err(|e| exceptions::PyOverflowError::new_err(e.to_string())) } @@ -146,7 +157,7 @@ int_fits_c_long!(i64); // manual implementation for i64 on systems with 32-bit long #[cfg(any(target_pointer_width = "32", target_os = "windows"))] -int_convert_u64_or_i64!(i64, ffi::PyLong_FromLongLong, ffi::PyLong_AsLongLong); +int_convert_u64_or_i64!(i64, ffi::PyLong_FromLongLong, ffi::PyLong_AsLongLong, false); #[cfg(all(target_pointer_width = "64", not(target_os = "windows")))] int_fits_c_long!(isize); @@ -159,7 +170,8 @@ int_fits_larger_int!(usize, u64); int_convert_u64_or_i64!( u64, ffi::PyLong_FromUnsignedLongLong, - ffi::PyLong_AsUnsignedLongLong + ffi::PyLong_AsUnsignedLongLong, + true ); #[cfg(not(Py_LIMITED_API))] @@ -738,4 +750,27 @@ mod tests { test_nonzero_common!(nonzero_usize, NonZeroUsize); test_nonzero_common!(nonzero_i128, NonZeroI128); test_nonzero_common!(nonzero_u128, NonZeroU128); + + #[test] + fn test_i64_bool() { + Python::with_gil(|py| { + let obj = true.to_object(py); + assert_eq!(1, obj.extract::(py).unwrap()); + let obj = false.to_object(py); + assert_eq!(0, obj.extract::(py).unwrap()); + }) + } + + #[test] + fn test_i64_f64() { + Python::with_gil(|py| { + let obj = 12.34f64.to_object(py); + let err = obj.extract::(py).unwrap_err(); + assert!(err.is_instance_of::(py)); + // with no remainder + let obj = 12f64.to_object(py); + let err = obj.extract::(py).unwrap_err(); + assert!(err.is_instance_of::(py)); + }) + } } From 06c95432c6a5b280e13c81a9802c6f7561a8fcf6 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 16 Jan 2024 22:53:38 +0000 Subject: [PATCH 043/349] set & frozenset bound constructors --- src/conversions/hashbrown.rs | 5 ++-- src/conversions/std/set.rs | 10 +++---- src/types/frozenset.rs | 58 +++++++++++++++++++++++++++--------- src/types/set.rs | 53 +++++++++++++++++++++++++------- 4 files changed, 95 insertions(+), 31 deletions(-) diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index 62a7e87b804..a315a27490d 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -109,6 +109,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::types::any::PyAnyMethods; #[test] fn test_hashbrown_hashmap_to_python() { @@ -178,11 +179,11 @@ mod tests { #[test] fn test_extract_hashbrown_hashset() { Python::with_gil(|py| { - let set = PySet::new(py, &[1, 2, 3, 4, 5]).unwrap(); + let set = PySet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); let hash_set: hashbrown::HashSet = set.extract().unwrap(); assert_eq!(hash_set, [1, 2, 3, 4, 5].iter().copied().collect()); - let set = PyFrozenSet::new(py, &[1, 2, 3, 4, 5]).unwrap(); + let set = PyFrozenSet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); let hash_set: hashbrown::HashSet = set.extract().unwrap(); assert_eq!(hash_set, [1, 2, 3, 4, 5].iter().copied().collect()); }); diff --git a/src/conversions/std/set.rs b/src/conversions/std/set.rs index 4221c1ff865..020b2505b11 100644 --- a/src/conversions/std/set.rs +++ b/src/conversions/std/set.rs @@ -113,18 +113,18 @@ where #[cfg(test)] mod tests { - use super::{PyFrozenSet, PySet}; + use crate::types::{any::PyAnyMethods, PyFrozenSet, PySet}; use crate::{IntoPy, PyObject, Python, ToPyObject}; use std::collections::{BTreeSet, HashSet}; #[test] fn test_extract_hashset() { Python::with_gil(|py| { - let set = PySet::new(py, &[1, 2, 3, 4, 5]).unwrap(); + let set = PySet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); let hash_set: HashSet = set.extract().unwrap(); assert_eq!(hash_set, [1, 2, 3, 4, 5].iter().copied().collect()); - let set = PyFrozenSet::new(py, &[1, 2, 3, 4, 5]).unwrap(); + let set = PyFrozenSet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); let hash_set: HashSet = set.extract().unwrap(); assert_eq!(hash_set, [1, 2, 3, 4, 5].iter().copied().collect()); }); @@ -133,11 +133,11 @@ mod tests { #[test] fn test_extract_btreeset() { Python::with_gil(|py| { - let set = PySet::new(py, &[1, 2, 3, 4, 5]).unwrap(); + let set = PySet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); let hash_set: BTreeSet = set.extract().unwrap(); assert_eq!(hash_set, [1, 2, 3, 4, 5].iter().copied().collect()); - let set = PyFrozenSet::new(py, &[1, 2, 3, 4, 5]).unwrap(); + let set = PyFrozenSet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); let hash_set: BTreeSet = set.extract().unwrap(); assert_eq!(hash_set, [1, 2, 3, 4, 5].iter().copied().collect()); }); diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index ff0be7b59a2..fa56a9c69f7 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -1,10 +1,12 @@ -#[cfg(not(Py_LIMITED_API))] -use crate::ffi_ptr_ext::FfiPtrExt; #[cfg(Py_LIMITED_API)] use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, - ffi, Bound, Py, PyAny, PyNativeType, PyObject, Python, ToPyObject, + ffi, + ffi_ptr_ext::FfiPtrExt, + py_result_ext::PyResultExt, + types::any::PyAnyMethods, + Bound, PyAny, PyNativeType, PyObject, Python, ToPyObject, }; use std::ptr; @@ -63,20 +65,45 @@ pyobject_native_type_core!( ); impl PyFrozenSet { + /// Deprecated form of [`PyFrozenSet::new_bound`]. + #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyFrozenSet::new` will be replaced by `PyFrozenSet::new_bound` in a future PyO3 version" + ) + )] + pub fn new<'a, 'p, T: ToPyObject + 'a>( + py: Python<'p>, + elements: impl IntoIterator, + ) -> PyResult<&'p PyFrozenSet> { + Self::new_bound(py, elements).map(Bound::into_gil_ref) + } + /// Creates a new frozenset. /// /// May panic when running out of memory. #[inline] - pub fn new<'a, 'p, T: ToPyObject + 'a>( + pub fn new_bound<'a, 'p, T: ToPyObject + 'a>( py: Python<'p>, elements: impl IntoIterator, - ) -> PyResult<&'p PyFrozenSet> { - new_from_iter(py, elements).map(|set| set.into_ref(py)) + ) -> PyResult> { + new_from_iter(py, elements) + } + + /// Deprecated form of [`PyFrozenSet::empty_bound`]. + pub fn empty(py: Python<'_>) -> PyResult<&'_ PyFrozenSet> { + Self::empty_bound(py).map(Bound::into_gil_ref) } /// Creates a new empty frozen set - pub fn empty(py: Python<'_>) -> PyResult<&PyFrozenSet> { - unsafe { py.from_owned_ptr_or_err(ffi::PyFrozenSet_New(ptr::null_mut())) } + pub fn empty_bound(py: Python<'_>) -> PyResult> { + unsafe { + ffi::PyFrozenSet_New(ptr::null_mut()) + .assume_owned_or_err(py) + .downcast_into_unchecked() + } } /// Return the number of items in the set. @@ -285,14 +312,16 @@ pub use impl_::*; pub(crate) fn new_from_iter( py: Python<'_>, elements: impl IntoIterator, -) -> PyResult> { - fn inner( - py: Python<'_>, +) -> PyResult> { + fn inner<'py>( + py: Python<'py>, elements: &mut dyn Iterator, - ) -> PyResult> { - let set: Py = unsafe { + ) -> PyResult> { + let set = unsafe { // We create the `Py` pointer because its Drop cleans up the set if user code panics. - Py::from_owned_ptr_or_err(py, ffi::PyFrozenSet_New(std::ptr::null_mut()))? + ffi::PyFrozenSet_New(std::ptr::null_mut()) + .assume_owned_or_err(py)? + .downcast_into_unchecked() }; let ptr = set.as_ptr(); @@ -308,6 +337,7 @@ pub(crate) fn new_from_iter( } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; diff --git a/src/types/set.rs b/src/types/set.rs index d59a9e3ffc4..3204e71f3e5 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -4,7 +4,9 @@ use crate::{ err::{self, PyErr, PyResult}, ffi_ptr_ext::FfiPtrExt, instance::Bound, - Py, PyNativeType, + py_result_ext::PyResultExt, + types::any::PyAnyMethods, + PyNativeType, }; use crate::{ffi, PyAny, PyObject, Python, ToPyObject}; use std::ptr; @@ -29,20 +31,45 @@ pyobject_native_type_core!( ); impl PySet { + /// Deprecated form of [`PySet::new_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PySet::new` will be replaced by `PySet::new_bound` in a future PyO3 version" + ) + )] + #[inline] + pub fn new<'a, 'p, T: ToPyObject + 'a>( + py: Python<'p>, + elements: impl IntoIterator, + ) -> PyResult<&'p PySet> { + Self::new_bound(py, elements).map(Bound::into_gil_ref) + } + /// Creates a new set with elements from the given slice. /// /// Returns an error if some element is not hashable. #[inline] - pub fn new<'a, 'p, T: ToPyObject + 'a>( + pub fn new_bound<'a, 'p, T: ToPyObject + 'a>( py: Python<'p>, elements: impl IntoIterator, - ) -> PyResult<&'p PySet> { - new_from_iter(py, elements).map(|set| set.into_ref(py)) + ) -> PyResult> { + new_from_iter(py, elements) + } + + /// Deprecated form of [`PySet::empty_bound`]. + pub fn empty(py: Python<'_>) -> PyResult<&'_ PySet> { + Self::empty_bound(py).map(Bound::into_gil_ref) } /// Creates a new empty set. - pub fn empty(py: Python<'_>) -> PyResult<&PySet> { - unsafe { py.from_owned_ptr_or_err(ffi::PySet_New(ptr::null_mut())) } + pub fn empty_bound(py: Python<'_>) -> PyResult> { + unsafe { + ffi::PySet_New(ptr::null_mut()) + .assume_owned_or_err(py) + .downcast_into_unchecked() + } } /// Removes all elements from the set. @@ -379,11 +406,16 @@ pub use impl_::*; pub(crate) fn new_from_iter( py: Python<'_>, elements: impl IntoIterator, -) -> PyResult> { - fn inner(py: Python<'_>, elements: &mut dyn Iterator) -> PyResult> { - let set: Py = unsafe { +) -> PyResult> { + fn inner<'py>( + py: Python<'py>, + elements: &mut dyn Iterator, + ) -> PyResult> { + let set = unsafe { // We create the `Py` pointer because its Drop cleans up the set if user code panics. - Py::from_owned_ptr_or_err(py, ffi::PySet_New(std::ptr::null_mut()))? + ffi::PySet_New(std::ptr::null_mut()) + .assume_owned_or_err(py)? + .downcast_into_unchecked() }; let ptr = set.as_ptr(); @@ -399,6 +431,7 @@ pub(crate) fn new_from_iter( } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::PySet; use crate::{Python, ToPyObject}; From 4e24e680e874832c61248b284e2b5de2a4a2c7c5 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 17 Jan 2024 10:49:21 +0000 Subject: [PATCH 044/349] update set benchmarks --- pyo3-benches/benches/bench_set.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/pyo3-benches/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs index 0753a2f9979..49243a63fd4 100644 --- a/pyo3-benches/benches/bench_set.rs +++ b/pyo3-benches/benches/bench_set.rs @@ -10,21 +10,17 @@ fn set_new(b: &mut Bencher<'_>) { // Create Python objects up-front, so that the benchmark doesn't need to include // the cost of allocating LEN Python integers let elements: Vec = (0..LEN).map(|i| i.into_py(py)).collect(); - b.iter(|| { - let pool = unsafe { py.new_pool() }; - PySet::new(py, &elements).unwrap(); - drop(pool); - }); + b.iter_with_large_drop(|| PySet::new_bound(py, &elements).unwrap()); }); } fn iter_set(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let set = PySet::new(py, &(0..LEN).collect::>()).unwrap(); + let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); let mut sum = 0; b.iter(|| { - for x in set { + for x in set.iter() { let i: u64 = x.extract().unwrap(); sum += i; } @@ -35,16 +31,16 @@ fn iter_set(b: &mut Bencher<'_>) { fn extract_hashset(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let set = PySet::new(py, &(0..LEN).collect::>()).unwrap(); - b.iter(|| HashSet::::extract(set)); + let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); + b.iter_with_large_drop(|| HashSet::::extract(set.as_gil_ref())); }); } fn extract_btreeset(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let set = PySet::new(py, &(0..LEN).collect::>()).unwrap(); - b.iter(|| BTreeSet::::extract(set)); + let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); + b.iter_with_large_drop(|| BTreeSet::::extract(set.as_gil_ref())); }); } @@ -52,8 +48,8 @@ fn extract_btreeset(b: &mut Bencher<'_>) { fn extract_hashbrown_set(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let set = PySet::new(py, &(0..LEN).collect::>()).unwrap(); - b.iter(|| hashbrown::HashSet::::extract(set)); + let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); + b.iter_with_large_drop(|| hashbrown::HashSet::::extract(set.as_gil_ref())); }); } From 3ed5ddb0ec6d06a94b41da5cb5b037d24a8b584c Mon Sep 17 00:00:00 2001 From: Mate Kovacs Date: Tue, 16 Jan 2024 00:57:42 +0900 Subject: [PATCH 045/349] feat: support pyclass on complex enums --- guide/src/class.md | 123 ++++- newsfragments/3582.added.md | 1 + pyo3-macros-backend/src/pyclass.rs | 644 +++++++++++++++++++++++--- pyo3-macros-backend/src/pymethod.rs | 3 +- pytests/noxfile.py | 8 +- pytests/src/enums.rs | 58 +++ pytests/src/lib.rs | 3 + pytests/tests/test_enums.py | 116 +++++ pytests/tests/test_enums_match.py | 59 +++ tests/test_compile_error.rs | 1 + tests/ui/invalid_pyclass_enum.rs | 12 + tests/ui/invalid_pyclass_enum.stderr | 16 + tests/ui/invalid_pymethod_enum.rs | 19 + tests/ui/invalid_pymethod_enum.stderr | 11 + 14 files changed, 1007 insertions(+), 67 deletions(-) create mode 100644 newsfragments/3582.added.md create mode 100644 pytests/src/enums.rs create mode 100644 pytests/tests/test_enums.py create mode 100644 pytests/tests/test_enums_match.py create mode 100644 tests/ui/invalid_pymethod_enum.rs create mode 100644 tests/ui/invalid_pymethod_enum.stderr diff --git a/guide/src/class.md b/guide/src/class.md index 25df536c6f7..356e7c71a7d 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -2,7 +2,7 @@ PyO3 exposes a group of attributes powered by Rust's proc macro system for defining Python classes as Rust structs. -The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` or a fieldless `enum` (a.k.a. C-like enum) to generate a Python type for it. They will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled, each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) `#[pymethods]` may also have implementations for Python magic methods such as `__str__`. +The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` or `enum` to generate a Python type for it. They will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled, each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) `#[pymethods]` may also have implementations for Python magic methods such as `__str__`. This chapter will discuss the functionality and configuration these attributes offer. Below is a list of links to the relevant section of this chapter for each: @@ -21,13 +21,13 @@ This chapter will discuss the functionality and configuration these attributes o ## Defining a new class -To define a custom Python class, add the `#[pyclass]` attribute to a Rust struct or a fieldless enum. +To define a custom Python class, add the `#[pyclass]` attribute to a Rust struct or enum. ```rust # #![allow(dead_code)] use pyo3::prelude::*; #[pyclass] -struct Integer { +struct MyClass { inner: i32, } @@ -35,7 +35,15 @@ struct Integer { #[pyclass] struct Number(i32); -// PyO3 supports custom discriminants in enums +// PyO3 supports unit-only enums (which contain only unit variants) +// These simple enums behave similarly to Python's enumerations (enum.Enum) +#[pyclass] +enum MyEnum { + Variant, + OtherVariant = 30, // PyO3 supports custom discriminants. +} + +// PyO3 supports custom discriminants in unit-only enums #[pyclass] enum HttpResponse { Ok = 200, @@ -44,14 +52,19 @@ enum HttpResponse { // ... } +// PyO3 also supports enums with non-unit variants +// These complex enums have sligtly different behavior from the simple enums above +// They are meant to work with instance checks and match statement patterns #[pyclass] -enum MyEnum { - Variant, - OtherVariant = 30, // PyO3 supports custom discriminants. +enum Shape { + Circle { radius: f64 }, + Rectangle { width: f64, height: f64 }, + RegularPolygon { side_count: u32, radius: f64 }, + Nothing { }, } ``` -The above example generates implementations for [`PyTypeInfo`] and [`PyClass`] for `MyClass` and `MyEnum`. To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter. +The above example generates implementations for [`PyTypeInfo`] and [`PyClass`] for `MyClass`, `Number`, `MyEnum`, `HttpResponse`, and `Shape`. To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter. ### Restrictions @@ -964,7 +977,13 @@ Note that `text_signature` on `#[new]` is not compatible with compilation in ## #[pyclass] enums -Currently PyO3 only supports fieldless enums. PyO3 adds a class attribute for each variant, so you can access them in Python without defining `#[new]`. PyO3 also provides default implementations of `__richcmp__` and `__int__`, so they can be compared using `==`: +Enum support in PyO3 comes in two flavors, depending on what kind of variants the enum has: simple and complex. + +### Simple enums + +A simple enum (a.k.a. C-like enum) has only unit variants. + +PyO3 adds a class attribute for each variant, so you can access them in Python without defining `#[new]`. PyO3 also provides default implementations of `__richcmp__` and `__int__`, so they can be compared using `==`: ```rust # use pyo3::prelude::*; @@ -986,7 +1005,7 @@ Python::with_gil(|py| { }) ``` -You can also convert your enums into `int`: +You can also convert your simple enums into `int`: ```rust # use pyo3::prelude::*; @@ -1094,6 +1113,90 @@ enum BadSubclass { `#[pyclass]` enums are currently not interoperable with `IntEnum` in Python. +### Complex enums + +An enum is complex if it has any non-unit (struct or tuple) variants. + +Currently PyO3 supports only struct variants in a complex enum. Support for unit and tuple variants is planned. + +PyO3 adds a class attribute for each variant, which may be used to construct values and in match patterns. PyO3 also provides getter methods for all fields of each variant. + +```rust +# use pyo3::prelude::*; +#[pyclass] +enum Shape { + Circle { radius: f64 }, + Rectangle { width: f64, height: f64 }, + RegularPolygon { side_count: u32, radius: f64 }, + Nothing { }, +} + +Python::with_gil(|py| { + let def_count_vertices = if py.version_info() >= (3, 10) { r#" + def count_vertices(cls, shape): + match shape: + case cls.Circle(): + return 0 + case cls.Rectangle(): + return 4 + case cls.RegularPolygon(side_count=n): + return n + case cls.Nothing(): + return 0 + "# } else { r#" + def count_vertices(cls, shape): + if isinstance(shape, cls.Circle): + return 0 + elif isinstance(shape, cls.Rectangle): + return 4 + elif isinstance(shape, cls.RegularPolygon): + n = shape.side_count + return n + elif isinstance(shape, cls.Nothing): + return 0 + "# }; + + let circle = Shape::Circle { radius: 10.0 }.into_py(py); + let square = Shape::RegularPolygon { side_count: 4, radius: 10.0 }.into_py(py); + let cls = py.get_type::(); + + pyo3::py_run!(py, circle square cls, &format!(r#" + assert isinstance(circle, cls) + assert isinstance(circle, cls.Circle) + assert circle.radius == 10.0 + + assert isinstance(square, cls) + assert isinstance(square, cls.RegularPolygon) + assert square.side_count == 4 + assert square.radius == 10.0 + + {} + + assert count_vertices(cls, circle) == 0 + assert count_vertices(cls, square) == 4 + "#, def_count_vertices)) +}) +``` + +WARNING: `Py::new` and `.into_py` are currently inconsistent. Note how the constructed value is _not_ an instance of the specific variant. For this reason, constructing values is only recommended using `.into_py`. + +```rust +# use pyo3::prelude::*; +#[pyclass] +enum MyEnum { + Variant { i: i32 }, +} + +Python::with_gil(|py| { + let x = Py::new(py, MyEnum::Variant { i: 42 }).unwrap(); + let cls = py.get_type::(); + pyo3::py_run!(py, x cls, r#" + assert isinstance(x, cls) + assert not isinstance(x, cls.Variant) + "#) +}) +``` + ## Implementation details The `#[pyclass]` macros rely on a lot of conditional code generation: each `#[pyclass]` can optionally have a `#[pymethods]` block. diff --git a/newsfragments/3582.added.md b/newsfragments/3582.added.md new file mode 100644 index 00000000000..59659a8819d --- /dev/null +++ b/newsfragments/3582.added.md @@ -0,0 +1 @@ +Support `#[pyclass]` on enums that have non-unit variants. diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 66f41504d12..e2f849492a0 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -7,7 +7,7 @@ use crate::attributes::{ }; use crate::deprecations::Deprecations; use crate::konst::{ConstAttributes, ConstSpec}; -use crate::method::FnSpec; +use crate::method::{FnArg, FnSpec}; use crate::pyimpl::{gen_py_const, PyClassMethodsType}; use crate::pymethod::{ impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, MethodAndSlotDef, PropertyType, @@ -16,7 +16,7 @@ use crate::pymethod::{ use crate::utils::{self, apply_renaming_rule, get_pyo3_crate, PythonDoc}; use crate::PyFunctionOptions; use proc_macro2::{Ident, Span, TokenStream}; -use quote::quote; +use quote::{format_ident, quote}; use syn::ext::IdentExt; use syn::parse::{Parse, ParseStream}; use syn::punctuated::Punctuated; @@ -30,6 +30,7 @@ pub enum PyClassKind { } /// The parsed arguments of the pyclass macro +#[derive(Clone)] pub struct PyClassArgs { pub class_kind: PyClassKind, pub options: PyClassPyO3Options, @@ -52,7 +53,7 @@ impl PyClassArgs { } } -#[derive(Default)] +#[derive(Clone, Default)] pub struct PyClassPyO3Options { pub krate: Option, pub dict: Option, @@ -128,7 +129,7 @@ impl Parse for PyClassPyO3Option { } } -impl PyClassPyO3Options { +impl Parse for PyClassPyO3Options { fn parse(input: ParseStream<'_>) -> syn::Result { let mut options: PyClassPyO3Options = Default::default(); @@ -138,7 +139,9 @@ impl PyClassPyO3Options { Ok(options) } +} +impl PyClassPyO3Options { pub fn take_pyo3_options(&mut self, attrs: &mut Vec) -> syn::Result<()> { take_pyo3_options(attrs)? .into_iter() @@ -369,38 +372,57 @@ fn impl_class( }) } -struct PyClassEnumVariant<'a> { - ident: &'a syn::Ident, - options: EnumVariantPyO3Options, +enum PyClassEnum<'a> { + Simple(PyClassSimpleEnum<'a>), + Complex(PyClassComplexEnum<'a>), } -impl<'a> PyClassEnumVariant<'a> { - fn python_name(&self, args: &PyClassArgs) -> Cow<'_, syn::Ident> { - self.options - .name - .as_ref() - .map(|name_attr| Cow::Borrowed(&name_attr.value.0)) - .unwrap_or_else(|| { - let name = self.ident.unraw(); - if let Some(attr) = &args.options.rename_all { - let new_name = apply_renaming_rule(attr.value.rule, &name.to_string()); - Cow::Owned(Ident::new(&new_name, Span::call_site())) - } else { - Cow::Owned(name) - } - }) +impl<'a> PyClassEnum<'a> { + fn new(enum_: &'a mut syn::ItemEnum) -> syn::Result { + let has_only_unit_variants = enum_ + .variants + .iter() + .all(|variant| matches!(variant.fields, syn::Fields::Unit)); + + Ok(if has_only_unit_variants { + let simple_enum = PyClassSimpleEnum::new(enum_)?; + Self::Simple(simple_enum) + } else { + let complex_enum = PyClassComplexEnum::new(enum_)?; + Self::Complex(complex_enum) + }) } } -struct PyClassEnum<'a> { +pub fn build_py_enum( + enum_: &mut syn::ItemEnum, + mut args: PyClassArgs, + method_type: PyClassMethodsType, +) -> syn::Result { + args.options.take_pyo3_options(&mut enum_.attrs)?; + + if let Some(extends) = &args.options.extends { + bail_spanned!(extends.span() => "enums can't extend from other classes"); + } else if let Some(subclass) = &args.options.subclass { + bail_spanned!(subclass.span() => "enums can't be inherited by other classes"); + } else if enum_.variants.is_empty() { + bail_spanned!(enum_.brace_token.span.join() => "#[pyclass] can't be used on enums without any variants"); + } + + let doc = utils::get_doc(&enum_.attrs, None); + let enum_ = PyClassEnum::new(enum_)?; + impl_enum(enum_, &args, doc, method_type) +} + +struct PyClassSimpleEnum<'a> { ident: &'a syn::Ident, // The underlying #[repr] of the enum, used to implement __int__ and __richcmp__. // This matters when the underlying representation may not fit in `isize`. repr_type: syn::Ident, - variants: Vec>, + variants: Vec>, } -impl<'a> PyClassEnum<'a> { +impl<'a> PyClassSimpleEnum<'a> { fn new(enum_: &'a mut syn::ItemEnum) -> syn::Result { fn is_numeric_type(t: &syn::Ident) -> bool { [ @@ -410,7 +432,21 @@ impl<'a> PyClassEnum<'a> { .iter() .any(|&s| t == s) } + + fn extract_unit_variant_data( + variant: &mut syn::Variant, + ) -> syn::Result> { + use syn::Fields; + let ident = match &variant.fields { + Fields::Unit => &variant.ident, + _ => bail_spanned!(variant.span() => "Must be a unit variant."), + }; + let options = EnumVariantPyO3Options::take_pyo3_options(&mut variant.attrs)?; + Ok(PyClassEnumUnitVariant { ident, options }) + } + let ident = &enum_.ident; + // According to the [reference](https://doc.rust-lang.org/reference/items/enumerations.html), // "Under the default representation, the specified discriminant is interpreted as an isize // value", so `isize` should be enough by default. @@ -427,10 +463,10 @@ impl<'a> PyClassEnum<'a> { } } - let variants = enum_ + let variants: Vec<_> = enum_ .variants .iter_mut() - .map(extract_variant_data) + .map(extract_unit_variant_data) .collect::>()?; Ok(Self { ident, @@ -440,24 +476,144 @@ impl<'a> PyClassEnum<'a> { } } -pub fn build_py_enum( - enum_: &mut syn::ItemEnum, - mut args: PyClassArgs, - method_type: PyClassMethodsType, -) -> syn::Result { - args.options.take_pyo3_options(&mut enum_.attrs)?; +struct PyClassComplexEnum<'a> { + ident: &'a syn::Ident, + variants: Vec>, +} - if let Some(extends) = &args.options.extends { - bail_spanned!(extends.span() => "enums can't extend from other classes"); - } else if let Some(subclass) = &args.options.subclass { - bail_spanned!(subclass.span() => "enums can't be inherited by other classes"); - } else if enum_.variants.is_empty() { - bail_spanned!(enum_.brace_token.span.join() => "#[pyclass] can't be used on enums without any variants"); +impl<'a> PyClassComplexEnum<'a> { + fn new(enum_: &'a mut syn::ItemEnum) -> syn::Result { + let witness = enum_ + .variants + .iter() + .find(|variant| !matches!(variant.fields, syn::Fields::Unit)) + .expect("complex enum has a non-unit variant") + .ident + .to_owned(); + + let extract_variant_data = + |variant: &'a mut syn::Variant| -> syn::Result> { + use syn::Fields; + let ident = &variant.ident; + let options = EnumVariantPyO3Options::take_pyo3_options(&mut variant.attrs)?; + + let variant = match &variant.fields { + Fields::Unit => { + bail_spanned!(variant.span() => format!( + "Unit variant `{ident}` is not yet supported in a complex enum\n\ + = help: change to a struct variant with no fields: `{ident} {{ }}`\n\ + = note: the enum is complex because of non-unit variant `{witness}`", + ident=ident, witness=witness)) + } + Fields::Named(fields) => { + let fields = fields + .named + .iter() + .map(|field| PyClassEnumVariantNamedField { + ident: field.ident.as_ref().expect("named field has an identifier"), + ty: &field.ty, + span: field.span(), + }) + .collect(); + + PyClassEnumVariant::Struct(PyClassEnumStructVariant { + ident, + fields, + options, + }) + } + Fields::Unnamed(_) => { + bail_spanned!(variant.span() => format!( + "Tuple variant `{ident}` is not yet supported in a complex enum\n\ + = help: change to a struct variant with named fields: `{ident} {{ /* fields */ }}`\n\ + = note: the enum is complex because of non-unit variant `{witness}`", + ident=ident, witness=witness)) + } + }; + + Ok(variant) + }; + + let ident = &enum_.ident; + + let variants: Vec<_> = enum_ + .variants + .iter_mut() + .map(extract_variant_data) + .collect::>()?; + + Ok(Self { ident, variants }) } +} - let doc = utils::get_doc(&enum_.attrs, None); - let enum_ = PyClassEnum::new(enum_)?; - impl_enum(enum_, &args, doc, method_type) +enum PyClassEnumVariant<'a> { + // TODO(mkovaxx): Unit(PyClassEnumUnitVariant<'a>), + Struct(PyClassEnumStructVariant<'a>), + // TODO(mkovaxx): Tuple(PyClassEnumTupleVariant<'a>), +} + +trait EnumVariant { + fn get_ident(&self) -> &syn::Ident; + fn get_options(&self) -> &EnumVariantPyO3Options; + + fn get_python_name(&self, args: &PyClassArgs) -> Cow<'_, syn::Ident> { + self.get_options() + .name + .as_ref() + .map(|name_attr| Cow::Borrowed(&name_attr.value.0)) + .unwrap_or_else(|| { + let name = self.get_ident().unraw(); + if let Some(attr) = &args.options.rename_all { + let new_name = apply_renaming_rule(attr.value.rule, &name.to_string()); + Cow::Owned(Ident::new(&new_name, Span::call_site())) + } else { + Cow::Owned(name) + } + }) + } +} + +impl<'a> EnumVariant for PyClassEnumVariant<'a> { + fn get_ident(&self) -> &syn::Ident { + match self { + PyClassEnumVariant::Struct(struct_variant) => struct_variant.ident, + } + } + + fn get_options(&self) -> &EnumVariantPyO3Options { + match self { + PyClassEnumVariant::Struct(struct_variant) => &struct_variant.options, + } + } +} + +/// A unit variant has no fields +struct PyClassEnumUnitVariant<'a> { + ident: &'a syn::Ident, + options: EnumVariantPyO3Options, +} + +impl<'a> EnumVariant for PyClassEnumUnitVariant<'a> { + fn get_ident(&self) -> &syn::Ident { + self.ident + } + + fn get_options(&self) -> &EnumVariantPyO3Options { + &self.options + } +} + +/// A struct variant has named fields +struct PyClassEnumStructVariant<'a> { + ident: &'a syn::Ident, + fields: Vec>, + options: EnumVariantPyO3Options, +} + +struct PyClassEnumVariantNamedField<'a> { + ident: &'a syn::Ident, + ty: &'a syn::Type, + span: Span, } /// `#[pyo3()]` options for pyclass enum variants @@ -505,11 +661,25 @@ fn impl_enum( args: &PyClassArgs, doc: PythonDoc, methods_type: PyClassMethodsType, +) -> Result { + match enum_ { + PyClassEnum::Simple(simple_enum) => impl_simple_enum(simple_enum, args, doc, methods_type), + PyClassEnum::Complex(complex_enum) => { + impl_complex_enum(complex_enum, args, doc, methods_type) + } + } +} + +fn impl_simple_enum( + simple_enum: PyClassSimpleEnum<'_>, + args: &PyClassArgs, + doc: PythonDoc, + methods_type: PyClassMethodsType, ) -> Result { let krate = get_pyo3_crate(&args.options.krate); - let cls = enum_.ident; + let cls = simple_enum.ident; let ty: syn::Type = syn::parse_quote!(#cls); - let variants = enum_.variants; + let variants = simple_enum.variants; let pytypeinfo = impl_pytypeinfo(cls, args, None); let (default_repr, default_repr_slot) = { @@ -519,7 +689,7 @@ fn impl_enum( let repr = format!( "{}.{}", get_class_python_name(cls, args), - variant.python_name(args), + variant.get_python_name(args), ); quote! { #cls::#variant_name => #repr, } }); @@ -534,7 +704,7 @@ fn impl_enum( (repr_impl, repr_slot) }; - let repr_type = &enum_.repr_type; + let repr_type = &simple_enum.repr_type; let (default_int, default_int_slot) = { // This implementation allows us to convert &T to #repr_type without implementing `Copy` @@ -601,7 +771,10 @@ fn impl_enum( cls, args, methods_type, - enum_default_methods(cls, variants.iter().map(|v| (v.ident, v.python_name(args)))), + simple_enum_default_methods( + cls, + variants.iter().map(|v| (v.ident, v.get_python_name(args))), + ), default_slots, ) .doc(doc) @@ -626,6 +799,214 @@ fn impl_enum( }) } +fn impl_complex_enum( + complex_enum: PyClassComplexEnum<'_>, + args: &PyClassArgs, + doc: PythonDoc, + methods_type: PyClassMethodsType, +) -> Result { + // Need to rig the enum PyClass options + let args = { + let mut rigged_args = args.clone(); + // Needs to be frozen to disallow `&mut self` methods, which could break a runtime invariant + rigged_args.options.frozen = parse_quote!(frozen); + // Needs to be subclassable by the variant PyClasses + rigged_args.options.subclass = parse_quote!(subclass); + rigged_args + }; + + let krate = get_pyo3_crate(&args.options.krate); + let cls = complex_enum.ident; + let variants = complex_enum.variants; + let pytypeinfo = impl_pytypeinfo(cls, &args, None); + + let default_slots = vec![]; + + let impl_builder = PyClassImplsBuilder::new( + cls, + &args, + methods_type, + complex_enum_default_methods( + cls, + variants + .iter() + .map(|v| (v.get_ident(), v.get_python_name(&args))), + ), + default_slots, + ) + .doc(doc); + + // Need to customize the into_py impl so that it returns the variant PyClass + let enum_into_py_impl = { + let match_arms: Vec = variants + .iter() + .map(|variant| { + let variant_ident = variant.get_ident(); + let variant_cls = gen_complex_enum_variant_class_ident(cls, variant.get_ident()); + quote! { + #cls::#variant_ident { .. } => { + let pyclass_init = _pyo3::PyClassInitializer::from(self).add_subclass(#variant_cls); + let variant_value = _pyo3::Py::new(py, pyclass_init).unwrap(); + _pyo3::IntoPy::into_py(variant_value, py) + } + } + }) + .collect(); + + quote! { + impl _pyo3::IntoPy<_pyo3::PyObject> for #cls { + fn into_py(self, py: _pyo3::Python) -> _pyo3::PyObject { + match self { + #(#match_arms)* + } + } + } + } + }; + + let pyclass_impls: TokenStream = vec![ + impl_builder.impl_pyclass(), + impl_builder.impl_extractext(), + enum_into_py_impl, + impl_builder.impl_pyclassimpl()?, + impl_builder.impl_freelist(), + ] + .into_iter() + .collect(); + + let mut variant_cls_zsts = vec![]; + let mut variant_cls_pytypeinfos = vec![]; + let mut variant_cls_pyclass_impls = vec![]; + let mut variant_cls_impls = vec![]; + for variant in &variants { + let variant_cls = gen_complex_enum_variant_class_ident(cls, variant.get_ident()); + + let variant_cls_zst = quote! { + #[doc(hidden)] + #[allow(non_camel_case_types)] + struct #variant_cls; + }; + variant_cls_zsts.push(variant_cls_zst); + + let variant_args = PyClassArgs { + class_kind: PyClassKind::Struct, + // TODO(mkovaxx): propagate variant.options + options: parse_quote!(extends = #cls, frozen), + }; + + let variant_cls_pytypeinfo = impl_pytypeinfo(&variant_cls, &variant_args, None); + variant_cls_pytypeinfos.push(variant_cls_pytypeinfo); + + let variant_new = complex_enum_variant_new(cls, variant)?; + + let (variant_cls_impl, field_getters) = impl_complex_enum_variant_cls(cls, variant)?; + variant_cls_impls.push(variant_cls_impl); + + let pyclass_impl = PyClassImplsBuilder::new( + &variant_cls, + &variant_args, + methods_type, + field_getters, + vec![variant_new], + ) + .impl_all()?; + + variant_cls_pyclass_impls.push(pyclass_impl); + } + + Ok(quote! { + const _: () = { + use #krate as _pyo3; + + #pytypeinfo + + #pyclass_impls + + #[doc(hidden)] + #[allow(non_snake_case)] + impl #cls {} + + #(#variant_cls_zsts)* + + #(#variant_cls_pytypeinfos)* + + #(#variant_cls_pyclass_impls)* + + #(#variant_cls_impls)* + }; + }) +} + +fn impl_complex_enum_variant_cls( + enum_name: &syn::Ident, + variant: &PyClassEnumVariant<'_>, +) -> Result<(TokenStream, Vec)> { + match variant { + PyClassEnumVariant::Struct(struct_variant) => { + impl_complex_enum_struct_variant_cls(enum_name, struct_variant) + } + } +} + +fn impl_complex_enum_struct_variant_cls( + enum_name: &syn::Ident, + variant: &PyClassEnumStructVariant<'_>, +) -> Result<(TokenStream, Vec)> { + let variant_ident = &variant.ident; + let variant_cls = gen_complex_enum_variant_class_ident(enum_name, variant.ident); + let variant_cls_type = parse_quote!(#variant_cls); + + let mut field_names: Vec = vec![]; + let mut fields_with_types: Vec = vec![]; + let mut field_getters = vec![]; + let mut field_getter_impls: Vec = vec![]; + for field in &variant.fields { + let field_name = field.ident; + let field_type = field.ty; + let field_with_type = quote! { #field_name: #field_type }; + + let field_getter = complex_enum_variant_field_getter( + &variant_cls_type, + field_name, + field_type, + field.span, + )?; + + let field_getter_impl = quote! { + fn #field_name(slf: _pyo3::PyRef) -> _pyo3::PyResult<#field_type> { + match &*slf.into_super() { + #enum_name::#variant_ident { #field_name, .. } => Ok(#field_name.clone()), + _ => unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), + } + } + }; + + field_names.push(field_name.clone()); + fields_with_types.push(field_with_type); + field_getters.push(field_getter); + field_getter_impls.push(field_getter_impl); + } + + let cls_impl = quote! { + #[doc(hidden)] + #[allow(non_snake_case)] + impl #variant_cls { + fn __pymethod_constructor__(py: _pyo3::Python<'_>, #(#fields_with_types,)*) -> _pyo3::PyClassInitializer<#variant_cls> { + let base_value = #enum_name::#variant_ident { #(#field_names,)* }; + _pyo3::PyClassInitializer::from(base_value).add_subclass(#variant_cls) + } + + #(#field_getter_impls)* + } + }; + + Ok((cls_impl, field_getters)) +} + +fn gen_complex_enum_variant_class_ident(enum_: &syn::Ident, variant: &syn::Ident) -> syn::Ident { + format_ident!("{}_{}", enum_, variant) +} + fn generate_default_protocol_slot( cls: &syn::Type, method: &mut syn::ImplItemFn, @@ -645,7 +1026,7 @@ fn generate_default_protocol_slot( ) } -fn enum_default_methods<'a>( +fn simple_enum_default_methods<'a>( cls: &'a syn::Ident, unit_variant_names: impl IntoIterator)>, ) -> Vec { @@ -667,14 +1048,167 @@ fn enum_default_methods<'a>( .collect() } -fn extract_variant_data(variant: &mut syn::Variant) -> syn::Result> { - use syn::Fields; - let ident = match variant.fields { - Fields::Unit => &variant.ident, - _ => bail_spanned!(variant.span() => "Currently only support unit variants."), +fn complex_enum_default_methods<'a>( + cls: &'a syn::Ident, + variant_names: impl IntoIterator)>, +) -> Vec { + let cls_type = syn::parse_quote!(#cls); + let variant_to_attribute = |var_ident: &syn::Ident, py_ident: &syn::Ident| ConstSpec { + rust_ident: var_ident.clone(), + attributes: ConstAttributes { + is_class_attr: true, + name: Some(NameAttribute { + kw: syn::parse_quote! { name }, + value: NameLitStr(py_ident.clone()), + }), + deprecations: Default::default(), + }, + }; + variant_names + .into_iter() + .map(|(var, py_name)| { + gen_complex_enum_variant_attr(cls, &cls_type, &variant_to_attribute(var, &py_name)) + }) + .collect() +} + +pub fn gen_complex_enum_variant_attr( + cls: &syn::Ident, + cls_type: &syn::Type, + spec: &ConstSpec, +) -> MethodAndMethodDef { + let member = &spec.rust_ident; + let wrapper_ident = format_ident!("__pymethod_variant_cls_{}__", member); + let deprecations = &spec.attributes.deprecations; + let python_name = &spec.null_terminated_python_name(); + + let variant_cls = format_ident!("{}_{}", cls, member); + let associated_method = quote! { + fn #wrapper_ident(py: _pyo3::Python<'_>) -> _pyo3::PyResult<_pyo3::PyObject> { + #deprecations + ::std::result::Result::Ok(py.get_type::<#variant_cls>().into()) + } + }; + + let method_def = quote! { + _pyo3::class::PyMethodDefType::ClassAttribute({ + _pyo3::class::PyClassAttributeDef::new( + #python_name, + _pyo3::impl_::pymethods::PyClassAttributeFactory(#cls_type::#wrapper_ident) + ) + }) + }; + + MethodAndMethodDef { + associated_method, + method_def, + } +} + +fn complex_enum_variant_new<'a>( + cls: &'a syn::Ident, + variant: &'a PyClassEnumVariant<'a>, +) -> Result { + match variant { + PyClassEnumVariant::Struct(struct_variant) => { + complex_enum_struct_variant_new(cls, struct_variant) + } + } +} + +fn complex_enum_struct_variant_new<'a>( + cls: &'a syn::Ident, + variant: &'a PyClassEnumStructVariant<'a>, +) -> Result { + let variant_cls = format_ident!("{}_{}", cls, variant.ident); + let variant_cls_type: syn::Type = parse_quote!(#variant_cls); + + let arg_py_ident: syn::Ident = parse_quote!(py); + let arg_py_type: syn::Type = parse_quote!(_pyo3::Python<'_>); + + let args = { + let mut no_pyo3_attrs = vec![]; + let attrs = crate::pyfunction::PyFunctionArgPyO3Attributes::from_attrs(&mut no_pyo3_attrs)?; + + let mut args = vec![ + // py: Python<'_> + FnArg { + name: &arg_py_ident, + ty: &arg_py_type, + optional: None, + default: None, + py: true, + attrs: attrs.clone(), + is_varargs: false, + is_kwargs: false, + is_cancel_handle: false, + }, + ]; + + for field in &variant.fields { + args.push(FnArg { + name: field.ident, + ty: field.ty, + optional: None, + default: None, + py: false, + attrs: attrs.clone(), + is_varargs: false, + is_kwargs: false, + is_cancel_handle: false, + }); + } + args + }; + let signature = crate::pyfunction::FunctionSignature::from_arguments(args)?; + + let spec = FnSpec { + tp: crate::method::FnType::FnNew, + name: &format_ident!("__pymethod_constructor__"), + python_name: format_ident!("__new__"), + signature, + output: variant_cls_type.clone(), + convention: crate::method::CallingConvention::TpNew, + text_signature: None, + asyncness: None, + unsafety: None, + deprecations: Deprecations::default(), + }; + + crate::pymethod::impl_py_method_def_new(&variant_cls_type, &spec) +} + +fn complex_enum_variant_field_getter<'a>( + variant_cls_type: &'a syn::Type, + field_name: &'a syn::Ident, + field_type: &'a syn::Type, + field_span: Span, +) -> Result { + let signature = crate::pyfunction::FunctionSignature::from_arguments(vec![])?; + + let self_type = crate::method::SelfType::TryFromPyCell(field_span); + + let spec = FnSpec { + tp: crate::method::FnType::Getter(self_type.clone()), + name: field_name, + python_name: field_name.clone(), + signature, + output: field_type.clone(), + convention: crate::method::CallingConvention::Noargs, + text_signature: None, + asyncness: None, + unsafety: None, + deprecations: Deprecations::default(), + }; + + let property_type = crate::pymethod::PropertyType::Function { + self_type: &self_type, + spec: &spec, + doc: crate::get_doc(&[], None), }; - let options = EnumVariantPyO3Options::take_pyo3_options(&mut variant.attrs)?; - Ok(PyClassEnumVariant { ident, options }) + + let getter = crate::pymethod::impl_py_getter_def(variant_cls_type, property_type)?; + Ok(getter) } fn descriptors_to_items( diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 8a97c712cac..d45d2e12f26 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -324,7 +324,8 @@ pub fn impl_py_method_def( }) } -fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec<'_>) -> Result { +/// Also used by pyclass. +pub fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec<'_>) -> Result { let wrapper_ident = syn::Ident::new("__pymethod___new____", Span::call_site()); let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls))?; // Use just the text_signature_call_signature() because the class' Python name diff --git a/pytests/noxfile.py b/pytests/noxfile.py index 57d9d63a044..7c681ab1aa8 100644 --- a/pytests/noxfile.py +++ b/pytests/noxfile.py @@ -1,4 +1,5 @@ import nox +import sys from nox.command import CommandFailed nox.options.sessions = ["test"] @@ -13,7 +14,12 @@ def test(session: nox.Session): except CommandFailed: # No binary wheel for numpy available on this platform pass - session.run("pytest", *session.posargs) + ignored_paths = [] + if sys.version_info < (3, 10): + # Match syntax is only available in Python >= 3.10 + ignored_paths.append("tests/test_enums_match.py") + ignore_args = [f"--ignore={path}" for path in ignored_paths] + session.run("pytest", *ignore_args, *session.posargs) @nox.session diff --git a/pytests/src/enums.rs b/pytests/src/enums.rs new file mode 100644 index 00000000000..11b592d3563 --- /dev/null +++ b/pytests/src/enums.rs @@ -0,0 +1,58 @@ +use pyo3::{pyclass, pyfunction, pymodule, types::PyModule, wrap_pyfunction, PyResult, Python}; + +#[pymodule] +pub fn enums(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_wrapped(wrap_pyfunction!(do_simple_stuff))?; + m.add_wrapped(wrap_pyfunction!(do_complex_stuff))?; + Ok(()) +} + +#[pyclass] +pub enum SimpleEnum { + Sunday, + Monday, + Tuesday, + Wednesday, + Thursday, + Friday, + Saturday, +} + +#[pyfunction] +pub fn do_simple_stuff(thing: &SimpleEnum) -> SimpleEnum { + match thing { + SimpleEnum::Sunday => SimpleEnum::Monday, + SimpleEnum::Monday => SimpleEnum::Tuesday, + SimpleEnum::Tuesday => SimpleEnum::Wednesday, + SimpleEnum::Wednesday => SimpleEnum::Thursday, + SimpleEnum::Thursday => SimpleEnum::Friday, + SimpleEnum::Friday => SimpleEnum::Saturday, + SimpleEnum::Saturday => SimpleEnum::Sunday, + } +} + +#[pyclass] +pub enum ComplexEnum { + Int { i: i32 }, + Float { f: f64 }, + Str { s: String }, + EmptyStruct {}, + MultiFieldStruct { a: i32, b: f64, c: bool }, +} + +#[pyfunction] +pub fn do_complex_stuff(thing: &ComplexEnum) -> ComplexEnum { + match thing { + ComplexEnum::Int { i } => ComplexEnum::Str { s: i.to_string() }, + ComplexEnum::Float { f } => ComplexEnum::Float { f: f * f }, + ComplexEnum::Str { s } => ComplexEnum::Int { i: s.len() as i32 }, + ComplexEnum::EmptyStruct {} => ComplexEnum::EmptyStruct {}, + ComplexEnum::MultiFieldStruct { a, b, c } => ComplexEnum::MultiFieldStruct { + a: *a, + b: *b, + c: *c, + }, + } +} diff --git a/pytests/src/lib.rs b/pytests/src/lib.rs index dbcd3ca4113..e65385bf679 100644 --- a/pytests/src/lib.rs +++ b/pytests/src/lib.rs @@ -7,6 +7,7 @@ pub mod buf_and_str; pub mod comparisons; pub mod datetime; pub mod dict_iter; +pub mod enums; pub mod misc; pub mod objstore; pub mod othermod; @@ -25,6 +26,7 @@ fn pyo3_pytests(py: Python<'_>, m: &PyModule) -> PyResult<()> { #[cfg(not(Py_LIMITED_API))] m.add_wrapped(wrap_pymodule!(datetime::datetime))?; m.add_wrapped(wrap_pymodule!(dict_iter::dict_iter))?; + m.add_wrapped(wrap_pymodule!(enums::enums))?; m.add_wrapped(wrap_pymodule!(misc::misc))?; m.add_wrapped(wrap_pymodule!(objstore::objstore))?; m.add_wrapped(wrap_pymodule!(othermod::othermod))?; @@ -44,6 +46,7 @@ fn pyo3_pytests(py: Python<'_>, m: &PyModule) -> PyResult<()> { sys_modules.set_item("pyo3_pytests.comparisons", m.getattr("comparisons")?)?; sys_modules.set_item("pyo3_pytests.datetime", m.getattr("datetime")?)?; sys_modules.set_item("pyo3_pytests.dict_iter", m.getattr("dict_iter")?)?; + sys_modules.set_item("pyo3_pytests.enums", m.getattr("enums")?)?; sys_modules.set_item("pyo3_pytests.misc", m.getattr("misc")?)?; sys_modules.set_item("pyo3_pytests.objstore", m.getattr("objstore")?)?; sys_modules.set_item("pyo3_pytests.othermod", m.getattr("othermod")?)?; diff --git a/pytests/tests/test_enums.py b/pytests/tests/test_enums.py new file mode 100644 index 00000000000..04b0cdca431 --- /dev/null +++ b/pytests/tests/test_enums.py @@ -0,0 +1,116 @@ +import pytest +from pyo3_pytests import enums + + +def test_complex_enum_variant_constructors(): + int_variant = enums.ComplexEnum.Int(42) + assert isinstance(int_variant, enums.ComplexEnum.Int) + + float_variant = enums.ComplexEnum.Float(3.14) + assert isinstance(float_variant, enums.ComplexEnum.Float) + + str_variant = enums.ComplexEnum.Str("hello") + assert isinstance(str_variant, enums.ComplexEnum.Str) + + empty_struct_variant = enums.ComplexEnum.EmptyStruct() + assert isinstance(empty_struct_variant, enums.ComplexEnum.EmptyStruct) + + multi_field_struct_variant = enums.ComplexEnum.MultiFieldStruct(42, 3.14, True) + assert isinstance(multi_field_struct_variant, enums.ComplexEnum.MultiFieldStruct) + + +@pytest.mark.parametrize( + "variant", + [ + enums.ComplexEnum.Int(42), + enums.ComplexEnum.Float(3.14), + enums.ComplexEnum.Str("hello"), + enums.ComplexEnum.EmptyStruct(), + enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + ], +) +def test_complex_enum_variant_subclasses(variant: enums.ComplexEnum): + assert isinstance(variant, enums.ComplexEnum) + + +def test_complex_enum_field_getters(): + int_variant = enums.ComplexEnum.Int(42) + assert int_variant.i == 42 + + float_variant = enums.ComplexEnum.Float(3.14) + assert float_variant.f == 3.14 + + str_variant = enums.ComplexEnum.Str("hello") + assert str_variant.s == "hello" + + multi_field_struct_variant = enums.ComplexEnum.MultiFieldStruct(42, 3.14, True) + assert multi_field_struct_variant.a == 42 + assert multi_field_struct_variant.b == 3.14 + assert multi_field_struct_variant.c is True + + +@pytest.mark.parametrize( + "variant", + [ + enums.ComplexEnum.Int(42), + enums.ComplexEnum.Float(3.14), + enums.ComplexEnum.Str("hello"), + enums.ComplexEnum.EmptyStruct(), + enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + ], +) +def test_complex_enum_desugared_match(variant: enums.ComplexEnum): + if isinstance(variant, enums.ComplexEnum.Int): + x = variant.i + assert x == 42 + elif isinstance(variant, enums.ComplexEnum.Float): + x = variant.f + assert x == 3.14 + elif isinstance(variant, enums.ComplexEnum.Str): + x = variant.s + assert x == "hello" + elif isinstance(variant, enums.ComplexEnum.EmptyStruct): + assert True + elif isinstance(variant, enums.ComplexEnum.MultiFieldStruct): + x = variant.a + y = variant.b + z = variant.c + assert x == 42 + assert y == 3.14 + assert z is True + else: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.ComplexEnum.Int(42), + enums.ComplexEnum.Float(3.14), + enums.ComplexEnum.Str("hello"), + enums.ComplexEnum.EmptyStruct(), + enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + ], +) +def test_complex_enum_pyfunction_in_out_desugared_match(variant: enums.ComplexEnum): + variant = enums.do_complex_stuff(variant) + if isinstance(variant, enums.ComplexEnum.Int): + x = variant.i + assert x == 5 + elif isinstance(variant, enums.ComplexEnum.Float): + x = variant.f + assert x == 9.8596 + elif isinstance(variant, enums.ComplexEnum.Str): + x = variant.s + assert x == "42" + elif isinstance(variant, enums.ComplexEnum.EmptyStruct): + assert True + elif isinstance(variant, enums.ComplexEnum.MultiFieldStruct): + x = variant.a + y = variant.b + z = variant.c + assert x == 42 + assert y == 3.14 + assert z is True + else: + assert False diff --git a/pytests/tests/test_enums_match.py b/pytests/tests/test_enums_match.py new file mode 100644 index 00000000000..4d55bbbe351 --- /dev/null +++ b/pytests/tests/test_enums_match.py @@ -0,0 +1,59 @@ +# This file is only collected when Python >= 3.10, because it tests match syntax. +import pytest +from pyo3_pytests import enums + + +@pytest.mark.parametrize( + "variant", + [ + enums.ComplexEnum.Int(42), + enums.ComplexEnum.Float(3.14), + enums.ComplexEnum.Str("hello"), + enums.ComplexEnum.EmptyStruct(), + enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + ], +) +def test_complex_enum_match_statement(variant: enums.ComplexEnum): + match variant: + case enums.ComplexEnum.Int(i=x): + assert x == 42 + case enums.ComplexEnum.Float(f=x): + assert x == 3.14 + case enums.ComplexEnum.Str(s=x): + assert x == "hello" + case enums.ComplexEnum.EmptyStruct(): + assert True + case enums.ComplexEnum.MultiFieldStruct(a=x, b=y, c=z): + assert x == 42 + assert y == 3.14 + assert z is True + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.ComplexEnum.Int(42), + enums.ComplexEnum.Float(3.14), + enums.ComplexEnum.Str("hello"), + enums.ComplexEnum.EmptyStruct(), + enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + ], +) +def test_complex_enum_pyfunction_in_out(variant: enums.ComplexEnum): + match enums.do_complex_stuff(variant): + case enums.ComplexEnum.Int(i=x): + assert x == 5 + case enums.ComplexEnum.Float(f=x): + assert x == 9.8596 + case enums.ComplexEnum.Str(s=x): + assert x == "42" + case enums.ComplexEnum.EmptyStruct(): + assert True + case enums.ComplexEnum.MultiFieldStruct(a=x, b=y, c=z): + assert x == 42 + assert y == 3.14 + assert z is True + case _: + assert False diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 0154f3f12bc..adcef887f5c 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -14,6 +14,7 @@ fn test_compile_errors() { #[cfg(any(not(Py_LIMITED_API), Py_3_11))] t.compile_fail("tests/ui/invalid_pymethods_buffer.rs"); t.compile_fail("tests/ui/invalid_pymethods_duplicates.rs"); + t.compile_fail("tests/ui/invalid_pymethod_enum.rs"); t.compile_fail("tests/ui/invalid_pymethod_names.rs"); t.compile_fail("tests/ui/invalid_pymodule_args.rs"); t.compile_fail("tests/ui/reject_generics.rs"); diff --git a/tests/ui/invalid_pyclass_enum.rs b/tests/ui/invalid_pyclass_enum.rs index 4bc53238a2c..95879c2fbd1 100644 --- a/tests/ui/invalid_pyclass_enum.rs +++ b/tests/ui/invalid_pyclass_enum.rs @@ -15,4 +15,16 @@ enum NotDrivedClass { #[pyclass] enum NoEmptyEnum {} +#[pyclass] +enum NoUnitVariants { + StructVariant { field: i32 }, + UnitVariant, +} + +#[pyclass] +enum NoTupleVariants { + StructVariant { field: i32 }, + TupleVariant(i32), +} + fn main() {} diff --git a/tests/ui/invalid_pyclass_enum.stderr b/tests/ui/invalid_pyclass_enum.stderr index 8f340a762bb..a03a0ae2814 100644 --- a/tests/ui/invalid_pyclass_enum.stderr +++ b/tests/ui/invalid_pyclass_enum.stderr @@ -15,3 +15,19 @@ error: #[pyclass] can't be used on enums without any variants | 16 | enum NoEmptyEnum {} | ^^ + +error: Unit variant `UnitVariant` is not yet supported in a complex enum + = help: change to a struct variant with no fields: `UnitVariant { }` + = note: the enum is complex because of non-unit variant `StructVariant` + --> tests/ui/invalid_pyclass_enum.rs:21:5 + | +21 | UnitVariant, + | ^^^^^^^^^^^ + +error: Tuple variant `TupleVariant` is not yet supported in a complex enum + = help: change to a struct variant with named fields: `TupleVariant { /* fields */ }` + = note: the enum is complex because of non-unit variant `StructVariant` + --> tests/ui/invalid_pyclass_enum.rs:27:5 + | +27 | TupleVariant(i32), + | ^^^^^^^^^^^^ diff --git a/tests/ui/invalid_pymethod_enum.rs b/tests/ui/invalid_pymethod_enum.rs new file mode 100644 index 00000000000..9b596e087ff --- /dev/null +++ b/tests/ui/invalid_pymethod_enum.rs @@ -0,0 +1,19 @@ +use pyo3::prelude::*; + +#[pyclass] +enum ComplexEnum { + Int { int: i32 }, + Str { string: String }, +} + +#[pymethods] +impl ComplexEnum { + fn mutate_in_place(&mut self) { + *self = match self { + ComplexEnum::Int { int } => ComplexEnum::Str { string: int.to_string() }, + ComplexEnum::Str { string } => ComplexEnum::Int { int: string.len() as i32 }, + } + } +} + +fn main() {} diff --git a/tests/ui/invalid_pymethod_enum.stderr b/tests/ui/invalid_pymethod_enum.stderr new file mode 100644 index 00000000000..bb327dccb5d --- /dev/null +++ b/tests/ui/invalid_pymethod_enum.stderr @@ -0,0 +1,11 @@ +error[E0271]: type mismatch resolving `::Frozen == False` + --> tests/ui/invalid_pymethod_enum.rs:11:24 + | +11 | fn mutate_in_place(&mut self) { + | ^ expected `False`, found `True` + | +note: required by a bound in `extract_pyclass_ref_mut` + --> src/impl_/extract_argument.rs + | + | pub fn extract_pyclass_ref_mut<'a, 'py: 'a, T: PyClass>( + | ^^^^^^^^^^^^^^ required by this bound in `extract_pyclass_ref_mut` From f32becacc700fae53e3f848ca7c0dd6e5b204187 Mon Sep 17 00:00:00 2001 From: Mate Kovacs Date: Sat, 20 Jan 2024 11:47:27 +0900 Subject: [PATCH 046/349] fix ugly example --- guide/src/class.md | 41 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index 356e7c71a7d..37265aaa2bc 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -1131,36 +1131,12 @@ enum Shape { Nothing { }, } +# #[cfg(Py_3_10)] Python::with_gil(|py| { - let def_count_vertices = if py.version_info() >= (3, 10) { r#" - def count_vertices(cls, shape): - match shape: - case cls.Circle(): - return 0 - case cls.Rectangle(): - return 4 - case cls.RegularPolygon(side_count=n): - return n - case cls.Nothing(): - return 0 - "# } else { r#" - def count_vertices(cls, shape): - if isinstance(shape, cls.Circle): - return 0 - elif isinstance(shape, cls.Rectangle): - return 4 - elif isinstance(shape, cls.RegularPolygon): - n = shape.side_count - return n - elif isinstance(shape, cls.Nothing): - return 0 - "# }; - let circle = Shape::Circle { radius: 10.0 }.into_py(py); let square = Shape::RegularPolygon { side_count: 4, radius: 10.0 }.into_py(py); let cls = py.get_type::(); - - pyo3::py_run!(py, circle square cls, &format!(r#" + pyo3::py_run!(py, circle square cls, r#" assert isinstance(circle, cls) assert isinstance(circle, cls.Circle) assert circle.radius == 10.0 @@ -1170,11 +1146,20 @@ Python::with_gil(|py| { assert square.side_count == 4 assert square.radius == 10.0 - {} + def count_vertices(cls, shape): + match shape: + case cls.Circle(): + return 0 + case cls.Rectangle(): + return 4 + case cls.RegularPolygon(side_count=n): + return n + case cls.Nothing(): + return 0 assert count_vertices(cls, circle) == 0 assert count_vertices(cls, square) == 4 - "#, def_count_vertices)) + "#) }) ``` From bcfbbf198dd5cb1b447a2570a5fd097cd9b11383 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 20:07:15 +0000 Subject: [PATCH 047/349] Bump actions/cache from 3 to 4 Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/benches.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/gh-pages.yml | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 5c6e781dae1..01ab5f802dd 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -19,7 +19,7 @@ jobs: with: components: rust-src - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: | ~/.cargo/registry diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fe7af2747a..79e6b9ee78c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -407,7 +407,7 @@ jobs: with: node-version: 14 - run: python -m pip install --upgrade pip && pip install nox - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: cache with: path: | diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index b151c6e18db..ec8874d5841 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -63,7 +63,7 @@ jobs: - uses: actions/setup-python@v5 - uses: dtolnay/rust-toolchain@stable - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: | ~/.cargo/registry @@ -82,7 +82,7 @@ jobs: # Download previous benchmark result from cache (if exists) - name: Download previous benchmark data - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./cache key: ${{ runner.os }}-benchmark @@ -110,7 +110,7 @@ jobs: - uses: actions/setup-python@v5 - uses: dtolnay/rust-toolchain@stable - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: | ~/.cargo/registry @@ -120,7 +120,7 @@ jobs: continue-on-error: true - name: Download previous benchmark data - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./cache key: ${{ runner.os }}-pytest-benchmark From 4d40f4183f4e69ba816532cc50df196fdbb0fcb3 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 23 Jan 2024 14:39:38 +0800 Subject: [PATCH 048/349] docs: Update opendal's repo name Apache OpenDAL is now a graduated project, remove the incubator prefix in the repo name. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa95ede4c6c..98eeeb80242 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ about this topic. - [inline-python](https://github.com/fusion-engineering/inline-python) _Inline Python code directly in your Rust code._ - [jsonschema-rs](https://github.com/Stranger6667/jsonschema-rs/tree/master/bindings/python) _Fast JSON Schema validation library._ - [mocpy](https://github.com/cds-astro/mocpy) _Astronomical Python library offering data structures for describing any arbitrary coverage regions on the unit sphere._ -- [opendal](https://github.com/apache/incubator-opendal/tree/main/bindings/python) _A data access layer that allows users to easily and efficiently retrieve data from various storage services in a unified way._ +- [opendal](https://github.com/apache/opendal/tree/main/bindings/python) _A data access layer that allows users to easily and efficiently retrieve data from various storage services in a unified way._ - [orjson](https://github.com/ijl/orjson) _Fast Python JSON library._ - [ormsgpack](https://github.com/aviramha/ormsgpack) _Fast Python msgpack library._ - [point-process](https://github.com/ManifoldFR/point-process-rust/tree/master/pylib) _High level API for pointprocesses as a Python library._ From 11b5ae7f5f848039c1cca9ee6cf1a59754868fa7 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 23 Jan 2024 20:34:19 +0000 Subject: [PATCH 049/349] update ffi structures for PyPy 7.3.15 --- newsfragments/3757.fixed.md | 1 + pyo3-ffi/src/cpython/object.rs | 13 +++++-------- pyo3-ffi/src/pybuffer.rs | 13 +++++-------- 3 files changed, 11 insertions(+), 16 deletions(-) create mode 100644 newsfragments/3757.fixed.md diff --git a/newsfragments/3757.fixed.md b/newsfragments/3757.fixed.md new file mode 100644 index 00000000000..103a634af9f --- /dev/null +++ b/newsfragments/3757.fixed.md @@ -0,0 +1 @@ +Match PyPy 7.3.14 in removing PyPy-only symbol `Py_MAX_NDIMS` in favour of `PyBUF_MAX_NDIM`. diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index ce4f56f61d5..161fb50cf24 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -22,9 +22,6 @@ mod bufferinfo { use std::os::raw::{c_char, c_int, c_void}; use std::ptr; - #[cfg(PyPy)] - const Py_MAX_NDIMS: usize = if cfg!(Py_3_9) { 64 } else { 36 }; - #[repr(C)] #[derive(Copy, Clone)] pub struct Py_buffer { @@ -43,9 +40,9 @@ mod bufferinfo { #[cfg(PyPy)] pub flags: c_int, #[cfg(PyPy)] - pub _strides: [Py_ssize_t; Py_MAX_NDIMS], + pub _strides: [Py_ssize_t; PyBUF_MAX_NDIM as usize], #[cfg(PyPy)] - pub _shape: [Py_ssize_t; Py_MAX_NDIMS], + pub _shape: [Py_ssize_t; PyBUF_MAX_NDIM as usize], } impl Py_buffer { @@ -65,9 +62,9 @@ mod bufferinfo { #[cfg(PyPy)] flags: 0, #[cfg(PyPy)] - _strides: [0; Py_MAX_NDIMS], + _strides: [0; PyBUF_MAX_NDIM as usize], #[cfg(PyPy)] - _shape: [0; Py_MAX_NDIMS], + _shape: [0; PyBUF_MAX_NDIM as usize], } } } @@ -81,7 +78,7 @@ mod bufferinfo { unsafe extern "C" fn(arg1: *mut crate::PyObject, arg2: *mut Py_buffer); /// Maximum number of dimensions - pub const PyBUF_MAX_NDIM: c_int = 64; + pub const PyBUF_MAX_NDIM: c_int = if cfg!(PyPy) { 36 } else { 64 }; /* Flags for getting buffers */ pub const PyBUF_SIMPLE: c_int = 0; diff --git a/pyo3-ffi/src/pybuffer.rs b/pyo3-ffi/src/pybuffer.rs index 20f92fb6d2b..a414f333ce6 100644 --- a/pyo3-ffi/src/pybuffer.rs +++ b/pyo3-ffi/src/pybuffer.rs @@ -3,9 +3,6 @@ use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int, c_void}; use std::ptr; -#[cfg(PyPy)] -const Py_MAX_NDIMS: usize = 36; - #[repr(C)] #[derive(Copy, Clone)] pub struct Py_buffer { @@ -24,9 +21,9 @@ pub struct Py_buffer { #[cfg(PyPy)] pub flags: c_int, #[cfg(PyPy)] - pub _strides: [Py_ssize_t; Py_MAX_NDIMS], + pub _strides: [Py_ssize_t; PyBUF_MAX_NDIM], #[cfg(PyPy)] - pub _shape: [Py_ssize_t; Py_MAX_NDIMS], + pub _shape: [Py_ssize_t; PyBUF_MAX_NDIM], } impl Py_buffer { @@ -46,9 +43,9 @@ impl Py_buffer { #[cfg(PyPy)] flags: 0, #[cfg(PyPy)] - _strides: [0; Py_MAX_NDIMS], + _strides: [0; PyBUF_MAX_NDIM], #[cfg(PyPy)] - _shape: [0; Py_MAX_NDIMS], + _shape: [0; PyBUF_MAX_NDIM], } } } @@ -105,7 +102,7 @@ extern "C" { } /// Maximum number of dimensions -pub const PyBUF_MAX_NDIM: c_int = 64; +pub const PyBUF_MAX_NDIM: c_int = if cfg!(PyPy) { 36 } else { 64 }; /* Flags for getting buffers */ pub const PyBUF_SIMPLE: c_int = 0; From f86053e2c2528642f0dd15703805063a128204d8 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 26 Jan 2024 22:20:24 +0100 Subject: [PATCH 050/349] implement `PyTracebackMethods` --- guide/src/python_from_rust.md | 2 +- src/err/err_state.rs | 16 ++++++--- src/err/mod.rs | 34 +++++++++++++----- src/exceptions.rs | 3 +- src/prelude.rs | 1 + src/types/mod.rs | 2 +- src/types/traceback.rs | 68 ++++++++++++++++++++++++++++++----- 7 files changed, 101 insertions(+), 25 deletions(-) diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index b81ba919637..99da2e15434 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -479,7 +479,7 @@ class House(object): } Err(e) => { house - .call_method1("__exit__", (e.get_type(py), e.value(py), e.traceback(py))) + .call_method1("__exit__", (e.get_type(py), e.value(py), e.traceback_bound(py))) .unwrap(); } } diff --git a/src/err/err_state.rs b/src/err/err_state.rs index b9bda9b7013..8b31c2b7476 100644 --- a/src/err/err_state.rs +++ b/src/err/err_state.rs @@ -2,7 +2,7 @@ use crate::{ exceptions::{PyBaseException, PyTypeError}, ffi, types::{PyTraceback, PyType}, - IntoPy, Py, PyAny, PyObject, PyTypeInfo, Python, + Bound, IntoPy, Py, PyAny, PyObject, PyTypeInfo, Python, }; #[derive(Clone)] @@ -26,15 +26,21 @@ impl PyErrStateNormalized { } #[cfg(not(Py_3_12))] - pub(crate) fn ptraceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { + pub(crate) fn ptraceback<'py>(&self, py: Python<'py>) -> Option> { self.ptraceback .as_ref() - .map(|traceback| traceback.as_ref(py)) + .map(|traceback| traceback.bind(py).clone()) } #[cfg(Py_3_12)] - pub(crate) fn ptraceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { - unsafe { py.from_owned_ptr_or_opt(ffi::PyException_GetTraceback(self.pvalue.as_ptr())) } + pub(crate) fn ptraceback<'py>(&self, py: Python<'py>) -> Option> { + use crate::ffi_ptr_ext::FfiPtrExt; + use crate::types::any::PyAnyMethods; + unsafe { + ffi::PyException_GetTraceback(self.pvalue.as_ptr()) + .assume_owned_or_opt(py) + .map(|b| b.downcast_into_unchecked()) + } } #[cfg(Py_3_12)] diff --git a/src/err/mod.rs b/src/err/mod.rs index 39975183f07..e6bcd521887 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -272,6 +272,18 @@ impl PyErr { exc } + /// Deprecated form of [`PyErr::traceback_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyErr::traceback` will be replaced by `PyErr::traceback_bound` in a future PyO3 version" + ) + )] + pub fn traceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { + self.normalized(py).ptraceback(py).map(|b| b.into_gil_ref()) + } + /// Returns the traceback of this exception object. /// /// # Examples @@ -280,10 +292,10 @@ impl PyErr { /// /// Python::with_gil(|py| { /// let err = PyTypeError::new_err(("some type error",)); - /// assert!(err.traceback(py).is_none()); + /// assert!(err.traceback_bound(py).is_none()); /// }); /// ``` - pub fn traceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { + pub fn traceback_bound<'py>(&self, py: Python<'py>) -> Option> { self.normalized(py).ptraceback(py) } @@ -476,10 +488,16 @@ impl PyErr { #[cfg(not(Py_3_12))] unsafe { + // keep the bound `traceback` alive for entire duration of + // PyErr_Display. if we inline this, the `Bound` will be dropped + // after the argument got evaluated, leading to call with a dangling + // pointer. + let traceback = self.traceback_bound(py); ffi::PyErr_Display( self.get_type(py).as_ptr(), self.value(py).as_ptr(), - self.traceback(py) + traceback + .as_ref() .map_or(std::ptr::null_mut(), |traceback| traceback.as_ptr()), ) } @@ -658,15 +676,15 @@ impl PyErr { /// /// # Examples /// ```rust - /// use pyo3::{exceptions::PyTypeError, PyErr, Python}; + /// use pyo3::{exceptions::PyTypeError, PyErr, Python, prelude::PyAnyMethods}; /// Python::with_gil(|py| { /// let err: PyErr = PyTypeError::new_err(("some type error",)); /// let err_clone = err.clone_ref(py); /// assert!(err.get_type(py).is(err_clone.get_type(py))); /// assert!(err.value(py).is(err_clone.value(py))); - /// match err.traceback(py) { - /// None => assert!(err_clone.traceback(py).is_none()), - /// Some(tb) => assert!(err_clone.traceback(py).unwrap().is(tb)), + /// match err.traceback_bound(py) { + /// None => assert!(err_clone.traceback_bound(py).is_none()), + /// Some(tb) => assert!(err_clone.traceback_bound(py).unwrap().is(&tb)), /// } /// }); /// ``` @@ -747,7 +765,7 @@ impl std::fmt::Debug for PyErr { f.debug_struct("PyErr") .field("type", self.get_type(py)) .field("value", self.value(py)) - .field("traceback", &self.traceback(py)) + .field("traceback", &self.traceback_bound(py)) .finish() }) } diff --git a/src/exceptions.rs b/src/exceptions.rs index 6ff662b4ae4..19f59742435 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -100,6 +100,7 @@ macro_rules! import_exception { impl $name { fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { use $crate::sync::GILOnceCell; + use $crate::prelude::PyTracebackMethods; static TYPE_OBJECT: GILOnceCell<$crate::Py<$crate::types::PyType>> = GILOnceCell::new(); @@ -109,7 +110,7 @@ macro_rules! import_exception { .import(stringify!($module)) .unwrap_or_else(|err| { let traceback = err - .traceback(py) + .traceback_bound(py) .map(|tb| tb.format().expect("raised exception will have a traceback")) .unwrap_or_default(); ::std::panic!("Can not import module {}: {}\n{}", stringify!($module), err, traceback); diff --git a/src/prelude.rs b/src/prelude.rs index 6be820ae92a..06d283c91d7 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -38,4 +38,5 @@ pub use crate::types::module::PyModuleMethods; pub use crate::types::sequence::PySequenceMethods; pub use crate::types::set::PySetMethods; pub use crate::types::string::PyStringMethods; +pub use crate::types::traceback::PyTracebackMethods; pub use crate::types::tuple::PyTupleMethods; diff --git a/src/types/mod.rs b/src/types/mod.rs index 00fe81cf37e..fcf843ec6c7 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -304,6 +304,6 @@ pub(crate) mod sequence; pub(crate) mod set; mod slice; pub(crate) mod string; -mod traceback; +pub(crate) mod traceback; pub(crate) mod tuple; mod typeobject; diff --git a/src/types/traceback.rs b/src/types/traceback.rs index b9909435976..84ecda747eb 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -1,7 +1,7 @@ use crate::err::{error_on_minusone, PyResult}; -use crate::ffi; use crate::types::PyString; -use crate::PyAny; +use crate::{ffi, Bound}; +use crate::{PyAny, PyNativeType}; /// Represents a Python traceback. #[repr(transparent)] @@ -24,14 +24,14 @@ impl PyTraceback { /// The following code formats a Python traceback and exception pair from Rust: /// /// ```rust - /// # use pyo3::{Python, PyResult}; + /// # use pyo3::{Python, PyResult, prelude::PyTracebackMethods}; /// # let result: PyResult<()> = /// Python::with_gil(|py| { /// let err = py /// .run("raise Exception('banana')", None, None) /// .expect_err("raise will create a Python error"); /// - /// let traceback = err.traceback(py).expect("raised exception will have a traceback"); + /// let traceback = err.traceback_bound(py).expect("raised exception will have a traceback"); /// assert_eq!( /// format!("{}{}", traceback.format()?, err), /// "\ @@ -46,6 +46,53 @@ impl PyTraceback { /// # result.expect("example failed"); /// ``` pub fn format(&self) -> PyResult { + self.as_borrowed().format() + } +} + +/// Implementation of functionality for [`PyTraceback`]. +/// +/// These methods are defined for the `Bound<'py, PyTraceback>` smart pointer, so to use method call +/// syntax these methods are separated into a trait, because stable Rust does not yet support +/// `arbitrary_self_types`. +#[doc(alias = "PyTraceback")] +pub trait PyTracebackMethods<'py> { + /// Formats the traceback as a string. + /// + /// This does not include the exception type and value. The exception type and value can be + /// formatted using the `Display` implementation for `PyErr`. + /// + /// # Example + /// + /// The following code formats a Python traceback and exception pair from Rust: + /// + /// ```rust + /// # use pyo3::{Python, PyResult, prelude::PyTracebackMethods}; + /// # let result: PyResult<()> = + /// Python::with_gil(|py| { + /// let err = py + /// .run("raise Exception('banana')", None, None) + /// .expect_err("raise will create a Python error"); + /// + /// let traceback = err.traceback_bound(py).expect("raised exception will have a traceback"); + /// assert_eq!( + /// format!("{}{}", traceback.format()?, err), + /// "\ + /// Traceback (most recent call last): + /// File \"\", line 1, in + /// Exception: banana\ + /// " + /// ); + /// Ok(()) + /// }) + /// # ; + /// # result.expect("example failed"); + /// ``` + fn format(&self) -> PyResult; +} + +impl<'py> PyTracebackMethods<'py> for Bound<'py, PyTraceback> { + fn format(&self) -> PyResult { let py = self.py(); let string_io = py .import(intern!(py, "io"))? @@ -65,7 +112,10 @@ impl PyTraceback { #[cfg(test)] mod tests { - use crate::{prelude::*, types::PyDict}; + use crate::{ + prelude::*, + types::{traceback::PyTracebackMethods, PyDict}, + }; #[test] fn format_traceback() { @@ -75,7 +125,7 @@ mod tests { .expect_err("raising should have given us an error"); assert_eq!( - err.traceback(py).unwrap().format().unwrap(), + err.traceback_bound(py).unwrap().format().unwrap(), "Traceback (most recent call last):\n File \"\", line 1, in \n" ); }) @@ -99,7 +149,7 @@ except Exception as e: .unwrap(); let err = PyErr::from_value(locals.get_item("err").unwrap().unwrap()); let traceback = err.value(py).getattr("__traceback__").unwrap(); - assert!(err.traceback(py).unwrap().is(traceback)); + assert!(err.traceback_bound(py).unwrap().is(traceback)); }) } @@ -119,10 +169,10 @@ def f(): .unwrap(); let f = locals.get_item("f").unwrap().unwrap(); let err = f.call0().unwrap_err(); - let traceback = err.traceback(py).unwrap(); + let traceback = err.traceback_bound(py).unwrap(); let err_object = err.clone_ref(py).into_py(py).into_ref(py); - assert!(err_object.getattr("__traceback__").unwrap().is(traceback)); + assert!(err_object.getattr("__traceback__").unwrap().is(&traceback)); }) } } From 7918815cee4ffa7f0c5dd8ad1436d2f7a2a1671e Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 24 Jan 2024 23:51:00 +0100 Subject: [PATCH 051/349] implement `PySliceMethods` --- src/types/slice.rs | 68 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/src/types/slice.rs b/src/types/slice.rs index 53e4f4af992..8e86ff7ceee 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -1,6 +1,8 @@ use crate::err::{PyErr, PyResult}; use crate::ffi::{self, Py_ssize_t}; -use crate::{PyAny, PyObject, Python, ToPyObject}; +use crate::ffi_ptr_ext::FfiPtrExt; +use crate::types::any::PyAnyMethods; +use crate::{Bound, PyAny, PyNativeType, PyObject, Python, ToPyObject}; use std::os::raw::c_long; /// Represents a Python `slice`. @@ -42,19 +44,39 @@ impl PySliceIndices { } impl PySlice { - /// Constructs a new slice with the given elements. + /// Deprecated form of `PySlice::new_bound`. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PySlice::new` will be replaced by `PySlice::new_bound` in a future PyO3 version" + ) + )] pub fn new(py: Python<'_>, start: isize, stop: isize, step: isize) -> &PySlice { + Self::new_bound(py, start, stop, step).into_gil_ref() + } + + /// Constructs a new slice with the given elements. + pub fn new_bound(py: Python<'_>, start: isize, stop: isize, step: isize) -> Bound<'_, PySlice> { unsafe { - let ptr = ffi::PySlice_New( + ffi::PySlice_New( ffi::PyLong_FromSsize_t(start), ffi::PyLong_FromSsize_t(stop), ffi::PyLong_FromSsize_t(step), - ); - py.from_owned_ptr(ptr) + ) + .assume_owned(py) + .downcast_into_unchecked() } } - /// Constructs a new full slice that is equivalent to `::`. + /// Deprecated form of `PySlice::full_bound`. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PySlice::full` will be replaced by `PySlice::full_bound` in a future PyO3 version" + ) + )] pub fn full(py: Python<'_>) -> &PySlice { unsafe { let ptr = ffi::PySlice_New(ffi::Py_None(), ffi::Py_None(), ffi::Py_None()); @@ -62,11 +84,39 @@ impl PySlice { } } + /// Constructs a new full slice that is equivalent to `::`. + pub fn full_bound(py: Python<'_>) -> Bound<'_, PySlice> { + unsafe { + ffi::PySlice_New(ffi::Py_None(), ffi::Py_None(), ffi::Py_None()) + .assume_owned(py) + .downcast_into_unchecked() + } + } + /// Retrieves the start, stop, and step indices from the slice object, /// assuming a sequence of length `length`, and stores the length of the /// slice in its `slicelength` member. #[inline] pub fn indices(&self, length: c_long) -> PyResult { + self.as_borrowed().indices(length) + } +} + +/// Implementation of functionality for [`PySlice`]. +/// +/// These methods are defined for the `Bound<'py, PyTuple>` smart pointer, so to use method call +/// syntax these methods are separated into a trait, because stable Rust does not yet support +/// `arbitrary_self_types`. +#[doc(alias = "PySlice")] +pub trait PySliceMethods<'py> { + /// Retrieves the start, stop, and step indices from the slice object, + /// assuming a sequence of length `length`, and stores the length of the + /// slice in its `slicelength` member. + fn indices(&self, length: c_long) -> PyResult; +} + +impl<'py> PySliceMethods<'py> for Bound<'py, PySlice> { + fn indices(&self, length: c_long) -> PyResult { // non-negative Py_ssize_t should always fit into Rust usize unsafe { let mut slicelength: isize = 0; @@ -97,7 +147,7 @@ impl PySlice { impl ToPyObject for PySliceIndices { fn to_object(&self, py: Python<'_>) -> PyObject { - PySlice::new(py, self.start, self.stop, self.step).into() + PySlice::new_bound(py, self.start, self.stop, self.step).into() } } @@ -108,7 +158,7 @@ mod tests { #[test] fn test_py_slice_new() { Python::with_gil(|py| { - let slice = PySlice::new(py, isize::MIN, isize::MAX, 1); + let slice = PySlice::new_bound(py, isize::MIN, isize::MAX, 1); assert_eq!( slice.getattr("start").unwrap().extract::().unwrap(), isize::MIN @@ -127,7 +177,7 @@ mod tests { #[test] fn test_py_slice_full() { Python::with_gil(|py| { - let slice = PySlice::full(py); + let slice = PySlice::full_bound(py); assert!(slice.getattr("start").unwrap().is_none(),); assert!(slice.getattr("stop").unwrap().is_none(),); assert!(slice.getattr("step").unwrap().is_none(),); From 7fddd983b416ef8c2c5b4a58bb4adaf6e5b5cff7 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 27 Jan 2024 12:37:26 +0100 Subject: [PATCH 052/349] update `test_compile_error` ui test output --- tests/ui/invalid_result_conversion.stderr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ui/invalid_result_conversion.stderr b/tests/ui/invalid_result_conversion.stderr index 2720c71db63..b3e65517e36 100644 --- a/tests/ui/invalid_result_conversion.stderr +++ b/tests/ui/invalid_result_conversion.stderr @@ -5,8 +5,8 @@ error[E0277]: the trait bound `PyErr: From` is not satisfied | ^^^^^^^^^^^^^ the trait `From` is not implemented for `PyErr` | = help: the following other types implement trait `From`: - > > + > > >> >> From 87e0610b58dafb15a6c11b1afa8e28a7592f3ef6 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 27 Jan 2024 11:27:36 +0000 Subject: [PATCH 053/349] remove internal uses of `_Py_NewRef` --- src/impl_/pyclass.rs | 36 +++++++++++++++++------------------ src/types/any.rs | 4 +++- tests/test_buffer.rs | 4 ++-- tests/test_buffer_protocol.rs | 2 +- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 23cebb26705..fa03b81e455 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -510,11 +510,11 @@ macro_rules! define_pyclass_binary_operator_slot { #[inline] unsafe fn $lhs( self, - _py: Python<'_>, + py: Python<'_>, _slf: *mut ffi::PyObject, _other: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + Ok(py.NotImplemented().into_ptr()) } } @@ -525,11 +525,11 @@ macro_rules! define_pyclass_binary_operator_slot { #[inline] unsafe fn $rhs( self, - _py: Python<'_>, + py: Python<'_>, _slf: *mut ffi::PyObject, _other: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + Ok(py.NotImplemented().into_ptr()) } } @@ -700,12 +700,12 @@ slot_fragment_trait! { #[inline] unsafe fn __pow__( self, - _py: Python<'_>, + py: Python<'_>, _slf: *mut ffi::PyObject, _other: *mut ffi::PyObject, _mod: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + Ok(py.NotImplemented().into_ptr()) } } @@ -716,12 +716,12 @@ slot_fragment_trait! { #[inline] unsafe fn __rpow__( self, - _py: Python<'_>, + py: Python<'_>, _slf: *mut ffi::PyObject, _other: *mut ffi::PyObject, _mod: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + Ok(py.NotImplemented().into_ptr()) } } @@ -761,11 +761,11 @@ slot_fragment_trait! { #[inline] unsafe fn __lt__( self, - _py: Python<'_>, + py: Python<'_>, _slf: *mut ffi::PyObject, _other: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + Ok(py.NotImplemented().into_ptr()) } } @@ -776,11 +776,11 @@ slot_fragment_trait! { #[inline] unsafe fn __le__( self, - _py: Python<'_>, + py: Python<'_>, _slf: *mut ffi::PyObject, _other: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + Ok(py.NotImplemented().into_ptr()) } } @@ -791,11 +791,11 @@ slot_fragment_trait! { #[inline] unsafe fn __eq__( self, - _py: Python<'_>, + py: Python<'_>, _slf: *mut ffi::PyObject, _other: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + Ok(py.NotImplemented().into_ptr()) } } @@ -824,11 +824,11 @@ slot_fragment_trait! { #[inline] unsafe fn __gt__( self, - _py: Python<'_>, + py: Python<'_>, _slf: *mut ffi::PyObject, _other: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + Ok(py.NotImplemented().into_ptr()) } } @@ -839,11 +839,11 @@ slot_fragment_trait! { #[inline] unsafe fn __ge__( self, - _py: Python<'_>, + py: Python<'_>, _slf: *mut ffi::PyObject, _other: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + Ok(py.NotImplemented().into_ptr()) } } diff --git a/src/types/any.rs b/src/types/any.rs index 9cdf78d6c6b..4536c2b889f 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -965,7 +965,9 @@ impl PyAny { #[inline] pub fn into_ptr(&self) -> *mut ffi::PyObject { // Safety: self.as_ptr() returns a valid non-null pointer - unsafe { ffi::_Py_NewRef(self.as_ptr()) } + let ptr = self.as_ptr(); + unsafe { ffi::Py_INCREF(ptr) }; + ptr } /// Return a proxy object that delegates method calls to a parent or sibling class of type. diff --git a/tests/test_buffer.rs b/tests/test_buffer.rs index 12d1756fd92..de8638baa7e 100644 --- a/tests/test_buffer.rs +++ b/tests/test_buffer.rs @@ -41,8 +41,6 @@ impl TestBufferErrors { return Err(PyBufferError::new_err("Object is not writable")); } - (*view).obj = ffi::_Py_NewRef(slf.as_ptr()); - let bytes = &slf.buf; (*view).buf = bytes.as_ptr() as *mut c_void; @@ -80,6 +78,8 @@ impl TestBufferErrors { } } + (*view).obj = slf.into_ptr(); + Ok(()) } } diff --git a/tests/test_buffer_protocol.rs b/tests/test_buffer_protocol.rs index 4d5dc69f942..b9f3861c65a 100644 --- a/tests/test_buffer_protocol.rs +++ b/tests/test_buffer_protocol.rs @@ -155,7 +155,7 @@ unsafe fn fill_view_from_readonly_data( return Err(PyBufferError::new_err("Object is not writable")); } - (*view).obj = ffi::_Py_NewRef(owner.as_ptr()); + (*view).obj = owner.into_ptr(); (*view).buf = data.as_ptr() as *mut c_void; (*view).len = data.len() as isize; From f83544910f55bdf741c689241b3e00058f833660 Mon Sep 17 00:00:00 2001 From: Tpt Date: Thu, 11 Jan 2024 10:00:49 +0100 Subject: [PATCH 054/349] Adds conversion between SystemTime and datetime --- guide/src/conversions/tables.md | 2 +- newsfragments/3736.added.md | 1 + src/conversions/std/duration.rs | 196 ----------------- src/conversions/std/mod.rs | 2 +- src/conversions/std/time.rs | 374 ++++++++++++++++++++++++++++++++ 5 files changed, 377 insertions(+), 198 deletions(-) create mode 100644 newsfragments/3736.added.md delete mode 100755 src/conversions/std/duration.rs create mode 100755 src/conversions/std/time.rs diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index f9d716b324d..6b7c3bfbb80 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -29,7 +29,7 @@ The table below contains the Python type and the corresponding function argument | `type` | - | `&PyType` | | `module` | - | `&PyModule` | | `collections.abc.Buffer` | - | `PyBuffer` | -| `datetime.datetime` | - | `&PyDateTime` | +| `datetime.datetime` | `SystemTime` | `&PyDateTime` | | `datetime.date` | - | `&PyDate` | | `datetime.time` | - | `&PyTime` | | `datetime.tzinfo` | - | `&PyTzInfo` | diff --git a/newsfragments/3736.added.md b/newsfragments/3736.added.md new file mode 100644 index 00000000000..0d3a4a08c1a --- /dev/null +++ b/newsfragments/3736.added.md @@ -0,0 +1 @@ +Conversion between `std::time::SystemTime` and `datetime.datetime` \ No newline at end of file diff --git a/src/conversions/std/duration.rs b/src/conversions/std/duration.rs deleted file mode 100755 index e4540bd0aaa..00000000000 --- a/src/conversions/std/duration.rs +++ /dev/null @@ -1,196 +0,0 @@ -use crate::exceptions::PyValueError; -#[cfg(Py_LIMITED_API)] -use crate::sync::GILOnceCell; -#[cfg(Py_LIMITED_API)] -use crate::types::PyType; -#[cfg(not(Py_LIMITED_API))] -use crate::types::{PyDelta, PyDeltaAccess}; -#[cfg(Py_LIMITED_API)] -use crate::{intern, Py}; -use crate::{FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject}; -use std::time::Duration; - -const SECONDS_PER_DAY: u64 = 24 * 60 * 60; - -impl FromPyObject<'_> for Duration { - fn extract(obj: &PyAny) -> PyResult { - #[cfg(not(Py_LIMITED_API))] - let (days, seconds, microseconds) = { - let delta: &PyDelta = obj.downcast()?; - ( - delta.get_days(), - delta.get_seconds(), - delta.get_microseconds(), - ) - }; - #[cfg(Py_LIMITED_API)] - let (days, seconds, microseconds): (i32, i32, i32) = { - ( - obj.getattr(intern!(obj.py(), "days"))?.extract()?, - obj.getattr(intern!(obj.py(), "seconds"))?.extract()?, - obj.getattr(intern!(obj.py(), "microseconds"))?.extract()?, - ) - }; - - // We cast - let days = u64::try_from(days).map_err(|_| { - PyValueError::new_err( - "It is not possible to convert a negative timedelta to a Rust Duration", - ) - })?; - let seconds = u64::try_from(seconds).unwrap(); // 0 <= seconds < 3600*24 - let microseconds = u32::try_from(microseconds).unwrap(); // 0 <= microseconds < 1000000 - - // We convert - let total_seconds = days * SECONDS_PER_DAY + seconds; // We casted from i32, this can't overflow - let nanoseconds = microseconds.checked_mul(1_000).unwrap(); // 0 <= microseconds < 1000000 - - Ok(Duration::new(total_seconds, nanoseconds)) - } -} - -impl ToPyObject for Duration { - fn to_object(&self, py: Python<'_>) -> PyObject { - let days = self.as_secs() / SECONDS_PER_DAY; - let seconds = self.as_secs() % SECONDS_PER_DAY; - let microseconds = self.subsec_micros(); - - #[cfg(not(Py_LIMITED_API))] - { - PyDelta::new( - py, - days.try_into() - .expect("Too large Rust duration for timedelta"), - seconds.try_into().unwrap(), - microseconds.try_into().unwrap(), - false, - ) - .expect("failed to construct timedelta (overflow?)") - .into() - } - #[cfg(Py_LIMITED_API)] - { - static TIMEDELTA: GILOnceCell> = GILOnceCell::new(); - TIMEDELTA - .get_or_try_init_type_ref(py, "datetime", "timedelta") - .unwrap() - .call1((days, seconds, microseconds)) - .unwrap() - .into() - } - } -} - -impl IntoPy for Duration { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::panic; - - #[test] - fn test_frompyobject() { - Python::with_gil(|py| { - assert_eq!( - new_timedelta(py, 0, 0, 0).extract::().unwrap(), - Duration::new(0, 0) - ); - assert_eq!( - new_timedelta(py, 1, 0, 0).extract::().unwrap(), - Duration::new(86400, 0) - ); - assert_eq!( - new_timedelta(py, 0, 1, 0).extract::().unwrap(), - Duration::new(1, 0) - ); - assert_eq!( - new_timedelta(py, 0, 0, 1).extract::().unwrap(), - Duration::new(0, 1_000) - ); - assert_eq!( - new_timedelta(py, 1, 1, 1).extract::().unwrap(), - Duration::new(86401, 1_000) - ); - assert_eq!( - timedelta_class(py) - .getattr("max") - .unwrap() - .extract::() - .unwrap(), - Duration::new(86399999999999, 999999000) - ); - }); - } - - #[test] - fn test_frompyobject_negative() { - Python::with_gil(|py| { - assert_eq!( - new_timedelta(py, 0, -1, 0) - .extract::() - .unwrap_err() - .to_string(), - "ValueError: It is not possible to convert a negative timedelta to a Rust Duration" - ); - }) - } - - #[test] - fn test_topyobject() { - Python::with_gil(|py| { - let assert_eq = |l: PyObject, r: &PyAny| { - assert!(l.as_ref(py).eq(r).unwrap()); - }; - - assert_eq( - Duration::new(0, 0).to_object(py), - new_timedelta(py, 0, 0, 0), - ); - assert_eq( - Duration::new(86400, 0).to_object(py), - new_timedelta(py, 1, 0, 0), - ); - assert_eq( - Duration::new(1, 0).to_object(py), - new_timedelta(py, 0, 1, 0), - ); - assert_eq( - Duration::new(0, 1_000).to_object(py), - new_timedelta(py, 0, 0, 1), - ); - assert_eq( - Duration::new(0, 1).to_object(py), - new_timedelta(py, 0, 0, 0), - ); - assert_eq( - Duration::new(86401, 1_000).to_object(py), - new_timedelta(py, 1, 1, 1), - ); - assert_eq( - Duration::new(86399999999999, 999999000).to_object(py), - timedelta_class(py).getattr("max").unwrap(), - ); - }); - } - - #[test] - fn test_topyobject_overflow() { - Python::with_gil(|py| { - assert!(panic::catch_unwind(|| Duration::MAX.to_object(py)).is_err()); - }) - } - - fn new_timedelta(py: Python<'_>, days: i32, seconds: i32, microseconds: i32) -> &PyAny { - timedelta_class(py) - .call1((days, seconds, microseconds)) - .unwrap() - } - - fn timedelta_class(py: Python<'_>) -> &PyAny { - py.import("datetime").unwrap().getattr("timedelta").unwrap() - } -} diff --git a/src/conversions/std/mod.rs b/src/conversions/std/mod.rs index ebe1c955cc6..9b10b59fd3f 100644 --- a/src/conversions/std/mod.rs +++ b/src/conversions/std/mod.rs @@ -1,5 +1,4 @@ mod array; -mod duration; mod ipaddr; mod map; mod num; @@ -8,4 +7,5 @@ mod path; mod set; mod slice; mod string; +mod time; mod vec; diff --git a/src/conversions/std/time.rs b/src/conversions/std/time.rs new file mode 100755 index 00000000000..71cdfa23bdd --- /dev/null +++ b/src/conversions/std/time.rs @@ -0,0 +1,374 @@ +use crate::exceptions::{PyOverflowError, PyValueError}; +use crate::sync::GILOnceCell; +#[cfg(Py_LIMITED_API)] +use crate::types::PyType; +#[cfg(not(Py_LIMITED_API))] +use crate::types::{timezone_utc, PyDateTime, PyDelta, PyDeltaAccess}; +#[cfg(Py_LIMITED_API)] +use crate::Py; +use crate::{intern, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +const SECONDS_PER_DAY: u64 = 24 * 60 * 60; + +impl FromPyObject<'_> for Duration { + fn extract(obj: &PyAny) -> PyResult { + #[cfg(not(Py_LIMITED_API))] + let (days, seconds, microseconds) = { + let delta: &PyDelta = obj.downcast()?; + ( + delta.get_days(), + delta.get_seconds(), + delta.get_microseconds(), + ) + }; + #[cfg(Py_LIMITED_API)] + let (days, seconds, microseconds): (i32, i32, i32) = { + ( + obj.getattr(intern!(obj.py(), "days"))?.extract()?, + obj.getattr(intern!(obj.py(), "seconds"))?.extract()?, + obj.getattr(intern!(obj.py(), "microseconds"))?.extract()?, + ) + }; + + // We cast + let days = u64::try_from(days).map_err(|_| { + PyValueError::new_err( + "It is not possible to convert a negative timedelta to a Rust Duration", + ) + })?; + let seconds = u64::try_from(seconds).unwrap(); // 0 <= seconds < 3600*24 + let microseconds = u32::try_from(microseconds).unwrap(); // 0 <= microseconds < 1000000 + + // We convert + let total_seconds = days * SECONDS_PER_DAY + seconds; // We casted from i32, this can't overflow + let nanoseconds = microseconds.checked_mul(1_000).unwrap(); // 0 <= microseconds < 1000000 + + Ok(Duration::new(total_seconds, nanoseconds)) + } +} + +impl ToPyObject for Duration { + fn to_object(&self, py: Python<'_>) -> PyObject { + let days = self.as_secs() / SECONDS_PER_DAY; + let seconds = self.as_secs() % SECONDS_PER_DAY; + let microseconds = self.subsec_micros(); + + #[cfg(not(Py_LIMITED_API))] + { + PyDelta::new( + py, + days.try_into() + .expect("Too large Rust duration for timedelta"), + seconds.try_into().unwrap(), + microseconds.try_into().unwrap(), + false, + ) + .expect("failed to construct timedelta (overflow?)") + .into() + } + #[cfg(Py_LIMITED_API)] + { + static TIMEDELTA: GILOnceCell> = GILOnceCell::new(); + TIMEDELTA + .get_or_try_init_type_ref(py, "datetime", "timedelta") + .unwrap() + .call1((days, seconds, microseconds)) + .unwrap() + .into() + } + } +} + +impl IntoPy for Duration { + fn into_py(self, py: Python<'_>) -> PyObject { + self.to_object(py) + } +} + +// Conversions between SystemTime and datetime do not rely on the floating point timestamp of the +// timestamp/fromtimestamp APIs to avoid possible precision loss but goes through the +// timedelta/std::time::Duration types by taking for reference point the UNIX epoch. +// +// TODO: it might be nice to investigate using timestamps anyway, at least when the datetime is a safe range. + +impl FromPyObject<'_> for SystemTime { + fn extract(obj: &PyAny) -> PyResult { + let duration_since_unix_epoch: Duration = obj + .call_method1(intern!(obj.py(), "__sub__"), (unix_epoch_py(obj.py()),))? + .extract()?; + UNIX_EPOCH + .checked_add(duration_since_unix_epoch) + .ok_or_else(|| { + PyOverflowError::new_err("Overflow error when converting the time to Rust") + }) + } +} + +impl ToPyObject for SystemTime { + fn to_object(&self, py: Python<'_>) -> PyObject { + let duration_since_unix_epoch = self.duration_since(UNIX_EPOCH).unwrap().into_py(py); + unix_epoch_py(py) + .call_method1(py, intern!(py, "__add__"), (duration_since_unix_epoch,)) + .unwrap() + } +} + +impl IntoPy for SystemTime { + fn into_py(self, py: Python<'_>) -> PyObject { + self.to_object(py) + } +} + +fn unix_epoch_py(py: Python<'_>) -> &PyObject { + static UNIX_EPOCH: GILOnceCell = GILOnceCell::new(); + UNIX_EPOCH + .get_or_try_init(py, || { + #[cfg(not(Py_LIMITED_API))] + { + Ok::<_, PyErr>( + PyDateTime::new(py, 1970, 1, 1, 0, 0, 0, 0, Some(timezone_utc(py)))?.into(), + ) + } + #[cfg(Py_LIMITED_API)] + { + let datetime = py.import("datetime")?; + let utc = datetime.getattr("timezone")?.getattr("utc")?; + Ok::<_, PyErr>( + datetime + .getattr("datetime")? + .call1((1970, 1, 1, 0, 0, 0, 0, utc)) + .unwrap() + .into(), + ) + } + }) + .unwrap() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::PyDict; + use std::panic; + + #[test] + fn test_duration_frompyobject() { + Python::with_gil(|py| { + assert_eq!( + new_timedelta(py, 0, 0, 0).extract::().unwrap(), + Duration::new(0, 0) + ); + assert_eq!( + new_timedelta(py, 1, 0, 0).extract::().unwrap(), + Duration::new(86400, 0) + ); + assert_eq!( + new_timedelta(py, 0, 1, 0).extract::().unwrap(), + Duration::new(1, 0) + ); + assert_eq!( + new_timedelta(py, 0, 0, 1).extract::().unwrap(), + Duration::new(0, 1_000) + ); + assert_eq!( + new_timedelta(py, 1, 1, 1).extract::().unwrap(), + Duration::new(86401, 1_000) + ); + assert_eq!( + timedelta_class(py) + .getattr("max") + .unwrap() + .extract::() + .unwrap(), + Duration::new(86399999999999, 999999000) + ); + }); + } + + #[test] + fn test_duration_frompyobject_negative() { + Python::with_gil(|py| { + assert_eq!( + new_timedelta(py, 0, -1, 0) + .extract::() + .unwrap_err() + .to_string(), + "ValueError: It is not possible to convert a negative timedelta to a Rust Duration" + ); + }) + } + + #[test] + fn test_duration_topyobject() { + Python::with_gil(|py| { + let assert_eq = |l: PyObject, r: &PyAny| { + assert!(l.as_ref(py).eq(r).unwrap()); + }; + + assert_eq( + Duration::new(0, 0).to_object(py), + new_timedelta(py, 0, 0, 0), + ); + assert_eq( + Duration::new(86400, 0).to_object(py), + new_timedelta(py, 1, 0, 0), + ); + assert_eq( + Duration::new(1, 0).to_object(py), + new_timedelta(py, 0, 1, 0), + ); + assert_eq( + Duration::new(0, 1_000).to_object(py), + new_timedelta(py, 0, 0, 1), + ); + assert_eq( + Duration::new(0, 1).to_object(py), + new_timedelta(py, 0, 0, 0), + ); + assert_eq( + Duration::new(86401, 1_000).to_object(py), + new_timedelta(py, 1, 1, 1), + ); + assert_eq( + Duration::new(86399999999999, 999999000).to_object(py), + timedelta_class(py).getattr("max").unwrap(), + ); + }); + } + + #[test] + fn test_duration_topyobject_overflow() { + Python::with_gil(|py| { + assert!(panic::catch_unwind(|| Duration::MAX.to_object(py)).is_err()); + }) + } + + #[test] + fn test_time_frompyobject() { + Python::with_gil(|py| { + assert_eq!( + new_datetime(py, 1970, 1, 1, 0, 0, 0, 0) + .extract::() + .unwrap(), + UNIX_EPOCH + ); + assert_eq!( + new_datetime(py, 2020, 2, 3, 4, 5, 6, 7) + .extract::() + .unwrap(), + UNIX_EPOCH + .checked_add(Duration::new(1580702706, 7000)) + .unwrap() + ); + assert_eq!( + max_datetime(py).extract::().unwrap(), + UNIX_EPOCH + .checked_add(Duration::new(253402300799, 999999000)) + .unwrap() + ); + }); + } + + #[test] + fn test_time_frompyobject_before_epoch() { + Python::with_gil(|py| { + assert_eq!( + new_datetime(py, 1950, 1, 1, 0, 0, 0, 0) + .extract::() + .unwrap_err() + .to_string(), + "ValueError: It is not possible to convert a negative timedelta to a Rust Duration" + ); + }) + } + + #[test] + fn test_time_topyobject() { + Python::with_gil(|py| { + let assert_eq = |l: PyObject, r: &PyAny| { + assert!(l.as_ref(py).eq(r).unwrap()); + }; + + assert_eq( + UNIX_EPOCH + .checked_add(Duration::new(1580702706, 7123)) + .unwrap() + .into_py(py), + new_datetime(py, 2020, 2, 3, 4, 5, 6, 7), + ); + assert_eq( + UNIX_EPOCH + .checked_add(Duration::new(253402300799, 999999000)) + .unwrap() + .into_py(py), + max_datetime(py), + ); + }); + } + + #[allow(clippy::too_many_arguments)] + fn new_datetime( + py: Python<'_>, + year: i32, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, + microsecond: u32, + ) -> &PyAny { + datetime_class(py) + .call1(( + year, + month, + day, + hour, + minute, + second, + microsecond, + tz_utc(py), + )) + .unwrap() + } + + fn max_datetime(py: Python<'_>) -> &PyAny { + let naive_max = datetime_class(py).getattr("max").unwrap(); + let kargs = PyDict::new(py); + kargs.set_item("tzinfo", tz_utc(py)).unwrap(); + naive_max.call_method("replace", (), Some(kargs)).unwrap() + } + + #[test] + fn test_time_topyobject_overflow() { + let big_system_time = UNIX_EPOCH + .checked_add(Duration::new(300000000000, 0)) + .unwrap(); + Python::with_gil(|py| { + assert!(panic::catch_unwind(|| big_system_time.into_py(py)).is_err()); + }) + } + + fn tz_utc(py: Python<'_>) -> &PyAny { + py.import("datetime") + .unwrap() + .getattr("timezone") + .unwrap() + .getattr("utc") + .unwrap() + } + + fn new_timedelta(py: Python<'_>, days: i32, seconds: i32, microseconds: i32) -> &PyAny { + timedelta_class(py) + .call1((days, seconds, microseconds)) + .unwrap() + } + + fn datetime_class(py: Python<'_>) -> &PyAny { + py.import("datetime").unwrap().getattr("datetime").unwrap() + } + + fn timedelta_class(py: Python<'_>) -> &PyAny { + py.import("datetime").unwrap().getattr("timedelta").unwrap() + } +} From eed196329d8ae8129aa399f40d529dd1c08ca270 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 23 Jan 2024 08:39:19 +0000 Subject: [PATCH 055/349] add list bound constructors --- pyo3-benches/benches/bench_frompyobject.rs | 20 +++------ pyo3-benches/benches/bench_list.rs | 15 +++---- src/conversions/smallvec.rs | 8 ++-- src/marker.rs | 10 ++--- src/tests/hygiene/pymethods.rs | 8 ++-- src/types/dict.rs | 1 + src/types/list.rs | 47 ++++++++++++++++++++-- src/types/module.rs | 2 +- src/types/sequence.rs | 1 + tests/test_frompyobject.rs | 3 +- tests/test_getter_setter.rs | 4 +- tests/test_methods.rs | 4 +- tests/test_proto_methods.rs | 4 +- 13 files changed, 79 insertions(+), 48 deletions(-) diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs index 1ad53c717f3..c94cb114322 100644 --- a/pyo3-benches/benches/bench_frompyobject.rs +++ b/pyo3-benches/benches/bench_frompyobject.rs @@ -23,21 +23,17 @@ fn enum_from_pyobject(b: &mut Bencher<'_>) { fn list_via_downcast(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &PyAny = PyList::empty(py).into(); + let any: &Bound<'_, PyAny> = &PyList::empty_bound(py); - b.iter(|| { - let _list: &PyList = black_box(any).downcast().unwrap(); - }); + b.iter(|| black_box(any).downcast::().unwrap()); }) } fn list_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &PyAny = PyList::empty(py).into(); + let any: &Bound<'_, PyAny> = &PyList::empty_bound(py); - b.iter(|| { - let _list: &PyList = black_box(any).extract().unwrap(); - }); + b.iter(|| black_box(any).extract::>().unwrap()); }) } @@ -45,9 +41,7 @@ fn not_a_list_via_downcast(b: &mut Bencher<'_>) { Python::with_gil(|py| { let any: &PyAny = PyString::new(py, "foobar").into(); - b.iter(|| { - black_box(any).downcast::().unwrap_err(); - }); + b.iter(|| black_box(any).downcast::().unwrap_err()); }) } @@ -55,9 +49,7 @@ fn not_a_list_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { let any: &PyAny = PyString::new(py, "foobar").into(); - b.iter(|| { - black_box(any).extract::<&PyList>().unwrap_err(); - }); + b.iter(|| black_box(any).extract::<&PyList>().unwrap_err()); }) } diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs index 34bf9d855e5..6f8a678b339 100644 --- a/pyo3-benches/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -6,10 +6,10 @@ use pyo3::types::{PyList, PySequence}; fn iter_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let list = PyList::new(py, 0..LEN); + let list = PyList::new_bound(py, 0..LEN); let mut sum = 0; b.iter(|| { - for x in list { + for x in list.iter() { let i: u64 = x.extract().unwrap(); sum += i; } @@ -20,14 +20,14 @@ fn iter_list(b: &mut Bencher<'_>) { fn list_new(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - b.iter(|| PyList::new(py, 0..LEN)); + b.iter(|| PyList::new_bound(py, 0..LEN)); }); } fn list_get_item(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let list = PyList::new(py, 0..LEN); + let list = PyList::new_bound(py, 0..LEN); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -41,7 +41,7 @@ fn list_get_item(b: &mut Bencher<'_>) { fn list_get_item_unchecked(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let list = PyList::new(py, 0..LEN); + let list = PyList::new_bound(py, 0..LEN); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -56,9 +56,10 @@ fn list_get_item_unchecked(b: &mut Bencher<'_>) { fn sequence_from_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let list = PyList::new(py, 0..LEN).to_object(py); + let list = PyList::new_bound(py, 0..LEN).to_object(py); b.iter(|| { - let _: &PySequence = list.extract(py).unwrap(); + let seq: &PySequence = list.extract(py).unwrap(); + seq }); }); } diff --git a/src/conversions/smallvec.rs b/src/conversions/smallvec.rs index d2e84421da3..37ae09225e7 100644 --- a/src/conversions/smallvec.rs +++ b/src/conversions/smallvec.rs @@ -95,14 +95,14 @@ where #[cfg(test)] mod tests { use super::*; - use crate::types::{PyDict, PyList}; + use crate::types::{any::PyAnyMethods, PyDict, PyList}; #[test] fn test_smallvec_into_py() { Python::with_gil(|py| { let sv: SmallVec<[u64; 8]> = [1, 2, 3, 4, 5].iter().cloned().collect(); let hso: PyObject = sv.clone().into_py(py); - let l = PyList::new(py, [1, 2, 3, 4, 5]); + let l = PyList::new_bound(py, [1, 2, 3, 4, 5]); assert!(l.eq(hso).unwrap()); }); } @@ -110,7 +110,7 @@ mod tests { #[test] fn test_smallvec_from_py_object() { Python::with_gil(|py| { - let l = PyList::new(py, [1, 2, 3, 4, 5]); + let l = PyList::new_bound(py, [1, 2, 3, 4, 5]); let sv: SmallVec<[u64; 8]> = l.extract().unwrap(); assert_eq!(sv.as_slice(), [1, 2, 3, 4, 5]); }); @@ -133,7 +133,7 @@ mod tests { Python::with_gil(|py| { let sv: SmallVec<[u64; 8]> = [1, 2, 3, 4, 5].iter().cloned().collect(); let hso: PyObject = sv.to_object(py); - let l = PyList::new(py, [1, 2, 3, 4, 5]); + let l = PyList::new_bound(py, [1, 2, 3, 4, 5]); assert!(l.eq(hso).unwrap()); }); } diff --git a/src/marker.rs b/src/marker.rs index 97f826e571f..74c7095fa4b 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -1067,8 +1067,7 @@ impl<'unbound> Python<'unbound> { #[cfg(test)] mod tests { use super::*; - use crate::types::{IntoPyDict, PyDict, PyList}; - use crate::Py; + use crate::types::{any::PyAnyMethods, IntoPyDict, PyDict, PyList}; use std::sync::Arc; #[test] @@ -1150,17 +1149,14 @@ mod tests { // If allow_threads is implemented correctly, this thread still owns the GIL here // so the following Python calls should not cause crashes. - let list = PyList::new(py, [1, 2, 3, 4]); + let list = PyList::new_bound(py, [1, 2, 3, 4]); assert_eq!(list.extract::>().unwrap(), vec![1, 2, 3, 4]); }); } #[test] fn test_allow_threads_pass_stuff_in() { - let list: Py = Python::with_gil(|py| { - let list = PyList::new(py, vec!["foo", "bar"]); - list.into() - }); + let list = Python::with_gil(|py| PyList::new_bound(py, vec!["foo", "bar"]).unbind()); let mut v = vec![1, 2, 3]; let a = Arc::new(String::from("foo")); diff --git a/src/tests/hygiene/pymethods.rs b/src/tests/hygiene/pymethods.rs index 8e5bce8eefe..51657185cd1 100644 --- a/src/tests/hygiene/pymethods.rs +++ b/src/tests/hygiene/pymethods.rs @@ -76,8 +76,8 @@ impl Dummy { fn __delattr__(&mut self, name: ::std::string::String) {} - fn __dir__<'py>(&self, py: crate::Python<'py>) -> &'py crate::types::PyList { - crate::types::PyList::new(py, ::std::vec![0_u8]) + fn __dir__<'py>(&self, py: crate::Python<'py>) -> crate::Bound<'py, crate::types::PyList> { + crate::types::PyList::new_bound(py, ::std::vec![0_u8]) } ////////////////////// @@ -469,8 +469,8 @@ impl Dummy { fn __delattr__(&mut self, name: ::std::string::String) {} - fn __dir__<'py>(&self, py: crate::Python<'py>) -> &'py crate::types::PyList { - crate::types::PyList::new(py, ::std::vec![0_u8]) + fn __dir__<'py>(&self, py: crate::Python<'py>) -> crate::Bound<'py, crate::types::PyList> { + crate::types::PyList::new_bound(py, ::std::vec![0_u8]) } ////////////////////// diff --git a/src/types/dict.rs b/src/types/dict.rs index 0fe48662780..898c980f8a1 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -697,6 +697,7 @@ where } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; #[cfg(not(PyPy))] diff --git a/src/types/list.rs b/src/types/list.rs index f8db0e62ee6..140d1e208c3 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -56,6 +56,24 @@ pub(crate) fn new_from_iter<'py>( } impl PyList { + /// Deprecated form of [`PyList::new_bound`]. + #[inline] + #[track_caller] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyList::new` will be replaced by `PyList::new_bound` in a future PyO3 version" + ) + )] + pub fn new(py: Python<'_>, elements: impl IntoIterator) -> &PyList + where + T: ToPyObject, + U: ExactSizeIterator, + { + Self::new_bound(py, elements).into_gil_ref() + } + /// Constructs a new list with the given elements. /// /// If you want to create a [`PyList`] with elements of different or unknown types, or from an @@ -82,18 +100,38 @@ impl PyList { /// All standard library structures implement this trait correctly, if they do, so calling this /// function with (for example) [`Vec`]`` or `&[T]` will always succeed. #[track_caller] - pub fn new(py: Python<'_>, elements: impl IntoIterator) -> &PyList + pub fn new_bound( + py: Python<'_>, + elements: impl IntoIterator, + ) -> Bound<'_, PyList> where T: ToPyObject, U: ExactSizeIterator, { let mut iter = elements.into_iter().map(|e| e.to_object(py)); - new_from_iter(py, &mut iter).into_gil_ref() + new_from_iter(py, &mut iter) } - /// Constructs a new empty list. + /// Deprecated form of [`PyList::empty_bound`]. + #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyList::empty` will be replaced by `PyList::empty_bound` in a future PyO3 version" + ) + )] pub fn empty(py: Python<'_>) -> &PyList { - unsafe { py.from_owned_ptr(ffi::PyList_New(0)) } + Self::empty_bound(py).into_gil_ref() + } + + /// Constructs a new empty list. + pub fn empty_bound(py: Python<'_>) -> Bound<'_, PyList> { + unsafe { + ffi::PyList_New(0) + .assume_owned(py) + .downcast_into_unchecked() + } } /// Returns the length of the list. @@ -667,6 +705,7 @@ impl<'py> IntoIterator for Bound<'py, PyList> { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use crate::types::{PyList, PyTuple}; use crate::Python; diff --git a/src/types/module.rs b/src/types/module.rs index aa6649a9533..416090a187b 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -560,7 +560,7 @@ impl<'py> PyModuleMethods<'py> for Bound<'py, PyModule> { Ok(idx) => idx.downcast_into().map_err(PyErr::from), Err(err) => { if err.is_instance_of::(self.py()) { - let l = PyList::empty(self.py()).as_borrowed().to_owned(); + let l = PyList::empty_bound(self.py()); self.setattr(__all__, &l).map_err(PyErr::from)?; Ok(l) } else { diff --git a/src/types/sequence.rs b/src/types/sequence.rs index fa15fe06858..64e8f316c74 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -566,6 +566,7 @@ impl<'v> crate::PyTryFrom<'v> for PySequence { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use crate::types::{PyList, PySequence, PyTuple}; use crate::{PyObject, Python, ToPyObject}; diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index aa0c1cefc09..8eea9b896b2 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -564,7 +564,8 @@ pub struct TransparentFromPyWith { #[test] fn test_transparent_from_py_with() { Python::with_gil(|py| { - let result = TransparentFromPyWith::extract(PyList::new(py, [1, 2, 3])).unwrap(); + let result = + TransparentFromPyWith::extract(PyList::new_bound(py, [1, 2, 3]).as_gil_ref()).unwrap(); let expected = TransparentFromPyWith { len: 3 }; assert_eq!(result, expected); diff --git a/tests/test_getter_setter.rs b/tests/test_getter_setter.rs index f16ce61166a..148380ecf67 100644 --- a/tests/test_getter_setter.rs +++ b/tests/test_getter_setter.rs @@ -42,8 +42,8 @@ impl ClassWithProperties { } #[getter] - fn get_data_list<'py>(&self, py: Python<'py>) -> &'py PyList { - PyList::new(py, [self.num]) + fn get_data_list<'py>(&self, py: Python<'py>) -> Bound<'py, PyList> { + PyList::new_bound(py, [self.num]) } } diff --git a/tests/test_methods.rs b/tests/test_methods.rs index 6e42ad66960..50b179fd911 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -689,12 +689,12 @@ struct MethodWithLifeTime {} #[pymethods] impl MethodWithLifeTime { - fn set_to_list<'py>(&self, py: Python<'py>, set: &'py PySet) -> PyResult<&'py PyList> { + fn set_to_list<'py>(&self, py: Python<'py>, set: &'py PySet) -> PyResult> { let mut items = vec![]; for _ in 0..set.len() { items.push(set.pop().unwrap()); } - let list = PyList::new(py, items); + let list = PyList::new_bound(py, items); list.sort()?; Ok(list) } diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index caaae751354..a328ab9e24d 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -850,7 +850,7 @@ struct DefaultedContains; #[pymethods] impl DefaultedContains { fn __iter__(&self, py: Python<'_>) -> PyObject { - PyList::new(py, ["a", "b", "c"]) + PyList::new_bound(py, ["a", "b", "c"]) .as_ref() .iter() .unwrap() @@ -864,7 +864,7 @@ struct NoContains; #[pymethods] impl NoContains { fn __iter__(&self, py: Python<'_>) -> PyObject { - PyList::new(py, ["a", "b", "c"]) + PyList::new_bound(py, ["a", "b", "c"]) .as_ref() .iter() .unwrap() From 674f7282d800d32a06e2a230af820fe2f26c0f22 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 23 Jan 2024 08:43:40 +0000 Subject: [PATCH 056/349] `ToPyObject` and `IntoPy` for `Borrowed` --- src/instance.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/instance.rs b/src/instance.rs index 5779e9ada6f..4d449993462 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -355,6 +355,20 @@ impl Clone for Borrowed<'_, '_, T> { impl Copy for Borrowed<'_, '_, T> {} +impl ToPyObject for Borrowed<'_, '_, T> { + /// Converts `Py` instance -> PyObject. + fn to_object(&self, py: Python<'_>) -> PyObject { + (*self).into_py(py) + } +} + +impl IntoPy for Borrowed<'_, '_, T> { + /// Converts `Py` instance -> PyObject. + fn into_py(self, py: Python<'_>) -> PyObject { + self.to_owned().into_py(py) + } +} + /// A GIL-independent reference to an object allocated on the Python heap. /// /// This type does not auto-dereference to the inner object because you must prove you hold the GIL to access it. From 57a49a2b12c7b8b0f610cf5977eefcdf7bc78f8b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 23 Jan 2024 08:44:47 +0000 Subject: [PATCH 057/349] update tuple benchmarks for bound API --- pyo3-benches/benches/bench_list.rs | 2 +- pyo3-benches/benches/bench_tuple.rs | 26 ++++++++++---------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs index 6f8a678b339..f4f746f6968 100644 --- a/pyo3-benches/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -20,7 +20,7 @@ fn iter_list(b: &mut Bencher<'_>) { fn list_new(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - b.iter(|| PyList::new_bound(py, 0..LEN)); + b.iter_with_large_drop(|| PyList::new_bound(py, 0..LEN)); }); } diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs index f224ee1bc4d..57920c25a8d 100644 --- a/pyo3-benches/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -6,10 +6,10 @@ use pyo3::types::{PyList, PySequence, PyTuple}; fn iter_tuple(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let tuple = PyTuple::new(py, 0..LEN); + let tuple = PyTuple::new_bound(py, 0..LEN); let mut sum = 0; b.iter(|| { - for x in tuple { + for x in tuple.iter_borrowed() { let i: u64 = x.extract().unwrap(); sum += i; } @@ -20,14 +20,14 @@ fn iter_tuple(b: &mut Bencher<'_>) { fn tuple_new(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - b.iter(|| PyTuple::new(py, 0..LEN)); + b.iter_with_large_drop(|| PyTuple::new_bound(py, 0..LEN)); }); } fn tuple_get_item(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new(py, 0..LEN); + let tuple = PyTuple::new_bound(py, 0..LEN); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -41,7 +41,7 @@ fn tuple_get_item(b: &mut Bencher<'_>) { fn tuple_get_item_unchecked(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new(py, 0..LEN); + let tuple = PyTuple::new_bound(py, 0..LEN); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -56,7 +56,7 @@ fn tuple_get_item_unchecked(b: &mut Bencher<'_>) { fn sequence_from_tuple(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new(py, 0..LEN).to_object(py); + let tuple = PyTuple::new_bound(py, 0..LEN).to_object(py); b.iter(|| tuple.extract::<&PySequence>(py).unwrap()); }); } @@ -64,22 +64,16 @@ fn sequence_from_tuple(b: &mut Bencher<'_>) { fn tuple_new_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new(py, 0..LEN); - b.iter(|| { - let _pool = unsafe { py.new_pool() }; - let _ = PyList::new(py, tuple); - }); + let tuple = PyTuple::new_bound(py, 0..LEN); + b.iter_with_large_drop(|| PyList::new_bound(py, tuple.iter_borrowed())); }); } fn tuple_to_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new(py, 0..LEN); - b.iter(|| { - let _pool = unsafe { py.new_pool() }; - let _ = tuple.to_list(); - }); + let tuple = PyTuple::new_bound(py, 0..LEN); + b.iter_with_large_drop(|| tuple.to_list()); }); } From 1657109ae0dbb29c1e22a9838a16d51266876e19 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 23 Jan 2024 09:23:30 +0000 Subject: [PATCH 058/349] documentation updates for `PyList::new_bound` --- guide/src/conversions/traits.md | 2 +- guide/src/exception.md | 4 ++-- guide/src/migration.md | 20 ++++++++++++++++---- guide/src/types.md | 4 +++- src/instance.rs | 7 +++---- src/macros.rs | 2 +- src/sync.rs | 6 +++--- src/types/list.rs | 8 ++++---- 8 files changed, 33 insertions(+), 20 deletions(-) diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index 68304753b05..746f098ba5d 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -13,7 +13,7 @@ fails, so usually you will use something like # use pyo3::types::PyList; # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let list = PyList::new(py, b"foo"); +# let list = PyList::new_bound(py, b"foo"); let v: Vec = list.extract()?; # assert_eq!(&v, &[102, 111, 111]); # Ok(()) diff --git a/guide/src/exception.md b/guide/src/exception.md index 756477957b8..56e343f7c08 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -75,12 +75,12 @@ Python has an [`isinstance`](https://docs.python.org/3/library/functions.html#is In PyO3 every object has the [`PyAny::is_instance`] and [`PyAny::is_instance_of`] methods which do the same thing. ```rust -use pyo3::Python; +use pyo3::prelude::*; use pyo3::types::{PyBool, PyList}; Python::with_gil(|py| { assert!(PyBool::new(py, true).is_instance_of::()); - let list = PyList::new(py, &[1, 2, 3, 4]); + let list = PyList::new_bound(py, &[1, 2, 3, 4]); assert!(!list.is_instance_of::()); assert!(list.is_instance_of::()); }); diff --git a/guide/src/migration.md b/guide/src/migration.md index ec195ad1bc3..ec066e993fe 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -45,7 +45,7 @@ pyo3 = { version = "0.21", features = ["gil-refs"] } The `PyTryFrom` trait has aged poorly, its [`try_from`] method now conflicts with `try_from` in the 2021 edition prelude. A lot of its functionality was also duplicated with `PyTypeInfo`. -To tighten up the PyO3 traits ahead of [a proposed upcoming API change](https://github.com/PyO3/pyo3/issues/3382) the `PyTypeInfo` trait has had a simpler companion `PyTypeCheck`. The methods [`PyAny::downcast`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast) and [`PyAny::downcast_exact`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast_exact) no longer use `PyTryFrom` as a bound, instead using `PyTypeCheck` and `PyTypeInfo` respectively. +To tighten up the PyO3 traits as part of the deprecation of the GIL Refs API the `PyTypeInfo` trait has had a simpler companion `PyTypeCheck`. The methods [`PyAny::downcast`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast) and [`PyAny::downcast_exact`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast_exact) no longer use `PyTryFrom` as a bound, instead using `PyTypeCheck` and `PyTypeInfo` respectively. To migrate, switch all type casts to use `obj.downcast()` instead of `try_from(obj)` (and similar for `downcast_exact`). @@ -71,6 +71,9 @@ After: # use pyo3::types::{PyInt, PyList}; # fn main() -> PyResult<()> { Python::with_gil(|py| { + // Note that PyList::new is deprecated for PyList::new_bound as part of the GIL Refs API removal, + // see the section below on migration to Bound. + #[allow(deprecated)] let list = PyList::new(py, 0..5); let b = list.get_item(0).unwrap().downcast::()?; Ok(()) @@ -234,7 +237,15 @@ impl PyClassAsyncIter { ### Migrating from the GIL-Refs API to `Bound` -TODO +To minimise breakage of code using the GIL-Refs API, the `Bound` smart pointer has been introduced by adding complements to all functions which accept or return GIL Refs. This allows code to migrate by replacing the deprecated APIs with the new ones. + +For example, the following APIs have gained updated variants: +- `PyList::new`, `PyTyple::new` and similar constructors have replacements `PyList::new_bound`, `PyTuple::new_bound` etc. + +Because the new `Bound` API brings ownership out of the PyO3 framework and into user code, there are a few places where user code is expected to need to adjust while switching to the new API: +- Code will need to add the occasional `&` to borrow the new smart pointer as `&Bound` to pass these types around (or use `.clone()` at the very small cost of increasing the Python reference count) +- `Bound` and `Bound` cannot support indexing with `list[0]`, you should use `list.get_item(0)` instead. +- `Bound::iter_borrowed` is slightly more efficient than `Bound::iter`. The default iteration of `Bound` cannot return borrowed references because Rust does not (yet) have "lending iterators". ## from 0.19.* to 0.20 @@ -853,6 +864,7 @@ that these types can now also support Rust's indexing operators as part of a consistent API: ```rust +#![allow(deprecated)] use pyo3::{Python, types::PyList}; Python::with_gil(|py| { @@ -1073,7 +1085,7 @@ This should require no code changes except removing `use pyo3::AsPyRef` for code `pyo3::prelude::*`. Before: -```rust,compile_fail +```rust,ignore use pyo3::{AsPyRef, Py, types::PyList}; # pyo3::Python::with_gil(|py| { let list_py: Py = PyList::empty(py).into(); @@ -1082,7 +1094,7 @@ let list_ref: &PyList = list_py.as_ref(py); ``` After: -```rust +```rust,ignore use pyo3::{Py, types::PyList}; # pyo3::Python::with_gil(|py| { let list_py: Py = PyList::empty(py).into(); diff --git a/guide/src/types.md b/guide/src/types.md index 8bbc48d3aa4..882baddb33c 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -71,6 +71,7 @@ a list: # use pyo3::prelude::*; # use pyo3::types::PyList; # Python::with_gil(|py| -> PyResult<()> { +#[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. let obj: &PyAny = PyList::empty(py); // To &PyList with PyAny::downcast @@ -133,6 +134,7 @@ To see all Python types exposed by `PyO3` you should consult the # use pyo3::prelude::*; # use pyo3::types::PyList; # Python::with_gil(|py| -> PyResult<()> { +#[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. let list = PyList::empty(py); // Use methods from PyAny on all Python types with Deref implementation @@ -173,7 +175,7 @@ For a `Py`, the conversions are as below: # use pyo3::prelude::*; # use pyo3::types::PyList; # Python::with_gil(|py| { -let list: Py = PyList::empty(py).into(); +let list: Py = PyList::empty_bound(py).unbind(); // To &PyList with Py::as_ref() (borrows from the Py) let _: &PyList = list.as_ref(py); diff --git a/src/instance.rs b/src/instance.rs index 4d449993462..ececf090b4f 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -615,10 +615,9 @@ where /// # use pyo3::types::PyList; /// # /// Python::with_gil(|py| { - /// let list: Py = PyList::empty(py).into(); - /// // FIXME as_ref() no longer makes sense with new Py API, remove this doc - /// // let list: &PyList = list.as_ref(py); - /// // assert_eq!(list.len(), 0); + /// let list: Py = PyList::empty_bound(py).into(); + /// let list: &PyList = list.as_ref(py); + /// assert_eq!(list.len(), 0); /// }); /// ``` /// diff --git a/src/macros.rs b/src/macros.rs index 560d43da1ca..41de9079c40 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -12,7 +12,7 @@ /// use pyo3::{prelude::*, py_run, types::PyList}; /// /// Python::with_gil(|py| { -/// let list = PyList::new(py, &[1, 2, 3]); +/// let list = PyList::new_bound(py, &[1, 2, 3]); /// py_run!(py, list, "assert list == [1, 2, 3]"); /// }); /// ``` diff --git a/src/sync.rs b/src/sync.rs index 62843113a8e..e747269ae70 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -75,10 +75,10 @@ unsafe impl Sync for GILProtected where T: Send {} /// /// static LIST_CELL: GILOnceCell> = GILOnceCell::new(); /// -/// pub fn get_shared_list(py: Python<'_>) -> &PyList { +/// pub fn get_shared_list(py: Python<'_>) -> &Bound<'_, PyList> { /// LIST_CELL -/// .get_or_init(py, || PyList::empty(py).into()) -/// .as_ref(py) +/// .get_or_init(py, || PyList::empty_bound(py).unbind()) +/// .bind(py) /// } /// # Python::with_gil(|py| assert_eq!(get_shared_list(py).len(), 0)); /// ``` diff --git a/src/types/list.rs b/src/types/list.rs index 140d1e208c3..1389d25ed29 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -88,7 +88,7 @@ impl PyList { /// # fn main() { /// Python::with_gil(|py| { /// let elements: Vec = vec![0, 1, 2, 3, 4, 5]; - /// let list: &PyList = PyList::new(py, elements); + /// let list = PyList::new_bound(py, elements); /// assert_eq!(format!("{:?}", list), "[0, 1, 2, 3, 4, 5]"); /// }); /// # } @@ -154,7 +154,7 @@ impl PyList { /// ``` /// use pyo3::{prelude::*, types::PyList}; /// Python::with_gil(|py| { - /// let list = PyList::new(py, [2, 3, 5, 7]); + /// let list = PyList::new_bound(py, [2, 3, 5, 7]); /// let obj = list.get_item(0); /// assert_eq!(obj.unwrap().extract::().unwrap(), 2); /// }); @@ -301,7 +301,7 @@ pub trait PyListMethods<'py> { /// ``` /// use pyo3::{prelude::*, types::PyList}; /// Python::with_gil(|py| { - /// let list = PyList::new(py, [2, 3, 5, 7]); + /// let list = PyList::new_bound(py, [2, 3, 5, 7]); /// let obj = list.get_item(0); /// assert_eq!(obj.unwrap().extract::().unwrap(), 2); /// }); @@ -414,7 +414,7 @@ impl<'py> PyListMethods<'py> for Bound<'py, PyList> { /// ``` /// use pyo3::{prelude::*, types::PyList}; /// Python::with_gil(|py| { - /// let list = PyList::new(py, [2, 3, 5, 7]); + /// let list = PyList::new_bound(py, [2, 3, 5, 7]); /// let obj = list.get_item(0); /// assert_eq!(obj.unwrap().extract::().unwrap(), 2); /// }); From 7927a2e211b3231e36a7d326ec08ac62d2df1f7a Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 23 Jan 2024 09:59:37 +0000 Subject: [PATCH 059/349] add bench for tuple `get_borrowed_item` --- guide/src/migration.md | 2 +- pyo3-benches/benches/bench_tuple.rs | 42 +++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index ec066e993fe..e04b477a345 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -245,7 +245,7 @@ For example, the following APIs have gained updated variants: Because the new `Bound` API brings ownership out of the PyO3 framework and into user code, there are a few places where user code is expected to need to adjust while switching to the new API: - Code will need to add the occasional `&` to borrow the new smart pointer as `&Bound` to pass these types around (or use `.clone()` at the very small cost of increasing the Python reference count) - `Bound` and `Bound` cannot support indexing with `list[0]`, you should use `list.get_item(0)` instead. -- `Bound::iter_borrowed` is slightly more efficient than `Bound::iter`. The default iteration of `Bound` cannot return borrowed references because Rust does not (yet) have "lending iterators". +- `Bound::iter_borrowed` is slightly more efficient than `Bound::iter`. The default iteration of `Bound` cannot return borrowed references because Rust does not (yet) have "lending iterators". Similarly `Bound::get_borrowed_item` is more efficient than `Bound::get_item` for the same reason. ## from 0.19.* to 0.20 diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs index 57920c25a8d..9af95efcab2 100644 --- a/pyo3-benches/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -53,6 +53,42 @@ fn tuple_get_item_unchecked(b: &mut Bencher<'_>) { }); } +fn tuple_get_borrowed_item(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + const LEN: usize = 50_000; + let tuple = PyTuple::new_bound(py, 0..LEN); + let mut sum = 0; + b.iter(|| { + for i in 0..LEN { + sum += tuple + .get_borrowed_item(i) + .unwrap() + .extract::() + .unwrap(); + } + }); + }); +} + +#[cfg(not(any(Py_LIMITED_API, PyPy)))] +fn tuple_get_borrowed_item_unchecked(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + const LEN: usize = 50_000; + let tuple = PyTuple::new_bound(py, 0..LEN); + let mut sum = 0; + b.iter(|| { + for i in 0..LEN { + unsafe { + sum += tuple + .get_borrowed_item_unchecked(i) + .extract::() + .unwrap(); + } + } + }); + }); +} + fn sequence_from_tuple(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; @@ -89,6 +125,12 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("tuple_get_item", tuple_get_item); #[cfg(not(any(Py_LIMITED_API, PyPy)))] c.bench_function("tuple_get_item_unchecked", tuple_get_item_unchecked); + c.bench_function("tuple_get_borrowed_item", tuple_get_borrowed_item); + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + c.bench_function( + "tuple_get_borrowed_item_unchecked", + tuple_get_borrowed_item_unchecked, + ); c.bench_function("sequence_from_tuple", sequence_from_tuple); c.bench_function("tuple_new_list", tuple_new_list); c.bench_function("tuple_to_list", tuple_to_list); From 37e2a4d9c9a4e350bd82211e1d4327a8603a6516 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 27 Jan 2024 02:02:58 +0100 Subject: [PATCH 060/349] implement `PyComplexMethods` --- guide/src/class/numeric.md | 8 +- src/conversions/num_complex.rs | 8 +- src/tests/hygiene/pymethods.rs | 14 +- src/types/complex.rs | 238 ++++++++++++++++++++++++--------- 4 files changed, 190 insertions(+), 78 deletions(-) diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index 10a06e7c02c..ee17ea10bd9 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -170,8 +170,8 @@ impl Number { self.0 as f64 } - fn __complex__<'py>(&self, py: Python<'py>) -> &'py PyComplex { - PyComplex::from_doubles(py, self.0 as f64, 0.0) + fn __complex__<'py>(&self, py: Python<'py>) -> Bound<'py, PyComplex> { + PyComplex::from_doubles_bound(py, self.0 as f64, 0.0) } } ``` @@ -321,8 +321,8 @@ impl Number { self.0 as f64 } - fn __complex__<'py>(&self, py: Python<'py>) -> &'py PyComplex { - PyComplex::from_doubles(py, self.0 as f64, 0.0) + fn __complex__<'py>(&self, py: Python<'py>) -> Bound<'py, PyComplex> { + PyComplex::from_doubles_bound(py, self.0 as f64, 0.0) } } diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index a6d76d0f34e..d488cfd466d 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -59,10 +59,10 @@ //! # //! # module.add_function(wrap_pyfunction!(get_eigenvalues, module)?)?; //! # -//! # let m11 = PyComplex::from_doubles(py, 0_f64, -1_f64); -//! # let m12 = PyComplex::from_doubles(py, 1_f64, 0_f64); -//! # let m21 = PyComplex::from_doubles(py, 2_f64, -1_f64); -//! # let m22 = PyComplex::from_doubles(py, -1_f64, 0_f64); +//! # let m11 = PyComplex::from_doubles_bound(py, 0_f64, -1_f64); +//! # let m12 = PyComplex::from_doubles_bound(py, 1_f64, 0_f64); +//! # let m21 = PyComplex::from_doubles_bound(py, 2_f64, -1_f64); +//! # let m22 = PyComplex::from_doubles_bound(py, -1_f64, 0_f64); //! # //! # let result = module //! # .getattr("get_eigenvalues")? diff --git a/src/tests/hygiene/pymethods.rs b/src/tests/hygiene/pymethods.rs index 8e5bce8eefe..59ff9285273 100644 --- a/src/tests/hygiene/pymethods.rs +++ b/src/tests/hygiene/pymethods.rs @@ -281,8 +281,11 @@ impl Dummy { slf } - fn __complex__<'py>(&self, py: crate::Python<'py>) -> &'py crate::types::PyComplex { - crate::types::PyComplex::from_doubles(py, 0.0, 0.0) + fn __complex__<'py>( + &self, + py: crate::Python<'py>, + ) -> crate::Bound<'py, crate::types::PyComplex> { + crate::types::PyComplex::from_doubles_bound(py, 0.0, 0.0) } fn __int__(&self) -> u32 { @@ -673,8 +676,11 @@ impl Dummy { slf } - fn __complex__<'py>(&self, py: crate::Python<'py>) -> &'py crate::types::PyComplex { - crate::types::PyComplex::from_doubles(py, 0.0, 0.0) + fn __complex__<'py>( + &self, + py: crate::Python<'py>, + ) -> crate::Bound<'py, crate::types::PyComplex> { + crate::types::PyComplex::from_doubles_bound(py, 0.0, 0.0) } fn __int__(&self) -> u32 { diff --git a/src/types/complex.rs b/src/types/complex.rs index b722508befa..dafda97dbd5 100644 --- a/src/types/complex.rs +++ b/src/types/complex.rs @@ -1,4 +1,4 @@ -use crate::{ffi, PyAny, Python}; +use crate::{ffi, types::any::PyAnyMethods, Bound, PyAny, PyNativeType, Python}; use std::os::raw::c_double; /// Represents a Python [`complex`](https://docs.python.org/3/library/functions.html#complex) object. @@ -19,118 +19,173 @@ pyobject_native_type!( ); impl PyComplex { - /// Creates a new `PyComplex` from the given real and imaginary values. + /// Deprecated form of [`PyComplex::from_doubles_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyComplex::from_doubles` will be replaced by `PyComplex::from_doubles_bound` in a future PyO3 version" + ) + )] pub fn from_doubles(py: Python<'_>, real: c_double, imag: c_double) -> &PyComplex { + Self::from_doubles_bound(py, real, imag).into_gil_ref() + } + /// Creates a new `PyComplex` from the given real and imaginary values. + pub fn from_doubles_bound( + py: Python<'_>, + real: c_double, + imag: c_double, + ) -> Bound<'_, PyComplex> { + use crate::ffi_ptr_ext::FfiPtrExt; unsafe { - let ptr = ffi::PyComplex_FromDoubles(real, imag); - py.from_owned_ptr(ptr) + ffi::PyComplex_FromDoubles(real, imag) + .assume_owned(py) + .downcast_into_unchecked() } } /// Returns the real part of the complex number. pub fn real(&self) -> c_double { - unsafe { ffi::PyComplex_RealAsDouble(self.as_ptr()) } + self.as_borrowed().real() } /// Returns the imaginary part of the complex number. pub fn imag(&self) -> c_double { - unsafe { ffi::PyComplex_ImagAsDouble(self.as_ptr()) } + self.as_borrowed().imag() } } #[cfg(not(any(Py_LIMITED_API, PyPy)))] mod not_limited_impls { + use crate::ffi_ptr_ext::FfiPtrExt; + use crate::Borrowed; + use super::*; use std::ops::{Add, Div, Mul, Neg, Sub}; impl PyComplex { /// Returns `|self|`. pub fn abs(&self) -> c_double { - unsafe { - let val = (*(self.as_ptr() as *mut ffi::PyComplexObject)).cval; - ffi::_Py_c_abs(val) - } + self.as_borrowed().abs() } /// Returns `self` raised to the power of `other`. - pub fn pow(&self, other: &PyComplex) -> &PyComplex { - unsafe { - self.py() - .from_owned_ptr(complex_operation(self, other, ffi::_Py_c_pow)) - } + pub fn pow<'py>(&'py self, other: &'py PyComplex) -> &'py PyComplex { + self.as_borrowed().pow(&other.as_borrowed()).into_gil_ref() } } #[inline(always)] - unsafe fn complex_operation( - l: &PyComplex, - r: &PyComplex, + pub(super) unsafe fn complex_operation<'py>( + l: Borrowed<'_, 'py, PyComplex>, + r: Borrowed<'_, 'py, PyComplex>, operation: unsafe extern "C" fn(ffi::Py_complex, ffi::Py_complex) -> ffi::Py_complex, ) -> *mut ffi::PyObject { - let l_val = (*(l.as_ptr() as *mut ffi::PyComplexObject)).cval; - let r_val = (*(r.as_ptr() as *mut ffi::PyComplexObject)).cval; + let l_val = (*l.as_ptr().cast::()).cval; + let r_val = (*r.as_ptr().cast::()).cval; ffi::PyComplex_FromCComplex(operation(l_val, r_val)) } - impl<'py> Add for &'py PyComplex { - type Output = &'py PyComplex; - fn add(self, other: &'py PyComplex) -> &'py PyComplex { - unsafe { - self.py() - .from_owned_ptr(complex_operation(self, other, ffi::_Py_c_sum)) + macro_rules! bin_ops { + ($trait:ident, $fn:ident, $op:tt, $ffi:path) => { + impl<'py> $trait for Borrowed<'_, 'py, PyComplex> { + type Output = Bound<'py, PyComplex>; + fn $fn(self, other: Self) -> Self::Output { + unsafe { + complex_operation(self, other, $ffi) + .assume_owned(self.py()) + .downcast_into_unchecked() + } + } } - } + + impl<'py> $trait for &'py PyComplex { + type Output = &'py PyComplex; + fn $fn(self, other: &'py PyComplex) -> &'py PyComplex { + (self.as_borrowed() $op other.as_borrowed()).into_gil_ref() + } + } + + impl<'py> $trait for &Bound<'py, PyComplex> { + type Output = Bound<'py, PyComplex>; + fn $fn(self, other: &Bound<'py, PyComplex>) -> Bound<'py, PyComplex> { + self.as_borrowed() $op other.as_borrowed() + } + } + + impl<'py> $trait> for &Bound<'py, PyComplex> { + type Output = Bound<'py, PyComplex>; + fn $fn(self, other: Bound<'py, PyComplex>) -> Bound<'py, PyComplex> { + self.as_borrowed() $op other.as_borrowed() + } + } + + impl<'py> $trait for Bound<'py, PyComplex> { + type Output = Bound<'py, PyComplex>; + fn $fn(self, other: Bound<'py, PyComplex>) -> Bound<'py, PyComplex> { + self.as_borrowed() $op other.as_borrowed() + } + } + + impl<'py> $trait<&Self> for Bound<'py, PyComplex> { + type Output = Bound<'py, PyComplex>; + fn $fn(self, other: &Bound<'py, PyComplex>) -> Bound<'py, PyComplex> { + self.as_borrowed() $op other.as_borrowed() + } + } + }; } - impl<'py> Sub for &'py PyComplex { + bin_ops!(Add, add, +, ffi::_Py_c_sum); + bin_ops!(Sub, sub, -, ffi::_Py_c_diff); + bin_ops!(Mul, mul, *, ffi::_Py_c_prod); + bin_ops!(Div, div, /, ffi::_Py_c_quot); + + impl<'py> Neg for &'py PyComplex { type Output = &'py PyComplex; - fn sub(self, other: &'py PyComplex) -> &'py PyComplex { + fn neg(self) -> &'py PyComplex { unsafe { + let val = (*self.as_ptr().cast::()).cval; self.py() - .from_owned_ptr(complex_operation(self, other, ffi::_Py_c_diff)) + .from_owned_ptr(ffi::PyComplex_FromCComplex(ffi::_Py_c_neg(val))) } } } - impl<'py> Mul for &'py PyComplex { - type Output = &'py PyComplex; - fn mul(self, other: &'py PyComplex) -> &'py PyComplex { + impl<'py> Neg for Borrowed<'_, 'py, PyComplex> { + type Output = Bound<'py, PyComplex>; + fn neg(self) -> Self::Output { unsafe { - self.py() - .from_owned_ptr(complex_operation(self, other, ffi::_Py_c_prod)) + let val = (*self.as_ptr().cast::()).cval; + ffi::PyComplex_FromCComplex(ffi::_Py_c_neg(val)) + .assume_owned(self.py()) + .downcast_into_unchecked() } } } - impl<'py> Div for &'py PyComplex { - type Output = &'py PyComplex; - fn div(self, other: &'py PyComplex) -> &'py PyComplex { - unsafe { - self.py() - .from_owned_ptr(complex_operation(self, other, ffi::_Py_c_quot)) - } + impl<'py> Neg for &Bound<'py, PyComplex> { + type Output = Bound<'py, PyComplex>; + fn neg(self) -> Bound<'py, PyComplex> { + -self.as_borrowed() } } - impl<'py> Neg for &'py PyComplex { - type Output = &'py PyComplex; - fn neg(self) -> &'py PyComplex { - unsafe { - let val = (*(self.as_ptr() as *mut ffi::PyComplexObject)).cval; - self.py() - .from_owned_ptr(ffi::PyComplex_FromCComplex(ffi::_Py_c_neg(val))) - } + impl<'py> Neg for Bound<'py, PyComplex> { + type Output = Bound<'py, PyComplex>; + fn neg(self) -> Bound<'py, PyComplex> { + -self.as_borrowed() } } #[cfg(test)] mod tests { use super::PyComplex; - use crate::Python; + use crate::{types::complex::PyComplexMethods, Python}; use assert_approx_eq::assert_approx_eq; #[test] fn test_add() { Python::with_gil(|py| { - let l = PyComplex::from_doubles(py, 3.0, 1.2); - let r = PyComplex::from_doubles(py, 1.0, 2.6); + let l = PyComplex::from_doubles_bound(py, 3.0, 1.2); + let r = PyComplex::from_doubles_bound(py, 1.0, 2.6); let res = l + r; assert_approx_eq!(res.real(), 4.0); assert_approx_eq!(res.imag(), 3.8); @@ -140,8 +195,8 @@ mod not_limited_impls { #[test] fn test_sub() { Python::with_gil(|py| { - let l = PyComplex::from_doubles(py, 3.0, 1.2); - let r = PyComplex::from_doubles(py, 1.0, 2.6); + let l = PyComplex::from_doubles_bound(py, 3.0, 1.2); + let r = PyComplex::from_doubles_bound(py, 1.0, 2.6); let res = l - r; assert_approx_eq!(res.real(), 2.0); assert_approx_eq!(res.imag(), -1.4); @@ -151,8 +206,8 @@ mod not_limited_impls { #[test] fn test_mul() { Python::with_gil(|py| { - let l = PyComplex::from_doubles(py, 3.0, 1.2); - let r = PyComplex::from_doubles(py, 1.0, 2.6); + let l = PyComplex::from_doubles_bound(py, 3.0, 1.2); + let r = PyComplex::from_doubles_bound(py, 1.0, 2.6); let res = l * r; assert_approx_eq!(res.real(), -0.12); assert_approx_eq!(res.imag(), 9.0); @@ -162,8 +217,8 @@ mod not_limited_impls { #[test] fn test_div() { Python::with_gil(|py| { - let l = PyComplex::from_doubles(py, 3.0, 1.2); - let r = PyComplex::from_doubles(py, 1.0, 2.6); + let l = PyComplex::from_doubles_bound(py, 3.0, 1.2); + let r = PyComplex::from_doubles_bound(py, 1.0, 2.6); let res = l / r; assert_approx_eq!(res.real(), 0.788_659_793_814_432_9); assert_approx_eq!(res.imag(), -0.850_515_463_917_525_7); @@ -173,7 +228,7 @@ mod not_limited_impls { #[test] fn test_neg() { Python::with_gil(|py| { - let val = PyComplex::from_doubles(py, 3.0, 1.2); + let val = PyComplex::from_doubles_bound(py, 3.0, 1.2); let res = -val; assert_approx_eq!(res.real(), -3.0); assert_approx_eq!(res.imag(), -1.2); @@ -183,7 +238,7 @@ mod not_limited_impls { #[test] fn test_abs() { Python::with_gil(|py| { - let val = PyComplex::from_doubles(py, 3.0, 1.2); + let val = PyComplex::from_doubles_bound(py, 3.0, 1.2); assert_approx_eq!(val.abs(), 3.231_098_884_280_702_2); }); } @@ -191,9 +246,9 @@ mod not_limited_impls { #[test] fn test_pow() { Python::with_gil(|py| { - let l = PyComplex::from_doubles(py, 3.0, 1.2); - let r = PyComplex::from_doubles(py, 1.2, 2.6); - let val = l.pow(r); + let l = PyComplex::from_doubles_bound(py, 3.0, 1.2); + let r = PyComplex::from_doubles_bound(py, 1.2, 2.6); + let val = l.pow(&r); assert_approx_eq!(val.real(), -1.419_309_997_016_603_7); assert_approx_eq!(val.imag(), -0.541_297_466_033_544_6); }); @@ -201,10 +256,61 @@ mod not_limited_impls { } } +/// Implementation of functionality for [`PyComplex`]. +/// +/// These methods are defined for the `Bound<'py, PyComplex>` smart pointer, so to use method call +/// syntax these methods are separated into a trait, because stable Rust does not yet support +/// `arbitrary_self_types`. +#[doc(alias = "PyComplex")] +pub trait PyComplexMethods<'py> { + /// Returns the real part of the complex number. + fn real(&self) -> c_double; + /// Returns the imaginary part of the complex number. + fn imag(&self) -> c_double; + /// Returns `|self|`. + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + fn abs(&self) -> c_double; + /// Returns `self` raised to the power of `other`. + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + fn pow(&self, other: &Bound<'py, PyComplex>) -> Bound<'py, PyComplex>; +} + +impl<'py> PyComplexMethods<'py> for Bound<'py, PyComplex> { + fn real(&self) -> c_double { + unsafe { ffi::PyComplex_RealAsDouble(self.as_ptr()) } + } + + fn imag(&self) -> c_double { + unsafe { ffi::PyComplex_ImagAsDouble(self.as_ptr()) } + } + + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + fn abs(&self) -> c_double { + unsafe { + let val = (*self.as_ptr().cast::()).cval; + ffi::_Py_c_abs(val) + } + } + + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + fn pow(&self, other: &Bound<'py, PyComplex>) -> Bound<'py, PyComplex> { + use crate::ffi_ptr_ext::FfiPtrExt; + unsafe { + not_limited_impls::complex_operation( + self.as_borrowed(), + other.as_borrowed(), + ffi::_Py_c_pow, + ) + .assume_owned(self.py()) + .downcast_into_unchecked() + } + } +} + #[cfg(test)] mod tests { use super::PyComplex; - use crate::Python; + use crate::{types::complex::PyComplexMethods, Python}; use assert_approx_eq::assert_approx_eq; #[test] @@ -212,7 +318,7 @@ mod tests { use assert_approx_eq::assert_approx_eq; Python::with_gil(|py| { - let complex = PyComplex::from_doubles(py, 3.0, 1.2); + let complex = PyComplex::from_doubles_bound(py, 3.0, 1.2); assert_approx_eq!(complex.real(), 3.0); assert_approx_eq!(complex.imag(), 1.2); }); From 5ccc46e459a96113dbe6d5337bc9cbfef09eb54f Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 27 Jan 2024 21:57:43 +0000 Subject: [PATCH 061/349] remove `test_pep_587` --- Cargo.toml | 1 - tests/test_pep_587.rs | 63 ------------------------------------------- 2 files changed, 64 deletions(-) delete mode 100644 tests/test_pep_587.rs diff --git a/Cargo.toml b/Cargo.toml index 31179e59a7c..c4ce8b6b033 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,6 @@ send_wrapper = "0.6" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.61" rayon = "1.6.1" -widestring = "0.5.1" futures = "0.3.28" [build-dependencies] diff --git a/tests/test_pep_587.rs b/tests/test_pep_587.rs deleted file mode 100644 index 24e1f07d2d8..00000000000 --- a/tests/test_pep_587.rs +++ /dev/null @@ -1,63 +0,0 @@ -#![cfg(all(Py_3_8, not(any(PyPy, Py_LIMITED_API))))] - -use pyo3::ffi; - -#[cfg(Py_3_10)] -use widestring::WideCString; - -#[test] -fn test_default_interpreter() { - macro_rules! ensure { - ($py_call:expr) => {{ - let status = $py_call; - unsafe { - if ffi::PyStatus_Exception(status) != 0 { - ffi::Py_ExitStatusException(status); - } - } - }}; - } - - let mut preconfig = unsafe { std::mem::zeroed() }; - - unsafe { ffi::PyPreConfig_InitPythonConfig(&mut preconfig) }; - preconfig.utf8_mode = 1; - - ensure!(unsafe { ffi::Py_PreInitialize(&preconfig) }); - - let mut config = unsafe { std::mem::zeroed() }; - unsafe { ffi::PyConfig_InitPythonConfig(&mut config) }; - - // Require manually calling _Py_InitializeMain to exercise more ffi code - #[allow(clippy::used_underscore_binding)] - { - config._init_main = 0; - } - - #[cfg(Py_3_10)] - unsafe { - ffi::PyConfig_SetBytesString( - &mut config, - &mut config.program_name, - "some_test\0".as_ptr().cast(), - ); - } - - ensure!(unsafe { ffi::Py_InitializeFromConfig(&config) }); - - // The GIL is held. - assert_eq!(unsafe { ffi::PyGILState_Check() }, 1); - - // Now proceed with the Python main initialization. - ensure!(unsafe { ffi::_Py_InitializeMain() }); - - // The GIL is held after finishing initialization. - assert_eq!(unsafe { ffi::PyGILState_Check() }, 1); - - // Confirm program name set above was picked up correctly - #[cfg(Py_3_10)] - { - let program_name = unsafe { WideCString::from_ptr_str(ffi::Py_GetProgramName().cast()) }; - assert_eq!(program_name.to_string().unwrap(), "some_test"); - } -} From 595ca4b3c17863ad3b36dc5659df7e2f2aab087b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 27 Dec 2023 08:56:01 +0000 Subject: [PATCH 062/349] Add `extract_bound` method to `FromPyObject` --- guide/src/migration.md | 30 ++++++++++++++++++++++++++++++ newsfragments/3706.added.md | 1 + src/conversion.rs | 25 ++++++++++++++++++++----- 3 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 newsfragments/3706.added.md diff --git a/guide/src/migration.md b/guide/src/migration.md index e04b477a345..9fb5adba387 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -241,12 +241,42 @@ To minimise breakage of code using the GIL-Refs API, the `Bound` smart pointe For example, the following APIs have gained updated variants: - `PyList::new`, `PyTyple::new` and similar constructors have replacements `PyList::new_bound`, `PyTuple::new_bound` etc. +- `FromPyObject::extract` has a new `FromPyObject::extract_bound` (see the section below) Because the new `Bound` API brings ownership out of the PyO3 framework and into user code, there are a few places where user code is expected to need to adjust while switching to the new API: - Code will need to add the occasional `&` to borrow the new smart pointer as `&Bound` to pass these types around (or use `.clone()` at the very small cost of increasing the Python reference count) - `Bound` and `Bound` cannot support indexing with `list[0]`, you should use `list.get_item(0)` instead. - `Bound::iter_borrowed` is slightly more efficient than `Bound::iter`. The default iteration of `Bound` cannot return borrowed references because Rust does not (yet) have "lending iterators". Similarly `Bound::get_borrowed_item` is more efficient than `Bound::get_item` for the same reason. +#### Migrating `FromPyObject` implementations + +`FromPyObject` has had a new method `extract_bound` which takes `&Bound<'py, PyAny>` as an argument instead of `&PyAny`. Both `extract` and `extract_bound` have been given default implementations in terms of the other, to avoid breaking code immediately on update to 0.21. + +All implementations of `FromPyObject` should be switched from `extract` to `extract_bound`. + +Before: + +```rust,ignore +impl<'py> FromPyObject<'py> for MyType { + fn extract(obj: &'py PyAny) -> PyResult { + /* ... */ + } +} +``` + +After: + +```rust,ignore +impl<'py> FromPyObject<'py> for MyType { + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { + /* ... */ + } +} +``` + + +The expectation is that in 0.22 `extract_bound` will have the default implementation removed and in 0.23 `extract` will be removed. + ## from 0.19.* to 0.20 ### Drop support for older technologies diff --git a/newsfragments/3706.added.md b/newsfragments/3706.added.md new file mode 100644 index 00000000000..31db8b96cef --- /dev/null +++ b/newsfragments/3706.added.md @@ -0,0 +1 @@ +Add `FromPyObject::extract_bound` method, which can be implemented to avoid using the GIL Ref API in `FromPyObject` implementations. diff --git a/src/conversion.rs b/src/conversion.rs index 429ca9e8779..46b39436e88 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -6,7 +6,7 @@ use crate::pyclass::boolean_struct::False; use crate::type_object::PyTypeInfo; use crate::types::PyTuple; use crate::{ - ffi, gil, Py, PyAny, PyCell, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, + ffi, gil, Bound, Py, PyAny, PyCell, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, }; use std::cell::Cell; use std::ptr::NonNull; @@ -215,11 +215,26 @@ pub trait IntoPy: Sized { /// Since which case applies depends on the runtime type of the Python object, /// both the `obj` and `prepared` variables must outlive the resulting string slice. /// -/// The trait's conversion method takes a `&PyAny` argument but is called -/// `FromPyObject` for historical reasons. +/// During the migration of PyO3 from the "GIL Refs" API to the `Bound` smart pointer, this trait +/// has two methods `extract` and `extract_bound` which are defaulted to call each other. To avoid +/// infinite recursion, implementors must implement at least one of these methods. The recommendation +/// is to implement `extract_bound` and leave `extract` as the default implementation. pub trait FromPyObject<'source>: Sized { - /// Extracts `Self` from the source `PyObject`. - fn extract(ob: &'source PyAny) -> PyResult; + /// Extracts `Self` from the source GIL Ref `obj`. + /// + /// Implementors are encouraged to implement `extract_bound` and leave this method as the + /// default implementation, which will forward calls to `extract_bound`. + fn extract(ob: &'source PyAny) -> PyResult { + Self::extract_bound(&ob.as_borrowed()) + } + + /// Extracts `Self` from the bound smart pointer `obj`. + /// + /// Implementors are encouraged to implement this method and leave `extract` defaulted, as + /// this will be most compatible with PyO3's future API. + fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { + Self::extract(ob.clone().into_gil_ref()) + } /// Extracts the type hint information for this type when it appears as an argument. /// From ffaa03e3f1959aa6c714896920dd7830a12fe964 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 27 Dec 2023 08:56:26 +0000 Subject: [PATCH 063/349] Migrate some conversions to `extract_bound` --- src/conversions/num_complex.rs | 13 ++-- src/instance.rs | 7 +- src/types/any.rs | 123 ++++++++++++++++++--------------- src/types/boolobject.rs | 10 +-- src/types/tuple.rs | 12 ++-- 5 files changed, 90 insertions(+), 75 deletions(-) diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index d488cfd466d..ba741323611 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -93,8 +93,11 @@ //! result = get_eigenvalues(m11,m12,m21,m22) //! assert result == [complex(1,-1), complex(-2,0)] //! ``` +#[cfg(any(Py_LIMITED_API, PyPy))] +use crate::types::any::PyAnyMethods; use crate::{ - ffi, types::PyComplex, FromPyObject, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, + ffi, types::PyComplex, Bound, FromPyObject, PyAny, PyErr, PyObject, PyResult, Python, + ToPyObject, }; use num_complex::Complex; use std::os::raw::c_double; @@ -131,8 +134,8 @@ macro_rules! complex_conversion { } #[cfg_attr(docsrs, doc(cfg(feature = "num-complex")))] - impl<'source> FromPyObject<'source> for Complex<$float> { - fn extract(obj: &'source PyAny) -> PyResult> { + impl FromPyObject<'_> for Complex<$float> { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult> { #[cfg(not(any(Py_LIMITED_API, PyPy)))] unsafe { let val = ffi::PyComplex_AsCComplex(obj.as_ptr()); @@ -146,12 +149,14 @@ macro_rules! complex_conversion { #[cfg(any(Py_LIMITED_API, PyPy))] unsafe { + let complex; let obj = if obj.is_instance_of::() { obj } else if let Some(method) = obj.lookup_special(crate::intern!(obj.py(), "__complex__"))? { - method.call0()? + complex = method.call0()?; + &complex } else { // `obj` might still implement `__float__` or `__index__`, which will be // handled by `PyComplex_{Real,Imag}AsDouble`, including propagating any diff --git a/src/instance.rs b/src/instance.rs index ececf090b4f..d7edfe06365 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1425,11 +1425,8 @@ where T: PyTypeInfo, { /// Extracts `Self` from the source `PyObject`. - fn extract(ob: &'a PyAny) -> PyResult { - ob.as_borrowed() - .downcast() - .map(Clone::clone) - .map_err(Into::into) + fn extract_bound(ob: &Bound<'a, PyAny>) -> PyResult { + ob.downcast().map(Clone::clone).map_err(Into::into) } } diff --git a/src/types/any.rs b/src/types/any.rs index 4536c2b889f..901b3451c3c 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -136,51 +136,6 @@ impl PyAny { .map(Bound::into_gil_ref) } - /// Retrieve an attribute value, skipping the instance dictionary during the lookup but still - /// binding the object to the instance. - /// - /// This is useful when trying to resolve Python's "magic" methods like `__getitem__`, which - /// are looked up starting from the type object. This returns an `Option` as it is not - /// typically a direct error for the special lookup to fail, as magic methods are optional in - /// many situations in which they might be called. - /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used - /// to intern `attr_name`. - #[allow(dead_code)] // Currently only used with num-complex+abi3, so dead without that. - pub(crate) fn lookup_special(&self, attr_name: N) -> PyResult> - where - N: IntoPy>, - { - let py = self.py(); - let self_type = self.get_type(); - let attr = if let Ok(attr) = self_type.getattr(attr_name) { - attr - } else { - return Ok(None); - }; - - // Manually resolve descriptor protocol. - if cfg!(Py_3_10) - || unsafe { ffi::PyType_HasFeature(attr.get_type_ptr(), ffi::Py_TPFLAGS_HEAPTYPE) } != 0 - { - // This is the preferred faster path, but does not work on static types (generally, - // types defined in extension modules) before Python 3.10. - unsafe { - let descr_get_ptr = ffi::PyType_GetSlot(attr.get_type_ptr(), ffi::Py_tp_descr_get); - if descr_get_ptr.is_null() { - return Ok(Some(attr)); - } - let descr_get: ffi::descrgetfunc = std::mem::transmute(descr_get_ptr); - let ret = descr_get(attr.as_ptr(), self.as_ptr(), self_type.as_ptr()); - py.from_owned_ptr_or_err(ret).map(Some) - } - } else if let Ok(descr_get) = attr.get_type().getattr(crate::intern!(py, "__get__")) { - descr_get.call1((attr, self, self_type)).map(Some) - } else { - Ok(Some(attr)) - } - } - /// Sets an attribute value. /// /// This is equivalent to the Python expression `self.attr_name = value`. @@ -1666,9 +1621,9 @@ pub trait PyAnyMethods<'py> { /// Extracts some type from the Python object. /// /// This is a wrapper function around [`FromPyObject::extract()`]. - fn extract<'a, D>(&'a self) -> PyResult + fn extract(&self) -> PyResult where - D: FromPyObject<'a>; + D: FromPyObject<'py>; /// Returns the reference count for the Python object. fn get_refcnt(&self) -> isize; @@ -2202,11 +2157,11 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { std::mem::transmute(self) } - fn extract<'a, D>(&'a self) -> PyResult + fn extract(&self) -> PyResult where - D: FromPyObject<'a>, + D: FromPyObject<'py>, { - FromPyObject::extract(self.as_gil_ref()) + FromPyObject::extract_bound(self) } fn get_refcnt(&self) -> isize { @@ -2293,13 +2248,64 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { } } +impl<'py> Bound<'py, PyAny> { + /// Retrieve an attribute value, skipping the instance dictionary during the lookup but still + /// binding the object to the instance. + /// + /// This is useful when trying to resolve Python's "magic" methods like `__getitem__`, which + /// are looked up starting from the type object. This returns an `Option` as it is not + /// typically a direct error for the special lookup to fail, as magic methods are optional in + /// many situations in which they might be called. + /// + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// to intern `attr_name`. + #[allow(dead_code)] // Currently only used with num-complex+abi3, so dead without that. + pub(crate) fn lookup_special(&self, attr_name: N) -> PyResult>> + where + N: IntoPy>, + { + let py = self.py(); + let self_type = self.get_type().as_borrowed(); + let attr = if let Ok(attr) = self_type.getattr(attr_name) { + attr + } else { + return Ok(None); + }; + + // Manually resolve descriptor protocol. + if cfg!(Py_3_10) + || unsafe { ffi::PyType_HasFeature(attr.get_type_ptr(), ffi::Py_TPFLAGS_HEAPTYPE) } != 0 + { + // This is the preferred faster path, but does not work on static types (generally, + // types defined in extension modules) before Python 3.10. + unsafe { + let descr_get_ptr = ffi::PyType_GetSlot(attr.get_type_ptr(), ffi::Py_tp_descr_get); + if descr_get_ptr.is_null() { + return Ok(Some(attr)); + } + let descr_get: ffi::descrgetfunc = std::mem::transmute(descr_get_ptr); + let ret = descr_get(attr.as_ptr(), self.as_ptr(), self_type.as_ptr()); + ret.assume_owned_or_err(py).map(Some) + } + } else if let Ok(descr_get) = attr + .get_type() + .as_borrowed() + .getattr(crate::intern!(py, "__get__")) + { + descr_get.call1((attr, self, self_type)).map(Some) + } else { + Ok(Some(attr)) + } + } +} + #[cfg(test)] #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use crate::{ basic::CompareOp, - types::{IntoPyDict, PyAny, PyBool, PyList, PyLong, PyModule}, - PyTypeInfo, Python, ToPyObject, + types::{any::PyAnyMethods, IntoPyDict, PyAny, PyBool, PyList, PyLong, PyModule}, + PyNativeType, PyTypeInfo, Python, ToPyObject, }; #[test] @@ -2344,8 +2350,13 @@ class NonHeapNonDescriptorInt: .unwrap(); let int = crate::intern!(py, "__int__"); - let eval_int = - |obj: &PyAny| obj.lookup_special(int)?.unwrap().call0()?.extract::(); + let eval_int = |obj: &PyAny| { + obj.as_borrowed() + .lookup_special(int)? + .unwrap() + .call0()? + .extract::() + }; let simple = module.getattr("SimpleInt").unwrap().call0().unwrap(); assert_eq!(eval_int(simple).unwrap(), 1); @@ -2354,7 +2365,7 @@ class NonHeapNonDescriptorInt: let no_descriptor = module.getattr("NoDescriptorInt").unwrap().call0().unwrap(); assert_eq!(eval_int(no_descriptor).unwrap(), 1); let missing = module.getattr("NoInt").unwrap().call0().unwrap(); - assert!(missing.lookup_special(int).unwrap().is_none()); + assert!(missing.as_borrowed().lookup_special(int).unwrap().is_none()); // Note the instance override should _not_ call the instance method that returns 2, // because that's not how special lookups are meant to work. let instance_override = module.getattr("instance_override").unwrap(); @@ -2364,7 +2375,7 @@ class NonHeapNonDescriptorInt: .unwrap() .call0() .unwrap(); - assert!(descriptor_error.lookup_special(int).is_err()); + assert!(descriptor_error.as_borrowed().lookup_special(int).is_err()); let nonheap_nondescriptor = module .getattr("NonHeapNonDescriptorInt") .unwrap() diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 5c246de380d..0decc0b4c80 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -5,6 +5,8 @@ use crate::{ PyObject, PyResult, Python, ToPyObject, }; +use super::any::PyAnyMethods; + /// Represents a Python `bool`. #[repr(transparent)] pub struct PyBool(PyAny); @@ -75,8 +77,8 @@ impl IntoPy for bool { /// Converts a Python `bool` to a Rust `bool`. /// /// Fails with `TypeError` if the input is not a Python `bool`. -impl<'source> FromPyObject<'source> for bool { - fn extract(obj: &'source PyAny) -> PyResult { +impl FromPyObject<'_> for bool { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { let err = match obj.downcast::() { Ok(obj) => return Ok(obj.is_true()), Err(err) => err, @@ -87,7 +89,7 @@ impl<'source> FromPyObject<'source> for bool { .name() .map_or(false, |name| name == "numpy.bool_") { - let missing_conversion = |obj: &PyAny| { + let missing_conversion = |obj: &Bound<'_, PyAny>| { PyTypeError::new_err(format!( "object of type '{}' does not define a '__bool__' conversion", obj.get_type() @@ -117,7 +119,7 @@ impl<'source> FromPyObject<'source> for bool { .lookup_special(crate::intern!(obj.py(), "__bool__"))? .ok_or_else(|| missing_conversion(obj))?; - let obj = meth.call0()?.downcast::()?; + let obj = meth.call0()?.downcast_into::()?; return Ok(obj.is_true()); } } diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 4860b8de30e..38f5f7e94e4 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -917,13 +917,13 @@ mod tests { assert_eq!(iter.size_hint(), (3, Some(3))); - assert_eq!(1_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(1, iter.next().unwrap().extract::().unwrap()); assert_eq!(iter.size_hint(), (2, Some(2))); - assert_eq!(2_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(2, iter.next().unwrap().extract::().unwrap()); assert_eq!(iter.size_hint(), (1, Some(1))); - assert_eq!(3_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(3, iter.next().unwrap().extract::().unwrap()); assert_eq!(iter.size_hint(), (0, Some(0))); assert!(iter.next().is_none()); @@ -940,13 +940,13 @@ mod tests { assert_eq!(iter.size_hint(), (3, Some(3))); - assert_eq!(3_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(3, iter.next().unwrap().extract::().unwrap()); assert_eq!(iter.size_hint(), (2, Some(2))); - assert_eq!(2_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(2, iter.next().unwrap().extract::().unwrap()); assert_eq!(iter.size_hint(), (1, Some(1))); - assert_eq!(1_i32, iter.next().unwrap().extract::<'_, i32>().unwrap()); + assert_eq!(1, iter.next().unwrap().extract::().unwrap()); assert_eq!(iter.size_hint(), (0, Some(0))); assert!(iter.next().is_none()); From d4d08b24b0f3bfd8b8c45d055cf98c0c81837fbe Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 29 Jan 2024 10:17:54 +0000 Subject: [PATCH 064/349] add `PyDict::new_bound` without deprecation --- src/types/dict.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/types/dict.rs b/src/types/dict.rs index 898c980f8a1..3a54e742ed5 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -56,9 +56,22 @@ pyobject_native_type_core!( ); impl PyDict { - /// Creates a new empty dictionary. + /// Deprecated form of [`new_bound`][PyDict::new_bound]. + #[inline] pub fn new(py: Python<'_>) -> &PyDict { - unsafe { py.from_owned_ptr(ffi::PyDict_New()) } + Self::new_bound(py).into_gil_ref() + } + + /// Creates a new empty dictionary. + pub fn new_bound(py: Python<'_>) -> Bound<'_, PyDict> { + unsafe { ffi::PyDict_New().assume_owned(py).downcast_into_unchecked() } + } + + /// Deprecated form of [`from_sequence_bound`][PyDict::from_sequence_bound]. + #[inline] + #[cfg(not(PyPy))] + pub fn from_sequence(seq: &PyAny) -> PyResult<&PyDict> { + Self::from_sequence_bound(&seq.as_borrowed()).map(Bound::into_gil_ref) } /// Creates a new dictionary from the sequence given. @@ -69,11 +82,11 @@ impl PyDict { /// Returns an error on invalid input. In the case of key collisions, /// this keeps the last entry seen. #[cfg(not(PyPy))] - pub fn from_sequence(seq: &PyAny) -> PyResult<&PyDict> { + pub fn from_sequence_bound<'py>(seq: &Bound<'py, PyAny>) -> PyResult> { let py = seq.py(); - let dict = Self::new(py); + let dict = Self::new_bound(py); err::error_on_minusone(py, unsafe { - ffi::PyDict_MergeFromSeq2(dict.into_ptr(), seq.into_ptr(), 1) + ffi::PyDict_MergeFromSeq2(dict.as_ptr(), seq.as_ptr(), 1) })?; Ok(dict) } From 345e122bbf649741b6bfe15316a8af0b599e2869 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 29 Jan 2024 10:25:40 +0000 Subject: [PATCH 065/349] add `PyFloat::new_bound` --- pyo3-benches/benches/bench_frompyobject.rs | 6 ++-- src/types/any.rs | 12 ++++---- src/types/float.rs | 36 +++++++++++++++++----- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs index c94cb114322..5a6cbcf76e1 100644 --- a/pyo3-benches/benches/bench_frompyobject.rs +++ b/pyo3-benches/benches/bench_frompyobject.rs @@ -73,10 +73,8 @@ fn not_a_list_via_extract_enum(b: &mut Bencher<'_>) { fn f64_from_pyobject(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let obj = PyFloat::new(py, 1.234); - b.iter(|| { - let _: f64 = obj.extract().unwrap(); - }); + let obj = &PyFloat::new_bound(py, 1.234); + b.iter(|| black_box(obj).extract::().unwrap()); }) } diff --git a/src/types/any.rs b/src/types/any.rs index 901b3451c3c..cd19e847e45 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -202,8 +202,8 @@ impl PyAny { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let a = PyFloat::new(py, 0_f64); - /// let b = PyFloat::new(py, 42_f64); + /// let a = PyFloat::new_bound(py, 0_f64); + /// let b = PyFloat::new_bound(py, 42_f64); /// assert_eq!(a.compare(b)?, Ordering::Less); /// Ok(()) /// })?; @@ -218,7 +218,7 @@ impl PyAny { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let a = PyFloat::new(py, 0_f64); + /// let a = PyFloat::new_bound(py, 0_f64); /// let b = PyString::new(py, "zero"); /// assert!(a.compare(b).is_err()); /// Ok(()) @@ -1058,8 +1058,8 @@ pub trait PyAnyMethods<'py> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let a = PyFloat::new(py, 0_f64); - /// let b = PyFloat::new(py, 42_f64); + /// let a = PyFloat::new_bound(py, 0_f64); + /// let b = PyFloat::new_bound(py, 42_f64); /// assert_eq!(a.compare(b)?, Ordering::Less); /// Ok(()) /// })?; @@ -1074,7 +1074,7 @@ pub trait PyAnyMethods<'py> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let a = PyFloat::new(py, 0_f64); + /// let a = PyFloat::new_bound(py, 0_f64); /// let b = PyString::new(py, "zero"); /// assert!(a.compare(b).is_err()); /// Ok(()) diff --git a/src/types/float.rs b/src/types/float.rs index b6dabdc48e8..4b4dc93d906 100644 --- a/src/types/float.rs +++ b/src/types/float.rs @@ -1,11 +1,13 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - ffi, instance::Bound, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, - Python, ToPyObject, + ffi, ffi_ptr_ext::FfiPtrExt, instance::Bound, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, + PyObject, PyResult, Python, ToPyObject, }; use std::os::raw::c_double; +use super::any::PyAnyMethods; + /// Represents a Python `float` object. /// /// You can usually avoid directly working with this type @@ -22,9 +24,26 @@ pyobject_native_type!( ); impl PyFloat { + /// Deprecated form of [`PyFloat::new_bound`]. + #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyFloat::new` will be replaced by `PyFloat::new_bound` in a future PyO3 version" + ) + )] + pub fn new(py: Python<'_>, val: f64) -> &'_ Self { + Self::new_bound(py, val).into_gil_ref() + } + /// Creates a new Python `float` object. - pub fn new(py: Python<'_>, val: c_double) -> &PyFloat { - unsafe { py.from_owned_ptr(ffi::PyFloat_FromDouble(val)) } + pub fn new_bound(py: Python<'_>, val: c_double) -> Bound<'_, PyFloat> { + unsafe { + ffi::PyFloat_FromDouble(val) + .assume_owned(py) + .downcast_into_unchecked() + } } /// Gets the value of this float. @@ -61,13 +80,13 @@ impl<'py> PyFloatMethods<'py> for Bound<'py, PyFloat> { impl ToPyObject for f64 { fn to_object(&self, py: Python<'_>) -> PyObject { - PyFloat::new(py, *self).into() + PyFloat::new_bound(py, *self).into() } } impl IntoPy for f64 { fn into_py(self, py: Python<'_>) -> PyObject { - PyFloat::new(py, self).into() + PyFloat::new_bound(py, self).into() } #[cfg(feature = "experimental-inspect")] @@ -108,13 +127,13 @@ impl<'source> FromPyObject<'source> for f64 { impl ToPyObject for f32 { fn to_object(&self, py: Python<'_>) -> PyObject { - PyFloat::new(py, f64::from(*self)).into() + PyFloat::new_bound(py, f64::from(*self)).into() } } impl IntoPy for f32 { fn into_py(self, py: Python<'_>) -> PyObject { - PyFloat::new(py, f64::from(self)).into() + PyFloat::new_bound(py, f64::from(self)).into() } #[cfg(feature = "experimental-inspect")] @@ -135,6 +154,7 @@ impl<'source> FromPyObject<'source> for f32 { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use crate::{types::PyFloat, Python, ToPyObject}; From c47565666d4b6431259994ea4ad72e8e48965d86 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 21 Nov 2023 11:41:43 +0000 Subject: [PATCH 066/349] add `PyString::new_bound` --- guide/src/conversions/traits.md | 4 +-- pyo3-benches/benches/bench_extract.rs | 15 ++++---- pyo3-benches/benches/bench_frompyobject.rs | 21 ++++++----- src/conversion.rs | 2 +- src/conversions/std/ipaddr.rs | 4 +-- src/conversions/std/string.rs | 16 ++++----- src/err/mod.rs | 2 +- src/ffi/tests.rs | 8 ++--- src/instance.rs | 2 +- src/marker.rs | 15 ++++---- src/types/any.rs | 4 +-- src/types/string.rs | 21 +++++++++-- tests/test_pyself.rs | 2 +- tests/ui/not_send2.rs | 4 +-- tests/ui/not_send2.stderr | 41 +++++++++++++++------- 15 files changed, 96 insertions(+), 65 deletions(-) diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index 746f098ba5d..f657fe073fc 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -236,7 +236,7 @@ struct RustyTransparentStruct { # use pyo3::types::PyString; # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let s = PyString::new(py, "test"); +# let s = PyString::new_bound(py, "test"); # # let tup: RustyTransparentTupleStruct = s.extract()?; # assert_eq!(tup.0, "test"); @@ -303,7 +303,7 @@ enum RustyEnum<'a> { # ); # } # { -# let thing = PyString::new(py, "text"); +# let thing = PyString::new_bound(py, "text"); # let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs index 6ff22d3834d..479bd0fd547 100644 --- a/pyo3-benches/benches/bench_extract.rs +++ b/pyo3-benches/benches/bench_extract.rs @@ -1,18 +1,16 @@ use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ + prelude::*, types::{PyDict, PyFloat, PyInt, PyString}, IntoPy, PyAny, PyObject, Python, }; fn extract_str_extract_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let s = PyString::new(py, "Hello, World!") as &PyAny; + let s = &PyString::new_bound(py, "Hello, World!"); - bench.iter(|| { - let v = black_box(s).extract::<&str>().unwrap(); - black_box(v); - }); + bench.iter(|| black_box(s).extract::<&str>().unwrap()); }); } @@ -27,14 +25,14 @@ fn extract_str_extract_fail(bench: &mut Bencher<'_>) { }); } +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] fn extract_str_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let s = PyString::new(py, "Hello, World!") as &PyAny; + let s = &PyString::new_bound(py, "Hello, World!"); bench.iter(|| { let py_str = black_box(s).downcast::().unwrap(); - let v = py_str.to_str().unwrap(); - black_box(v); + py_str.to_str().unwrap() }); }); } @@ -147,6 +145,7 @@ fn extract_float_downcast_fail(bench: &mut Bencher<'_>) { fn criterion_benchmark(c: &mut Criterion) { c.bench_function("extract_str_extract_success", extract_str_extract_success); c.bench_function("extract_str_extract_fail", extract_str_extract_fail); + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] c.bench_function("extract_str_downcast_success", extract_str_downcast_success); c.bench_function("extract_str_downcast_fail", extract_str_downcast_fail); c.bench_function("extract_int_extract_success", extract_int_extract_success); diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs index 5a6cbcf76e1..8114ee5a802 100644 --- a/pyo3-benches/benches/bench_frompyobject.rs +++ b/pyo3-benches/benches/bench_frompyobject.rs @@ -14,10 +14,9 @@ enum ManyTypes { fn enum_from_pyobject(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let obj = PyString::new(py, "hello world"); - b.iter(|| { - let _: ManyTypes = obj.extract().unwrap(); - }); + let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "hello world"); + + b.iter(|| any.extract::().unwrap()); }) } @@ -39,7 +38,7 @@ fn list_via_extract(b: &mut Bencher<'_>) { fn not_a_list_via_downcast(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &PyAny = PyString::new(py, "foobar").into(); + let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "foobar"); b.iter(|| black_box(any).downcast::().unwrap_err()); }) @@ -47,25 +46,25 @@ fn not_a_list_via_downcast(b: &mut Bencher<'_>) { fn not_a_list_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &PyAny = PyString::new(py, "foobar").into(); + let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "foobar"); - b.iter(|| black_box(any).extract::<&PyList>().unwrap_err()); + b.iter(|| black_box(any).extract::>().unwrap_err()); }) } #[derive(FromPyObject)] enum ListOrNotList<'a> { - List(&'a PyList), - NotList(&'a PyAny), + List(Bound<'a, PyList>), + NotList(Bound<'a, PyAny>), } fn not_a_list_via_extract_enum(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &PyAny = PyString::new(py, "foobar").into(); + let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "foobar"); b.iter(|| match black_box(any).extract::>() { Ok(ListOrNotList::List(_list)) => panic!(), - Ok(ListOrNotList::NotList(_any)) => (), + Ok(ListOrNotList::NotList(any)) => any, Err(_) => panic!(), }); }) diff --git a/src/conversion.rs b/src/conversion.rs index 46b39436e88..98ef98310c7 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -190,7 +190,7 @@ pub trait IntoPy: Sized { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { -/// let obj: Py = PyString::new(py, "blah").into(); +/// let obj: Py = PyString::new_bound(py, "blah").into(); /// /// // Straight from an owned reference /// let s: &str = obj.extract(py)?; diff --git a/src/conversions/std/ipaddr.rs b/src/conversions/std/ipaddr.rs index ca3c8728f9b..713de0afd1a 100755 --- a/src/conversions/std/ipaddr.rs +++ b/src/conversions/std/ipaddr.rs @@ -99,11 +99,11 @@ mod test_ipaddr { #[test] fn test_from_pystring() { Python::with_gil(|py| { - let py_str = PyString::new(py, "0:0:0:0:0:0:0:1"); + let py_str = PyString::new_bound(py, "0:0:0:0:0:0:0:1"); let ip: IpAddr = py_str.to_object(py).extract(py).unwrap(); assert_eq!(ip, IpAddr::from_str("::1").unwrap()); - let py_str = PyString::new(py, "invalid"); + let py_str = PyString::new_bound(py, "invalid"); assert!(py_str.to_object(py).extract::(py).is_err()); }); } diff --git a/src/conversions/std/string.rs b/src/conversions/std/string.rs index b4bc8c26099..b43e5422981 100644 --- a/src/conversions/std/string.rs +++ b/src/conversions/std/string.rs @@ -11,14 +11,14 @@ use crate::{ impl ToPyObject for str { #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { - PyString::new(py, self).into() + PyString::new_bound(py, self).into() } } impl<'a> IntoPy for &'a str { #[inline] fn into_py(self, py: Python<'_>) -> PyObject { - PyString::new(py, self).into() + PyString::new_bound(py, self).into() } #[cfg(feature = "experimental-inspect")] @@ -30,7 +30,7 @@ impl<'a> IntoPy for &'a str { impl<'a> IntoPy> for &'a str { #[inline] fn into_py(self, py: Python<'_>) -> Py { - PyString::new(py, self).into() + PyString::new_bound(py, self).into() } #[cfg(feature = "experimental-inspect")] @@ -44,7 +44,7 @@ impl<'a> IntoPy> for &'a str { impl ToPyObject for Cow<'_, str> { #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { - PyString::new(py, self).into() + PyString::new_bound(py, self).into() } } @@ -65,7 +65,7 @@ impl IntoPy for Cow<'_, str> { impl ToPyObject for String { #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { - PyString::new(py, self).into() + PyString::new_bound(py, self).into() } } @@ -78,7 +78,7 @@ impl ToPyObject for char { impl IntoPy for char { fn into_py(self, py: Python<'_>) -> PyObject { let mut bytes = [0u8; 4]; - PyString::new(py, self.encode_utf8(&mut bytes)).into() + PyString::new_bound(py, self.encode_utf8(&mut bytes)).into() } #[cfg(feature = "experimental-inspect")] @@ -89,7 +89,7 @@ impl IntoPy for char { impl IntoPy for String { fn into_py(self, py: Python<'_>) -> PyObject { - PyString::new(py, &self).into() + PyString::new_bound(py, &self).into() } #[cfg(feature = "experimental-inspect")] @@ -101,7 +101,7 @@ impl IntoPy for String { impl<'a> IntoPy for &'a String { #[inline] fn into_py(self, py: Python<'_>) -> PyObject { - PyString::new(py, self).into() + PyString::new_bound(py, self).into() } #[cfg(feature = "experimental-inspect")] diff --git a/src/err/mod.rs b/src/err/mod.rs index e6bcd521887..13e154b3860 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -206,7 +206,7 @@ impl PyErr { /// assert_eq!(err.to_string(), "TypeError: "); /// /// // Case #3: Invalid exception value - /// let err = PyErr::from_value(PyString::new(py, "foo").into()); + /// let err = PyErr::from_value(PyString::new_bound(py, "foo").as_gil_ref()); /// assert_eq!( /// err.to_string(), /// "TypeError: exceptions must derive from BaseException" diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index 2cbd84dc6db..cb0972590ff 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -3,7 +3,7 @@ use crate::Python; #[cfg(not(Py_LIMITED_API))] use crate::{ - types::{PyDict, PyString}, + types::{any::PyAnyMethods, PyDict, PyString}, IntoPy, Py, PyAny, }; #[cfg(not(any(Py_3_12, Py_LIMITED_API)))] @@ -98,7 +98,7 @@ fn test_timezone_from_offset_and_name() { Python::with_gil(|py| { let delta = PyDelta::new(py, 0, 100, 0, false).unwrap(); - let tzname = PyString::new(py, "testtz"); + let tzname = PyString::new_bound(py, "testtz"); let tz: &PyAny = unsafe { py.from_borrowed_ptr(PyTimeZone_FromOffsetAndName( delta.as_ptr(), @@ -167,7 +167,7 @@ fn ascii_object_bitfield() { fn ascii() { Python::with_gil(|py| { // This test relies on implementation details of PyString. - let s = PyString::new(py, "hello, world"); + let s = PyString::new_bound(py, "hello, world"); let ptr = s.as_ptr(); unsafe { @@ -209,7 +209,7 @@ fn ascii() { fn ucs4() { Python::with_gil(|py| { let s = "哈哈🐈"; - let py_string = PyString::new(py, s); + let py_string = PyString::new_bound(py, s); let ptr = py_string.as_ptr(); unsafe { diff --git a/src/instance.rs b/src/instance.rs index d7edfe06365..c1ab45926b3 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1624,7 +1624,7 @@ a = A() assert!(instance .getattr(py, "foo")? .as_ref(py) - .eq(PyString::new(py, "bar"))?); + .eq(PyString::new_bound(py, "bar"))?); instance.getattr(py, "foo")?; Ok(()) diff --git a/src/marker.rs b/src/marker.rs index 74c7095fa4b..836d0109eeb 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -72,7 +72,7 @@ //! use send_wrapper::SendWrapper; //! //! Python::with_gil(|py| { -//! let string = PyString::new(py, "foo"); +//! let string = PyString::new_bound(py, "foo"); //! //! let wrapped = SendWrapper::new(string); //! @@ -80,7 +80,7 @@ //! # #[cfg(not(feature = "nightly"))] //! # { //! // 💥 Unsound! 💥 -//! let smuggled: &PyString = *wrapped; +//! let smuggled: &Bound<'_, PyString> = &*wrapped; //! println!("{:?}", smuggled); //! # } //! }); @@ -164,12 +164,12 @@ use std::os::raw::c_int; /// use send_wrapper::SendWrapper; /// /// Python::with_gil(|py| { -/// let string = PyString::new(py, "foo"); +/// let string = PyString::new_bound(py, "foo"); /// /// let wrapped = SendWrapper::new(string); /// /// py.allow_threads(|| { -/// let sneaky: &PyString = *wrapped; +/// let sneaky: &Bound<'_, PyString> = &*wrapped; /// /// println!("{:?}", sneaky); /// }); @@ -210,7 +210,7 @@ mod nightly { /// # use pyo3::prelude::*; /// # use pyo3::types::PyString; /// Python::with_gil(|py| { - /// let string = PyString::new(py, "foo"); + /// let string = PyString::new_bound(py, "foo"); /// /// py.allow_threads(|| { /// println!("{:?}", string); @@ -238,7 +238,7 @@ mod nightly { /// use send_wrapper::SendWrapper; /// /// Python::with_gil(|py| { - /// let string = PyString::new(py, "foo"); + /// let string = PyString::new_bound(py, "foo"); /// /// let wrapped = SendWrapper::new(string); /// @@ -521,7 +521,7 @@ impl<'py> Python<'py> { /// use pyo3::types::PyString; /// /// fn parallel_print(py: Python<'_>) { - /// let s = PyString::new(py, "This object cannot be accessed without holding the GIL >_<"); + /// let s = PyString::new_bound(py, "This object cannot be accessed without holding the GIL >_<"); /// py.allow_threads(move || { /// println!("{:?}", s); // This causes a compile error. /// }); @@ -1004,6 +1004,7 @@ impl Python<'_> { /// The `Ungil` bound on the closure does prevent hanging on to existing GIL-bound references /// /// ```compile_fail + /// # #![allow(deprecated)] /// # use pyo3::prelude::*; /// # use pyo3::types::PyString; /// diff --git a/src/types/any.rs b/src/types/any.rs index cd19e847e45..14102d80a8d 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -219,7 +219,7 @@ impl PyAny { /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { /// let a = PyFloat::new_bound(py, 0_f64); - /// let b = PyString::new(py, "zero"); + /// let b = PyString::new_bound(py, "zero"); /// assert!(a.compare(b).is_err()); /// Ok(()) /// })?; @@ -1075,7 +1075,7 @@ pub trait PyAnyMethods<'py> { /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { /// let a = PyFloat::new_bound(py, 0_f64); - /// let b = PyString::new(py, "zero"); + /// let b = PyString::new_bound(py, "zero"); /// assert!(a.compare(b).is_err()); /// Ok(()) /// })?; diff --git a/src/types/string.rs b/src/types/string.rs index 0ee64be53f9..fd20aab8c45 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -134,13 +134,29 @@ pub struct PyString(PyAny); pyobject_native_type_core!(PyString, pyobject_native_static_type_object!(ffi::PyUnicode_Type), #checkfunction=ffi::PyUnicode_Check); impl PyString { + /// Deprecated form of [`PyString::new_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyString::new` will be replaced by `PyString::new_bound` in a future PyO3 version" + ) + )] + pub fn new<'py>(py: Python<'py>, s: &str) -> &'py Self { + Self::new_bound(py, s).into_gil_ref() + } + /// Creates a new Python string object. /// /// Panics if out of memory. - pub fn new<'p>(py: Python<'p>, s: &str) -> &'p PyString { + pub fn new_bound<'py>(py: Python<'py>, s: &str) -> Bound<'py, PyString> { let ptr = s.as_ptr() as *const c_char; let len = s.len() as ffi::Py_ssize_t; - unsafe { py.from_owned_ptr(ffi::PyUnicode_FromStringAndSize(ptr, len)) } + unsafe { + ffi::PyUnicode_FromStringAndSize(ptr, len) + .assume_owned(py) + .downcast_into_unchecked() + } } /// Intern the given string @@ -452,6 +468,7 @@ impl IntoPy> for &'_ Py { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; use crate::Python; diff --git a/tests/test_pyself.rs b/tests/test_pyself.rs index ed922d3e2b0..7abec02b0a2 100644 --- a/tests/test_pyself.rs +++ b/tests/test_pyself.rs @@ -73,7 +73,7 @@ impl Iter { let res = reader_ref .inner .get(&b) - .map(|s| PyString::new(py, s).into()); + .map(|s| PyString::new_bound(py, s).into()); Ok(res) } None => Ok(None), diff --git a/tests/ui/not_send2.rs b/tests/ui/not_send2.rs index 4eb0a9f0f09..fa99e602ba0 100644 --- a/tests/ui/not_send2.rs +++ b/tests/ui/not_send2.rs @@ -3,10 +3,10 @@ use pyo3::types::PyString; fn main() { Python::with_gil(|py| { - let string = PyString::new(py, "foo"); + let string = PyString::new_bound(py, "foo"); py.allow_threads(|| { println!("{:?}", string); }); }); -} \ No newline at end of file +} diff --git a/tests/ui/not_send2.stderr b/tests/ui/not_send2.stderr index d3a60a1f708..30189e09b9f 100644 --- a/tests/ui/not_send2.stderr +++ b/tests/ui/not_send2.stderr @@ -1,4 +1,4 @@ -error[E0277]: `UnsafeCell` cannot be shared between threads safely +error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safely --> tests/ui/not_send2.rs:8:26 | 8 | py.allow_threads(|| { @@ -7,21 +7,36 @@ error[E0277]: `UnsafeCell` cannot be shared between threads safely | | required by a bound introduced by this call 9 | | println!("{:?}", string); 10 | | }); - | |_________^ `UnsafeCell` cannot be shared between threads safely + | |_________^ `*mut pyo3::Python<'static>` cannot be shared between threads safely | - = help: within `&PyString`, the trait `Sync` is not implemented for `UnsafeCell` -note: required because it appears within the type `PyAny` - --> src/types/any.rs + = help: within `pyo3::Bound<'_, PyString>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` +note: required because it appears within the type `PhantomData<*mut Python<'static>>` + --> $RUST/core/src/marker.rs | - | pub struct PyAny(UnsafeCell); - | ^^^^^ -note: required because it appears within the type `PyString` - --> src/types/string.rs + | pub struct PhantomData; + | ^^^^^^^^^^^ +note: required because it appears within the type `NotSend` + --> src/impl_/not_send.rs + | + | pub(crate) struct NotSend(PhantomData<*mut Python<'static>>); + | ^^^^^^^ + = note: required because it appears within the type `(&GILGuard, NotSend)` +note: required because it appears within the type `PhantomData<(&GILGuard, NotSend)>` + --> $RUST/core/src/marker.rs + | + | pub struct PhantomData; + | ^^^^^^^^^^^ +note: required because it appears within the type `Python<'_>` + --> src/marker.rs + | + | pub struct Python<'py>(PhantomData<(&'py GILGuard, NotSend)>); + | ^^^^^^ +note: required because it appears within the type `Bound<'_, PyString>` + --> src/instance.rs | - | pub struct PyString(PyAny); - | ^^^^^^^^ - = note: required because it appears within the type `&PyString` - = note: required for `&&PyString` to implement `Send` + | pub struct Bound<'py, T>(Python<'py>, ManuallyDrop>); + | ^^^^^ + = note: required for `&pyo3::Bound<'_, PyString>` to implement `Send` note: required because it's used within this closure --> tests/ui/not_send2.rs:8:26 | From a3eb328378e17d7fa79f0876393269dd4e511ab8 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 29 Jan 2024 13:31:04 +0000 Subject: [PATCH 067/349] migrate `FromPyObject` for `Bound` and `Py` to `extract_bound` --- newsfragments/3776.changed.md | 1 + src/instance.rs | 19 +++++++------------ 2 files changed, 8 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3776.changed.md diff --git a/newsfragments/3776.changed.md b/newsfragments/3776.changed.md new file mode 100644 index 00000000000..71ffd893a18 --- /dev/null +++ b/newsfragments/3776.changed.md @@ -0,0 +1 @@ +Relax bound of `FromPyObject` for `Py` to just `T: PyTypeCheck`. diff --git a/src/instance.rs b/src/instance.rs index d7edfe06365..e29f46356fc 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1405,27 +1405,22 @@ impl Drop for Py { } } -impl<'a, T> FromPyObject<'a> for Py +impl FromPyObject<'_> for Py where - T: PyTypeInfo, - &'a T::AsRefTarget: FromPyObject<'a>, - T::AsRefTarget: 'a + AsPyPointer, + T: PyTypeCheck, { /// Extracts `Self` from the source `PyObject`. - fn extract(ob: &'a PyAny) -> PyResult { - unsafe { - ob.extract::<&T::AsRefTarget>() - .map(|val| Py::from_borrowed_ptr(ob.py(), val.as_ptr())) - } + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { + ob.extract::>().map(Bound::unbind) } } -impl<'a, T> FromPyObject<'a> for Bound<'a, T> +impl<'py, T> FromPyObject<'py> for Bound<'py, T> where - T: PyTypeInfo, + T: PyTypeCheck, { /// Extracts `Self` from the source `PyObject`. - fn extract_bound(ob: &Bound<'a, PyAny>) -> PyResult { + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { ob.downcast().map(Clone::clone).map_err(Into::into) } } From e323fcbb9ebfd5030a5375e8cff8f3f2c604852c Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 27 Jan 2024 20:17:12 +0100 Subject: [PATCH 068/349] implement `PyCapsuleMethods` --- src/prelude.rs | 1 + src/types/capsule.rs | 273 +++++++++++++++++++++++++++++++----------- src/types/function.rs | 3 +- src/types/mod.rs | 2 +- 4 files changed, 209 insertions(+), 70 deletions(-) diff --git a/src/prelude.rs b/src/prelude.rs index 06d283c91d7..47951aedcce 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -29,6 +29,7 @@ pub use crate::types::any::PyAnyMethods; pub use crate::types::boolobject::PyBoolMethods; pub use crate::types::bytearray::PyByteArrayMethods; pub use crate::types::bytes::PyBytesMethods; +pub use crate::types::capsule::PyCapsuleMethods; pub use crate::types::dict::PyDictMethods; pub use crate::types::float::PyFloatMethods; pub use crate::types::frozenset::PyFrozenSetMethods; diff --git a/src/types/capsule.rs b/src/types/capsule.rs index f97320eb474..30e77391171 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -1,6 +1,8 @@ -use crate::Python; -use crate::{ffi, PyAny}; +use crate::ffi_ptr_ext::FfiPtrExt; +use crate::py_result_ext::PyResultExt; +use crate::{ffi, PyAny, PyNativeType}; use crate::{pyobject_native_type_core, PyErr, PyResult}; +use crate::{Bound, Python}; use std::ffi::{CStr, CString}; use std::os::raw::{c_char, c_int, c_void}; @@ -27,7 +29,7 @@ use std::os::raw::{c_char, c_int, c_void}; /// let foo = Foo { val: 123 }; /// let name = CString::new("builtins.capsule").unwrap(); /// -/// let capsule = PyCapsule::new(py, foo, Some(name.clone()))?; +/// let capsule = PyCapsule::new_bound(py, foo, Some(name.clone()))?; /// /// let module = PyModule::import(py, "builtins")?; /// module.add("capsule", capsule)?; @@ -44,6 +46,22 @@ pub struct PyCapsule(PyAny); pyobject_native_type_core!(PyCapsule, pyobject_native_static_type_object!(ffi::PyCapsule_Type), #checkfunction=ffi::PyCapsule_CheckExact); impl PyCapsule { + /// Deprecated form of [`PyCapsule::new_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyCapsule::new` will be replaced by `PyCapsule::new_bound` in a future PyO3 version" + ) + )] + pub fn new( + py: Python<'_>, + value: T, + name: Option, + ) -> PyResult<&Self> { + Self::new_bound(py, value, name).map(Bound::into_gil_ref) + } + /// Constructs a new capsule whose contents are `value`, associated with `name`. /// `name` is the identifier for the capsule; if it is stored as an attribute of a module, /// the name should be in the format `"modulename.attribute"`. @@ -59,7 +77,7 @@ impl PyCapsule { /// /// Python::with_gil(|py| { /// let name = CString::new("foo").unwrap(); - /// let capsule = PyCapsule::new(py, 123_u32, Some(name)).unwrap(); + /// let capsule = PyCapsule::new_bound(py, 123_u32, Some(name)).unwrap(); /// let val = unsafe { capsule.reference::() }; /// assert_eq!(*val, 123); /// }); @@ -72,15 +90,35 @@ impl PyCapsule { /// use std::ffi::CString; /// /// Python::with_gil(|py| { - /// let capsule = PyCapsule::new(py, (), None).unwrap(); // Oops! `()` is zero sized! + /// let capsule = PyCapsule::new_bound(py, (), None).unwrap(); // Oops! `()` is zero sized! /// }); /// ``` - pub fn new( + pub fn new_bound( py: Python<'_>, value: T, name: Option, - ) -> PyResult<&Self> { - Self::new_with_destructor(py, value, name, |_, _| {}) + ) -> PyResult> { + Self::new_bound_with_destructor(py, value, name, |_, _| {}) + } + + /// Deprecated form of [`PyCapsule::new_bound_with_destructor`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyCapsule::new_with_destructor` will be replaced by `PyCapsule::new_bound_with_destructor` in a future PyO3 version" + ) + )] + pub fn new_with_destructor< + T: 'static + Send + AssertNotZeroSized, + F: FnOnce(T, *mut c_void) + Send, + >( + py: Python<'_>, + value: T, + name: Option, + destructor: F, + ) -> PyResult<&'_ Self> { + Self::new_bound_with_destructor(py, value, name, destructor).map(Bound::into_gil_ref) } /// Constructs a new capsule whose contents are `value`, associated with `name`. @@ -90,7 +128,7 @@ impl PyCapsule { /// /// The `destructor` must be `Send`, because there is no guarantee which thread it will eventually /// be called from. - pub fn new_with_destructor< + pub fn new_bound_with_destructor< T: 'static + Send + AssertNotZeroSized, F: FnOnce(T, *mut c_void) + Send, >( @@ -98,7 +136,7 @@ impl PyCapsule { value: T, name: Option, destructor: F, - ) -> PyResult<&'_ Self> { + ) -> PyResult> { AssertNotZeroSized::assert_not_zero_sized(&value); // Sanity check for capsule layout @@ -112,12 +150,13 @@ impl PyCapsule { }); unsafe { - let cap_ptr = ffi::PyCapsule_New( - Box::into_raw(val) as *mut c_void, + ffi::PyCapsule_New( + Box::into_raw(val).cast(), name_ptr, Some(capsule_destructor::), - ); - py.from_owned_ptr_or_err(cap_ptr) + ) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } @@ -134,7 +173,7 @@ impl PyCapsule { if ptr.is_null() { Err(PyErr::fetch(py)) } else { - Ok(&*(ptr as *const T)) + Ok(&*ptr.cast::()) } } @@ -160,29 +199,23 @@ impl PyCapsule { /// let (tx, rx) = channel::(); /// /// fn destructor(val: u32, context: *mut c_void) { - /// let ctx = unsafe { *Box::from_raw(context as *mut Sender) }; + /// let ctx = unsafe { *Box::from_raw(context.cast::>()) }; /// ctx.send("Destructor called!".to_string()).unwrap(); /// } /// /// Python::with_gil(|py| { /// let capsule = - /// PyCapsule::new_with_destructor(py, 123, None, destructor as fn(u32, *mut c_void)) + /// PyCapsule::new_bound_with_destructor(py, 123, None, destructor as fn(u32, *mut c_void)) /// .unwrap(); /// let context = Box::new(tx); // `Sender` is our context, box it up and ship it! - /// capsule.set_context(Box::into_raw(context) as *mut c_void).unwrap(); + /// capsule.set_context(Box::into_raw(context).cast()).unwrap(); /// // This scope will end, causing our destructor to be called... /// }); /// /// assert_eq!(rx.recv(), Ok("Destructor called!".to_string())); /// ``` - #[allow(clippy::not_unsafe_ptr_arg_deref)] pub fn set_context(&self, context: *mut c_void) -> PyResult<()> { - let result = unsafe { ffi::PyCapsule_SetContext(self.as_ptr(), context) }; - if result != 0 { - Err(PyErr::fetch(self.py())) - } else { - Ok(()) - } + self.as_borrowed().set_context(context) } /// Gets the current context stored in the capsule. If there is no context, the pointer @@ -190,11 +223,7 @@ impl PyCapsule { /// /// Returns an error if this capsule is not valid. pub fn context(&self) -> PyResult<*mut c_void> { - let ctx = unsafe { ffi::PyCapsule_GetContext(self.as_ptr()) }; - if ctx.is_null() { - ensure_no_error(self.py())? - } - Ok(ctx) + self.as_borrowed().context() } /// Obtains a reference to the value of this capsule. @@ -203,15 +232,132 @@ impl PyCapsule { /// /// It must be known that this capsule is valid and its pointer is to an item of type `T`. pub unsafe fn reference(&self) -> &T { - &*(self.pointer() as *const T) + self.as_borrowed().reference() } /// Gets the raw `c_void` pointer to the value in this capsule. /// /// Returns null if this capsule is not valid. pub fn pointer(&self) -> *mut c_void { + self.as_borrowed().pointer() + } + + /// Checks if this is a valid capsule. + /// + /// Returns true if the stored `pointer()` is non-null. + pub fn is_valid(&self) -> bool { + self.as_borrowed().is_valid() + } + + /// Retrieves the name of this capsule, if set. + /// + /// Returns an error if this capsule is not valid. + pub fn name(&self) -> PyResult> { + self.as_borrowed().name() + } +} + +/// Implementation of functionality for [`PyCapsule`]. +/// +/// These methods are defined for the `Bound<'py, PyCapsule>` smart pointer, so to use method call +/// syntax these methods are separated into a trait, because stable Rust does not yet support +/// `arbitrary_self_types`. +#[doc(alias = "PyCapsule")] +pub trait PyCapsuleMethods<'py> { + /// Sets the context pointer in the capsule. + /// + /// Returns an error if this capsule is not valid. + /// + /// # Notes + /// + /// The context is treated much like the value of the capsule, but should likely act as + /// a place to store any state management when using the capsule. + /// + /// If you want to store a Rust value as the context, and drop it from the destructor, use + /// `Box::into_raw` to convert it into a pointer, see the example. + /// + /// # Example + /// + /// ``` + /// use std::sync::mpsc::{channel, Sender}; + /// use libc::c_void; + /// use pyo3::{prelude::*, types::PyCapsule}; + /// + /// let (tx, rx) = channel::(); + /// + /// fn destructor(val: u32, context: *mut c_void) { + /// let ctx = unsafe { *Box::from_raw(context.cast::>()) }; + /// ctx.send("Destructor called!".to_string()).unwrap(); + /// } + /// + /// Python::with_gil(|py| { + /// let capsule = + /// PyCapsule::new_bound_with_destructor(py, 123, None, destructor as fn(u32, *mut c_void)) + /// .unwrap(); + /// let context = Box::new(tx); // `Sender` is our context, box it up and ship it! + /// capsule.set_context(Box::into_raw(context).cast()).unwrap(); + /// // This scope will end, causing our destructor to be called... + /// }); + /// + /// assert_eq!(rx.recv(), Ok("Destructor called!".to_string())); + /// ``` + fn set_context(&self, context: *mut c_void) -> PyResult<()>; + + /// Gets the current context stored in the capsule. If there is no context, the pointer + /// will be null. + /// + /// Returns an error if this capsule is not valid. + fn context(&self) -> PyResult<*mut c_void>; + + /// Obtains a reference to the value of this capsule. + /// + /// # Safety + /// + /// It must be known that this capsule is valid and its pointer is to an item of type `T`. + unsafe fn reference(&self) -> &'py T; + + /// Gets the raw `c_void` pointer to the value in this capsule. + /// + /// Returns null if this capsule is not valid. + fn pointer(&self) -> *mut c_void; + + /// Checks if this is a valid capsule. + /// + /// Returns true if the stored `pointer()` is non-null. + fn is_valid(&self) -> bool; + + /// Retrieves the name of this capsule, if set. + /// + /// Returns an error if this capsule is not valid. + fn name(&self) -> PyResult>; +} + +impl<'py> PyCapsuleMethods<'py> for Bound<'py, PyCapsule> { + #[allow(clippy::not_unsafe_ptr_arg_deref)] + fn set_context(&self, context: *mut c_void) -> PyResult<()> { + let result = unsafe { ffi::PyCapsule_SetContext(self.as_ptr(), context) }; + if result != 0 { + Err(PyErr::fetch(self.py())) + } else { + Ok(()) + } + } + + fn context(&self) -> PyResult<*mut c_void> { + let ctx = unsafe { ffi::PyCapsule_GetContext(self.as_ptr()) }; + if ctx.is_null() { + ensure_no_error(self.py())? + } + Ok(ctx) + } + + unsafe fn reference(&self) -> &'py T { + &*self.pointer().cast() + } + + fn pointer(&self) -> *mut c_void { unsafe { - let ptr = ffi::PyCapsule_GetPointer(self.0.as_ptr(), self.name_ptr_ignore_error()); + let ptr = ffi::PyCapsule_GetPointer(self.as_ptr(), name_ptr_ignore_error(self)); if ptr.is_null() { ffi::PyErr_Clear(); } @@ -219,21 +365,15 @@ impl PyCapsule { } } - /// Checks if this is a valid capsule. - /// - /// Returns true if the stored `pointer()` is non-null. - pub fn is_valid(&self) -> bool { + fn is_valid(&self) -> bool { // As well as if the stored pointer is null, PyCapsule_IsValid also returns false if // self.as_ptr() is null or not a ptr to a PyCapsule object. Both of these are guaranteed // to not be the case thanks to invariants of this PyCapsule struct. - let r = unsafe { ffi::PyCapsule_IsValid(self.as_ptr(), self.name_ptr_ignore_error()) }; + let r = unsafe { ffi::PyCapsule_IsValid(self.as_ptr(), name_ptr_ignore_error(self)) }; r != 0 } - /// Retrieves the name of this capsule, if set. - /// - /// Returns an error if this capsule is not valid. - pub fn name(&self) -> PyResult> { + fn name(&self) -> PyResult> { unsafe { let ptr = ffi::PyCapsule_GetName(self.as_ptr()); if ptr.is_null() { @@ -244,18 +384,6 @@ impl PyCapsule { } } } - - /// Attempts to retrieve the raw name pointer of this capsule. - /// - /// On error, clears the error indicator and returns NULL. This is a private function and next - /// use of this capsule will error anyway. - fn name_ptr_ignore_error(&self) -> *const c_char { - let ptr = unsafe { ffi::PyCapsule_GetName(self.as_ptr()) }; - if ptr.is_null() { - unsafe { ffi::PyErr_Clear() }; - } - ptr - } } // C layout, as PyCapsule::get_reference depends on `T` being first. @@ -277,7 +405,7 @@ unsafe extern "C" fn capsule_destructor); + } = *Box::from_raw(ptr.cast::>()); destructor(value, ctx) } @@ -304,11 +432,20 @@ fn ensure_no_error(py: Python<'_>) -> PyResult<()> { } } +fn name_ptr_ignore_error(slf: &Bound<'_, PyCapsule>) -> *const c_char { + let ptr = unsafe { ffi::PyCapsule_GetName(slf.as_ptr()) }; + if ptr.is_null() { + unsafe { ffi::PyErr_Clear() }; + } + ptr +} + #[cfg(test)] mod tests { use libc::c_void; use crate::prelude::PyModule; + use crate::types::capsule::PyCapsuleMethods; use crate::{types::PyCapsule, Py, PyResult, Python}; use std::ffi::CString; use std::sync::mpsc::{channel, Sender}; @@ -330,7 +467,7 @@ mod tests { let foo = Foo { val: 123 }; let name = CString::new("foo").unwrap(); - let cap = PyCapsule::new(py, foo, Some(name.clone()))?; + let cap = PyCapsule::new_bound(py, foo, Some(name.clone()))?; assert!(cap.is_valid()); let foo_capi = unsafe { cap.reference::() }; @@ -349,7 +486,7 @@ mod tests { let cap: Py = Python::with_gil(|py| { let name = CString::new("foo").unwrap(); - let cap = PyCapsule::new(py, foo as fn(u32) -> u32, Some(name)).unwrap(); + let cap = PyCapsule::new_bound(py, foo as fn(u32) -> u32, Some(name)).unwrap(); cap.into() }); @@ -363,16 +500,16 @@ mod tests { fn test_pycapsule_context() -> PyResult<()> { Python::with_gil(|py| { let name = CString::new("foo").unwrap(); - let cap = PyCapsule::new(py, 0, Some(name))?; + let cap = PyCapsule::new_bound(py, 0, Some(name))?; let c = cap.context()?; assert!(c.is_null()); let ctx = Box::new(123_u32); - cap.set_context(Box::into_raw(ctx) as _)?; + cap.set_context(Box::into_raw(ctx).cast())?; let ctx_ptr: *mut c_void = cap.context()?; - let ctx = unsafe { *Box::from_raw(ctx_ptr as *mut u32) }; + let ctx = unsafe { *Box::from_raw(ctx_ptr.cast::()) }; assert_eq!(ctx, 123); Ok(()) }) @@ -389,7 +526,7 @@ mod tests { let foo = Foo { val: 123 }; let name = CString::new("builtins.capsule").unwrap(); - let capsule = PyCapsule::new(py, foo, Some(name.clone()))?; + let capsule = PyCapsule::new_bound(py, foo, Some(name.clone()))?; let module = PyModule::import(py, "builtins")?; module.add("capsule", capsule)?; @@ -412,7 +549,7 @@ mod tests { let name = CString::new("foo").unwrap(); let stuff: Vec = vec![1, 2, 3, 4]; - let cap = PyCapsule::new(py, stuff, Some(name)).unwrap(); + let cap = PyCapsule::new_bound(py, stuff, Some(name)).unwrap(); cap.into() }); @@ -429,8 +566,8 @@ mod tests { let cap: Py = Python::with_gil(|py| { let name = CString::new("foo").unwrap(); - let cap = PyCapsule::new(py, 0, Some(name)).unwrap(); - cap.set_context(Box::into_raw(Box::new(&context)) as _) + let cap = PyCapsule::new_bound(py, 0, Some(name)).unwrap(); + cap.set_context(Box::into_raw(Box::new(&context)).cast()) .unwrap(); cap.into() @@ -438,7 +575,7 @@ mod tests { Python::with_gil(|py| { let ctx_ptr: *mut c_void = cap.as_ref(py).context().unwrap(); - let ctx = unsafe { *Box::from_raw(ctx_ptr as *mut &Vec) }; + let ctx = unsafe { *Box::from_raw(ctx_ptr.cast::<&Vec>()) }; assert_eq!(ctx, &vec![1_u8, 2, 3, 4]); }) } @@ -449,14 +586,14 @@ mod tests { fn destructor(_val: u32, ctx: *mut c_void) { assert!(!ctx.is_null()); - let context = unsafe { *Box::from_raw(ctx as *mut Sender) }; + let context = unsafe { *Box::from_raw(ctx.cast::>()) }; context.send(true).unwrap(); } Python::with_gil(|py| { let name = CString::new("foo").unwrap(); - let cap = PyCapsule::new_with_destructor(py, 0, Some(name), destructor).unwrap(); - cap.set_context(Box::into_raw(Box::new(tx)) as _).unwrap(); + let cap = PyCapsule::new_bound_with_destructor(py, 0, Some(name), destructor).unwrap(); + cap.set_context(Box::into_raw(Box::new(tx)).cast()).unwrap(); }); // the destructor was called. @@ -466,7 +603,7 @@ mod tests { #[test] fn test_pycapsule_no_name() { Python::with_gil(|py| { - let cap = PyCapsule::new(py, 0usize, None).unwrap(); + let cap = PyCapsule::new_bound(py, 0usize, None).unwrap(); assert_eq!(unsafe { cap.reference::() }, &0usize); assert_eq!(cap.name().unwrap(), None); diff --git a/src/types/function.rs b/src/types/function.rs index b4492b58c65..c6f0f6b0b02 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -1,6 +1,7 @@ use crate::derive_utils::PyFunctionArguments; use crate::methods::PyMethodDefDestructor; use crate::prelude::*; +use crate::types::capsule::PyCapsuleMethods; use crate::{ ffi, impl_::pymethods::{self, PyMethodDef}, @@ -80,7 +81,7 @@ impl PyCFunction { ); let (def, def_destructor) = method_def.as_method_def()?; - let capsule = PyCapsule::new( + let capsule = PyCapsule::new_bound( py, ClosureDestructor:: { closure, diff --git a/src/types/mod.rs b/src/types/mod.rs index fcf843ec6c7..e77526a2520 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -277,7 +277,7 @@ pub(crate) mod any; pub(crate) mod boolobject; pub(crate) mod bytearray; pub(crate) mod bytes; -mod capsule; +pub(crate) mod capsule; #[cfg(not(Py_LIMITED_API))] mod code; mod complex; From ab90403953f22a0a42b4c5e36181bafefa4b97c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 20:31:27 +0000 Subject: [PATCH 069/349] Bump dorny/paths-filter from 2 to 3 Bumps [dorny/paths-filter](https://github.com/dorny/paths-filter) from 2 to 3. - [Release notes](https://github.com/dorny/paths-filter/releases) - [Changelog](https://github.com/dorny/paths-filter/blob/master/CHANGELOG.md) - [Commits](https://github.com/dorny/paths-filter/compare/v2...v3) --- updated-dependencies: - dependency-name: dorny/paths-filter dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bf98dd485ad..af05eb20376 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -119,7 +119,7 @@ jobs: env: CARGO_TARGET_DIR: ${{ github.workspace }}/target - - uses: dorny/paths-filter@v2 + - uses: dorny/paths-filter@v3 # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' }} id: ffi-changes From fed8bcadafe7bfc50c092b0ed84d55b403f46c88 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 30 Jan 2024 09:13:24 +0000 Subject: [PATCH 070/349] add remaining bound string constructors --- src/types/string.rs | 48 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/src/types/string.rs b/src/types/string.rs index fd20aab8c45..b1457950bb8 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -2,6 +2,7 @@ use crate::exceptions::PyUnicodeDecodeError; use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::Borrowed; +use crate::py_result_ext::PyResultExt; use crate::types::any::PyAnyMethods; use crate::types::bytes::PyBytesMethods; use crate::types::PyBytes; @@ -159,6 +160,18 @@ impl PyString { } } + /// Deprecated form of [`PyString::intern_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyString::intern` will be replaced by `PyString::intern_bound` in a future PyO3 version" + ) + )] + pub fn intern<'py>(py: Python<'py>, s: &str) -> &'py Self { + Self::intern_bound(py, s).into_gil_ref() + } + /// Intern the given string /// /// This will return a reference to the same Python string object if called repeatedly with the same string. @@ -167,7 +180,7 @@ impl PyString { /// temporary Python string object and is thereby slower than [`PyString::new`]. /// /// Panics if out of memory. - pub fn intern<'p>(py: Python<'p>, s: &str) -> &'p PyString { + pub fn intern_bound<'py>(py: Python<'py>, s: &str) -> Bound<'py, PyString> { let ptr = s.as_ptr() as *const c_char; let len = s.len() as ffi::Py_ssize_t; unsafe { @@ -175,21 +188,38 @@ impl PyString { if !ob.is_null() { ffi::PyUnicode_InternInPlace(&mut ob); } - py.from_owned_ptr(ob) + ob.assume_owned(py).downcast_into_unchecked() } } + /// Deprecated form of [`PyString::from_object_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyString::from_object` will be replaced by `PyString::from_object_bound` in a future PyO3 version" + ) + )] + pub fn from_object<'py>(src: &'py PyAny, encoding: &str, errors: &str) -> PyResult<&'py Self> { + Self::from_object_bound(&src.as_borrowed(), encoding, errors).map(Bound::into_gil_ref) + } + /// Attempts to create a Python string from a Python [bytes-like object]. /// /// [bytes-like object]: (https://docs.python.org/3/glossary.html#term-bytes-like-object). - pub fn from_object<'p>(src: &'p PyAny, encoding: &str, errors: &str) -> PyResult<&'p PyString> { + pub fn from_object_bound<'py>( + src: &Bound<'py, PyAny>, + encoding: &str, + errors: &str, + ) -> PyResult> { unsafe { - src.py() - .from_owned_ptr_or_err(ffi::PyUnicode_FromEncodedObject( - src.as_ptr(), - encoding.as_ptr() as *const c_char, - errors.as_ptr() as *const c_char, - )) + ffi::PyUnicode_FromEncodedObject( + src.as_ptr(), + encoding.as_ptr() as *const c_char, + errors.as_ptr() as *const c_char, + ) + .assume_owned_or_err(src.py()) + .downcast_into_unchecked() } } From aa139ad422a629903d61139e72c78aed53de8cd3 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 30 Jan 2024 09:13:42 +0000 Subject: [PATCH 071/349] add `intern_bound!` macro --- guide/src/migration.md | 2 +- pyo3-benches/benches/bench_intern.rs | 4 +- pyo3-macros-backend/src/frompyobject.rs | 8 +-- pyo3-macros-backend/src/method.rs | 2 +- src/conversions/chrono.rs | 35 ++++++++----- src/conversions/chrono_tz.rs | 6 ++- src/conversions/num_bigint.rs | 14 ++--- src/conversions/num_complex.rs | 2 +- src/conversions/rust_decimal.rs | 8 +-- src/conversions/std/duration.rs | 9 ++-- src/conversions/std/ipaddr.rs | 6 ++- src/coroutine/waker.rs | 11 ++-- src/impl_/coroutine.rs | 10 +++- src/instance.rs | 30 +++++------ src/marker.rs | 2 +- src/sync.rs | 37 +++++++++---- src/tests/hygiene/misc.rs | 4 +- src/types/any.rs | 70 ++++++++++++------------- src/types/boolobject.rs | 2 +- src/types/mod.rs | 2 +- src/types/module.rs | 8 +-- src/types/traceback.rs | 6 +-- src/types/typeobject.rs | 8 +-- tests/ui/invalid_intern_arg.rs | 2 +- tests/ui/invalid_intern_arg.stderr | 10 ++-- 25 files changed, 171 insertions(+), 127 deletions(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index 9fb5adba387..b3d577548dc 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -242,6 +242,7 @@ To minimise breakage of code using the GIL-Refs API, the `Bound` smart pointe For example, the following APIs have gained updated variants: - `PyList::new`, `PyTyple::new` and similar constructors have replacements `PyList::new_bound`, `PyTuple::new_bound` etc. - `FromPyObject::extract` has a new `FromPyObject::extract_bound` (see the section below) +- `pyo3::intern!` macro has a new replacement `pyo3::intern_bound!` Because the new `Bound` API brings ownership out of the PyO3 framework and into user code, there are a few places where user code is expected to need to adjust while switching to the new API: - Code will need to add the occasional `&` to borrow the new smart pointer as `&Bound` to pass these types around (or use `.clone()` at the very small cost of increasing the Python reference count) @@ -274,7 +275,6 @@ impl<'py> FromPyObject<'py> for MyType { } ``` - The expectation is that in 0.22 `extract_bound` will have the default implementation removed and in 0.23 `extract` will be removed. ## from 0.19.* to 0.20 diff --git a/pyo3-benches/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs index d8dd1b8fd30..11e6395220c 100644 --- a/pyo3-benches/benches/bench_intern.rs +++ b/pyo3-benches/benches/bench_intern.rs @@ -2,7 +2,7 @@ use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criter use pyo3::prelude::*; -use pyo3::intern; +use pyo3::intern_bound; fn getattr_direct(b: &mut Bencher<'_>) { Python::with_gil(|py| { @@ -16,7 +16,7 @@ fn getattr_intern(b: &mut Bencher<'_>) { Python::with_gil(|py| { let sys = py.import("sys").unwrap(); - b.iter(|| sys.getattr(intern!(py, "version")).unwrap()); + b.iter(|| sys.getattr(intern_bound!(py, "version")).unwrap()); }); } diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 2b527dc8a29..7a825f7d492 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -324,17 +324,17 @@ impl<'a> Container<'a> { let field_name = ident.to_string(); let getter = match field.getter.as_ref().unwrap_or(&FieldGetter::GetAttr(None)) { FieldGetter::GetAttr(Some(name)) => { - quote!(getattr(_pyo3::intern!(obj.py(), #name))) + quote!(getattr(_pyo3::intern_bound!(obj.py(), #name))) } FieldGetter::GetAttr(None) => { - quote!(getattr(_pyo3::intern!(obj.py(), #field_name))) + quote!(getattr(_pyo3::intern_bound!(obj.py(), #field_name))) } FieldGetter::GetItem(Some(syn::Lit::Str(key))) => { - quote!(get_item(_pyo3::intern!(obj.py(), #key))) + quote!(get_item(_pyo3::intern_bound!(obj.py(), #key))) } FieldGetter::GetItem(Some(key)) => quote!(get_item(#key)), FieldGetter::GetItem(None) => { - quote!(get_item(_pyo3::intern!(obj.py(), #field_name))) + quote!(get_item(_pyo3::intern_bound!(obj.py(), #field_name))) } }; let extractor = match &field.from_py_with { diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 7050be23d5c..e0d61672982 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -522,7 +522,7 @@ impl<'a> FnSpec<'a> { let mut call = quote! {{ let future = #future; _pyo3::impl_::coroutine::new_coroutine( - _pyo3::intern!(py, stringify!(#python_name)), + _pyo3::intern_bound!(py, stringify!(#python_name)), #qualname_prefix, #throw_callback, async move { _pyo3::impl_::wrap::OkWrap::wrap(future.await) }, diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 0e95375b5bd..7b74c596472 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -52,7 +52,7 @@ use crate::types::{ PyTzInfo, PyTzInfoAccess, }; #[cfg(Py_LIMITED_API)] -use crate::{intern, PyDowncastError}; +use crate::{intern_bound, PyDowncastError}; use crate::{ FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, ToPyObject, }; @@ -127,9 +127,10 @@ impl FromPyObject<'_> for Duration { let (days, seconds, microseconds) = { check_type(ob, &DatetimeTypes::get(ob.py()).timedelta, "PyDelta")?; ( - ob.getattr(intern!(ob.py(), "days"))?.extract()?, - ob.getattr(intern!(ob.py(), "seconds"))?.extract()?, - ob.getattr(intern!(ob.py(), "microseconds"))?.extract()?, + ob.getattr(intern_bound!(ob.py(), "days"))?.extract()?, + ob.getattr(intern_bound!(ob.py(), "seconds"))?.extract()?, + ob.getattr(intern_bound!(ob.py(), "microseconds"))? + .extract()?, ) }; Ok( @@ -250,7 +251,7 @@ impl FromPyObject<'_> for NaiveDateTime { #[cfg(not(Py_LIMITED_API))] let has_tzinfo = dt.get_tzinfo_bound().is_some(); #[cfg(Py_LIMITED_API)] - let has_tzinfo = !dt.getattr(intern!(dt.py(), "tzinfo"))?.is_none(); + let has_tzinfo = !dt.getattr(intern_bound!(dt.py(), "tzinfo"))?.is_none(); if has_tzinfo { return Err(PyTypeError::new_err("expected a datetime without tzinfo")); } @@ -286,7 +287,7 @@ impl FromPyObject<'a>> FromPyObject<'_> for DateTime #[cfg(not(Py_LIMITED_API))] let tzinfo = dt.get_tzinfo_bound(); #[cfg(Py_LIMITED_API)] - let tzinfo: Option<&PyAny> = dt.getattr(intern!(dt.py(), "tzinfo"))?.extract()?; + let tzinfo: Option<&PyAny> = dt.getattr(intern_bound!(dt.py(), "tzinfo"))?.extract()?; let tz = if let Some(tzinfo) = tzinfo { tzinfo.extract()? @@ -482,9 +483,15 @@ fn py_date_to_naive_date(py_date: &impl PyDateAccess) -> PyResult { #[cfg(Py_LIMITED_API)] fn py_date_to_naive_date(py_date: &PyAny) -> PyResult { NaiveDate::from_ymd_opt( - py_date.getattr(intern!(py_date.py(), "year"))?.extract()?, - py_date.getattr(intern!(py_date.py(), "month"))?.extract()?, - py_date.getattr(intern!(py_date.py(), "day"))?.extract()?, + py_date + .getattr(intern_bound!(py_date.py(), "year"))? + .extract()?, + py_date + .getattr(intern_bound!(py_date.py(), "month"))? + .extract()?, + py_date + .getattr(intern_bound!(py_date.py(), "day"))? + .extract()?, ) .ok_or_else(|| PyValueError::new_err("invalid or out-of-range date")) } @@ -503,15 +510,17 @@ fn py_time_to_naive_time(py_time: &impl PyTimeAccess) -> PyResult { #[cfg(Py_LIMITED_API)] fn py_time_to_naive_time(py_time: &PyAny) -> PyResult { NaiveTime::from_hms_micro_opt( - py_time.getattr(intern!(py_time.py(), "hour"))?.extract()?, py_time - .getattr(intern!(py_time.py(), "minute"))? + .getattr(intern_bound!(py_time.py(), "hour"))? + .extract()?, + py_time + .getattr(intern_bound!(py_time.py(), "minute"))? .extract()?, py_time - .getattr(intern!(py_time.py(), "second"))? + .getattr(intern_bound!(py_time.py(), "second"))? .extract()?, py_time - .getattr(intern!(py_time.py(), "microsecond"))? + .getattr(intern_bound!(py_time.py(), "microsecond"))? .extract()?, ) .ok_or_else(|| PyValueError::new_err("invalid or out-of-range time")) diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index 8740d0bdd98..980ac03caff 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -36,7 +36,9 @@ use crate::exceptions::PyValueError; use crate::sync::GILOnceCell; use crate::types::PyType; -use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; +use crate::{ + intern_bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, +}; use chrono_tz::Tz; use std::str::FromStr; @@ -60,7 +62,7 @@ impl IntoPy for Tz { impl FromPyObject<'_> for Tz { fn extract(ob: &PyAny) -> PyResult { - Tz::from_str(ob.getattr(intern!(ob.py(), "key"))?.extract()?) + Tz::from_str(ob.getattr(intern_bound!(ob.py(), "key"))?.extract()?) .map_err(|e| PyValueError::new_err(e.to_string())) } } diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 5cc2157d446..b7d2f11c036 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -81,7 +81,9 @@ macro_rules! bigint_conversion { let bytes_obj = PyBytes::new(py, &bytes); let kwargs = if $is_signed > 0 { let kwargs = PyDict::new(py); - kwargs.set_item(crate::intern!(py, "signed"), true).unwrap(); + kwargs + .set_item(crate::intern_bound!(py, "signed"), true) + .unwrap(); Some(kwargs) } else { None @@ -208,18 +210,18 @@ fn int_to_u32_vec(long: &PyLong, n_digits: usize, is_signed: bool) -> PyResult PyResult<&PyBytes> { - use crate::intern; + use crate::intern_bound; let py = long.py(); let kwargs = if is_signed { let kwargs = PyDict::new(py); - kwargs.set_item(intern!(py, "signed"), true)?; + kwargs.set_item(intern_bound!(py, "signed"), true)?; Some(kwargs) } else { None }; let bytes = long.call_method( - intern!(py, "to_bytes"), - (n_bytes, intern!(py, "little")), + intern_bound!(py, "to_bytes"), + (n_bytes, intern_bound!(py, "little")), kwargs, )?; Ok(bytes.downcast()?) @@ -241,7 +243,7 @@ fn int_n_bits(long: &PyLong) -> PyResult { #[cfg(Py_LIMITED_API)] { // slow path - long.call_method0(crate::intern!(py, "bit_length")) + long.call_method0(crate::intern_bound!(py, "bit_length")) .and_then(PyAny::extract) } } diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index ba741323611..ad8ea27397d 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -153,7 +153,7 @@ macro_rules! complex_conversion { let obj = if obj.is_instance_of::() { obj } else if let Some(method) = - obj.lookup_special(crate::intern!(obj.py(), "__complex__"))? + obj.lookup_special(crate::intern_bound!(obj.py(), "__complex__"))? { complex = method.call0()?; &complex diff --git a/src/conversions/rust_decimal.rs b/src/conversions/rust_decimal.rs index 173e57851c9..b42488e9f11 100644 --- a/src/conversions/rust_decimal.rs +++ b/src/conversions/rust_decimal.rs @@ -52,7 +52,9 @@ use crate::exceptions::PyValueError; use crate::sync::GILOnceCell; use crate::types::PyType; -use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; +use crate::{ + intern_bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, +}; use rust_decimal::Decimal; use std::str::FromStr; @@ -73,8 +75,8 @@ static DECIMAL_CLS: GILOnceCell> = GILOnceCell::new(); fn get_decimal_cls(py: Python<'_>) -> PyResult<&PyType> { DECIMAL_CLS .get_or_try_init(py, || { - py.import(intern!(py, "decimal"))? - .getattr(intern!(py, "Decimal"))? + py.import(intern_bound!(py, "decimal"))? + .getattr(intern_bound!(py, "Decimal"))? .extract() }) .map(|ty| ty.as_ref(py)) diff --git a/src/conversions/std/duration.rs b/src/conversions/std/duration.rs index e4540bd0aaa..24f183a65ef 100755 --- a/src/conversions/std/duration.rs +++ b/src/conversions/std/duration.rs @@ -6,7 +6,7 @@ use crate::types::PyType; #[cfg(not(Py_LIMITED_API))] use crate::types::{PyDelta, PyDeltaAccess}; #[cfg(Py_LIMITED_API)] -use crate::{intern, Py}; +use crate::{intern_bound, Py}; use crate::{FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject}; use std::time::Duration; @@ -26,9 +26,10 @@ impl FromPyObject<'_> for Duration { #[cfg(Py_LIMITED_API)] let (days, seconds, microseconds): (i32, i32, i32) = { ( - obj.getattr(intern!(obj.py(), "days"))?.extract()?, - obj.getattr(intern!(obj.py(), "seconds"))?.extract()?, - obj.getattr(intern!(obj.py(), "microseconds"))?.extract()?, + obj.getattr(intern_bound!(obj.py(), "days"))?.extract()?, + obj.getattr(intern_bound!(obj.py(), "seconds"))?.extract()?, + obj.getattr(intern_bound!(obj.py(), "microseconds"))? + .extract()?, ) }; diff --git a/src/conversions/std/ipaddr.rs b/src/conversions/std/ipaddr.rs index 713de0afd1a..d3cbda2c216 100755 --- a/src/conversions/std/ipaddr.rs +++ b/src/conversions/std/ipaddr.rs @@ -3,11 +3,13 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use crate::exceptions::PyValueError; use crate::sync::GILOnceCell; use crate::types::PyType; -use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; +use crate::{ + intern_bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, +}; impl FromPyObject<'_> for IpAddr { fn extract(obj: &PyAny) -> PyResult { - match obj.getattr(intern!(obj.py(), "packed")) { + match obj.getattr(intern_bound!(obj.py(), "packed")) { Ok(packed) => { if let Ok(packed) = packed.extract::<[u8; 4]>() { Ok(IpAddr::V4(Ipv4Addr::from(packed))) diff --git a/src/coroutine/waker.rs b/src/coroutine/waker.rs index 8a1166ce3fb..d4a63d35669 100644 --- a/src/coroutine/waker.rs +++ b/src/coroutine/waker.rs @@ -1,6 +1,6 @@ use crate::sync::GILOnceCell; use crate::types::PyCFunction; -use crate::{intern, wrap_pyfunction, Py, PyAny, PyObject, PyResult, Python}; +use crate::{intern_bound, wrap_pyfunction, Py, PyAny, PyObject, PyResult, Python}; use pyo3_macros::pyfunction; use std::sync::Arc; use std::task::Wake; @@ -72,7 +72,7 @@ impl LoopAndFuture { // so it requires `call_soon_threadsafe` let call_soon_threadsafe = self.event_loop.call_method1( py, - intern!(py, "call_soon_threadsafe"), + intern_bound!(py, "call_soon_threadsafe"), (release_waiter, self.future.as_ref(py)), ); if let Err(err) = call_soon_threadsafe { @@ -93,9 +93,12 @@ impl LoopAndFuture { /// See #[pyfunction(crate = "crate")] fn release_waiter(future: &PyAny) -> PyResult<()> { - let done = future.call_method0(intern!(future.py(), "done"))?; + let done = future.call_method0(intern_bound!(future.py(), "done"))?; if !done.extract::()? { - future.call_method1(intern!(future.py(), "set_result"), (future.py().None(),))?; + future.call_method1( + intern_bound!(future.py(), "set_result"), + (future.py().None(),), + )?; } Ok(()) } diff --git a/src/impl_/coroutine.rs b/src/impl_/coroutine.rs index 32f3e94ad8a..c9ca4873c6c 100644 --- a/src/impl_/coroutine.rs +++ b/src/impl_/coroutine.rs @@ -6,13 +6,14 @@ use std::{ use crate::{ coroutine::{cancel::ThrowCallback, Coroutine}, + instance::Bound, pyclass::boolean_struct::False, types::PyString, IntoPy, Py, PyAny, PyCell, PyClass, PyErr, PyObject, PyResult, Python, }; pub fn new_coroutine( - name: &PyString, + name: &Bound<'_, PyString>, qualname_prefix: Option<&'static str>, throw_callback: Option, future: F, @@ -22,7 +23,12 @@ where T: IntoPy, E: Into, { - Coroutine::new(Some(name.into()), qualname_prefix, throw_callback, future) + Coroutine::new( + Some(name.clone().into()), + qualname_prefix, + throw_callback, + future, + ) } fn get_ptr(obj: &Py) -> *mut T { diff --git a/src/instance.rs b/src/instance.rs index c1ab45926b3..ff51cb30293 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -66,7 +66,7 @@ impl<'py> Bound<'py, PyAny> { /// /// # Safety /// - /// `ptr`` must be a valid pointer to a Python object. + /// `ptr` must be a valid pointer to a Python object. pub(crate) unsafe fn from_owned_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self(py, ManuallyDrop::new(Py::from_owned_ptr(py, ptr))) } @@ -75,7 +75,7 @@ impl<'py> Bound<'py, PyAny> { /// /// # Safety /// - /// `ptr`` must be a valid pointer to a Python object, or NULL. + /// `ptr` must be a valid pointer to a Python object, or NULL. pub(crate) unsafe fn from_owned_ptr_or_opt( py: Python<'py>, ptr: *mut ffi::PyObject, @@ -971,18 +971,18 @@ impl Py { /// /// This is equivalent to the Python expression `self.attr_name`. /// - /// If calling this method becomes performance-critical, the [`intern!`](crate::intern) macro + /// If calling this method becomes performance-critical, the [`intern_bound!`](crate::intern_bound) macro /// can be used to intern `attr_name`, thereby avoiding repeated temporary allocations of /// Python strings. /// - /// # Example: `intern!`ing the attribute name + /// # Example: `intern_bound!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, IntoPy, Py, Python, PyObject, PyResult}; + /// # use pyo3::{intern_bound, pyfunction, types::PyModule, IntoPy, Py, Python, PyObject, PyResult}; /// # /// #[pyfunction] /// fn version(sys: Py, py: Python<'_>) -> PyResult { - /// sys.getattr(py, intern!(py, "version")) + /// sys.getattr(py, intern_bound!(py, "version")) /// } /// # /// # Python::with_gil(|py| { @@ -1001,17 +1001,17 @@ impl Py { /// /// This is equivalent to the Python expression `self.attr_name = value`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`](crate::intern) + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`](crate::intern_bound) /// macro can be used to intern `attr_name`. /// - /// # Example: `intern!`ing the attribute name + /// # Example: `intern_bound!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, IntoPy, PyObject, Python, PyResult}; + /// # use pyo3::{intern_bound, pyfunction, types::PyModule, IntoPy, PyObject, Python, PyResult}; /// # /// #[pyfunction] /// fn set_answer(ob: PyObject, py: Python<'_>) -> PyResult<()> { - /// ob.setattr(py, intern!(py, "answer"), 42) + /// ob.setattr(py, intern_bound!(py, "answer"), 42) /// } /// # /// # Python::with_gil(|py| { @@ -1098,7 +1098,7 @@ impl Py { /// /// This is equivalent to the Python expression `self.name(*args, **kwargs)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`](crate::intern) + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`](crate::intern_bound) /// macro can be used to intern `name`. pub fn call_method_bound( &self, @@ -1121,7 +1121,7 @@ impl Py { /// /// This is equivalent to the Python expression `self.name(*args)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`](crate::intern) + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`](crate::intern_bound) /// macro can be used to intern `name`. pub fn call_method1(&self, py: Python<'_>, name: N, args: A) -> PyResult where @@ -1138,7 +1138,7 @@ impl Py { /// /// This is equivalent to the Python expression `self.name()`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`](crate::intern) + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`](crate::intern_bound) /// macro can be used to intern `name`. pub fn call_method0(&self, py: Python<'_>, name: N) -> PyResult where @@ -1644,8 +1644,8 @@ a = A() let module = PyModule::from_code(py, CODE, "", "")?; let instance: Py = module.getattr("a")?.into(); - let foo = crate::intern!(py, "foo"); - let bar = crate::intern!(py, "bar"); + let foo = crate::intern_bound!(py, "foo"); + let bar = crate::intern_bound!(py, "bar"); instance.getattr(py, foo).unwrap_err(); instance.setattr(py, foo, bar)?; diff --git a/src/marker.rs b/src/marker.rs index 836d0109eeb..6a298db6a9a 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -651,7 +651,7 @@ impl<'py> Python<'py> { // See also: // - https://github.com/python/cpython/pull/24564 (the same fix in CPython 3.10) // - https://github.com/PyO3/pyo3/issues/3370 - let builtins_s = crate::intern!(self, "__builtins__").as_ptr(); + let builtins_s = crate::intern_bound!(self, "__builtins__").as_ptr(); let has_builtins = ffi::PyDict_Contains(globals, builtins_s); if has_builtins == -1 { return Err(PyErr::fetch(self)); diff --git a/src/sync.rs b/src/sync.rs index e747269ae70..36356c4fe5b 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,5 +1,5 @@ //! Synchronization mechanisms based on the Python GIL. -use crate::{types::PyString, types::PyType, Py, PyResult, PyVisit, Python}; +use crate::{instance::Bound, types::PyString, types::PyType, Py, PyResult, PyVisit, Python}; use std::cell::UnsafeCell; /// Value with concurrent access protected by the GIL. @@ -202,14 +202,29 @@ impl GILOnceCell> { } } +/// Deprecated form of [intern_bound!][crate::intern_bound]. +#[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`intern_bound!` will be replaced by `intern_bound!` in a future PyO3 version" + ) +)] +#[macro_export] +macro_rules! intern { + ($py: expr, $text: expr) => { + $crate::intern_bound!($py, $text).as_gil_ref() + }; +} + /// Interns `text` as a Python string and stores a reference to it in static storage. /// /// A reference to the same Python string is returned on each invocation. /// -/// # Example: Using `intern!` to avoid needlessly recreating the same Python string +/// # Example: Using `intern_bound!` to avoid needlessly recreating the same Python string /// /// ``` -/// use pyo3::intern; +/// use pyo3::intern_bound; /// # use pyo3::{pyfunction, types::PyDict, wrap_pyfunction, PyResult, Python}; /// /// #[pyfunction] @@ -226,7 +241,7 @@ impl GILOnceCell> { /// let dict = PyDict::new(py); /// // 👇 A `PyString` is created once and reused /// // for the lifetime of the program. -/// dict.set_item(intern!(py, "foo"), 42)?; +/// dict.set_item(intern_bound!(py, "foo"), 42)?; /// Ok(dict) /// } /// # @@ -240,14 +255,14 @@ impl GILOnceCell> { /// # }); /// ``` #[macro_export] -macro_rules! intern { +macro_rules! intern_bound { ($py: expr, $text: expr) => {{ static INTERNED: $crate::sync::Interned = $crate::sync::Interned::new($text); INTERNED.get($py) }}; } -/// Implementation detail for `intern!` macro. +/// Implementation detail for `intern_bound!` macro. #[doc(hidden)] pub struct Interned(&'static str, GILOnceCell>); @@ -259,10 +274,10 @@ impl Interned { /// Gets or creates the interned `str` value. #[inline] - pub fn get<'py>(&'py self, py: Python<'py>) -> &'py PyString { + pub fn get<'py>(&self, py: Python<'py>) -> &Bound<'py, PyString> { self.1 - .get_or_init(py, || PyString::intern(py, self.0).into()) - .as_ref(py) + .get_or_init(py, || PyString::intern_bound(py, self.0).into()) + .bind(py) } } @@ -276,8 +291,8 @@ mod tests { fn test_intern() { Python::with_gil(|py| { let foo1 = "foo"; - let foo2 = intern!(py, "foo"); - let foo3 = intern!(py, stringify!(foo)); + let foo2 = intern_bound!(py, "foo"); + let foo3 = intern_bound!(py, stringify!(foo)); let dict = PyDict::new(py); dict.set_item(foo1, 42_usize).unwrap(); diff --git a/src/tests/hygiene/misc.rs b/src/tests/hygiene/misc.rs index 66db7f3a28a..3008b5a3b9e 100644 --- a/src/tests/hygiene/misc.rs +++ b/src/tests/hygiene/misc.rs @@ -30,8 +30,8 @@ crate::import_exception!(socket, gaierror); #[allow(dead_code)] fn intern(py: crate::Python<'_>) { - let _foo = crate::intern!(py, "foo"); - let _bar = crate::intern!(py, stringify!(bar)); + let _foo = crate::intern_bound!(py, "foo"); + let _bar = crate::intern_bound!(py, stringify!(bar)); } #[allow(dead_code)] diff --git a/src/types/any.rs b/src/types/any.rs index 14102d80a8d..aec978f7fb7 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -80,17 +80,17 @@ impl PyAny { /// /// This is equivalent to the Python expression `hasattr(self, attr_name)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `attr_name`. /// - /// # Example: `intern!`ing the attribute name + /// # Example: `intern_bound!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, Python, PyResult}; + /// # use pyo3::{intern_bound, pyfunction, types::PyModule, Python, PyResult}; /// # /// #[pyfunction] /// fn has_version(sys: &PyModule) -> PyResult { - /// sys.hasattr(intern!(sys.py(), "version")) + /// sys.hasattr(intern_bound!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { @@ -109,17 +109,17 @@ impl PyAny { /// /// This is equivalent to the Python expression `self.attr_name`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `attr_name`. /// - /// # Example: `intern!`ing the attribute name + /// # Example: `intern_bound!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{intern_bound, pyfunction, types::PyModule, PyAny, Python, PyResult}; /// # /// #[pyfunction] /// fn version(sys: &PyModule) -> PyResult<&PyAny> { - /// sys.getattr(intern!(sys.py(), "version")) + /// sys.getattr(intern_bound!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { @@ -140,17 +140,17 @@ impl PyAny { /// /// This is equivalent to the Python expression `self.attr_name = value`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `name`. /// - /// # Example: `intern!`ing the attribute name + /// # Example: `intern_bound!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{intern_bound, pyfunction, types::PyModule, PyAny, Python, PyResult}; /// # /// #[pyfunction] /// fn set_answer(ob: &PyAny) -> PyResult<()> { - /// ob.setattr(intern!(ob.py(), "answer"), 42) + /// ob.setattr(intern_bound!(ob.py(), "answer"), 42) /// } /// # /// # Python::with_gil(|py| { @@ -170,7 +170,7 @@ impl PyAny { /// /// This is equivalent to the Python statement `del self.attr_name`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `attr_name`. pub fn delattr(&self, attr_name: N) -> PyResult<()> where @@ -465,7 +465,7 @@ impl PyAny { /// /// This is equivalent to the Python expression `self.name(*args, **kwargs)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `name`. /// /// # Examples @@ -510,7 +510,7 @@ impl PyAny { /// /// This is equivalent to the Python expression `self.name()`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `name`. /// /// # Examples @@ -550,7 +550,7 @@ impl PyAny { /// /// This is equivalent to the Python expression `self.name(*args)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `name`. /// /// # Examples @@ -950,17 +950,17 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `hasattr(self, attr_name)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `attr_name`. /// - /// # Example: `intern!`ing the attribute name + /// # Example: `intern_bound!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, Python, PyResult}; + /// # use pyo3::{intern_bound, pyfunction, types::PyModule, Python, PyResult}; /// # /// #[pyfunction] /// fn has_version(sys: &PyModule) -> PyResult { - /// sys.hasattr(intern!(sys.py(), "version")) + /// sys.hasattr(intern_bound!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { @@ -976,17 +976,17 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `self.attr_name`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `attr_name`. /// - /// # Example: `intern!`ing the attribute name + /// # Example: `intern_bound!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{intern_bound, pyfunction, types::PyModule, PyAny, Python, PyResult}; /// # /// #[pyfunction] /// fn version(sys: &PyModule) -> PyResult<&PyAny> { - /// sys.getattr(intern!(sys.py(), "version")) + /// sys.getattr(intern_bound!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { @@ -1002,17 +1002,17 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `self.attr_name = value`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `name`. /// - /// # Example: `intern!`ing the attribute name + /// # Example: `intern_bound!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{intern_bound, pyfunction, types::PyModule, PyAny, Python, PyResult}; /// # /// #[pyfunction] /// fn set_answer(ob: &PyAny) -> PyResult<()> { - /// ob.setattr(intern!(ob.py(), "answer"), 42) + /// ob.setattr(intern_bound!(ob.py(), "answer"), 42) /// } /// # /// # Python::with_gil(|py| { @@ -1029,7 +1029,7 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python statement `del self.attr_name`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `attr_name`. fn delattr(&self, attr_name: N) -> PyResult<()> where @@ -1337,7 +1337,7 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `self.name(*args, **kwargs)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `name`. /// /// # Examples @@ -1382,7 +1382,7 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `self.name()`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `name`. /// /// # Examples @@ -1417,7 +1417,7 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `self.name(*args)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `name`. /// /// # Examples @@ -2257,7 +2257,7 @@ impl<'py> Bound<'py, PyAny> { /// typically a direct error for the special lookup to fail, as magic methods are optional in /// many situations in which they might be called. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used /// to intern `attr_name`. #[allow(dead_code)] // Currently only used with num-complex+abi3, so dead without that. pub(crate) fn lookup_special(&self, attr_name: N) -> PyResult>> @@ -2290,7 +2290,7 @@ impl<'py> Bound<'py, PyAny> { } else if let Ok(descr_get) = attr .get_type() .as_borrowed() - .getattr(crate::intern!(py, "__get__")) + .getattr(crate::intern_bound!(py, "__get__")) { descr_get.call1((attr, self, self_type)).map(Some) } else { @@ -2349,7 +2349,7 @@ class NonHeapNonDescriptorInt: ) .unwrap(); - let int = crate::intern!(py, "__int__"); + let int = crate::intern_bound!(py, "__int__"); let eval_int = |obj: &PyAny| { obj.as_borrowed() .lookup_special(int)? diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 0decc0b4c80..b3384f8830b 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -116,7 +116,7 @@ impl FromPyObject<'_> for bool { #[cfg(any(Py_LIMITED_API, PyPy))] { let meth = obj - .lookup_special(crate::intern!(obj.py(), "__bool__"))? + .lookup_special(crate::intern_bound!(obj.py(), "__bool__"))? .ok_or_else(|| missing_conversion(obj))?; let obj = meth.call0()?.downcast_into::()?; diff --git a/src/types/mod.rs b/src/types/mod.rs index e77526a2520..5f88d57ce4d 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -290,7 +290,7 @@ pub(crate) mod float; mod frame; pub(crate) mod frozenset; mod function; -mod iterator; +pub(crate) mod iterator; pub(crate) mod list; pub(crate) mod mapping; mod memoryview; diff --git a/src/types/module.rs b/src/types/module.rs index 416090a187b..a44fe383f5e 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -660,12 +660,12 @@ impl<'py> PyModuleMethods<'py> for Bound<'py, PyModule> { } } -fn __all__(py: Python<'_>) -> &PyString { - intern!(py, "__all__") +fn __all__(py: Python<'_>) -> &Bound<'_, PyString> { + intern_bound!(py, "__all__") } -fn __name__(py: Python<'_>) -> &PyString { - intern!(py, "__name__") +fn __name__(py: Python<'_>) -> &Bound<'_, PyString> { + intern_bound!(py, "__name__") } #[cfg(test)] diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 84ecda747eb..3dd9e150e31 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -95,13 +95,13 @@ impl<'py> PyTracebackMethods<'py> for Bound<'py, PyTraceback> { fn format(&self) -> PyResult { let py = self.py(); let string_io = py - .import(intern!(py, "io"))? - .getattr(intern!(py, "StringIO"))? + .import(intern_bound!(py, "io"))? + .getattr(intern_bound!(py, "StringIO"))? .call0()?; let result = unsafe { ffi::PyTraceBack_Print(self.as_ptr(), string_io.as_ptr()) }; error_on_minusone(py, result)?; let formatted = string_io - .getattr(intern!(py, "getvalue"))? + .getattr(intern_bound!(py, "getvalue"))? .call0()? .downcast::()? .to_str()? diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index a1a053f93a4..b37f8ebe564 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -36,7 +36,9 @@ impl PyType { /// Gets the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name) of the `PyType`. pub fn qualname(&self) -> PyResult { #[cfg(any(Py_LIMITED_API, PyPy, not(Py_3_11)))] - let name = self.getattr(intern!(self.py(), "__qualname__"))?.extract(); + let name = self + .getattr(intern_bound!(self.py(), "__qualname__"))? + .extract(); #[cfg(not(any(Py_LIMITED_API, PyPy, not(Py_3_11))))] let name = { @@ -71,10 +73,10 @@ impl PyType { #[cfg(any(Py_LIMITED_API, PyPy))] { - let module = self.getattr(intern!(self.py(), "__module__"))?; + let module = self.getattr(intern_bound!(self.py(), "__module__"))?; #[cfg(not(Py_3_11))] - let name = self.getattr(intern!(self.py(), "__name__"))?; + let name = self.getattr(intern_bound!(self.py(), "__name__"))?; #[cfg(Py_3_11)] let name = { diff --git a/tests/ui/invalid_intern_arg.rs b/tests/ui/invalid_intern_arg.rs index fa9e1e59f0c..7bc29ddc65b 100644 --- a/tests/ui/invalid_intern_arg.rs +++ b/tests/ui/invalid_intern_arg.rs @@ -2,5 +2,5 @@ use pyo3::Python; fn main() { let foo = if true { "foo" } else { "bar" }; - Python::with_gil(|py| py.import(pyo3::intern!(py, foo)).unwrap()); + Python::with_gil(|py| py.import(pyo3::intern_bound!(py, foo)).unwrap()); } diff --git a/tests/ui/invalid_intern_arg.stderr b/tests/ui/invalid_intern_arg.stderr index bb84d00e15b..5d03cadf6ed 100644 --- a/tests/ui/invalid_intern_arg.stderr +++ b/tests/ui/invalid_intern_arg.stderr @@ -1,8 +1,8 @@ error[E0435]: attempt to use a non-constant value in a constant - --> tests/ui/invalid_intern_arg.rs:5:55 + --> tests/ui/invalid_intern_arg.rs:5:61 | -5 | Python::with_gil(|py| py.import(pyo3::intern!(py, foo)).unwrap()); - | ------------------^^^- - | | | - | | non-constant value +5 | Python::with_gil(|py| py.import(pyo3::intern_bound!(py, foo)).unwrap()); + | ------------------------^^^- + | | | + | | non-constant value | help: consider using `let` instead of `static`: `let INTERNED` From 0bb9de1aba54ab340d1ed946212f47de02be93dc Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 30 Jan 2024 11:45:06 +0000 Subject: [PATCH 072/349] remove bench of `GILPool::new` --- pyo3-benches/benches/bench_gil.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pyo3-benches/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs index e25345e1fe9..169c7c6cf9e 100644 --- a/pyo3-benches/benches/bench_gil.rs +++ b/pyo3-benches/benches/bench_gil.rs @@ -1,14 +1,6 @@ use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Bencher, Criterion}; -use pyo3::{prelude::*, GILPool}; - -fn bench_clean_gilpool_new(b: &mut Bencher<'_>) { - Python::with_gil(|_py| { - b.iter(|| { - let _ = unsafe { GILPool::new() }; - }); - }); -} +use pyo3::prelude::*; fn bench_clean_acquire_gil(b: &mut Bencher<'_>) { // Acquiring first GIL will also create a "clean" GILPool, so this measures the Python overhead. @@ -28,7 +20,6 @@ fn bench_dirty_acquire_gil(b: &mut Bencher<'_>) { } fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("clean_gilpool_new", bench_clean_gilpool_new); c.bench_function("clean_acquire_gil", bench_clean_acquire_gil); c.bench_function("dirty_acquire_gil", bench_dirty_acquire_gil); } From 2f00eb1423505698746a00024469f9a64bc2feb2 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 30 Jan 2024 13:28:07 +0000 Subject: [PATCH 073/349] for now just change return type of `intern!` --- guide/src/migration.md | 4 +- pyo3-benches/benches/bench_intern.rs | 4 +- pyo3-macros-backend/src/frompyobject.rs | 8 +-- pyo3-macros-backend/src/method.rs | 2 +- src/conversions/chrono.rs | 35 +++++-------- src/conversions/chrono_tz.rs | 6 +-- src/conversions/num_bigint.rs | 14 +++-- src/conversions/num_complex.rs | 2 +- src/conversions/rust_decimal.rs | 8 ++- src/conversions/std/duration.rs | 9 ++-- src/conversions/std/ipaddr.rs | 6 +-- src/coroutine/waker.rs | 11 ++-- src/instance.rs | 26 ++++----- src/marker.rs | 2 +- src/sync.rs | 29 +++------- src/tests/hygiene/misc.rs | 4 +- src/types/any.rs | 70 ++++++++++++------------- src/types/boolobject.rs | 2 +- src/types/module.rs | 4 +- src/types/traceback.rs | 6 +-- src/types/typeobject.rs | 8 ++- tests/ui/invalid_intern_arg.rs | 2 +- tests/ui/invalid_intern_arg.stderr | 10 ++-- 23 files changed, 117 insertions(+), 155 deletions(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index b3d577548dc..2eb4d3d6117 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -22,6 +22,8 @@ The following sections are laid out in this order. To make the transition for the PyO3 ecosystem away from the GIL Refs API as smooth as possible, in PyO3 0.21 no APIs consuming or producing GIL Refs have been altered. Instead, variants using `Bound` smart pointers have been introduced, for example `PyTuple::new_bound` which returns `Bound` is the replacement form of `PyTuple::new`. The GIL Ref APIs have been deprecated, but to make migration easier it is possible to disable these deprecation warnings by enabling the `gil-refs` feature. +> The one single exception where an existing API was changed in-place is the `pyo3::intern!` macro. Almost all uses of this macro did not need to update code to account it changing to return `&Bound` immediately, and adding an `intern_bound!` replacement was perceived as adding more work for users. + It is recommended that users do this as a first step of updating to PyO3 0.21 so that the deprecation warnings do not get in the way of resolving the rest of the migration steps. Before: @@ -40,7 +42,6 @@ After: pyo3 = { version = "0.21", features = ["gil-refs"] } ``` - ### `PyTypeInfo` and `PyTryFrom` have been adjusted The `PyTryFrom` trait has aged poorly, its [`try_from`] method now conflicts with `try_from` in the 2021 edition prelude. A lot of its functionality was also duplicated with `PyTypeInfo`. @@ -242,7 +243,6 @@ To minimise breakage of code using the GIL-Refs API, the `Bound` smart pointe For example, the following APIs have gained updated variants: - `PyList::new`, `PyTyple::new` and similar constructors have replacements `PyList::new_bound`, `PyTuple::new_bound` etc. - `FromPyObject::extract` has a new `FromPyObject::extract_bound` (see the section below) -- `pyo3::intern!` macro has a new replacement `pyo3::intern_bound!` Because the new `Bound` API brings ownership out of the PyO3 framework and into user code, there are a few places where user code is expected to need to adjust while switching to the new API: - Code will need to add the occasional `&` to borrow the new smart pointer as `&Bound` to pass these types around (or use `.clone()` at the very small cost of increasing the Python reference count) diff --git a/pyo3-benches/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs index 11e6395220c..d8dd1b8fd30 100644 --- a/pyo3-benches/benches/bench_intern.rs +++ b/pyo3-benches/benches/bench_intern.rs @@ -2,7 +2,7 @@ use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criter use pyo3::prelude::*; -use pyo3::intern_bound; +use pyo3::intern; fn getattr_direct(b: &mut Bencher<'_>) { Python::with_gil(|py| { @@ -16,7 +16,7 @@ fn getattr_intern(b: &mut Bencher<'_>) { Python::with_gil(|py| { let sys = py.import("sys").unwrap(); - b.iter(|| sys.getattr(intern_bound!(py, "version")).unwrap()); + b.iter(|| sys.getattr(intern!(py, "version")).unwrap()); }); } diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 7a825f7d492..2b527dc8a29 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -324,17 +324,17 @@ impl<'a> Container<'a> { let field_name = ident.to_string(); let getter = match field.getter.as_ref().unwrap_or(&FieldGetter::GetAttr(None)) { FieldGetter::GetAttr(Some(name)) => { - quote!(getattr(_pyo3::intern_bound!(obj.py(), #name))) + quote!(getattr(_pyo3::intern!(obj.py(), #name))) } FieldGetter::GetAttr(None) => { - quote!(getattr(_pyo3::intern_bound!(obj.py(), #field_name))) + quote!(getattr(_pyo3::intern!(obj.py(), #field_name))) } FieldGetter::GetItem(Some(syn::Lit::Str(key))) => { - quote!(get_item(_pyo3::intern_bound!(obj.py(), #key))) + quote!(get_item(_pyo3::intern!(obj.py(), #key))) } FieldGetter::GetItem(Some(key)) => quote!(get_item(#key)), FieldGetter::GetItem(None) => { - quote!(get_item(_pyo3::intern_bound!(obj.py(), #field_name))) + quote!(get_item(_pyo3::intern!(obj.py(), #field_name))) } }; let extractor = match &field.from_py_with { diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index e0d61672982..7050be23d5c 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -522,7 +522,7 @@ impl<'a> FnSpec<'a> { let mut call = quote! {{ let future = #future; _pyo3::impl_::coroutine::new_coroutine( - _pyo3::intern_bound!(py, stringify!(#python_name)), + _pyo3::intern!(py, stringify!(#python_name)), #qualname_prefix, #throw_callback, async move { _pyo3::impl_::wrap::OkWrap::wrap(future.await) }, diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 7b74c596472..0e95375b5bd 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -52,7 +52,7 @@ use crate::types::{ PyTzInfo, PyTzInfoAccess, }; #[cfg(Py_LIMITED_API)] -use crate::{intern_bound, PyDowncastError}; +use crate::{intern, PyDowncastError}; use crate::{ FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, ToPyObject, }; @@ -127,10 +127,9 @@ impl FromPyObject<'_> for Duration { let (days, seconds, microseconds) = { check_type(ob, &DatetimeTypes::get(ob.py()).timedelta, "PyDelta")?; ( - ob.getattr(intern_bound!(ob.py(), "days"))?.extract()?, - ob.getattr(intern_bound!(ob.py(), "seconds"))?.extract()?, - ob.getattr(intern_bound!(ob.py(), "microseconds"))? - .extract()?, + ob.getattr(intern!(ob.py(), "days"))?.extract()?, + ob.getattr(intern!(ob.py(), "seconds"))?.extract()?, + ob.getattr(intern!(ob.py(), "microseconds"))?.extract()?, ) }; Ok( @@ -251,7 +250,7 @@ impl FromPyObject<'_> for NaiveDateTime { #[cfg(not(Py_LIMITED_API))] let has_tzinfo = dt.get_tzinfo_bound().is_some(); #[cfg(Py_LIMITED_API)] - let has_tzinfo = !dt.getattr(intern_bound!(dt.py(), "tzinfo"))?.is_none(); + let has_tzinfo = !dt.getattr(intern!(dt.py(), "tzinfo"))?.is_none(); if has_tzinfo { return Err(PyTypeError::new_err("expected a datetime without tzinfo")); } @@ -287,7 +286,7 @@ impl FromPyObject<'a>> FromPyObject<'_> for DateTime #[cfg(not(Py_LIMITED_API))] let tzinfo = dt.get_tzinfo_bound(); #[cfg(Py_LIMITED_API)] - let tzinfo: Option<&PyAny> = dt.getattr(intern_bound!(dt.py(), "tzinfo"))?.extract()?; + let tzinfo: Option<&PyAny> = dt.getattr(intern!(dt.py(), "tzinfo"))?.extract()?; let tz = if let Some(tzinfo) = tzinfo { tzinfo.extract()? @@ -483,15 +482,9 @@ fn py_date_to_naive_date(py_date: &impl PyDateAccess) -> PyResult { #[cfg(Py_LIMITED_API)] fn py_date_to_naive_date(py_date: &PyAny) -> PyResult { NaiveDate::from_ymd_opt( - py_date - .getattr(intern_bound!(py_date.py(), "year"))? - .extract()?, - py_date - .getattr(intern_bound!(py_date.py(), "month"))? - .extract()?, - py_date - .getattr(intern_bound!(py_date.py(), "day"))? - .extract()?, + py_date.getattr(intern!(py_date.py(), "year"))?.extract()?, + py_date.getattr(intern!(py_date.py(), "month"))?.extract()?, + py_date.getattr(intern!(py_date.py(), "day"))?.extract()?, ) .ok_or_else(|| PyValueError::new_err("invalid or out-of-range date")) } @@ -510,17 +503,15 @@ fn py_time_to_naive_time(py_time: &impl PyTimeAccess) -> PyResult { #[cfg(Py_LIMITED_API)] fn py_time_to_naive_time(py_time: &PyAny) -> PyResult { NaiveTime::from_hms_micro_opt( + py_time.getattr(intern!(py_time.py(), "hour"))?.extract()?, py_time - .getattr(intern_bound!(py_time.py(), "hour"))? - .extract()?, - py_time - .getattr(intern_bound!(py_time.py(), "minute"))? + .getattr(intern!(py_time.py(), "minute"))? .extract()?, py_time - .getattr(intern_bound!(py_time.py(), "second"))? + .getattr(intern!(py_time.py(), "second"))? .extract()?, py_time - .getattr(intern_bound!(py_time.py(), "microsecond"))? + .getattr(intern!(py_time.py(), "microsecond"))? .extract()?, ) .ok_or_else(|| PyValueError::new_err("invalid or out-of-range time")) diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index 980ac03caff..8740d0bdd98 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -36,9 +36,7 @@ use crate::exceptions::PyValueError; use crate::sync::GILOnceCell; use crate::types::PyType; -use crate::{ - intern_bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, -}; +use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; use chrono_tz::Tz; use std::str::FromStr; @@ -62,7 +60,7 @@ impl IntoPy for Tz { impl FromPyObject<'_> for Tz { fn extract(ob: &PyAny) -> PyResult { - Tz::from_str(ob.getattr(intern_bound!(ob.py(), "key"))?.extract()?) + Tz::from_str(ob.getattr(intern!(ob.py(), "key"))?.extract()?) .map_err(|e| PyValueError::new_err(e.to_string())) } } diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index b7d2f11c036..5cc2157d446 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -81,9 +81,7 @@ macro_rules! bigint_conversion { let bytes_obj = PyBytes::new(py, &bytes); let kwargs = if $is_signed > 0 { let kwargs = PyDict::new(py); - kwargs - .set_item(crate::intern_bound!(py, "signed"), true) - .unwrap(); + kwargs.set_item(crate::intern!(py, "signed"), true).unwrap(); Some(kwargs) } else { None @@ -210,18 +208,18 @@ fn int_to_u32_vec(long: &PyLong, n_digits: usize, is_signed: bool) -> PyResult PyResult<&PyBytes> { - use crate::intern_bound; + use crate::intern; let py = long.py(); let kwargs = if is_signed { let kwargs = PyDict::new(py); - kwargs.set_item(intern_bound!(py, "signed"), true)?; + kwargs.set_item(intern!(py, "signed"), true)?; Some(kwargs) } else { None }; let bytes = long.call_method( - intern_bound!(py, "to_bytes"), - (n_bytes, intern_bound!(py, "little")), + intern!(py, "to_bytes"), + (n_bytes, intern!(py, "little")), kwargs, )?; Ok(bytes.downcast()?) @@ -243,7 +241,7 @@ fn int_n_bits(long: &PyLong) -> PyResult { #[cfg(Py_LIMITED_API)] { // slow path - long.call_method0(crate::intern_bound!(py, "bit_length")) + long.call_method0(crate::intern!(py, "bit_length")) .and_then(PyAny::extract) } } diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index ad8ea27397d..ba741323611 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -153,7 +153,7 @@ macro_rules! complex_conversion { let obj = if obj.is_instance_of::() { obj } else if let Some(method) = - obj.lookup_special(crate::intern_bound!(obj.py(), "__complex__"))? + obj.lookup_special(crate::intern!(obj.py(), "__complex__"))? { complex = method.call0()?; &complex diff --git a/src/conversions/rust_decimal.rs b/src/conversions/rust_decimal.rs index b42488e9f11..173e57851c9 100644 --- a/src/conversions/rust_decimal.rs +++ b/src/conversions/rust_decimal.rs @@ -52,9 +52,7 @@ use crate::exceptions::PyValueError; use crate::sync::GILOnceCell; use crate::types::PyType; -use crate::{ - intern_bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, -}; +use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; use rust_decimal::Decimal; use std::str::FromStr; @@ -75,8 +73,8 @@ static DECIMAL_CLS: GILOnceCell> = GILOnceCell::new(); fn get_decimal_cls(py: Python<'_>) -> PyResult<&PyType> { DECIMAL_CLS .get_or_try_init(py, || { - py.import(intern_bound!(py, "decimal"))? - .getattr(intern_bound!(py, "Decimal"))? + py.import(intern!(py, "decimal"))? + .getattr(intern!(py, "Decimal"))? .extract() }) .map(|ty| ty.as_ref(py)) diff --git a/src/conversions/std/duration.rs b/src/conversions/std/duration.rs index 24f183a65ef..e4540bd0aaa 100755 --- a/src/conversions/std/duration.rs +++ b/src/conversions/std/duration.rs @@ -6,7 +6,7 @@ use crate::types::PyType; #[cfg(not(Py_LIMITED_API))] use crate::types::{PyDelta, PyDeltaAccess}; #[cfg(Py_LIMITED_API)] -use crate::{intern_bound, Py}; +use crate::{intern, Py}; use crate::{FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject}; use std::time::Duration; @@ -26,10 +26,9 @@ impl FromPyObject<'_> for Duration { #[cfg(Py_LIMITED_API)] let (days, seconds, microseconds): (i32, i32, i32) = { ( - obj.getattr(intern_bound!(obj.py(), "days"))?.extract()?, - obj.getattr(intern_bound!(obj.py(), "seconds"))?.extract()?, - obj.getattr(intern_bound!(obj.py(), "microseconds"))? - .extract()?, + obj.getattr(intern!(obj.py(), "days"))?.extract()?, + obj.getattr(intern!(obj.py(), "seconds"))?.extract()?, + obj.getattr(intern!(obj.py(), "microseconds"))?.extract()?, ) }; diff --git a/src/conversions/std/ipaddr.rs b/src/conversions/std/ipaddr.rs index d3cbda2c216..713de0afd1a 100755 --- a/src/conversions/std/ipaddr.rs +++ b/src/conversions/std/ipaddr.rs @@ -3,13 +3,11 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use crate::exceptions::PyValueError; use crate::sync::GILOnceCell; use crate::types::PyType; -use crate::{ - intern_bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, -}; +use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; impl FromPyObject<'_> for IpAddr { fn extract(obj: &PyAny) -> PyResult { - match obj.getattr(intern_bound!(obj.py(), "packed")) { + match obj.getattr(intern!(obj.py(), "packed")) { Ok(packed) => { if let Ok(packed) = packed.extract::<[u8; 4]>() { Ok(IpAddr::V4(Ipv4Addr::from(packed))) diff --git a/src/coroutine/waker.rs b/src/coroutine/waker.rs index d4a63d35669..8a1166ce3fb 100644 --- a/src/coroutine/waker.rs +++ b/src/coroutine/waker.rs @@ -1,6 +1,6 @@ use crate::sync::GILOnceCell; use crate::types::PyCFunction; -use crate::{intern_bound, wrap_pyfunction, Py, PyAny, PyObject, PyResult, Python}; +use crate::{intern, wrap_pyfunction, Py, PyAny, PyObject, PyResult, Python}; use pyo3_macros::pyfunction; use std::sync::Arc; use std::task::Wake; @@ -72,7 +72,7 @@ impl LoopAndFuture { // so it requires `call_soon_threadsafe` let call_soon_threadsafe = self.event_loop.call_method1( py, - intern_bound!(py, "call_soon_threadsafe"), + intern!(py, "call_soon_threadsafe"), (release_waiter, self.future.as_ref(py)), ); if let Err(err) = call_soon_threadsafe { @@ -93,12 +93,9 @@ impl LoopAndFuture { /// See #[pyfunction(crate = "crate")] fn release_waiter(future: &PyAny) -> PyResult<()> { - let done = future.call_method0(intern_bound!(future.py(), "done"))?; + let done = future.call_method0(intern!(future.py(), "done"))?; if !done.extract::()? { - future.call_method1( - intern_bound!(future.py(), "set_result"), - (future.py().None(),), - )?; + future.call_method1(intern!(future.py(), "set_result"), (future.py().None(),))?; } Ok(()) } diff --git a/src/instance.rs b/src/instance.rs index ff51cb30293..4dffe67f9b2 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -971,18 +971,18 @@ impl Py { /// /// This is equivalent to the Python expression `self.attr_name`. /// - /// If calling this method becomes performance-critical, the [`intern_bound!`](crate::intern_bound) macro + /// If calling this method becomes performance-critical, the [`intern!`](crate::intern) macro /// can be used to intern `attr_name`, thereby avoiding repeated temporary allocations of /// Python strings. /// - /// # Example: `intern_bound!`ing the attribute name + /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern_bound, pyfunction, types::PyModule, IntoPy, Py, Python, PyObject, PyResult}; + /// # use pyo3::{intern, pyfunction, types::PyModule, IntoPy, Py, Python, PyObject, PyResult}; /// # /// #[pyfunction] /// fn version(sys: Py, py: Python<'_>) -> PyResult { - /// sys.getattr(py, intern_bound!(py, "version")) + /// sys.getattr(py, intern!(py, "version")) /// } /// # /// # Python::with_gil(|py| { @@ -1001,17 +1001,17 @@ impl Py { /// /// This is equivalent to the Python expression `self.attr_name = value`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`](crate::intern_bound) + /// To avoid repeated temporary allocations of Python strings, the [`intern!`](crate::intern) /// macro can be used to intern `attr_name`. /// - /// # Example: `intern_bound!`ing the attribute name + /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern_bound, pyfunction, types::PyModule, IntoPy, PyObject, Python, PyResult}; + /// # use pyo3::{intern, pyfunction, types::PyModule, IntoPy, PyObject, Python, PyResult}; /// # /// #[pyfunction] /// fn set_answer(ob: PyObject, py: Python<'_>) -> PyResult<()> { - /// ob.setattr(py, intern_bound!(py, "answer"), 42) + /// ob.setattr(py, intern!(py, "answer"), 42) /// } /// # /// # Python::with_gil(|py| { @@ -1098,7 +1098,7 @@ impl Py { /// /// This is equivalent to the Python expression `self.name(*args, **kwargs)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`](crate::intern_bound) + /// To avoid repeated temporary allocations of Python strings, the [`intern!`](crate::intern) /// macro can be used to intern `name`. pub fn call_method_bound( &self, @@ -1121,7 +1121,7 @@ impl Py { /// /// This is equivalent to the Python expression `self.name(*args)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`](crate::intern_bound) + /// To avoid repeated temporary allocations of Python strings, the [`intern!`](crate::intern) /// macro can be used to intern `name`. pub fn call_method1(&self, py: Python<'_>, name: N, args: A) -> PyResult where @@ -1138,7 +1138,7 @@ impl Py { /// /// This is equivalent to the Python expression `self.name()`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`](crate::intern_bound) + /// To avoid repeated temporary allocations of Python strings, the [`intern!`](crate::intern) /// macro can be used to intern `name`. pub fn call_method0(&self, py: Python<'_>, name: N) -> PyResult where @@ -1644,8 +1644,8 @@ a = A() let module = PyModule::from_code(py, CODE, "", "")?; let instance: Py = module.getattr("a")?.into(); - let foo = crate::intern_bound!(py, "foo"); - let bar = crate::intern_bound!(py, "bar"); + let foo = crate::intern!(py, "foo"); + let bar = crate::intern!(py, "bar"); instance.getattr(py, foo).unwrap_err(); instance.setattr(py, foo, bar)?; diff --git a/src/marker.rs b/src/marker.rs index 6a298db6a9a..836d0109eeb 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -651,7 +651,7 @@ impl<'py> Python<'py> { // See also: // - https://github.com/python/cpython/pull/24564 (the same fix in CPython 3.10) // - https://github.com/PyO3/pyo3/issues/3370 - let builtins_s = crate::intern_bound!(self, "__builtins__").as_ptr(); + let builtins_s = crate::intern!(self, "__builtins__").as_ptr(); let has_builtins = ffi::PyDict_Contains(globals, builtins_s); if has_builtins == -1 { return Err(PyErr::fetch(self)); diff --git a/src/sync.rs b/src/sync.rs index 36356c4fe5b..a88a0f4b485 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -202,29 +202,14 @@ impl GILOnceCell> { } } -/// Deprecated form of [intern_bound!][crate::intern_bound]. -#[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`intern_bound!` will be replaced by `intern_bound!` in a future PyO3 version" - ) -)] -#[macro_export] -macro_rules! intern { - ($py: expr, $text: expr) => { - $crate::intern_bound!($py, $text).as_gil_ref() - }; -} - /// Interns `text` as a Python string and stores a reference to it in static storage. /// /// A reference to the same Python string is returned on each invocation. /// -/// # Example: Using `intern_bound!` to avoid needlessly recreating the same Python string +/// # Example: Using `intern!` to avoid needlessly recreating the same Python string /// /// ``` -/// use pyo3::intern_bound; +/// use pyo3::intern; /// # use pyo3::{pyfunction, types::PyDict, wrap_pyfunction, PyResult, Python}; /// /// #[pyfunction] @@ -241,7 +226,7 @@ macro_rules! intern { /// let dict = PyDict::new(py); /// // 👇 A `PyString` is created once and reused /// // for the lifetime of the program. -/// dict.set_item(intern_bound!(py, "foo"), 42)?; +/// dict.set_item(intern!(py, "foo"), 42)?; /// Ok(dict) /// } /// # @@ -255,14 +240,14 @@ macro_rules! intern { /// # }); /// ``` #[macro_export] -macro_rules! intern_bound { +macro_rules! intern { ($py: expr, $text: expr) => {{ static INTERNED: $crate::sync::Interned = $crate::sync::Interned::new($text); INTERNED.get($py) }}; } -/// Implementation detail for `intern_bound!` macro. +/// Implementation detail for `intern!` macro. #[doc(hidden)] pub struct Interned(&'static str, GILOnceCell>); @@ -291,8 +276,8 @@ mod tests { fn test_intern() { Python::with_gil(|py| { let foo1 = "foo"; - let foo2 = intern_bound!(py, "foo"); - let foo3 = intern_bound!(py, stringify!(foo)); + let foo2 = intern!(py, "foo"); + let foo3 = intern!(py, stringify!(foo)); let dict = PyDict::new(py); dict.set_item(foo1, 42_usize).unwrap(); diff --git a/src/tests/hygiene/misc.rs b/src/tests/hygiene/misc.rs index 3008b5a3b9e..66db7f3a28a 100644 --- a/src/tests/hygiene/misc.rs +++ b/src/tests/hygiene/misc.rs @@ -30,8 +30,8 @@ crate::import_exception!(socket, gaierror); #[allow(dead_code)] fn intern(py: crate::Python<'_>) { - let _foo = crate::intern_bound!(py, "foo"); - let _bar = crate::intern_bound!(py, stringify!(bar)); + let _foo = crate::intern!(py, "foo"); + let _bar = crate::intern!(py, stringify!(bar)); } #[allow(dead_code)] diff --git a/src/types/any.rs b/src/types/any.rs index aec978f7fb7..14102d80a8d 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -80,17 +80,17 @@ impl PyAny { /// /// This is equivalent to the Python expression `hasattr(self, attr_name)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `attr_name`. /// - /// # Example: `intern_bound!`ing the attribute name + /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern_bound, pyfunction, types::PyModule, Python, PyResult}; + /// # use pyo3::{intern, pyfunction, types::PyModule, Python, PyResult}; /// # /// #[pyfunction] /// fn has_version(sys: &PyModule) -> PyResult { - /// sys.hasattr(intern_bound!(sys.py(), "version")) + /// sys.hasattr(intern!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { @@ -109,17 +109,17 @@ impl PyAny { /// /// This is equivalent to the Python expression `self.attr_name`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `attr_name`. /// - /// # Example: `intern_bound!`ing the attribute name + /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern_bound, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; /// # /// #[pyfunction] /// fn version(sys: &PyModule) -> PyResult<&PyAny> { - /// sys.getattr(intern_bound!(sys.py(), "version")) + /// sys.getattr(intern!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { @@ -140,17 +140,17 @@ impl PyAny { /// /// This is equivalent to the Python expression `self.attr_name = value`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `name`. /// - /// # Example: `intern_bound!`ing the attribute name + /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern_bound, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; /// # /// #[pyfunction] /// fn set_answer(ob: &PyAny) -> PyResult<()> { - /// ob.setattr(intern_bound!(ob.py(), "answer"), 42) + /// ob.setattr(intern!(ob.py(), "answer"), 42) /// } /// # /// # Python::with_gil(|py| { @@ -170,7 +170,7 @@ impl PyAny { /// /// This is equivalent to the Python statement `del self.attr_name`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `attr_name`. pub fn delattr(&self, attr_name: N) -> PyResult<()> where @@ -465,7 +465,7 @@ impl PyAny { /// /// This is equivalent to the Python expression `self.name(*args, **kwargs)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `name`. /// /// # Examples @@ -510,7 +510,7 @@ impl PyAny { /// /// This is equivalent to the Python expression `self.name()`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `name`. /// /// # Examples @@ -550,7 +550,7 @@ impl PyAny { /// /// This is equivalent to the Python expression `self.name(*args)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `name`. /// /// # Examples @@ -950,17 +950,17 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `hasattr(self, attr_name)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `attr_name`. /// - /// # Example: `intern_bound!`ing the attribute name + /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern_bound, pyfunction, types::PyModule, Python, PyResult}; + /// # use pyo3::{intern, pyfunction, types::PyModule, Python, PyResult}; /// # /// #[pyfunction] /// fn has_version(sys: &PyModule) -> PyResult { - /// sys.hasattr(intern_bound!(sys.py(), "version")) + /// sys.hasattr(intern!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { @@ -976,17 +976,17 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `self.attr_name`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `attr_name`. /// - /// # Example: `intern_bound!`ing the attribute name + /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern_bound, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; /// # /// #[pyfunction] /// fn version(sys: &PyModule) -> PyResult<&PyAny> { - /// sys.getattr(intern_bound!(sys.py(), "version")) + /// sys.getattr(intern!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { @@ -1002,17 +1002,17 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `self.attr_name = value`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `name`. /// - /// # Example: `intern_bound!`ing the attribute name + /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern_bound, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; /// # /// #[pyfunction] /// fn set_answer(ob: &PyAny) -> PyResult<()> { - /// ob.setattr(intern_bound!(ob.py(), "answer"), 42) + /// ob.setattr(intern!(ob.py(), "answer"), 42) /// } /// # /// # Python::with_gil(|py| { @@ -1029,7 +1029,7 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python statement `del self.attr_name`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `attr_name`. fn delattr(&self, attr_name: N) -> PyResult<()> where @@ -1337,7 +1337,7 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `self.name(*args, **kwargs)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `name`. /// /// # Examples @@ -1382,7 +1382,7 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `self.name()`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `name`. /// /// # Examples @@ -1417,7 +1417,7 @@ pub trait PyAnyMethods<'py> { /// /// This is equivalent to the Python expression `self.name(*args)`. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `name`. /// /// # Examples @@ -2257,7 +2257,7 @@ impl<'py> Bound<'py, PyAny> { /// typically a direct error for the special lookup to fail, as magic methods are optional in /// many situations in which they might be called. /// - /// To avoid repeated temporary allocations of Python strings, the [`intern_bound!`] macro can be used + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `attr_name`. #[allow(dead_code)] // Currently only used with num-complex+abi3, so dead without that. pub(crate) fn lookup_special(&self, attr_name: N) -> PyResult>> @@ -2290,7 +2290,7 @@ impl<'py> Bound<'py, PyAny> { } else if let Ok(descr_get) = attr .get_type() .as_borrowed() - .getattr(crate::intern_bound!(py, "__get__")) + .getattr(crate::intern!(py, "__get__")) { descr_get.call1((attr, self, self_type)).map(Some) } else { @@ -2349,7 +2349,7 @@ class NonHeapNonDescriptorInt: ) .unwrap(); - let int = crate::intern_bound!(py, "__int__"); + let int = crate::intern!(py, "__int__"); let eval_int = |obj: &PyAny| { obj.as_borrowed() .lookup_special(int)? diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index b3384f8830b..0decc0b4c80 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -116,7 +116,7 @@ impl FromPyObject<'_> for bool { #[cfg(any(Py_LIMITED_API, PyPy))] { let meth = obj - .lookup_special(crate::intern_bound!(obj.py(), "__bool__"))? + .lookup_special(crate::intern!(obj.py(), "__bool__"))? .ok_or_else(|| missing_conversion(obj))?; let obj = meth.call0()?.downcast_into::()?; diff --git a/src/types/module.rs b/src/types/module.rs index a44fe383f5e..7edf164266f 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -661,11 +661,11 @@ impl<'py> PyModuleMethods<'py> for Bound<'py, PyModule> { } fn __all__(py: Python<'_>) -> &Bound<'_, PyString> { - intern_bound!(py, "__all__") + intern!(py, "__all__") } fn __name__(py: Python<'_>) -> &Bound<'_, PyString> { - intern_bound!(py, "__name__") + intern!(py, "__name__") } #[cfg(test)] diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 3dd9e150e31..84ecda747eb 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -95,13 +95,13 @@ impl<'py> PyTracebackMethods<'py> for Bound<'py, PyTraceback> { fn format(&self) -> PyResult { let py = self.py(); let string_io = py - .import(intern_bound!(py, "io"))? - .getattr(intern_bound!(py, "StringIO"))? + .import(intern!(py, "io"))? + .getattr(intern!(py, "StringIO"))? .call0()?; let result = unsafe { ffi::PyTraceBack_Print(self.as_ptr(), string_io.as_ptr()) }; error_on_minusone(py, result)?; let formatted = string_io - .getattr(intern_bound!(py, "getvalue"))? + .getattr(intern!(py, "getvalue"))? .call0()? .downcast::()? .to_str()? diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index b37f8ebe564..a1a053f93a4 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -36,9 +36,7 @@ impl PyType { /// Gets the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name) of the `PyType`. pub fn qualname(&self) -> PyResult { #[cfg(any(Py_LIMITED_API, PyPy, not(Py_3_11)))] - let name = self - .getattr(intern_bound!(self.py(), "__qualname__"))? - .extract(); + let name = self.getattr(intern!(self.py(), "__qualname__"))?.extract(); #[cfg(not(any(Py_LIMITED_API, PyPy, not(Py_3_11))))] let name = { @@ -73,10 +71,10 @@ impl PyType { #[cfg(any(Py_LIMITED_API, PyPy))] { - let module = self.getattr(intern_bound!(self.py(), "__module__"))?; + let module = self.getattr(intern!(self.py(), "__module__"))?; #[cfg(not(Py_3_11))] - let name = self.getattr(intern_bound!(self.py(), "__name__"))?; + let name = self.getattr(intern!(self.py(), "__name__"))?; #[cfg(Py_3_11)] let name = { diff --git a/tests/ui/invalid_intern_arg.rs b/tests/ui/invalid_intern_arg.rs index 7bc29ddc65b..fa9e1e59f0c 100644 --- a/tests/ui/invalid_intern_arg.rs +++ b/tests/ui/invalid_intern_arg.rs @@ -2,5 +2,5 @@ use pyo3::Python; fn main() { let foo = if true { "foo" } else { "bar" }; - Python::with_gil(|py| py.import(pyo3::intern_bound!(py, foo)).unwrap()); + Python::with_gil(|py| py.import(pyo3::intern!(py, foo)).unwrap()); } diff --git a/tests/ui/invalid_intern_arg.stderr b/tests/ui/invalid_intern_arg.stderr index 5d03cadf6ed..bb84d00e15b 100644 --- a/tests/ui/invalid_intern_arg.stderr +++ b/tests/ui/invalid_intern_arg.stderr @@ -1,8 +1,8 @@ error[E0435]: attempt to use a non-constant value in a constant - --> tests/ui/invalid_intern_arg.rs:5:61 + --> tests/ui/invalid_intern_arg.rs:5:55 | -5 | Python::with_gil(|py| py.import(pyo3::intern_bound!(py, foo)).unwrap()); - | ------------------------^^^- - | | | - | | non-constant value +5 | Python::with_gil(|py| py.import(pyo3::intern!(py, foo)).unwrap()); + | ------------------^^^- + | | | + | | non-constant value | help: consider using `let` instead of `static`: `let INTERNED` From e704a760b729c8b81b4da9083e99c5aa345a564f Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 30 Jan 2024 20:26:43 +0100 Subject: [PATCH 074/349] add `Bound` constructors for `PyByteArray` --- src/types/bytearray.rs | 131 +++++++++++++++++++++++++++++++---------- 1 file changed, 100 insertions(+), 31 deletions(-) diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 2514e987d4d..65aef27ee52 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -1,6 +1,9 @@ use crate::err::{PyErr, PyResult}; +use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::{Borrowed, Bound}; -use crate::{ffi, AsPyPointer, Py, PyAny, PyNativeType, Python}; +use crate::py_result_ext::PyResultExt; +use crate::types::any::PyAnyMethods; +use crate::{ffi, AsPyPointer, PyAny, PyNativeType, Python}; use std::os::raw::c_char; use std::slice; @@ -11,13 +14,44 @@ pub struct PyByteArray(PyAny); pyobject_native_type_core!(PyByteArray, pyobject_native_static_type_object!(ffi::PyByteArray_Type), #checkfunction=ffi::PyByteArray_Check); impl PyByteArray { + /// Deprecated form of [`PyByteArray::new_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyByteArray::new` will be replaced by `PyByteArray::new_bound` in a future PyO3 version" + ) + )] + pub fn new<'py>(py: Python<'py>, src: &[u8]) -> &'py PyByteArray { + Self::new_bound(py, src).into_gil_ref() + } + /// Creates a new Python bytearray object. /// /// The byte string is initialized by copying the data from the `&[u8]`. - pub fn new<'p>(py: Python<'p>, src: &[u8]) -> &'p PyByteArray { + pub fn new_bound<'py>(py: Python<'py>, src: &[u8]) -> Bound<'py, PyByteArray> { let ptr = src.as_ptr() as *const c_char; let len = src.len() as ffi::Py_ssize_t; - unsafe { py.from_owned_ptr::(ffi::PyByteArray_FromStringAndSize(ptr, len)) } + unsafe { + ffi::PyByteArray_FromStringAndSize(ptr, len) + .assume_owned(py) + .downcast_into_unchecked() + } + } + + /// Deprecated form of [`PyByteArray::new_bound_with`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyByteArray::new_with` will be replaced by `PyByteArray::new_bound_with` in a future PyO3 version" + ) + )] + pub fn new_with(py: Python<'_>, len: usize, init: F) -> PyResult<&PyByteArray> + where + F: FnOnce(&mut [u8]) -> PyResult<()>, + { + Self::new_bound_with(py, len, init).map(Bound::into_gil_ref) } /// Creates a new Python `bytearray` object with an `init` closure to write its contents. @@ -34,7 +68,7 @@ impl PyByteArray { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let py_bytearray = PyByteArray::new_with(py, 10, |bytes: &mut [u8]| { + /// let py_bytearray = PyByteArray::new_bound_with(py, 10, |bytes: &mut [u8]| { /// bytes.copy_from_slice(b"Hello Rust"); /// Ok(()) /// })?; @@ -44,27 +78,39 @@ impl PyByteArray { /// }) /// # } /// ``` - pub fn new_with(py: Python<'_>, len: usize, init: F) -> PyResult<&PyByteArray> + pub fn new_bound_with( + py: Python<'_>, + len: usize, + init: F, + ) -> PyResult> where F: FnOnce(&mut [u8]) -> PyResult<()>, { unsafe { - let pyptr = - ffi::PyByteArray_FromStringAndSize(std::ptr::null(), len as ffi::Py_ssize_t); - // Check for an allocation error and return it - let pypybytearray: Py = Py::from_owned_ptr_or_err(py, pyptr)?; - let buffer: *mut u8 = ffi::PyByteArray_AsString(pyptr).cast(); + // Allocate buffer and check for an error + let pybytearray: Bound<'_, Self> = + ffi::PyByteArray_FromStringAndSize(std::ptr::null(), len as ffi::Py_ssize_t) + .assume_owned_or_err(py)? + .downcast_into_unchecked(); + + let buffer: *mut u8 = ffi::PyByteArray_AsString(pybytearray.as_ptr()).cast(); debug_assert!(!buffer.is_null()); // Zero-initialise the uninitialised bytearray std::ptr::write_bytes(buffer, 0u8, len); // (Further) Initialise the bytearray in init // If init returns an Err, pypybytearray will automatically deallocate the buffer - init(std::slice::from_raw_parts_mut(buffer, len)).map(|_| pypybytearray.into_ref(py)) + init(std::slice::from_raw_parts_mut(buffer, len)).map(|_| pybytearray) } } - /// Creates a new Python `bytearray` object from another Python object that - /// implements the buffer protocol. + /// Deprecated form of [`PyByteArray::from_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyByteArray::from` will be replaced by `PyByteArray::from_bound` in a future PyO3 version" + ) + )] pub fn from(src: &PyAny) -> PyResult<&PyByteArray> { unsafe { src.py() @@ -72,6 +118,16 @@ impl PyByteArray { } } + /// Creates a new Python `bytearray` object from another Python object that + /// implements the buffer protocol. + pub fn from_bound<'py>(src: &Bound<'py, PyAny>) -> PyResult> { + unsafe { + ffi::PyByteArray_FromObject(src.as_ptr()) + .assume_owned_or_err(src.py()) + .downcast_into_unchecked() + } + } + /// Gets the length of the bytearray. #[inline] pub fn len(&self) -> usize { @@ -211,7 +267,7 @@ impl PyByteArray { /// # use pyo3::prelude::*; /// # use pyo3::types::PyByteArray; /// # Python::with_gil(|py| { - /// let bytearray = PyByteArray::new(py, b"Hello World."); + /// let bytearray = PyByteArray::new_bound(py, b"Hello World."); /// let mut copied_message = bytearray.to_vec(); /// assert_eq!(b"Hello World.", copied_message.as_slice()); /// @@ -369,7 +425,7 @@ pub trait PyByteArrayMethods<'py> { /// # use pyo3::prelude::*; /// # use pyo3::types::PyByteArray; /// # Python::with_gil(|py| { - /// let bytearray = PyByteArray::new(py, b"Hello World."); + /// let bytearray = PyByteArray::new_bound(py, b"Hello World."); /// let mut copied_message = bytearray.to_vec(); /// assert_eq!(b"Hello World.", copied_message.as_slice()); /// @@ -450,21 +506,34 @@ impl<'py> TryFrom<&'py PyAny> for &'py PyByteArray { /// Creates a new Python `bytearray` object from another Python object that /// implements the buffer protocol. fn try_from(value: &'py PyAny) -> Result { - PyByteArray::from(value) + PyByteArray::from_bound(&value.as_borrowed()).map(Bound::into_gil_ref) + } +} + +impl<'py> TryFrom<&Bound<'py, PyAny>> for Bound<'py, PyByteArray> { + type Error = crate::PyErr; + + /// Creates a new Python `bytearray` object from another Python object that + /// implements the buffer protocol. + fn try_from(value: &Bound<'py, PyAny>) -> Result { + PyByteArray::from_bound(value) } } #[cfg(test)] mod tests { + use crate::types::any::PyAnyMethods; + use crate::types::bytearray::PyByteArrayMethods; + use crate::types::string::PyStringMethods; use crate::types::PyByteArray; - use crate::{exceptions, PyAny}; + use crate::{exceptions, Bound, PyAny, PyNativeType}; use crate::{PyObject, Python}; #[test] fn test_len() { Python::with_gil(|py| { let src = b"Hello Python"; - let bytearray = PyByteArray::new(py, src); + let bytearray = PyByteArray::new_bound(py, src); assert_eq!(src.len(), bytearray.len()); }); } @@ -473,7 +542,7 @@ mod tests { fn test_as_bytes() { Python::with_gil(|py| { let src = b"Hello Python"; - let bytearray = PyByteArray::new(py, src); + let bytearray = PyByteArray::new_bound(py, src); let slice = unsafe { bytearray.as_bytes() }; assert_eq!(src, slice); @@ -485,7 +554,7 @@ mod tests { fn test_as_bytes_mut() { Python::with_gil(|py| { let src = b"Hello Python"; - let bytearray = PyByteArray::new(py, src); + let bytearray = PyByteArray::new_bound(py, src); let slice = unsafe { bytearray.as_bytes_mut() }; assert_eq!(src, slice); @@ -494,7 +563,7 @@ mod tests { slice[0..5].copy_from_slice(b"Hi..."); assert_eq!( - bytearray.str().unwrap().to_str().unwrap(), + bytearray.str().unwrap().to_cow().unwrap(), "bytearray(b'Hi... Python')" ); }); @@ -504,7 +573,7 @@ mod tests { fn test_to_vec() { Python::with_gil(|py| { let src = b"Hello Python"; - let bytearray = PyByteArray::new(py, src); + let bytearray = PyByteArray::new_bound(py, src); let vec = bytearray.to_vec(); assert_eq!(src, vec.as_slice()); @@ -515,10 +584,10 @@ mod tests { fn test_from() { Python::with_gil(|py| { let src = b"Hello Python"; - let bytearray = PyByteArray::new(py, src); + let bytearray = PyByteArray::new_bound(py, src); let ba: PyObject = bytearray.into(); - let bytearray = PyByteArray::from(ba.as_ref(py)).unwrap(); + let bytearray = PyByteArray::from_bound(ba.bind(py)).unwrap(); assert_eq!(src, unsafe { bytearray.as_bytes() }); }); @@ -527,7 +596,7 @@ mod tests { #[test] fn test_from_err() { Python::with_gil(|py| { - if let Err(err) = PyByteArray::from(py.None()) { + if let Err(err) = PyByteArray::from_bound(&py.None().as_borrowed()) { assert!(err.is_instance_of::(py)); } else { panic!("error"); @@ -539,8 +608,8 @@ mod tests { fn test_try_from() { Python::with_gil(|py| { let src = b"Hello Python"; - let bytearray: &PyAny = PyByteArray::new(py, src).into(); - let bytearray: &PyByteArray = TryInto::try_into(bytearray).unwrap(); + let bytearray: &Bound<'_, PyAny> = &PyByteArray::new_bound(py, src); + let bytearray: Bound<'_, PyByteArray> = TryInto::try_into(bytearray).unwrap(); assert_eq!(src, unsafe { bytearray.as_bytes() }); }); @@ -550,7 +619,7 @@ mod tests { fn test_resize() { Python::with_gil(|py| { let src = b"Hello Python"; - let bytearray = PyByteArray::new(py, src); + let bytearray = PyByteArray::new_bound(py, src); bytearray.resize(20).unwrap(); assert_eq!(20, bytearray.len()); @@ -560,7 +629,7 @@ mod tests { #[test] fn test_byte_array_new_with() -> super::PyResult<()> { Python::with_gil(|py| -> super::PyResult<()> { - let py_bytearray = PyByteArray::new_with(py, 10, |b: &mut [u8]| { + let py_bytearray = PyByteArray::new_bound_with(py, 10, |b: &mut [u8]| { b.copy_from_slice(b"Hello Rust"); Ok(()) })?; @@ -573,7 +642,7 @@ mod tests { #[test] fn test_byte_array_new_with_zero_initialised() -> super::PyResult<()> { Python::with_gil(|py| -> super::PyResult<()> { - let py_bytearray = PyByteArray::new_with(py, 10, |_b: &mut [u8]| Ok(()))?; + let py_bytearray = PyByteArray::new_bound_with(py, 10, |_b: &mut [u8]| Ok(()))?; let bytearray: &[u8] = unsafe { py_bytearray.as_bytes() }; assert_eq!(bytearray, &[0; 10]); Ok(()) @@ -584,7 +653,7 @@ mod tests { fn test_byte_array_new_with_error() { use crate::exceptions::PyValueError; Python::with_gil(|py| { - let py_bytearray_result = PyByteArray::new_with(py, 10, |_b: &mut [u8]| { + let py_bytearray_result = PyByteArray::new_bound_with(py, 10, |_b: &mut [u8]| { Err(PyValueError::new_err("Hello Crustaceans!")) }); assert!(py_bytearray_result.is_err()); From b14dbcf29f161a6dfbd62fcd0fef2a009f5d7331 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 30 Jan 2024 20:28:04 +0100 Subject: [PATCH 075/349] add `Bound` constructors for `PyMemoryView` --- src/types/memoryview.rs | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/types/memoryview.rs b/src/types/memoryview.rs index 0d115540689..414bfc69cfa 100644 --- a/src/types/memoryview.rs +++ b/src/types/memoryview.rs @@ -1,5 +1,7 @@ use crate::err::PyResult; -use crate::{ffi, AsPyPointer, PyAny}; +use crate::ffi_ptr_ext::FfiPtrExt; +use crate::py_result_ext::PyResultExt; +use crate::{ffi, AsPyPointer, Bound, PyAny, PyNativeType}; /// Represents a Python `memoryview`. #[repr(transparent)] @@ -8,14 +10,30 @@ pub struct PyMemoryView(PyAny); pyobject_native_type_core!(PyMemoryView, pyobject_native_static_type_object!(ffi::PyMemoryView_Type), #checkfunction=ffi::PyMemoryView_Check); impl PyMemoryView { - /// Creates a new Python `memoryview` object from another Python object that - /// implements the buffer protocol. + /// Deprecated form of [`PyMemoryView::from_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyMemoryView::from` will be replaced by `PyMemoryView::from_bound` in a future PyO3 version" + ) + )] pub fn from(src: &PyAny) -> PyResult<&PyMemoryView> { unsafe { src.py() .from_owned_ptr_or_err(ffi::PyMemoryView_FromObject(src.as_ptr())) } } + + /// Creates a new Python `memoryview` object from another Python object that + /// implements the buffer protocol. + pub fn from_bound<'py>(src: &Bound<'py, PyAny>) -> PyResult> { + unsafe { + ffi::PyMemoryView_FromObject(src.as_ptr()) + .assume_owned_or_err(src.py()) + .downcast_into_unchecked() + } + } } impl<'py> TryFrom<&'py PyAny> for &'py PyMemoryView { @@ -24,6 +42,16 @@ impl<'py> TryFrom<&'py PyAny> for &'py PyMemoryView { /// Creates a new Python `memoryview` object from another Python object that /// implements the buffer protocol. fn try_from(value: &'py PyAny) -> Result { - PyMemoryView::from(value) + PyMemoryView::from_bound(&value.as_borrowed()).map(Bound::into_gil_ref) + } +} + +impl<'py> TryFrom<&Bound<'py, PyAny>> for Bound<'py, PyMemoryView> { + type Error = crate::PyErr; + + /// Creates a new Python `memoryview` object from another Python object that + /// implements the buffer protocol. + fn try_from(value: &Bound<'py, PyAny>) -> Result { + PyMemoryView::from_bound(value) } } From 4c94be51a7b6885ad91b7287a4a75dacb3278102 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 27 Nov 2023 13:50:54 +0000 Subject: [PATCH 076/349] add `PyBytes::new_bound` --- guide/src/conversions/traits.md | 2 +- pytests/src/buf_and_str.rs | 7 ++- src/conversions/anyhow.rs | 2 +- src/conversions/eyre.rs | 2 +- src/conversions/num_bigint.rs | 2 +- src/conversions/std/slice.rs | 4 +- src/tests/hygiene/pymethods.rs | 8 +-- src/types/bytes.rs | 94 ++++++++++++++++++++++++++++----- tests/test_bytes.rs | 6 +-- 9 files changed, 97 insertions(+), 30 deletions(-) diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index f657fe073fc..4f3342f2ccc 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -388,7 +388,7 @@ enum RustyEnum<'a> { # } # # { -# let thing = PyBytes::new(py, b"text"); +# let thing = PyBytes::new_bound(py, b"text"); # let rust_thing: RustyEnum<'_> = thing.extract()?; # # assert_eq!( diff --git a/pytests/src/buf_and_str.rs b/pytests/src/buf_and_str.rs index 7cddc686c03..196126f7cc1 100644 --- a/pytests/src/buf_and_str.rs +++ b/pytests/src/buf_and_str.rs @@ -42,10 +42,9 @@ impl BytesExtractor { } #[pyfunction] -fn return_memoryview(py: Python<'_>) -> PyResult<&PyMemoryView> { - let bytes: &PyAny = PyBytes::new(py, b"hello world").into(); - let memoryview = TryInto::try_into(bytes)?; - Ok(memoryview) +fn return_memoryview(py: Python<'_>) -> PyResult> { + let bytes = PyBytes::new_bound(py, b"hello world"); + PyMemoryView::from_bound(&bytes) } #[pymodule] diff --git a/src/conversions/anyhow.rs b/src/conversions/anyhow.rs index b362faf479f..3b5aa053e84 100644 --- a/src/conversions/anyhow.rs +++ b/src/conversions/anyhow.rs @@ -77,7 +77,7 @@ //! let res = Python::with_gil(|py| { //! let zlib = PyModule::import(py, "zlib")?; //! let decompress = zlib.getattr("decompress")?; -//! let bytes = PyBytes::new(py, bytes); +//! let bytes = PyBytes::new_bound(py, bytes); //! let value = decompress.call1((bytes,))?; //! value.extract::>() //! })?; diff --git a/src/conversions/eyre.rs b/src/conversions/eyre.rs index f881599e2c3..b0559ad1469 100644 --- a/src/conversions/eyre.rs +++ b/src/conversions/eyre.rs @@ -76,7 +76,7 @@ //! let res = Python::with_gil(|py| { //! let zlib = PyModule::import(py, "zlib")?; //! let decompress = zlib.getattr("decompress")?; -//! let bytes = PyBytes::new(py, bytes); +//! let bytes = PyBytes::new_bound(py, bytes); //! let value = decompress.call1((bytes,))?; //! value.extract::>() //! })?; diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 5cc2157d446..893d3b9d226 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -78,7 +78,7 @@ macro_rules! bigint_conversion { #[cfg(Py_LIMITED_API)] fn to_object(&self, py: Python<'_>) -> PyObject { let bytes = $to_bytes(self); - let bytes_obj = PyBytes::new(py, &bytes); + let bytes_obj = PyBytes::new_bound(py, &bytes); let kwargs = if $is_signed > 0 { let kwargs = PyDict::new(py); kwargs.set_item(crate::intern!(py, "signed"), true).unwrap(); diff --git a/src/conversions/std/slice.rs b/src/conversions/std/slice.rs index fbe2d19e1ac..3d9351f8476 100644 --- a/src/conversions/std/slice.rs +++ b/src/conversions/std/slice.rs @@ -1,10 +1,10 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; -use crate::{types::PyBytes, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject}; +use crate::{types::PyBytes, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python}; impl<'a> IntoPy for &'a [u8] { fn into_py(self, py: Python<'_>) -> PyObject { - PyBytes::new(py, self).to_object(py) + PyBytes::new_bound(py, self).unbind().into() } #[cfg(feature = "experimental-inspect")] diff --git a/src/tests/hygiene/pymethods.rs b/src/tests/hygiene/pymethods.rs index 0e39634ab5d..15ea675943c 100644 --- a/src/tests/hygiene/pymethods.rs +++ b/src/tests/hygiene/pymethods.rs @@ -24,8 +24,8 @@ impl Dummy { "Dummy" } - fn __bytes__<'py>(&self, py: crate::Python<'py>) -> &'py crate::types::PyBytes { - crate::types::PyBytes::new(py, &[0]) + fn __bytes__<'py>(&self, py: crate::Python<'py>) -> crate::Bound<'py, crate::types::PyBytes> { + crate::types::PyBytes::new_bound(py, &[0]) } fn __format__(&self, format_spec: ::std::string::String) -> ::std::string::String { @@ -420,8 +420,8 @@ impl Dummy { "Dummy" } - fn __bytes__<'py>(&self, py: crate::Python<'py>) -> &'py crate::types::PyBytes { - crate::types::PyBytes::new(py, &[0]) + fn __bytes__<'py>(&self, py: crate::Python<'py>) -> crate::Bound<'py, crate::types::PyBytes> { + crate::types::PyBytes::new_bound(py, &[0]) } fn __format__(&self, format_spec: ::std::string::String) -> ::std::string::String { diff --git a/src/types/bytes.rs b/src/types/bytes.rs index d9d22dbb173..296925dceff 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -1,4 +1,6 @@ +use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::{Borrowed, Bound}; +use crate::types::any::PyAnyMethods; use crate::{ffi, FromPyObject, IntoPy, Py, PyAny, PyNativeType, PyResult, Python, ToPyObject}; use std::borrow::Cow; use std::ops::Index; @@ -17,14 +19,45 @@ pub struct PyBytes(PyAny); pyobject_native_type_core!(PyBytes, pyobject_native_static_type_object!(ffi::PyBytes_Type), #checkfunction=ffi::PyBytes_Check); impl PyBytes { + /// Deprecated form of [`PyBytes::new_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyBytes::new` will be replaced by `PyBytes::new_bound` in a future PyO3 version" + ) + )] + pub fn new<'p>(py: Python<'p>, s: &[u8]) -> &'p PyBytes { + Self::new_bound(py, s).into_gil_ref() + } + /// Creates a new Python bytestring object. /// The bytestring is initialized by copying the data from the `&[u8]`. /// /// Panics if out of memory. - pub fn new<'p>(py: Python<'p>, s: &[u8]) -> &'p PyBytes { + pub fn new_bound<'p>(py: Python<'p>, s: &[u8]) -> Bound<'p, PyBytes> { let ptr = s.as_ptr() as *const c_char; let len = s.len() as ffi::Py_ssize_t; - unsafe { py.from_owned_ptr(ffi::PyBytes_FromStringAndSize(ptr, len)) } + unsafe { + ffi::PyBytes_FromStringAndSize(ptr, len) + .assume_owned(py) + .downcast_into_unchecked() + } + } + + /// Deprecated form of [`PyBytes::new_bound_with`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyBytes::new_with` will be replaced by `PyBytes::new_bound_with` in a future PyO3 version" + ) + )] + pub fn new_with(py: Python<'_>, len: usize, init: F) -> PyResult<&PyBytes> + where + F: FnOnce(&mut [u8]) -> PyResult<()>, + { + Self::new_bound_with(py, len, init).map(Bound::into_gil_ref) } /// Creates a new Python `bytes` object with an `init` closure to write its contents. @@ -41,34 +74,49 @@ impl PyBytes { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let py_bytes = PyBytes::new_with(py, 10, |bytes: &mut [u8]| { + /// let py_bytes = PyBytes::new_bound_with(py, 10, |bytes: &mut [u8]| { /// bytes.copy_from_slice(b"Hello Rust"); /// Ok(()) /// })?; - /// let bytes: &[u8] = FromPyObject::extract(py_bytes)?; + /// let bytes: &[u8] = py_bytes.extract()?; /// assert_eq!(bytes, b"Hello Rust"); /// Ok(()) /// }) /// # } /// ``` - pub fn new_with(py: Python<'_>, len: usize, init: F) -> PyResult<&PyBytes> + pub fn new_bound_with(py: Python<'_>, len: usize, init: F) -> PyResult> where F: FnOnce(&mut [u8]) -> PyResult<()>, { unsafe { let pyptr = ffi::PyBytes_FromStringAndSize(std::ptr::null(), len as ffi::Py_ssize_t); // Check for an allocation error and return it - let pypybytes: Py = Py::from_owned_ptr_or_err(py, pyptr)?; + let pybytes = pyptr.assume_owned_or_err(py)?.downcast_into_unchecked(); let buffer: *mut u8 = ffi::PyBytes_AsString(pyptr).cast(); debug_assert!(!buffer.is_null()); // Zero-initialise the uninitialised bytestring std::ptr::write_bytes(buffer, 0u8, len); // (Further) Initialise the bytestring in init // If init returns an Err, pypybytearray will automatically deallocate the buffer - init(std::slice::from_raw_parts_mut(buffer, len)).map(|_| pypybytes.into_ref(py)) + init(std::slice::from_raw_parts_mut(buffer, len)).map(|_| pybytes) } } + /// Deprecated form of [`PyBytes::bound_from_ptr`]. + /// + /// # Safety + /// See [`PyBytes::bound_from_ptr`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyBytes::from_ptr` will be replaced by `PyBytes::bound_from_ptr` in a future PyO3 version" + ) + )] + pub unsafe fn from_ptr(py: Python<'_>, ptr: *const u8, len: usize) -> &PyBytes { + Self::bound_from_ptr(py, ptr, len).into_gil_ref() + } + /// Creates a new Python byte string object from a raw pointer and length. /// /// Panics if out of memory. @@ -79,11 +127,10 @@ impl PyBytes { /// leading pointer of a slice of length `len`. [As with /// `std::slice::from_raw_parts`, this is /// unsafe](https://doc.rust-lang.org/std/slice/fn.from_raw_parts.html#safety). - pub unsafe fn from_ptr(py: Python<'_>, ptr: *const u8, len: usize) -> &PyBytes { - py.from_owned_ptr(ffi::PyBytes_FromStringAndSize( - ptr as *const _, - len as isize, - )) + pub unsafe fn bound_from_ptr(py: Python<'_>, ptr: *const u8, len: usize) -> Bound<'_, PyBytes> { + ffi::PyBytes_FromStringAndSize(ptr as *const _, len as isize) + .assume_owned(py) + .downcast_into_unchecked() } /// Gets the Python string as a byte slice. @@ -142,6 +189,15 @@ impl> Index for PyBytes { } } +/// This is the same way [Vec] is indexed. +impl> Index for Bound<'_, PyBytes> { + type Output = I::Output; + + fn index(&self, index: I) -> &Self::Output { + &self.as_bytes()[index] + } +} + /// Special-purpose trait impl to efficiently handle both `bytes` and `bytearray` /// /// If the source object is a `bytes` object, the `Cow` will be borrowed and @@ -160,7 +216,7 @@ impl<'source> FromPyObject<'source> for Cow<'source, [u8]> { impl ToPyObject for Cow<'_, [u8]> { fn to_object(&self, py: Python<'_>) -> Py { - PyBytes::new(py, self.as_ref()).into() + PyBytes::new_bound(py, self.as_ref()).into() } } @@ -171,6 +227,7 @@ impl IntoPy> for Cow<'_, [u8]> { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; @@ -182,6 +239,17 @@ mod tests { }); } + #[test] + fn test_bound_bytes_index() { + Python::with_gil(|py| { + let bytes = PyBytes::new_bound(py, b"Hello World"); + assert_eq!(bytes[1], b'e'); + + let bytes = &bytes; + assert_eq!(bytes[1], b'e'); + }); + } + #[test] fn test_bytes_new_with() -> super::PyResult<()> { Python::with_gil(|py| -> super::PyResult<()> { diff --git a/tests/test_bytes.rs b/tests/test_bytes.rs index 3ad1352748c..cdb4ec15750 100644 --- a/tests/test_bytes.rs +++ b/tests/test_bytes.rs @@ -20,8 +20,8 @@ fn test_pybytes_bytes_conversion() { } #[pyfunction] -fn bytes_vec_conversion(py: Python<'_>, bytes: Vec) -> &PyBytes { - PyBytes::new(py, bytes.as_slice()) +fn bytes_vec_conversion(py: Python<'_>, bytes: Vec) -> Bound<'_, PyBytes> { + PyBytes::new_bound(py, bytes.as_slice()) } #[test] @@ -43,7 +43,7 @@ fn test_bytearray_vec_conversion() { #[test] fn test_py_as_bytes() { let pyobj: pyo3::Py = - Python::with_gil(|py| pyo3::types::PyBytes::new(py, b"abc").into_py(py)); + Python::with_gil(|py| pyo3::types::PyBytes::new_bound(py, b"abc").unbind()); let data = Python::with_gil(|py| pyobj.as_bytes(py)); From cbc97f8ea9c4f5993bc4eaa2f1b1176d59a30247 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 28 Nov 2023 10:40:37 +0000 Subject: [PATCH 077/349] add `Bound::as_any` and `Bound::into_any` --- src/instance.rs | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index 4dffe67f9b2..b388fb9dddf 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -96,14 +96,6 @@ impl<'py> Bound<'py, PyAny> { } } -impl<'py, T> Bound<'py, T> { - /// Helper to cast to Bound<'py, PyAny> - pub(crate) fn as_any(&self) -> &Bound<'py, PyAny> { - // Safety: all Bound have the same memory layout, and all Bound are valid Bound - unsafe { std::mem::transmute(self) } - } -} - impl<'py, T> std::fmt::Debug for Bound<'py, T> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { let any = self.as_any(); @@ -197,6 +189,22 @@ impl<'py, T> Bound<'py, T> { self.into_non_null().as_ptr() } + /// Helper to cast to `Bound<'py, PyAny>`. + pub fn as_any(&self) -> &Bound<'py, PyAny> { + // Safety: all Bound have the same memory layout, and all Bound are valid + // Bound, so pointer casting is valid. + unsafe { &*(self as *const Self).cast::>() } + } + + /// Helper to cast to `Bound<'py, PyAny>`, transferring ownership. + pub fn into_any(self) -> Bound<'py, PyAny> { + // Safety: all Bound are valid Bound + Bound( + self.0, + ManuallyDrop::new(unsafe { Py::from_non_null(self.into_non_null()) }), + ) + } + /// Casts this `Bound` to a `Borrowed` smart pointer. pub fn as_borrowed<'a>(&'a self) -> Borrowed<'a, 'py, T> { Borrowed( @@ -1292,7 +1300,7 @@ impl IntoPy for Bound<'_, T> { /// Consumes `self` without calling `Py_DECREF()`. #[inline] fn into_py(self, _py: Python<'_>) -> PyObject { - unsafe { PyObject::from_non_null(self.into_non_null()) } + self.into_any().unbind() } } @@ -1724,6 +1732,24 @@ a = A() }); } + #[test] + fn test_bound_as_any() { + Python::with_gil(|py| { + let obj = PyString::new_bound(py, "hello world"); + let any = obj.as_any(); + assert_eq!(any.as_ptr(), obj.as_ptr()); + }); + } + + #[test] + fn test_bound_into_any() { + Python::with_gil(|py| { + let obj = PyString::new_bound(py, "hello world"); + let any = obj.clone().into_any(); + assert_eq!(any.as_ptr(), obj.as_ptr()); + }); + } + #[cfg(feature = "macros")] mod using_macros { use crate::PyCell; From 4437e8f61637b30bfa0205ea4738f2029989305a Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 1 Feb 2024 09:07:36 +0000 Subject: [PATCH 078/349] add `Py::as_any` and `Py::into_any` --- newsfragments/3785.added.md | 1 + src/instance.rs | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3785.added.md diff --git a/newsfragments/3785.added.md b/newsfragments/3785.added.md new file mode 100644 index 00000000000..6af3bb999f8 --- /dev/null +++ b/newsfragments/3785.added.md @@ -0,0 +1 @@ +Add `Py::as_any` and `Py::into_any`. diff --git a/src/instance.rs b/src/instance.rs index b388fb9dddf..87880e09c8f 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -199,10 +199,7 @@ impl<'py, T> Bound<'py, T> { /// Helper to cast to `Bound<'py, PyAny>`, transferring ownership. pub fn into_any(self) -> Bound<'py, PyAny> { // Safety: all Bound are valid Bound - Bound( - self.0, - ManuallyDrop::new(unsafe { Py::from_non_null(self.into_non_null()) }), - ) + Bound(self.0, ManuallyDrop::new(self.unbind().into_any())) } /// Casts this `Bound` to a `Borrowed` smart pointer. @@ -721,6 +718,19 @@ impl Py { std::mem::forget(self); ptr } + + /// Helper to cast to `Py`. + pub fn as_any(&self) -> &Py { + // Safety: all Py have the same memory layout, and all Py are valid + // Py, so pointer casting is valid. + unsafe { &*(self as *const Self).cast::>() } + } + + /// Helper to cast to `Py`, transferring ownership. + pub fn into_any(self) -> Py { + // Safety: all Py are valid Py + unsafe { Py::from_non_null(self.into_non_null()) } + } } impl Py From 49a57dfd1852fc394321635a93d8d10d4592eeb4 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 1 Feb 2024 09:27:44 +0000 Subject: [PATCH 079/349] clean up implementations in `src/instance.rs` --- src/instance.rs | 100 +++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 52 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index 87880e09c8f..627c5e678f2 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -11,7 +11,7 @@ use crate::{ }; use crate::{gil, PyTypeCheck}; use std::marker::PhantomData; -use std::mem::{self, ManuallyDrop}; +use std::mem::ManuallyDrop; use std::ops::Deref; use std::ptr::NonNull; @@ -35,10 +35,15 @@ pub unsafe trait PyNativeType: Sized { /// /// This is available as a migration tool to adjust code from the deprecated "GIL Refs" /// API to the `Bound` smart pointer API. + #[inline] fn as_borrowed(&self) -> Borrowed<'_, '_, Self::AsRefSource> { // Safety: &'py Self is expected to be a Python pointer, // so has the same layout as Borrowed<'py, 'py, T> - unsafe { std::mem::transmute(self) } + Borrowed( + unsafe { NonNull::new_unchecked(self as *const Self as *mut _) }, + PhantomData, + self.py(), + ) } /// Returns a GIL marker constrained to the lifetime of this type. @@ -142,25 +147,29 @@ impl<'py, T> AsRef> for Bound<'py, T> where T: AsRef, { + #[inline] fn as_ref(&self) -> &Bound<'py, PyAny> { self.as_any() } } impl Clone for Bound<'_, T> { + #[inline] fn clone(&self) -> Self { Self(self.0, ManuallyDrop::new(self.1.clone_ref(self.0))) } } impl Drop for Bound<'_, T> { + #[inline] fn drop(&mut self) { - unsafe { ffi::Py_DECREF(self.1.as_ptr()) } + unsafe { ffi::Py_DECREF(self.as_ptr()) } } } impl<'py, T> Bound<'py, T> { /// Returns the GIL token associated with this object. + #[inline] pub fn py(&self) -> Python<'py> { self.0 } @@ -186,10 +195,11 @@ impl<'py, T> Bound<'py, T> { /// of the pointer or decrease the reference count (e.g. with [`pyo3::ffi::Py_DecRef`](crate::ffi::Py_DecRef)). #[inline] pub fn into_ptr(self) -> *mut ffi::PyObject { - self.into_non_null().as_ptr() + ManuallyDrop::new(self).as_ptr() } /// Helper to cast to `Bound<'py, PyAny>`. + #[inline] pub fn as_any(&self) -> &Bound<'py, PyAny> { // Safety: all Bound have the same memory layout, and all Bound are valid // Bound, so pointer casting is valid. @@ -197,12 +207,14 @@ impl<'py, T> Bound<'py, T> { } /// Helper to cast to `Bound<'py, PyAny>`, transferring ownership. + #[inline] pub fn into_any(self) -> Bound<'py, PyAny> { // Safety: all Bound are valid Bound Bound(self.0, ManuallyDrop::new(self.unbind().into_any())) } /// Casts this `Bound` to a `Borrowed` smart pointer. + #[inline] pub fn as_borrowed<'a>(&'a self) -> Borrowed<'a, 'py, T> { Borrowed( unsafe { NonNull::new_unchecked(self.as_ptr()) }, @@ -213,15 +225,18 @@ impl<'py, T> Bound<'py, T> { /// Removes the connection for this `Bound` from the GIL, allowing /// it to cross thread boundaries. + #[inline] pub fn unbind(self) -> Py { // Safety: the type T is known to be correct and the ownership of the // pointer is transferred to the new Py instance. - unsafe { Py::from_non_null(self.into_non_null()) } + let non_null = (ManuallyDrop::new(self).1).0; + unsafe { Py::from_non_null(non_null) } } /// Casts this `Bound` as the corresponding "GIL Ref" type. /// /// This is a helper to be used for migration from the deprecated "GIL Refs" API. + #[inline] pub fn as_gil_ref(&'py self) -> &'py T::AsRefTarget where T: HasPyGilRef, @@ -233,20 +248,13 @@ impl<'py, T> Bound<'py, T> { /// [release pool](Python::from_owned_ptr). /// /// This is a helper to be used for migration from the deprecated "GIL Refs" API. + #[inline] pub fn into_gil_ref(self) -> &'py T::AsRefTarget where T: HasPyGilRef, { unsafe { self.py().from_owned_ptr(self.into_ptr()) } } - - // Internal helper to convert `self` into a `NonNull` which owns the - // Python reference. - pub(crate) fn into_non_null(self) -> NonNull { - // wrap in ManuallyDrop to avoid running Drop for self and decreasing - // the reference count - ManuallyDrop::new(self).1 .0 - } } unsafe impl AsPyPointer for Bound<'_, T> { @@ -267,11 +275,7 @@ pub struct Borrowed<'a, 'py, T>(NonNull, PhantomData<&'a Py>, impl<'py, T> Borrowed<'_, 'py, T> { /// Creates a new owned `Bound` from this borrowed reference by increasing the reference count. pub(crate) fn to_owned(self) -> Bound<'py, T> { - unsafe { ffi::Py_INCREF(self.as_ptr()) }; - Bound( - self.py(), - ManuallyDrop::new(unsafe { Py::from_non_null(self.0) }), - ) + (*self).clone() } } @@ -353,6 +357,7 @@ impl<'py, T> Deref for Borrowed<'_, 'py, T> { } impl Clone for Borrowed<'_, '_, T> { + #[inline] fn clone(&self) -> Self { *self } @@ -362,6 +367,7 @@ impl Copy for Borrowed<'_, '_, T> {} impl ToPyObject for Borrowed<'_, '_, T> { /// Converts `Py` instance -> PyObject. + #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { (*self).into_py(py) } @@ -369,6 +375,7 @@ impl ToPyObject for Borrowed<'_, '_, T> { impl IntoPy for Borrowed<'_, '_, T> { /// Converts `Py` instance -> PyObject. + #[inline] fn into_py(self, py: Python<'_>) -> PyObject { self.to_owned().into_py(py) } @@ -714,12 +721,11 @@ impl Py { /// of the pointer or decrease the reference count (e.g. with [`pyo3::ffi::Py_DecRef`](crate::ffi::Py_DecRef)). #[inline] pub fn into_ptr(self) -> *mut ffi::PyObject { - let ptr = self.0.as_ptr(); - std::mem::forget(self); - ptr + ManuallyDrop::new(self).0.as_ptr() } /// Helper to cast to `Py`. + #[inline] pub fn as_any(&self) -> &Py { // Safety: all Py have the same memory layout, and all Py are valid // Py, so pointer casting is valid. @@ -727,9 +733,10 @@ impl Py { } /// Helper to cast to `Py`, transferring ownership. + #[inline] pub fn into_any(self) -> Py { // Safety: all Py are valid Py - unsafe { Py::from_non_null(self.into_non_null()) } + unsafe { Py::from_non_null(ManuallyDrop::new(self).0) } } } @@ -886,17 +893,20 @@ where impl Py { /// Attaches this `Py` to the given Python context, allowing access to further Python APIs. + #[inline] pub fn bind<'py>(&self, _py: Python<'py>) -> &Bound<'py, T> { // Safety: `Bound` has the same layout as `Py` unsafe { &*(self as *const Py).cast() } } /// Same as `bind` but takes ownership of `self`. + #[inline] pub fn into_bound(self, py: Python<'_>) -> Bound<'_, T> { Bound(py, ManuallyDrop::new(self)) } /// Same as `bind` but produces a `Borrowed` instead of a `Bound`. + #[inline] pub fn bind_borrowed<'a, 'py>(&'a self, py: Python<'py>) -> Borrowed<'a, 'py, T> { Borrowed(self.0, PhantomData, py) } @@ -1261,24 +1271,16 @@ impl Py { /// /// # Safety /// `ptr` must point to a Python object of type T. - #[inline] - pub(crate) unsafe fn from_non_null(ptr: NonNull) -> Self { + unsafe fn from_non_null(ptr: NonNull) -> Self { Self(ptr, PhantomData) } - - /// Returns the inner pointer without decreasing the refcount. - #[inline] - fn into_non_null(self) -> NonNull { - let pointer = self.0; - mem::forget(self); - pointer - } } impl ToPyObject for Py { /// Converts `Py` instance -> PyObject. + #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { - unsafe { PyObject::from_borrowed_ptr(py, self.as_ptr()) } + self.clone_ref(py).into_any() } } @@ -1287,7 +1289,7 @@ impl IntoPy for Py { /// Consumes `self` without calling `Py_DECREF()`. #[inline] fn into_py(self, _py: Python<'_>) -> PyObject { - unsafe { PyObject::from_non_null(self.into_non_null()) } + self.into_any() } } @@ -1299,15 +1301,15 @@ impl IntoPy for &'_ Py { } impl ToPyObject for Bound<'_, T> { - /// Converts `Py` instance -> PyObject. + /// Converts `&Bound` instance -> PyObject, increasing the reference count. + #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { - unsafe { PyObject::from_borrowed_ptr(py, self.as_ptr()) } + self.clone().into_py(py) } } impl IntoPy for Bound<'_, T> { - /// Converts a `Py` instance to `PyObject`. - /// Consumes `self` without calling `Py_DECREF()`. + /// Converts a `Bound` instance to `PyObject`. #[inline] fn into_py(self, _py: Python<'_>) -> PyObject { self.into_any().unbind() @@ -1315,11 +1317,10 @@ impl IntoPy for Bound<'_, T> { } impl IntoPy for &Bound<'_, T> { - /// Converts a `Py` instance to `PyObject`. - /// Consumes `self` without calling `Py_DECREF()`. + /// Converts `&Bound` instance -> PyObject, increasing the reference count. #[inline] - fn into_py(self, _py: Python<'_>) -> PyObject { - unsafe { PyObject::from_non_null(self.clone().into_non_null()) } + fn into_py(self, py: Python<'_>) -> PyObject { + self.to_object(py) } } @@ -1331,18 +1332,13 @@ unsafe impl crate::AsPyPointer for Py { } } -impl std::convert::From<&'_ PyAny> for PyObject { - fn from(obj: &PyAny) -> Self { - unsafe { Py::from_borrowed_ptr(obj.py(), obj.as_ptr()) } - } -} - impl std::convert::From<&'_ T> for PyObject where - T: PyNativeType + AsRef, + T: PyNativeType, { + #[inline] fn from(obj: &T) -> Self { - unsafe { Py::from_borrowed_ptr(obj.py(), obj.as_ref().as_ptr()) } + obj.as_borrowed().to_owned().into_any().unbind() } } @@ -1352,7 +1348,7 @@ where { #[inline] fn from(other: Py) -> Self { - unsafe { Self::from_non_null(other.into_non_null()) } + other.into_any() } } @@ -1380,7 +1376,7 @@ where T: PyClass, { fn from(cell: &PyCell) -> Self { - unsafe { Py::from_borrowed_ptr(cell.py(), cell.as_ptr()) } + cell.as_borrowed().to_owned().unbind() } } From a60c1821afb80f7775c2067b6210ceb765dfd1fb Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 29 Jan 2024 13:31:21 +0000 Subject: [PATCH 080/349] implement `PyFunctionArgument` for `&Bound` --- guide/src/migration.md | 1 + src/impl_/extract_argument.rs | 14 +++++++++++++- tests/test_pyfunction.rs | 17 +++++++++++++++++ tests/ui/invalid_argument_attributes.stderr | 1 + 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index 9fb5adba387..0174d0dc626 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -247,6 +247,7 @@ Because the new `Bound` API brings ownership out of the PyO3 framework and in - Code will need to add the occasional `&` to borrow the new smart pointer as `&Bound` to pass these types around (or use `.clone()` at the very small cost of increasing the Python reference count) - `Bound` and `Bound` cannot support indexing with `list[0]`, you should use `list.get_item(0)` instead. - `Bound::iter_borrowed` is slightly more efficient than `Bound::iter`. The default iteration of `Bound` cannot return borrowed references because Rust does not (yet) have "lending iterators". Similarly `Bound::get_borrowed_item` is more efficient than `Bound::get_item` for the same reason. +- `&Bound` does not implement `FromPyObject` (although it might be possible to do this in the future once the GIL Refs API is completely removed). Use `bound_any.downcast::()` instead of `bound_any.extract::<&Bound>()`. #### Migrating `FromPyObject` implementations diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index a18c8d4bc30..cd63fe270b0 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -3,7 +3,7 @@ use crate::{ ffi, pyclass::boolean_struct::False, types::{PyDict, PyString, PyTuple}, - FromPyObject, PyAny, PyClass, PyErr, PyRef, PyRefMut, PyResult, Python, + Bound, FromPyObject, PyAny, PyClass, PyErr, PyRef, PyRefMut, PyResult, PyTypeCheck, Python, }; /// A trait which is used to help PyO3 macros extract function arguments. @@ -31,6 +31,18 @@ where } } +impl<'a, 'py, T> PyFunctionArgument<'a, 'py> for &'a Bound<'py, T> +where + T: PyTypeCheck, +{ + type Holder = Option>; + + #[inline] + fn extract(obj: &'py PyAny, holder: &'a mut Option>) -> PyResult { + Ok(&*holder.insert(obj.extract()?)) + } +} + /// Trait for types which can be a function argument holder - they should /// to be able to const-initialize to an empty value. pub trait FunctionArgumentHolder: Sized { diff --git a/tests/test_pyfunction.rs b/tests/test_pyfunction.rs index a2f44be6bbd..f1116363432 100644 --- a/tests/test_pyfunction.rs +++ b/tests/test_pyfunction.rs @@ -529,3 +529,20 @@ fn test_some_wrap_arguments() { py_assert!(py, function, "function() == [1, 2, None, None]"); }) } + +#[test] +fn test_reference_to_bound_arguments() { + #[pyfunction] + fn reference_args<'py>( + x: &Bound<'py, PyAny>, + y: Option<&Bound<'py, PyAny>>, + ) -> PyResult> { + y.map_or_else(|| Ok(x.clone()), |y| y.add(x)) + } + + Python::with_gil(|py| { + let function = wrap_pyfunction!(reference_args, py).unwrap(); + py_assert!(py, function, "function(1) == 1"); + py_assert!(py, function, "function(1, 2) == 3"); + }) +} diff --git a/tests/ui/invalid_argument_attributes.stderr b/tests/ui/invalid_argument_attributes.stderr index ef8b59a8691..c122dd25f8c 100644 --- a/tests/ui/invalid_argument_attributes.stderr +++ b/tests/ui/invalid_argument_attributes.stderr @@ -93,6 +93,7 @@ error[E0277]: the trait bound `CancelHandle: Clone` is not satisfied | ^^^^ the trait `Clone` is not implemented for `CancelHandle` | = help: the following other types implement trait `PyFunctionArgument<'a, 'py>`: + &'a pyo3::Bound<'py, T> &'a pyo3::coroutine::Coroutine &'a mut pyo3::coroutine::Coroutine = note: required for `CancelHandle` to implement `FromPyObject<'_>` From af21a9dc74dbd8257a8bd6811afa993d0ada5b2a Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 30 Jan 2024 21:55:22 +0100 Subject: [PATCH 081/349] add `Bound` constructor for `PyBool` --- guide/src/exception.md | 2 +- src/impl_/pyclass.rs | 2 +- src/instance.rs | 8 ++++++ src/types/any.rs | 20 +++++++------- src/types/boolobject.rs | 49 +++++++++++++++++++++++++--------- tests/test_class_attributes.rs | 4 +-- 6 files changed, 59 insertions(+), 26 deletions(-) diff --git a/guide/src/exception.md b/guide/src/exception.md index 56e343f7c08..e1ce24980d3 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -79,7 +79,7 @@ use pyo3::prelude::*; use pyo3::types::{PyBool, PyList}; Python::with_gil(|py| { - assert!(PyBool::new(py, true).is_instance_of::()); + assert!(PyBool::new_bound(py, true).is_instance_of::()); let list = PyList::new_bound(py, &[1, 2, 3, 4]); assert!(!list.is_instance_of::()); assert!(list.is_instance_of::()); diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index fa03b81e455..2bd39dda1a0 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -813,7 +813,7 @@ slot_fragment_trait! { // By default `__ne__` will try `__eq__` and invert the result let slf: &PyAny = py.from_borrowed_ptr(slf); let other: &PyAny = py.from_borrowed_ptr(other); - slf.eq(other).map(|is_eq| PyBool::new(py, !is_eq).into_ptr()) + slf.eq(other).map(|is_eq| PyBool::new_bound(py, !is_eq).to_owned().into_ptr()) } } diff --git a/src/instance.rs b/src/instance.rs index 627c5e678f2..47c9d1705be 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -321,6 +321,14 @@ impl<'a, 'py> Borrowed<'a, 'py, PyAny> { pub(crate) unsafe fn from_ptr_unchecked(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self(NonNull::new_unchecked(ptr), PhantomData, py) } + + /// Converts this `PyAny` to a concrete Python type without checking validity. + /// + /// # Safety + /// Callers must ensure that the type is valid or risk type confusion. + pub(crate) unsafe fn downcast_unchecked(self) -> Borrowed<'a, 'py, T> { + Borrowed(self.0, PhantomData, self.2) + } } impl<'a, 'py, T> From<&'a Bound<'py, T>> for Borrowed<'a, 'py, T> { diff --git a/src/types/any.rs b/src/types/any.rs index 14102d80a8d..61c3c67217a 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -759,9 +759,9 @@ impl PyAny { /// use pyo3::types::{PyBool, PyLong}; /// /// Python::with_gil(|py| { - /// let b = PyBool::new(py, true); + /// let b = PyBool::new_bound(py, true); /// assert!(b.is_instance_of::()); - /// let any: &PyAny = b.as_ref(); + /// let any: &Bound<'_, PyAny> = b.as_any(); /// /// // `bool` is a subtype of `int`, so `downcast` will accept a `bool` as an `int` /// // but `downcast_exact` will not. @@ -1583,9 +1583,9 @@ pub trait PyAnyMethods<'py> { /// use pyo3::types::{PyBool, PyLong}; /// /// Python::with_gil(|py| { - /// let b = PyBool::new(py, true); + /// let b = PyBool::new_bound(py, true); /// assert!(b.is_instance_of::()); - /// let any: &PyAny = b.as_ref(); + /// let any: &Bound<'_, PyAny> = b.as_any(); /// /// // `bool` is a subtype of `int`, so `downcast` will accept a `bool` as an `int` /// // but `downcast_exact` will not. @@ -2530,7 +2530,7 @@ class SimpleClass: let x = 5.to_object(py).into_ref(py); assert!(x.is_exact_instance_of::()); - let t = PyBool::new(py, true); + let t = PyBool::new_bound(py, true); assert!(t.is_instance_of::()); assert!(!t.is_exact_instance_of::()); assert!(t.is_exact_instance_of::()); @@ -2543,10 +2543,12 @@ class SimpleClass: #[test] fn test_any_is_exact_instance() { Python::with_gil(|py| { - let t = PyBool::new(py, true); - assert!(t.is_instance(py.get_type::()).unwrap()); - assert!(!t.is_exact_instance(py.get_type::())); - assert!(t.is_exact_instance(py.get_type::())); + let t = PyBool::new_bound(py, true); + assert!(t + .is_instance(&py.get_type::().as_borrowed()) + .unwrap()); + assert!(!t.is_exact_instance(&py.get_type::().as_borrowed())); + assert!(t.is_exact_instance(&py.get_type::().as_borrowed())); }); } diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 0decc0b4c80..c0c4e4ba5c2 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -1,8 +1,8 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - exceptions::PyTypeError, ffi, instance::Bound, FromPyObject, IntoPy, PyAny, PyNativeType, - PyObject, PyResult, Python, ToPyObject, + exceptions::PyTypeError, ffi, ffi_ptr_ext::FfiPtrExt, instance::Bound, Borrowed, FromPyObject, + IntoPy, PyAny, PyNativeType, PyObject, PyResult, Python, ToPyObject, }; use super::any::PyAnyMethods; @@ -14,12 +14,33 @@ pub struct PyBool(PyAny); pyobject_native_type!(PyBool, ffi::PyObject, pyobject_native_static_type_object!(ffi::PyBool_Type), #checkfunction=ffi::PyBool_Check); impl PyBool { - /// Depending on `val`, returns `true` or `false`. + /// Deprecated form of [`PyBool::new_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyBool::new` will be replaced by `PyBool::new_bound` in a future PyO3 version" + ) + )] #[inline] pub fn new(py: Python<'_>, val: bool) -> &PyBool { unsafe { py.from_borrowed_ptr(if val { ffi::Py_True() } else { ffi::Py_False() }) } } + /// Depending on `val`, returns `true` or `false`. + /// + /// # Note + /// This returns a [`Borrowed`] reference to one of Pythons `True` or + /// `False` singletons + #[inline] + pub fn new_bound(py: Python<'_>, val: bool) -> Borrowed<'_, '_, Self> { + unsafe { + if val { ffi::Py_True() } else { ffi::Py_False() } + .assume_borrowed(py) + .downcast_unchecked() + } + } + /// Gets whether this boolean is `true`. #[inline] pub fn is_true(&self) -> bool { @@ -65,7 +86,7 @@ impl ToPyObject for bool { impl IntoPy for bool { #[inline] fn into_py(self, py: Python<'_>) -> PyObject { - PyBool::new(py, self).into() + PyBool::new_bound(py, self).into_py(py) } #[cfg(feature = "experimental-inspect")] @@ -135,27 +156,29 @@ impl FromPyObject<'_> for bool { #[cfg(test)] mod tests { - use crate::types::{PyAny, PyBool}; + use crate::types::any::PyAnyMethods; + use crate::types::boolobject::PyBoolMethods; + use crate::types::PyBool; use crate::Python; use crate::ToPyObject; #[test] fn test_true() { Python::with_gil(|py| { - assert!(PyBool::new(py, true).is_true()); - let t: &PyAny = PyBool::new(py, true).into(); - assert!(t.extract::().unwrap()); - assert!(true.to_object(py).is(PyBool::new(py, true))); + assert!(PyBool::new_bound(py, true).is_true()); + let t = PyBool::new_bound(py, true); + assert!(t.as_any().extract::().unwrap()); + assert!(true.to_object(py).is(&*PyBool::new_bound(py, true))); }); } #[test] fn test_false() { Python::with_gil(|py| { - assert!(!PyBool::new(py, false).is_true()); - let t: &PyAny = PyBool::new(py, false).into(); - assert!(!t.extract::().unwrap()); - assert!(false.to_object(py).is(PyBool::new(py, false))); + assert!(!PyBool::new_bound(py, false).is_true()); + let t = PyBool::new_bound(py, false); + assert!(!t.as_any().extract::().unwrap()); + assert!(false.to_object(py).is(&*PyBool::new_bound(py, false))); }); } } diff --git a/tests/test_class_attributes.rs b/tests/test_class_attributes.rs index 0ff0dd6d9d8..b589dd08809 100644 --- a/tests/test_class_attributes.rs +++ b/tests/test_class_attributes.rs @@ -190,12 +190,12 @@ fn test_renaming_all_struct_fields() { let struct_class = py.get_type::(); let struct_obj = struct_class.call0().unwrap(); assert!(struct_obj - .setattr("firstField", PyBool::new(py, false)) + .setattr("firstField", PyBool::new_bound(py, false)) .is_ok()); py_assert!(py, struct_obj, "struct_obj.firstField == False"); py_assert!(py, struct_obj, "struct_obj.secondField == 5"); assert!(struct_obj - .setattr("third_field", PyBool::new(py, true)) + .setattr("third_field", PyBool::new_bound(py, true)) .is_ok()); py_assert!(py, struct_obj, "struct_obj.third_field == True"); }); From 5e9d97d1c6170a816ea4ff1b0aafbcf8c3ea2089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20=C5=A0nuderl?= Date: Sat, 3 Feb 2024 19:01:18 +0100 Subject: [PATCH 082/349] Implement new API for PyNone #3684 --- guide/src/migration.md | 2 +- pyo3-benches/benches/bench_gil.rs | 2 +- pytests/src/awaitable.rs | 2 +- src/conversion.rs | 30 ++++++++++++++++-------------- src/coroutine.rs | 2 +- src/marker.rs | 4 ++-- src/types/none.rs | 19 ++++++++++++------- tests/test_arithmetics.rs | 2 +- tests/test_class_conversion.rs | 2 +- tests/test_frompyobject.rs | 2 +- tests/test_gc.rs | 8 ++++---- tests/test_no_imports.rs | 4 +++- 12 files changed, 44 insertions(+), 35 deletions(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index d5ce7f4ac1f..4c7700e7a76 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -108,7 +108,7 @@ After: # use pyo3::prelude::*; Python::with_gil(|py| { // For uses needing a PyObject, add `.into()` - let a: PyObject = py.None().into(); + let a: PyObject = py.None().into_py(py); // For uses needing &PyAny, remove `.as_ref(py)` let b: &PyAny = py.None(); diff --git a/pyo3-benches/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs index 169c7c6cf9e..55a5a04ab57 100644 --- a/pyo3-benches/benches/bench_gil.rs +++ b/pyo3-benches/benches/bench_gil.rs @@ -8,7 +8,7 @@ fn bench_clean_acquire_gil(b: &mut Bencher<'_>) { } fn bench_dirty_acquire_gil(b: &mut Bencher<'_>) { - let obj: PyObject = Python::with_gil(|py| py.None().into()); + let obj: PyObject = Python::with_gil(|py| py.None().into_py(py)); b.iter_batched( || { // Clone and drop an object so that the GILPool has work to do. diff --git a/pytests/src/awaitable.rs b/pytests/src/awaitable.rs index 0cc173334b9..55f1bce478c 100644 --- a/pytests/src/awaitable.rs +++ b/pytests/src/awaitable.rs @@ -37,7 +37,7 @@ impl IterAwaitable { Ok(v) => Err(PyStopIteration::new_err(v)), Err(err) => Err(err), }, - _ => Ok(py.None().into()), + _ => Ok(py.None().into_py(py)), } } } diff --git a/src/conversion.rs b/src/conversion.rs index 98ef98310c7..f75a82d4601 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -141,7 +141,7 @@ pub trait ToPyObject { /// match self { /// Self::Integer(val) => val.into_py(py), /// Self::String(val) => val.into_py(py), -/// Self::None => py.None().into(), +/// Self::None => py.None().into_py(py), /// } /// } /// } @@ -266,7 +266,7 @@ where { fn to_object(&self, py: Python<'_>) -> PyObject { self.as_ref() - .map_or_else(|| py.None().into(), |val| val.to_object(py)) + .map_or_else(|| py.None().into_py(py), |val| val.to_object(py)) } } @@ -275,7 +275,7 @@ where T: IntoPy, { fn into_py(self, py: Python<'_>) -> PyObject { - self.map_or_else(|| py.None().into(), |val| val.into_py(py)) + self.map_or_else(|| py.None().into_py(py), |val| val.into_py(py)) } } @@ -593,6 +593,8 @@ mod test_no_clone {} #[cfg(test)] mod tests { + use crate::conversion::IntoPy; + use crate::prelude::PyAnyMethods; use crate::{PyObject, Python}; #[allow(deprecated)] @@ -629,14 +631,14 @@ mod tests { }); } - #[test] - fn test_try_from_unchecked() { - Python::with_gil(|py| { - let list = PyList::new(py, [1, 2, 3]); - let val = unsafe { ::try_from_unchecked(list.as_ref()) }; - assert!(list.is(val)); - }); - } + // #[test] + // fn test_try_from_unchecked() { + // Python::with_gil(|py| { + // let list = PyList::new(py, [1, 2, 3]); + // let val = unsafe { ::try_from_unchecked(list.as_ref()) }; + // assert!(list.is(val)); + // }); + // } } #[test] @@ -647,13 +649,13 @@ mod tests { assert_eq!(option.as_ptr(), std::ptr::null_mut()); let none = py.None(); - option = Some(none.into()); + option = Some(none.into_py(py)); - let ref_cnt = none.get_refcnt(); + let ref_cnt = none.into_py(py).get_refcnt(py); assert_eq!(option.as_ptr(), none.as_ptr()); // Ensure ref count not changed by as_ptr call - assert_eq!(none.get_refcnt(), ref_cnt); + assert_eq!(none.into_py(py).get_refcnt(py), ref_cnt); }); } } diff --git a/src/coroutine.rs b/src/coroutine.rs index 415bbc88db9..c4b5ddf3185 100644 --- a/src/coroutine.rs +++ b/src/coroutine.rs @@ -118,7 +118,7 @@ impl Coroutine { } // if waker has been waken during future polling, this is roughly equivalent to // `await asyncio.sleep(0)`, so just yield `None`. - Ok(py.None().into()) + Ok(py.None().into_py(py)) } } diff --git a/src/marker.rs b/src/marker.rs index 836d0109eeb..06fbec7e790 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -123,7 +123,7 @@ use crate::types::{ PyAny, PyDict, PyEllipsis, PyModule, PyNone, PyNotImplemented, PyString, PyType, }; use crate::version::PythonVersionInfo; -use crate::{ffi, FromPyPointer, IntoPy, Py, PyObject, PyTypeCheck, PyTypeInfo}; +use crate::{ffi, Borrowed, FromPyPointer, IntoPy, Py, PyObject, PyTypeCheck, PyTypeInfo}; use std::ffi::{CStr, CString}; use std::marker::PhantomData; use std::os::raw::c_int; @@ -698,7 +698,7 @@ impl<'py> Python<'py> { /// Gets the Python builtin value `None`. #[allow(non_snake_case)] // the Python keyword starts with uppercase #[inline] - pub fn None(self) -> &'py PyNone { + pub fn None(self) -> Borrowed<'py, 'py, PyNone> { PyNone::get(self) } diff --git a/src/types/none.rs b/src/types/none.rs index 19ee80ede57..c645d9dff29 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -1,4 +1,5 @@ -use crate::{ffi, IntoPy, PyAny, PyObject, PyTypeInfo, Python, ToPyObject}; +use crate::ffi_ptr_ext::FfiPtrExt; +use crate::{ffi, Borrowed, IntoPy, PyAny, PyObject, PyTypeInfo, Python, ToPyObject}; /// Represents the Python `None` object. #[repr(transparent)] @@ -10,8 +11,11 @@ pyobject_native_type_extract!(PyNone); impl PyNone { /// Returns the `None` object. #[inline] - pub fn get(py: Python<'_>) -> &PyNone { - unsafe { py.from_borrowed_ptr(ffi::Py_None()) } + pub fn get<'py>(py: Python<'py>) -> Borrowed<'py, 'py, PyNone> { + unsafe { + let bound = ffi::Py_None().assume_borrowed(py); + std::mem::transmute(bound) + } } } @@ -32,29 +36,30 @@ unsafe impl PyTypeInfo for PyNone { #[inline] fn is_exact_type_of(object: &PyAny) -> bool { - object.is(Self::get(object.py())) + let none = Self::get(object.py()); + object.is(none.as_ref()) } } /// `()` is converted to Python `None`. impl ToPyObject for () { fn to_object(&self, py: Python<'_>) -> PyObject { - PyNone::get(py).into() + PyNone::get(py).into_py(py) } } impl IntoPy for () { #[inline] fn into_py(self, py: Python<'_>) -> PyObject { - PyNone::get(py).into() + PyNone::get(py).into_py(py) } } #[cfg(test)] mod tests { + use crate::types::any::PyAnyMethods; use crate::types::{PyDict, PyNone}; use crate::{IntoPy, PyObject, PyTypeInfo, Python, ToPyObject}; - #[test] fn test_none_is_itself() { Python::with_gil(|py| { diff --git a/tests/test_arithmetics.rs b/tests/test_arithmetics.rs index 456d21a3b62..1b399a1f41c 100644 --- a/tests/test_arithmetics.rs +++ b/tests/test_arithmetics.rs @@ -556,7 +556,7 @@ mod return_not_implemented { } fn __richcmp__(&self, other: PyRef<'_, Self>, _op: CompareOp) -> PyObject { - other.py().None().into() + other.py().None().into_py(other.py()) } fn __add__<'p>(slf: PyRef<'p, Self>, _other: PyRef<'p, Self>) -> PyRef<'p, Self> { diff --git a/tests/test_class_conversion.rs b/tests/test_class_conversion.rs index 81a3d7bfbfd..a0a16e21c88 100644 --- a/tests/test_class_conversion.rs +++ b/tests/test_class_conversion.rs @@ -119,7 +119,7 @@ fn test_polymorphic_container_does_not_accept_other_types() { let setattr = |value: PyObject| p.as_ref(py).setattr("inner", value); assert!(setattr(1i32.into_py(py)).is_err()); - assert!(setattr(py.None().into()).is_err()); + assert!(setattr(py.None().into_py(py)).is_err()); assert!(setattr((1i32, 2i32).into_py(py)).is_err()); }); } diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index 8eea9b896b2..c141b3e6e22 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -352,7 +352,7 @@ fn test_enum() { _ => panic!("Expected extracting Foo::TransparentTuple, got {:?}", f), } let none = py.None(); - let f = Foo::extract(none).expect("Failed to extract Foo from int"); + let f = Foo::extract_bound(none.as_ref()).expect("Failed to extract Foo from int"); match f { Foo::TransparentStructVar { a } => assert!(a.is_none()), _ => panic!("Expected extracting Foo::TransparentStructVar, got {:?}", f), diff --git a/tests/test_gc.rs b/tests/test_gc.rs index 54c3e1a100c..0f68bf0b2f6 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -89,7 +89,7 @@ impl GcIntegration { fn __clear__(&mut self) { Python::with_gil(|py| { - self.self_ref = py.None().into(); + self.self_ref = py.None().into_py(py); }); } } @@ -102,7 +102,7 @@ fn gc_integration() { let inst = PyCell::new( py, GcIntegration { - self_ref: py.None().into(), + self_ref: py.None().into_py(py), dropped: TestDropCall { drop_called: Arc::clone(&drop_called), }, @@ -287,7 +287,7 @@ struct PartialTraverse { impl PartialTraverse { fn new(py: Python<'_>) -> Self { Self { - member: py.None().into(), + member: py.None().into_py(py), } } } @@ -325,7 +325,7 @@ struct PanickyTraverse { impl PanickyTraverse { fn new(py: Python<'_>) -> Self { Self { - member: py.None().into(), + member: py.None().into_py(py), } } } diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index 4f04282702e..fe49b2d2b31 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -2,10 +2,12 @@ #![cfg(feature = "macros")] +use pyo3::IntoPy; + #[pyo3::pyfunction] #[pyo3(name = "identity", signature = (x = None))] fn basic_function(py: pyo3::Python<'_>, x: Option) -> pyo3::PyObject { - x.unwrap_or_else(|| py.None().into()) + x.unwrap_or_else(|| py.None().into_py(py)) } #[pyo3::pymodule] From 7e94da576db430d683a22e2371cdb53eb7a0f728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20=C5=A0nuderl?= Date: Sat, 3 Feb 2024 20:44:48 +0100 Subject: [PATCH 083/349] Fix doctests --- guide/src/migration.md | 4 ++-- guide/src/python_from_rust.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index 4c7700e7a76..7d65d3972c0 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -98,7 +98,7 @@ Before: Python::with_gil(|py| { let a: PyObject = py.None(); - let b: &PyAny = py.None().as_ref(py); // or into_ref(py) + // let b: &PyAny = py.None().as_ref(py); // or into_ref(py) }); ``` @@ -111,7 +111,7 @@ Python::with_gil(|py| { let a: PyObject = py.None().into_py(py); // For uses needing &PyAny, remove `.as_ref(py)` - let b: &PyAny = py.None(); + // let b: &PyAny = py.None(); }); ``` diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 99da2e15434..a5cd624ee69 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -474,7 +474,7 @@ class House(object): Ok(_) => { let none = py.None(); house - .call_method1("__exit__", (&none, &none, &none)) + .call_method1("__exit__", (none, none, none)) .unwrap(); } Err(e) => { From a2a6062adcba6482d05786029fd669dda45db3e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20=C5=A0nuderl?= Date: Sat, 3 Feb 2024 20:48:25 +0100 Subject: [PATCH 084/349] fmt --- src/conversion.rs | 1 - src/types/bytearray.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index f75a82d4601..28e15d17972 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -594,7 +594,6 @@ mod test_no_clone {} #[cfg(test)] mod tests { use crate::conversion::IntoPy; - use crate::prelude::PyAnyMethods; use crate::{PyObject, Python}; #[allow(deprecated)] diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 65aef27ee52..55cd75d570b 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -526,7 +526,7 @@ mod tests { use crate::types::bytearray::PyByteArrayMethods; use crate::types::string::PyStringMethods; use crate::types::PyByteArray; - use crate::{exceptions, Bound, PyAny, PyNativeType}; + use crate::{exceptions, Bound, PyAny}; use crate::{PyObject, Python}; #[test] From 9641b117529dbfe824b9d59f440a3267edd6a8db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20=C5=A0nuderl?= Date: Sat, 3 Feb 2024 20:57:46 +0100 Subject: [PATCH 085/349] hmm --- src/conversions/chrono.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 0e95375b5bd..8d201fb88c0 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -566,6 +566,7 @@ fn timezone_utc(py: Python<'_>) -> &PyAny { #[cfg(test)] mod tests { use super::*; + use crate::types::any::PyAnyMethods; use crate::{types::PyTuple, Py}; use std::{cmp::Ordering, panic}; From 507ea28b27962029459f3a2e6123ef083d755e08 Mon Sep 17 00:00:00 2001 From: Blaz Snuderl Date: Sat, 3 Feb 2024 21:14:31 +0100 Subject: [PATCH 086/349] test --- src/ffi/tests.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index cb0972590ff..a3160751875 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -311,6 +311,8 @@ fn test_inc_dec_ref() { #[test] #[cfg(Py_3_12)] fn test_inc_dec_ref_immortal() { + use crate::types::any::PyAnyMethods; + Python::with_gil(|py| { let obj = py.None(); From b1863c73df1fcb330b4c4b72467686fd1633bd1b Mon Sep 17 00:00:00 2001 From: Blaz Snuderl Date: Sat, 3 Feb 2024 21:25:47 +0100 Subject: [PATCH 087/349] clippy --- src/types/none.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types/none.rs b/src/types/none.rs index c645d9dff29..6d135785c53 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -11,7 +11,7 @@ pyobject_native_type_extract!(PyNone); impl PyNone { /// Returns the `None` object. #[inline] - pub fn get<'py>(py: Python<'py>) -> Borrowed<'py, 'py, PyNone> { + pub fn get(py: Python<'_>) -> Borrowed<'_, '_, PyNone> { unsafe { let bound = ffi::Py_None().assume_borrowed(py); std::mem::transmute(bound) From 76d1b34cd5dafe51a358827b84efa5d753a8d4f3 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 3 Feb 2024 20:56:23 +0000 Subject: [PATCH 088/349] Revert "Merge pull request #3578 from davidhewitt/typed-helpers" This reverts commit 7b07d6d21b90b52b15dc0517b664e24ef8f4163b, reversing changes made to 99858236bd8322fd3badff01b74a92763497a1ba. --- guide/src/class/protocols.md | 2 +- guide/src/migration.md | 39 ++------------------------ newsfragments/3578.changed.md | 1 - pyo3-benches/benches/bench_gil.rs | 2 +- pyo3-benches/benches/bench_pyobject.rs | 2 +- pyo3-macros-backend/src/pyclass.rs | 6 ++-- pytests/src/awaitable.rs | 2 +- src/conversion.rs | 12 ++++---- src/conversions/chrono.rs | 2 +- src/err/mod.rs | 2 +- src/ffi/tests.rs | 6 ++-- src/marker.rs | 12 ++++---- src/types/bytearray.rs | 4 +-- tests/test_arithmetics.rs | 4 +-- tests/test_class_conversion.rs | 2 +- tests/test_exceptions.rs | 2 +- tests/test_frompyobject.rs | 2 +- tests/test_gc.rs | 12 +++----- tests/test_no_imports.rs | 2 +- 19 files changed, 39 insertions(+), 77 deletions(-) delete mode 100644 newsfragments/3578.changed.md diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 4577a5102b6..411978f0567 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -103,7 +103,7 @@ given signatures should be interpreted as follows: match op { CompareOp::Eq => (self.0 == other.0).into_py(py), CompareOp::Ne => (self.0 != other.0).into_py(py), - _ => py.NotImplemented().into(), + _ => py.NotImplemented(), } } } diff --git a/guide/src/migration.md b/guide/src/migration.md index d5ce7f4ac1f..6a151b99164 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -82,39 +82,6 @@ Python::with_gil(|py| { # } ``` -### `py.None()`, `py.NotImplemented()` and `py.Ellipsis()` now return typed singletons - -Previously `py.None()`, `py.NotImplemented()` and `py.Ellipsis()` would return `PyObject`. This had a few downsides: - - `PyObject` does not carry static type information - - `PyObject` takes ownership of a reference to the singletons, adding refcounting performance overhead - - `PyObject` is not gil-bound, meaning follow up method calls might again need `py`, causing repetition - -To avoid these downsides, these methods now return typed gil-bound references to the singletons, e.g. `py.None()` returns `&PyNone`. These typed singletons all implement `Into`, so migration is straightforward. - -Before: - -```rust,compile_fail -# use pyo3::prelude::*; -Python::with_gil(|py| { - let a: PyObject = py.None(); - - let b: &PyAny = py.None().as_ref(py); // or into_ref(py) -}); -``` - -After: - -```rust -# use pyo3::prelude::*; -Python::with_gil(|py| { - // For uses needing a PyObject, add `.into()` - let a: PyObject = py.None().into(); - - // For uses needing &PyAny, remove `.as_ref(py)` - let b: &PyAny = py.None(); -}); -``` - ### `Iter(A)NextOutput` are deprecated The `__next__` and `__anext__` magic methods can now return any type convertible into Python objects directly just like all other `#[pymethods]`. The `IterNextOutput` used by `__next__` and `IterANextOutput` used by `__anext__` are subsequently deprecated. Most importantly, this change allows returning an awaitable from `__anext__` without non-sensically wrapping it into `Yield` or `Some`. Only the return types `Option` and `Result, E>` are still handled in a special manner where `Some(val)` yields `val` and `None` stops iteration. @@ -433,7 +400,7 @@ fn raise_err() -> anyhow::Result<()> { Err(PyValueError::new_err("original error message").into()) } -# fn main() { +fn main() { Python::with_gil(|py| { let rs_func = wrap_pyfunction!(raise_err, py).unwrap(); pyo3::py_run!( @@ -1212,14 +1179,14 @@ ensure that the Python GIL was held by the current thread). Technically, this wa To migrate, just pass a `py` argument to any calls to these methods. Before: -```rust,ignore +```rust,compile_fail # pyo3::Python::with_gil(|py| { py.None().get_refcnt(); # }) ``` After: -```rust,compile_fail +```rust # pyo3::Python::with_gil(|py| { py.None().get_refcnt(py); # }) diff --git a/newsfragments/3578.changed.md b/newsfragments/3578.changed.md deleted file mode 100644 index 53fc9943b43..00000000000 --- a/newsfragments/3578.changed.md +++ /dev/null @@ -1 +0,0 @@ -Change return type of `py.None()`, `py.NotImplemented()`, and `py.Ellipsis()` from `PyObject` to typed singletons (`&PyNone`, `&PyNotImplemented` and `PyEllipsis` respectively). diff --git a/pyo3-benches/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs index 169c7c6cf9e..59b9ff9686f 100644 --- a/pyo3-benches/benches/bench_gil.rs +++ b/pyo3-benches/benches/bench_gil.rs @@ -8,7 +8,7 @@ fn bench_clean_acquire_gil(b: &mut Bencher<'_>) { } fn bench_dirty_acquire_gil(b: &mut Bencher<'_>) { - let obj: PyObject = Python::with_gil(|py| py.None().into()); + let obj = Python::with_gil(|py| py.None()); b.iter_batched( || { // Clone and drop an object so that the GILPool has work to do. diff --git a/pyo3-benches/benches/bench_pyobject.rs b/pyo3-benches/benches/bench_pyobject.rs index 169cf1f0666..af25d61ce6a 100644 --- a/pyo3-benches/benches/bench_pyobject.rs +++ b/pyo3-benches/benches/bench_pyobject.rs @@ -6,7 +6,7 @@ fn drop_many_objects(b: &mut Bencher<'_>) { Python::with_gil(|py| { b.iter(|| { for _ in 0..1000 { - std::mem::drop(PyObject::from(py.None())); + std::mem::drop(py.None()); } }); }); diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index e2f849492a0..02024011366 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -743,7 +743,7 @@ fn impl_simple_enum( return Ok((self_val == other.__pyo3__int__()).to_object(py)); } - return Ok(::std::convert::Into::into(py.NotImplemented())); + return Ok(py.NotImplemented()); } _pyo3::basic::CompareOp::Ne => { let self_val = self.__pyo3__int__(); @@ -754,9 +754,9 @@ fn impl_simple_enum( return Ok((self_val != other.__pyo3__int__()).to_object(py)); } - return Ok(::std::convert::Into::into(py.NotImplemented())); + return Ok(py.NotImplemented()); } - _ => Ok(::std::convert::Into::into(py.NotImplemented())), + _ => Ok(py.NotImplemented()), } } }; diff --git a/pytests/src/awaitable.rs b/pytests/src/awaitable.rs index 0cc173334b9..e1a70b42bb0 100644 --- a/pytests/src/awaitable.rs +++ b/pytests/src/awaitable.rs @@ -37,7 +37,7 @@ impl IterAwaitable { Ok(v) => Err(PyStopIteration::new_err(v)), Err(err) => Err(err), }, - _ => Ok(py.None().into()), + _ => Ok(py.None()), } } } diff --git a/src/conversion.rs b/src/conversion.rs index 98ef98310c7..3af038af4d9 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -141,7 +141,7 @@ pub trait ToPyObject { /// match self { /// Self::Integer(val) => val.into_py(py), /// Self::String(val) => val.into_py(py), -/// Self::None => py.None().into(), +/// Self::None => py.None(), /// } /// } /// } @@ -266,7 +266,7 @@ where { fn to_object(&self, py: Python<'_>) -> PyObject { self.as_ref() - .map_or_else(|| py.None().into(), |val| val.to_object(py)) + .map_or_else(|| py.None(), |val| val.to_object(py)) } } @@ -275,7 +275,7 @@ where T: IntoPy, { fn into_py(self, py: Python<'_>) -> PyObject { - self.map_or_else(|| py.None().into(), |val| val.into_py(py)) + self.map_or_else(|| py.None(), |val| val.into_py(py)) } } @@ -647,13 +647,13 @@ mod tests { assert_eq!(option.as_ptr(), std::ptr::null_mut()); let none = py.None(); - option = Some(none.into()); + option = Some(none.clone()); - let ref_cnt = none.get_refcnt(); + let ref_cnt = none.get_refcnt(py); assert_eq!(option.as_ptr(), none.as_ptr()); // Ensure ref count not changed by as_ptr call - assert_eq!(none.get_refcnt(), ref_cnt); + assert_eq!(none.get_refcnt(py), ref_cnt); }); } } diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 0e95375b5bd..0d9a79c7be2 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -635,7 +635,7 @@ mod tests { // Test that if a user tries to convert a python's timezone aware datetime into a naive // one, the conversion fails. Python::with_gil(|py| { - let none = py.None(); + let none = py.None().into_ref(py); assert_eq!( none.extract::().unwrap_err().to_string(), "TypeError: 'NoneType' object cannot be converted to 'PyDelta'" diff --git a/src/err/mod.rs b/src/err/mod.rs index 13e154b3860..c50e377409e 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -219,7 +219,7 @@ impl PyErr { } else { // Assume obj is Type[Exception]; let later normalization handle if this // is not the case - PyErrState::lazy(obj, Option::::None) + PyErrState::lazy(obj, obj.py().None()) }; PyErr::from_state(state) diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index cb0972590ff..0e0d3688e11 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -314,15 +314,15 @@ fn test_inc_dec_ref_immortal() { Python::with_gil(|py| { let obj = py.None(); - let ref_count = obj.get_refcnt(); + let ref_count = obj.get_refcnt(py); let ptr = obj.as_ptr(); unsafe { Py_INCREF(ptr) }; - assert_eq!(obj.get_refcnt(), ref_count); + assert_eq!(obj.get_refcnt(py), ref_count); unsafe { Py_DECREF(ptr) }; - assert_eq!(obj.get_refcnt(), ref_count); + assert_eq!(obj.get_refcnt(py), ref_count); }) } diff --git a/src/marker.rs b/src/marker.rs index 836d0109eeb..026e3da4949 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -698,22 +698,22 @@ impl<'py> Python<'py> { /// Gets the Python builtin value `None`. #[allow(non_snake_case)] // the Python keyword starts with uppercase #[inline] - pub fn None(self) -> &'py PyNone { - PyNone::get(self) + pub fn None(self) -> PyObject { + PyNone::get(self).into() } /// Gets the Python builtin value `Ellipsis`, or `...`. #[allow(non_snake_case)] // the Python keyword starts with uppercase #[inline] - pub fn Ellipsis(self) -> &'py PyEllipsis { - PyEllipsis::get(self) + pub fn Ellipsis(self) -> PyObject { + PyEllipsis::get(self).into() } /// Gets the Python builtin value `NotImplemented`. #[allow(non_snake_case)] // the Python keyword starts with uppercase #[inline] - pub fn NotImplemented(self) -> &'py PyNotImplemented { - PyNotImplemented::get(self) + pub fn NotImplemented(self) -> PyObject { + PyNotImplemented::get(self).into() } /// Gets the running Python interpreter version as a string. diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 65aef27ee52..6303a87de3e 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -526,7 +526,7 @@ mod tests { use crate::types::bytearray::PyByteArrayMethods; use crate::types::string::PyStringMethods; use crate::types::PyByteArray; - use crate::{exceptions, Bound, PyAny, PyNativeType}; + use crate::{exceptions, Bound, PyAny}; use crate::{PyObject, Python}; #[test] @@ -596,7 +596,7 @@ mod tests { #[test] fn test_from_err() { Python::with_gil(|py| { - if let Err(err) = PyByteArray::from_bound(&py.None().as_borrowed()) { + if let Err(err) = PyByteArray::from_bound(py.None().bind(py)) { assert!(err.is_instance_of::(py)); } else { panic!("error"); diff --git a/tests/test_arithmetics.rs b/tests/test_arithmetics.rs index 456d21a3b62..f336140e432 100644 --- a/tests/test_arithmetics.rs +++ b/tests/test_arithmetics.rs @@ -486,7 +486,7 @@ impl RichComparisons2 { match op { CompareOp::Eq => true.into_py(other.py()), CompareOp::Ne => false.into_py(other.py()), - _ => other.py().NotImplemented().into(), + _ => other.py().NotImplemented(), } } } @@ -556,7 +556,7 @@ mod return_not_implemented { } fn __richcmp__(&self, other: PyRef<'_, Self>, _op: CompareOp) -> PyObject { - other.py().None().into() + other.py().None() } fn __add__<'p>(slf: PyRef<'p, Self>, _other: PyRef<'p, Self>) -> PyRef<'p, Self> { diff --git a/tests/test_class_conversion.rs b/tests/test_class_conversion.rs index 81a3d7bfbfd..27a8f604b3f 100644 --- a/tests/test_class_conversion.rs +++ b/tests/test_class_conversion.rs @@ -119,7 +119,7 @@ fn test_polymorphic_container_does_not_accept_other_types() { let setattr = |value: PyObject| p.as_ref(py).setattr("inner", value); assert!(setattr(1i32.into_py(py)).is_err()); - assert!(setattr(py.None().into()).is_err()); + assert!(setattr(py.None()).is_err()); assert!(setattr((1i32, 2i32).into_py(py)).is_err()); }); } diff --git a/tests/test_exceptions.rs b/tests/test_exceptions.rs index b783e887dcd..d6d6b46fcad 100644 --- a/tests/test_exceptions.rs +++ b/tests/test_exceptions.rs @@ -118,7 +118,7 @@ fn test_write_unraisable() { assert!(object.is_none(py)); let err = PyRuntimeError::new_err("bar"); - err.write_unraisable(py, Some(py.NotImplemented())); + err.write_unraisable(py, Some(py.NotImplemented().as_ref(py))); let (err, object) = capture.borrow_mut(py).capture.take().unwrap(); assert_eq!(err.to_string(), "RuntimeError: bar"); diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index 8eea9b896b2..c6f6e148943 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -352,7 +352,7 @@ fn test_enum() { _ => panic!("Expected extracting Foo::TransparentTuple, got {:?}", f), } let none = py.None(); - let f = Foo::extract(none).expect("Failed to extract Foo from int"); + let f = Foo::extract(none.as_ref(py)).expect("Failed to extract Foo from int"); match f { Foo::TransparentStructVar { a } => assert!(a.is_none()), _ => panic!("Expected extracting Foo::TransparentStructVar, got {:?}", f), diff --git a/tests/test_gc.rs b/tests/test_gc.rs index 54c3e1a100c..43cdb1b4811 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -89,7 +89,7 @@ impl GcIntegration { fn __clear__(&mut self) { Python::with_gil(|py| { - self.self_ref = py.None().into(); + self.self_ref = py.None(); }); } } @@ -102,7 +102,7 @@ fn gc_integration() { let inst = PyCell::new( py, GcIntegration { - self_ref: py.None().into(), + self_ref: py.None(), dropped: TestDropCall { drop_called: Arc::clone(&drop_called), }, @@ -286,9 +286,7 @@ struct PartialTraverse { impl PartialTraverse { fn new(py: Python<'_>) -> Self { - Self { - member: py.None().into(), - } + Self { member: py.None() } } } @@ -324,9 +322,7 @@ struct PanickyTraverse { impl PanickyTraverse { fn new(py: Python<'_>) -> Self { - Self { - member: py.None().into(), - } + Self { member: py.None() } } } diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index 4f04282702e..df73b9a27d8 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -5,7 +5,7 @@ #[pyo3::pyfunction] #[pyo3(name = "identity", signature = (x = None))] fn basic_function(py: pyo3::Python<'_>, x: Option) -> pyo3::PyObject { - x.unwrap_or_else(|| py.None().into()) + x.unwrap_or_else(|| py.None()) } #[pyo3::pymodule] From eca943ea35f502584d9013f144f9dadbfc222192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20=C5=A0nuderl?= Date: Sun, 4 Feb 2024 07:30:28 +0100 Subject: [PATCH 089/349] Add new get_bound and mark old get as deprecated --- src/marker.rs | 4 ++-- src/types/none.rs | 38 ++++++++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/marker.rs b/src/marker.rs index 55f5cf64694..74e3dde60cd 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -123,7 +123,7 @@ use crate::types::{ PyAny, PyDict, PyEllipsis, PyModule, PyNone, PyNotImplemented, PyString, PyType, }; use crate::version::PythonVersionInfo; -use crate::{ffi, Borrowed, FromPyPointer, IntoPy, Py, PyObject, PyTypeCheck, PyTypeInfo}; +use crate::{ffi, FromPyPointer, IntoPy, Py, PyObject, PyTypeCheck, PyTypeInfo}; use std::ffi::{CStr, CString}; use std::marker::PhantomData; use std::os::raw::c_int; @@ -699,7 +699,7 @@ impl<'py> Python<'py> { #[allow(non_snake_case)] // the Python keyword starts with uppercase #[inline] pub fn None(self) -> PyObject { - PyNone::get(self).into() + PyNone::get_bound(self).into_py(self) } /// Gets the Python builtin value `Ellipsis`, or `...`. diff --git a/src/types/none.rs b/src/types/none.rs index 6d135785c53..44c483440d2 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -10,12 +10,23 @@ pyobject_native_type_extract!(PyNone); impl PyNone { /// Returns the `None` object. + /// Deprecated form of [`PyNone::get_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyNone::get` will be replaced by `PyBool::get_bound` in a future PyO3 version" + ) + )] #[inline] - pub fn get(py: Python<'_>) -> Borrowed<'_, '_, PyNone> { - unsafe { - let bound = ffi::Py_None().assume_borrowed(py); - std::mem::transmute(bound) - } + pub fn get(py: Python<'_>) -> &PyNone { + unsafe { py.from_borrowed_ptr(ffi::Py_None()) } + } + + /// Returns the `None` object. + #[inline] + pub fn get_bound(py: Python<'_>) -> Borrowed<'_, '_, PyNone> { + unsafe { ffi::Py_None().assume_borrowed(py).downcast_unchecked() } } } @@ -36,7 +47,7 @@ unsafe impl PyTypeInfo for PyNone { #[inline] fn is_exact_type_of(object: &PyAny) -> bool { - let none = Self::get(object.py()); + let none = Self::get_bound(object.py()); object.is(none.as_ref()) } } @@ -44,14 +55,14 @@ unsafe impl PyTypeInfo for PyNone { /// `()` is converted to Python `None`. impl ToPyObject for () { fn to_object(&self, py: Python<'_>) -> PyObject { - PyNone::get(py).into_py(py) + PyNone::get_bound(py).into_py(py) } } impl IntoPy for () { #[inline] fn into_py(self, py: Python<'_>) -> PyObject { - PyNone::get(py).into_py(py) + PyNone::get_bound(py).into_py(py) } } @@ -63,22 +74,25 @@ mod tests { #[test] fn test_none_is_itself() { Python::with_gil(|py| { - assert!(PyNone::get(py).is_instance_of::()); - assert!(PyNone::get(py).is_exact_instance_of::()); + assert!(PyNone::get_bound(py).is_instance_of::()); + assert!(PyNone::get_bound(py).is_exact_instance_of::()); }) } #[test] fn test_none_type_object_consistent() { Python::with_gil(|py| { - assert!(PyNone::get(py).get_type().is(PyNone::type_object(py))); + assert!(PyNone::get_bound(py).get_type().is(PyNone::type_object(py))); }) } #[test] fn test_none_is_none() { Python::with_gil(|py| { - assert!(PyNone::get(py).downcast::().unwrap().is_none()); + assert!(PyNone::get_bound(py) + .downcast::() + .unwrap() + .is_none()); }) } From d1e967e9ea144fe2472de14b102d94aaf1d4da41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20=C5=A0nuderl?= Date: Sun, 4 Feb 2024 07:31:29 +0100 Subject: [PATCH 090/349] Uncomment a test --- src/conversion.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index 1370fb2a59e..3af038af4d9 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -629,14 +629,14 @@ mod tests { }); } - // #[test] - // fn test_try_from_unchecked() { - // Python::with_gil(|py| { - // let list = PyList::new(py, [1, 2, 3]); - // let val = unsafe { ::try_from_unchecked(list.as_ref()) }; - // assert!(list.is(val)); - // }); - // } + #[test] + fn test_try_from_unchecked() { + Python::with_gil(|py| { + let list = PyList::new(py, [1, 2, 3]); + let val = unsafe { ::try_from_unchecked(list.as_ref()) }; + assert!(list.is(val)); + }); + } } #[test] From 2a741a21e6384846654f70c3dc6296485ab7fea4 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 29 Nov 2023 11:10:36 +0000 Subject: [PATCH 091/349] migrate many `FromPyObject` implementations to `Bound` API --- pyo3-benches/benches/bench_dict.rs | 10 ++++---- pyo3-benches/benches/bench_list.rs | 9 +++---- pyo3-benches/benches/bench_tuple.rs | 2 +- src/buffer.rs | 5 ++-- src/conversions/chrono.rs | 39 ++++++++++++++--------------- src/conversions/chrono_tz.rs | 7 ++++-- src/conversions/either.rs | 5 ++-- src/conversions/hashbrown.rs | 21 +++++++++------- src/conversions/indexmap.rs | 12 +++++---- src/conversions/num_bigint.rs | 39 +++++++++++++++++++---------- src/conversions/rust_decimal.rs | 16 +++++++----- src/conversions/smallvec.rs | 14 ++++++----- src/conversions/std/array.rs | 19 ++++++++------ src/conversions/std/ipaddr.rs | 7 ++++-- src/conversions/std/map.rs | 20 ++++++++------- src/conversions/std/num.rs | 26 ++++++++++--------- src/conversions/std/osstr.rs | 12 ++++++--- src/conversions/std/path.rs | 13 +++++----- src/conversions/std/set.rs | 17 +++++++------ src/conversions/std/slice.rs | 3 +-- src/conversions/std/string.rs | 22 ++++++++-------- src/conversions/std/time.rs | 11 +++++--- src/coroutine.rs | 2 +- src/types/float.rs | 4 +-- src/types/mod.rs | 4 +-- src/types/module.rs | 4 +-- src/types/sequence.rs | 15 +++++------ src/types/tuple.rs | 4 +-- 28 files changed, 206 insertions(+), 156 deletions(-) diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs index d1c23466a39..06559519e7e 100644 --- a/pyo3-benches/benches/bench_dict.rs +++ b/pyo3-benches/benches/bench_dict.rs @@ -3,6 +3,7 @@ use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criter use pyo3::types::IntoPyDict; use pyo3::{prelude::*, types::PyMapping}; use std::collections::{BTreeMap, HashMap}; +use std::hint::black_box; fn iter_dict(b: &mut Bencher<'_>) { Python::with_gil(|py| { @@ -71,13 +72,12 @@ fn extract_hashbrown_map(b: &mut Bencher<'_>) { fn mapping_from_dict(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64) + let dict = &(0..LEN as u64) .map(|i| (i, i * 2)) .into_py_dict(py) - .to_object(py); - b.iter(|| { - let _: &PyMapping = dict.extract(py).unwrap(); - }); + .to_object(py) + .into_bound(py); + b.iter(|| black_box(dict).downcast::().unwrap()); }); } diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs index f4f746f6968..e0f238c5599 100644 --- a/pyo3-benches/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -1,3 +1,5 @@ +use std::hint::black_box; + use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; @@ -56,11 +58,8 @@ fn list_get_item_unchecked(b: &mut Bencher<'_>) { fn sequence_from_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let list = PyList::new_bound(py, 0..LEN).to_object(py); - b.iter(|| { - let seq: &PySequence = list.extract(py).unwrap(); - seq - }); + let list = &PyList::new_bound(py, 0..LEN); + b.iter(|| black_box(list).downcast::().unwrap()); }); } diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs index 9af95efcab2..24f32fac364 100644 --- a/pyo3-benches/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -93,7 +93,7 @@ fn sequence_from_tuple(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; let tuple = PyTuple::new_bound(py, 0..LEN).to_object(py); - b.iter(|| tuple.extract::<&PySequence>(py).unwrap()); + b.iter(|| tuple.downcast::(py).unwrap()); }); } diff --git a/src/buffer.rs b/src/buffer.rs index 4ff2c1d97eb..40bf1a1e99d 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -18,6 +18,7 @@ // DEALINGS IN THE SOFTWARE. //! `PyBuffer` implementation +use crate::instance::Bound; use crate::{err, exceptions::PyBufferError, ffi, FromPyObject, PyAny, PyResult, Python}; use std::marker::PhantomData; use std::os::raw; @@ -182,8 +183,8 @@ pub unsafe trait Element: Copy { } impl<'source, T: Element> FromPyObject<'source> for PyBuffer { - fn extract(obj: &PyAny) -> PyResult> { - Self::get(obj) + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult> { + Self::get(obj.as_gil_ref()) } } diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 0d9a79c7be2..4a00d8a3975 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -42,7 +42,6 @@ use crate::exceptions::{PyTypeError, PyUserWarning, PyValueError}; #[cfg(Py_LIMITED_API)] use crate::sync::GILOnceCell; -#[cfg(not(Py_LIMITED_API))] use crate::types::any::PyAnyMethods; #[cfg(not(Py_LIMITED_API))] use crate::types::datetime::timezone_from_offset; @@ -52,9 +51,9 @@ use crate::types::{ PyTzInfo, PyTzInfoAccess, }; #[cfg(Py_LIMITED_API)] -use crate::{intern, PyDowncastError}; +use crate::{intern, DowncastError}; use crate::{ - FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, ToPyObject, + Bound, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, ToPyObject, }; use chrono::offset::{FixedOffset, Utc}; use chrono::{ @@ -109,14 +108,14 @@ impl IntoPy for Duration { } impl FromPyObject<'_> for Duration { - fn extract(ob: &PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { // Python size are much lower than rust size so we do not need bound checks. // 0 <= microseconds < 1000000 // 0 <= seconds < 3600*24 // -999999999 <= days <= 999999999 #[cfg(not(Py_LIMITED_API))] let (days, seconds, microseconds) = { - let delta: &PyDelta = ob.downcast()?; + let delta = ob.downcast::()?; ( delta.get_days().into(), delta.get_seconds().into(), @@ -166,10 +165,10 @@ impl IntoPy for NaiveDate { } impl FromPyObject<'_> for NaiveDate { - fn extract(ob: &PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { #[cfg(not(Py_LIMITED_API))] { - let date: &PyDate = ob.downcast()?; + let date = ob.downcast::()?; py_date_to_naive_date(date) } #[cfg(Py_LIMITED_API)] @@ -211,10 +210,10 @@ impl IntoPy for NaiveTime { } impl FromPyObject<'_> for NaiveTime { - fn extract(ob: &PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { #[cfg(not(Py_LIMITED_API))] { - let time: &PyTime = ob.downcast()?; + let time = ob.downcast::()?; py_time_to_naive_time(time) } #[cfg(Py_LIMITED_API)] @@ -238,9 +237,9 @@ impl IntoPy for NaiveDateTime { } impl FromPyObject<'_> for NaiveDateTime { - fn extract(dt: &PyAny) -> PyResult { + fn extract_bound(dt: &Bound<'_, PyAny>) -> PyResult { #[cfg(not(Py_LIMITED_API))] - let dt: &PyDateTime = dt.downcast()?; + let dt = dt.downcast::()?; #[cfg(Py_LIMITED_API)] check_type(dt, &DatetimeTypes::get(dt.py()).datetime, "PyDateTime")?; @@ -277,9 +276,9 @@ impl IntoPy for DateTime { } impl FromPyObject<'a>> FromPyObject<'_> for DateTime { - fn extract(dt: &PyAny) -> PyResult> { + fn extract_bound(dt: &Bound<'_, PyAny>) -> PyResult> { #[cfg(not(Py_LIMITED_API))] - let dt: &PyDateTime = dt.downcast()?; + let dt = dt.downcast::()?; #[cfg(Py_LIMITED_API)] check_type(dt, &DatetimeTypes::get(dt.py()).datetime, "PyDateTime")?; @@ -339,7 +338,7 @@ impl FromPyObject<'_> for FixedOffset { /// /// Note that the conversion will result in precision lost in microseconds as chrono offset /// does not supports microseconds. - fn extract(ob: &PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { #[cfg(not(Py_LIMITED_API))] let ob: &PyTzInfo = ob.extract()?; #[cfg(Py_LIMITED_API)] @@ -378,7 +377,7 @@ impl IntoPy for Utc { } impl FromPyObject<'_> for Utc { - fn extract(ob: &PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { let py_utc = timezone_utc(ob.py()); if ob.eq(py_utc)? { Ok(Utc) @@ -480,7 +479,7 @@ fn py_date_to_naive_date(py_date: &impl PyDateAccess) -> PyResult { } #[cfg(Py_LIMITED_API)] -fn py_date_to_naive_date(py_date: &PyAny) -> PyResult { +fn py_date_to_naive_date(py_date: &Bound<'_, PyAny>) -> PyResult { NaiveDate::from_ymd_opt( py_date.getattr(intern!(py_date.py(), "year"))?.extract()?, py_date.getattr(intern!(py_date.py(), "month"))?.extract()?, @@ -501,7 +500,7 @@ fn py_time_to_naive_time(py_time: &impl PyTimeAccess) -> PyResult { } #[cfg(Py_LIMITED_API)] -fn py_time_to_naive_time(py_time: &PyAny) -> PyResult { +fn py_time_to_naive_time(py_time: &Bound<'_, PyAny>) -> PyResult { NaiveTime::from_hms_micro_opt( py_time.getattr(intern!(py_time.py(), "hour"))?.extract()?, py_time @@ -518,9 +517,9 @@ fn py_time_to_naive_time(py_time: &PyAny) -> PyResult { } #[cfg(Py_LIMITED_API)] -fn check_type(value: &PyAny, t: &PyObject, type_name: &'static str) -> PyResult<()> { - if !value.is_instance(t.as_ref(value.py()))? { - return Err(PyDowncastError::new(value, type_name).into()); +fn check_type(value: &Bound<'_, PyAny>, t: &PyObject, type_name: &'static str) -> PyResult<()> { + if !value.is_instance(t.bind(value.py()))? { + return Err(DowncastError::new(value, type_name).into()); } Ok(()) } diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index 8740d0bdd98..108dae253e3 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -35,8 +35,11 @@ //! ``` use crate::exceptions::PyValueError; use crate::sync::GILOnceCell; +use crate::types::any::PyAnyMethods; use crate::types::PyType; -use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; +use crate::{ + intern, Bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, +}; use chrono_tz::Tz; use std::str::FromStr; @@ -59,7 +62,7 @@ impl IntoPy for Tz { } impl FromPyObject<'_> for Tz { - fn extract(ob: &PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { Tz::from_str(ob.getattr(intern!(ob.py(), "key"))?.extract()?) .map_err(|e| PyValueError::new_err(e.to_string())) } diff --git a/src/conversions/either.rs b/src/conversions/either.rs index 759b282e416..897369e57ce 100644 --- a/src/conversions/either.rs +++ b/src/conversions/either.rs @@ -46,7 +46,8 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - exceptions::PyTypeError, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject, + exceptions::PyTypeError, types::any::PyAnyMethods, Bound, FromPyObject, IntoPy, PyAny, + PyObject, PyResult, Python, ToPyObject, }; use either::Either; @@ -87,7 +88,7 @@ where R: FromPyObject<'source>, { #[inline] - fn extract(obj: &'source PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'source, PyAny>) -> PyResult { if let Ok(l) = obj.extract::() { Ok(Either::Left(l)) } else if let Ok(r) = obj.extract::() { diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index a315a27490d..f8260037d3e 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -17,9 +17,12 @@ //! Note that you must use compatible versions of hashbrown and PyO3. //! The required hashbrown version may vary based on the version of PyO3. use crate::{ - types::set::new_from_iter, + types::any::PyAnyMethods, + types::dict::PyDictMethods, + types::frozenset::PyFrozenSetMethods, + types::set::{new_from_iter, PySetMethods}, types::{IntoPyDict, PyDict, PyFrozenSet, PySet}, - FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, + Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, }; use std::{cmp, hash}; @@ -54,11 +57,11 @@ where V: FromPyObject<'source>, S: hash::BuildHasher + Default, { - fn extract(ob: &'source PyAny) -> Result { - let dict: &PyDict = ob.downcast()?; + fn extract_bound(ob: &Bound<'source, PyAny>) -> Result { + let dict = ob.downcast::()?; let mut ret = hashbrown::HashMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict { - ret.insert(K::extract(k)?, V::extract(v)?); + for (k, v) in dict.iter() { + ret.insert(k.extract()?, v.extract()?); } Ok(ret) } @@ -92,12 +95,12 @@ where K: FromPyObject<'source> + cmp::Eq + hash::Hash, S: hash::BuildHasher + Default, { - fn extract(ob: &'source PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { match ob.downcast::() { - Ok(set) => set.iter().map(K::extract).collect(), + Ok(set) => set.iter().map(|any| any.extract()).collect(), Err(err) => { if let Ok(frozen_set) = ob.downcast::() { - frozen_set.iter().map(K::extract).collect() + frozen_set.iter().map(|any| any.extract()).collect() } else { Err(PyErr::from(err)) } diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index 7c7303e6d83..1bd638dad25 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -87,8 +87,10 @@ //! # if another hash table was used, the order could be random //! ``` +use crate::types::any::PyAnyMethods; +use crate::types::dict::PyDictMethods; use crate::types::*; -use crate::{FromPyObject, IntoPy, PyErr, PyObject, Python, ToPyObject}; +use crate::{Bound, FromPyObject, IntoPy, PyErr, PyObject, Python, ToPyObject}; use std::{cmp, hash}; impl ToPyObject for indexmap::IndexMap @@ -122,11 +124,11 @@ where V: FromPyObject<'source>, S: hash::BuildHasher + Default, { - fn extract(ob: &'source PyAny) -> Result { - let dict: &PyDict = ob.downcast()?; + fn extract_bound(ob: &Bound<'source, PyAny>) -> Result { + let dict = ob.downcast::()?; let mut ret = indexmap::IndexMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict { - ret.insert(K::extract(k)?, V::extract(v)?); + for (k, v) in dict.iter() { + ret.insert(k.extract()?, v.extract()?); } Ok(ret) } diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 893d3b9d226..418458143b7 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -47,8 +47,13 @@ //! assert n + 1 == value //! ``` +#[cfg(Py_LIMITED_API)] +use crate::types::bytes::PyBytesMethods; use crate::{ - ffi, types::*, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, + ffi, + instance::Bound, + types::{any::PyAnyMethods, *}, + FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, }; use num_bigint::{BigInt, BigUint}; @@ -107,7 +112,7 @@ bigint_conversion!(BigInt, 1, BigInt::to_signed_bytes_le); #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] impl<'source> FromPyObject<'source> for BigInt { - fn extract(ob: &'source PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { let py = ob.py(); // fast path - checking for subclass of `int` just checks a bit in the type object let num_owned: Py; @@ -115,7 +120,7 @@ impl<'source> FromPyObject<'source> for BigInt { long } else { num_owned = unsafe { Py::from_owned_ptr_or_err(py, ffi::PyNumber_Index(ob.as_ptr()))? }; - num_owned.as_ref(py) + num_owned.bind(py) }; let n_bits = int_n_bits(num)?; if n_bits == 0 { @@ -155,7 +160,7 @@ impl<'source> FromPyObject<'source> for BigInt { #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] impl<'source> FromPyObject<'source> for BigUint { - fn extract(ob: &'source PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { let py = ob.py(); // fast path - checking for subclass of `int` just checks a bit in the type object let num_owned: Py; @@ -163,7 +168,7 @@ impl<'source> FromPyObject<'source> for BigUint { long } else { num_owned = unsafe { Py::from_owned_ptr_or_err(py, ffi::PyNumber_Index(ob.as_ptr()))? }; - num_owned.as_ref(py) + num_owned.bind(py) }; let n_bits = int_n_bits(num)?; if n_bits == 0 { @@ -184,7 +189,11 @@ impl<'source> FromPyObject<'source> for BigUint { #[cfg(not(Py_LIMITED_API))] #[inline] -fn int_to_u32_vec(long: &PyLong, n_digits: usize, is_signed: bool) -> PyResult> { +fn int_to_u32_vec( + long: &Bound<'_, PyLong>, + n_digits: usize, + is_signed: bool, +) -> PyResult> { let mut buffer = Vec::with_capacity(n_digits); unsafe { crate::err::error_on_minusone( @@ -207,11 +216,15 @@ fn int_to_u32_vec(long: &PyLong, n_digits: usize, is_signed: bool) -> PyResult PyResult<&PyBytes> { +fn int_to_py_bytes<'py>( + long: &Bound<'py, PyLong>, + n_bytes: usize, + is_signed: bool, +) -> PyResult> { use crate::intern; let py = long.py(); let kwargs = if is_signed { - let kwargs = PyDict::new(py); + let kwargs = PyDict::new_bound(py); kwargs.set_item(intern!(py, "signed"), true)?; Some(kwargs) } else { @@ -220,13 +233,13 @@ fn int_to_py_bytes(long: &PyLong, n_bytes: usize, is_signed: bool) -> PyResult<& let bytes = long.call_method( intern!(py, "to_bytes"), (n_bytes, intern!(py, "little")), - kwargs, + kwargs.as_ref(), )?; - Ok(bytes.downcast()?) + Ok(bytes.downcast_into()?) } #[inline] -fn int_n_bits(long: &PyLong) -> PyResult { +fn int_n_bits(long: &Bound<'_, PyLong>) -> PyResult { let py = long.py(); #[cfg(not(Py_LIMITED_API))] { @@ -242,7 +255,7 @@ fn int_n_bits(long: &PyLong) -> PyResult { { // slow path long.call_method0(crate::intern!(py, "bit_length")) - .and_then(PyAny::extract) + .and_then(|any| any.extract()) } } @@ -330,7 +343,7 @@ mod tests { let locals = PyDict::new(py); locals.set_item("index", index).unwrap(); let ob = py.eval("index.C(10)", None, Some(locals)).unwrap(); - let _: BigInt = FromPyObject::extract(ob).unwrap(); + let _: BigInt = ob.extract().unwrap(); }); } diff --git a/src/conversions/rust_decimal.rs b/src/conversions/rust_decimal.rs index 173e57851c9..2e38e7808e5 100644 --- a/src/conversions/rust_decimal.rs +++ b/src/conversions/rust_decimal.rs @@ -51,18 +51,22 @@ use crate::exceptions::PyValueError; use crate::sync::GILOnceCell; +use crate::types::any::PyAnyMethods; +use crate::types::string::PyStringMethods; use crate::types::PyType; -use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; +use crate::{ + intern, Bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, +}; use rust_decimal::Decimal; use std::str::FromStr; impl FromPyObject<'_> for Decimal { - fn extract(obj: &PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { // use the string representation to not be lossy if let Ok(val) = obj.extract() { Ok(Decimal::new(val, 0)) } else { - Decimal::from_str(obj.str()?.to_str()?) + Decimal::from_str(&obj.str()?.to_cow()?) .map_err(|e| PyValueError::new_err(e.to_string())) } } @@ -131,7 +135,7 @@ mod test_rust_decimal { .unwrap(); // Checks if Python Decimal -> Rust Decimal conversion is correct let py_dec = locals.get_item("py_dec").unwrap().unwrap(); - let py_result: Decimal = FromPyObject::extract(py_dec).unwrap(); + let py_result: Decimal = py_dec.extract().unwrap(); assert_eq!(rs_orig, py_result); }) } @@ -193,7 +197,7 @@ mod test_rust_decimal { ) .unwrap(); let py_dec = locals.get_item("py_dec").unwrap().unwrap(); - let roundtripped: Result = FromPyObject::extract(py_dec); + let roundtripped: Result = py_dec.extract(); assert!(roundtripped.is_err()); }) } @@ -209,7 +213,7 @@ mod test_rust_decimal { ) .unwrap(); let py_dec = locals.get_item("py_dec").unwrap().unwrap(); - let roundtripped: Result = FromPyObject::extract(py_dec); + let roundtripped: Result = py_dec.extract(); assert!(roundtripped.is_err()); }) } diff --git a/src/conversions/smallvec.rs b/src/conversions/smallvec.rs index 37ae09225e7..d9ed84194b7 100644 --- a/src/conversions/smallvec.rs +++ b/src/conversions/smallvec.rs @@ -18,10 +18,12 @@ use crate::exceptions::PyTypeError; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; +use crate::types::any::PyAnyMethods; use crate::types::list::new_from_iter; use crate::types::{PySequence, PyString}; use crate::{ - ffi, FromPyObject, IntoPy, PyAny, PyDowncastError, PyObject, PyResult, Python, ToPyObject, + err::DowncastError, ffi, Bound, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, + ToPyObject, }; use smallvec::{Array, SmallVec}; @@ -57,7 +59,7 @@ where A: Array, A::Item: FromPyObject<'a>, { - fn extract(obj: &'a PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'a, PyAny>) -> PyResult { if obj.is_instance_of::() { return Err(PyTypeError::new_err("Can't extract `str` to `SmallVec`")); } @@ -70,18 +72,18 @@ where } } -fn extract_sequence<'s, A>(obj: &'s PyAny) -> PyResult> +fn extract_sequence<'s, A>(obj: &Bound<'s, PyAny>) -> PyResult> where A: Array, A::Item: FromPyObject<'s>, { // Types that pass `PySequence_Check` usually implement enough of the sequence protocol // to support this function and if not, we will only fail extraction safely. - let seq: &PySequence = unsafe { + let seq = unsafe { if ffi::PySequence_Check(obj.as_ptr()) != 0 { - obj.downcast_unchecked() + obj.downcast_unchecked::() } else { - return Err(PyDowncastError::new(obj, "Sequence").into()); + return Err(DowncastError::new(obj, "Sequence").into()); } }; diff --git a/src/conversions/std/array.rs b/src/conversions/std/array.rs index 167f8070632..e5e1744e74b 100644 --- a/src/conversions/std/array.rs +++ b/src/conversions/std/array.rs @@ -1,8 +1,11 @@ +use crate::instance::Bound; +use crate::types::any::PyAnyMethods; use crate::types::PySequence; -use crate::{exceptions, PyErr}; use crate::{ - ffi, FromPyObject, IntoPy, Py, PyAny, PyDowncastError, PyObject, PyResult, Python, ToPyObject, + err::DowncastError, ffi, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, + ToPyObject, }; +use crate::{exceptions, PyErr}; impl IntoPy for [T; N] where @@ -46,29 +49,29 @@ impl<'a, T, const N: usize> FromPyObject<'a> for [T; N] where T: FromPyObject<'a>, { - fn extract(obj: &'a PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'a, PyAny>) -> PyResult { create_array_from_obj(obj) } } -fn create_array_from_obj<'s, T, const N: usize>(obj: &'s PyAny) -> PyResult<[T; N]> +fn create_array_from_obj<'s, T, const N: usize>(obj: &Bound<'s, PyAny>) -> PyResult<[T; N]> where T: FromPyObject<'s>, { // Types that pass `PySequence_Check` usually implement enough of the sequence protocol // to support this function and if not, we will only fail extraction safely. - let seq: &PySequence = unsafe { + let seq = unsafe { if ffi::PySequence_Check(obj.as_ptr()) != 0 { - obj.downcast_unchecked() + obj.downcast_unchecked::() } else { - return Err(PyDowncastError::new(obj, "Sequence").into()); + return Err(DowncastError::new(obj, "Sequence").into()); } }; let seq_len = seq.len()?; if seq_len != N { return Err(invalid_sequence_length(N, seq_len)); } - array_try_from_fn(|idx| seq.get_item(idx).and_then(PyAny::extract)) + array_try_from_fn(|idx| seq.get_item(idx).and_then(|any| any.extract())) } // TODO use std::array::try_from_fn, if that stabilises: diff --git a/src/conversions/std/ipaddr.rs b/src/conversions/std/ipaddr.rs index 713de0afd1a..5e313cd7cdd 100755 --- a/src/conversions/std/ipaddr.rs +++ b/src/conversions/std/ipaddr.rs @@ -1,12 +1,15 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use crate::exceptions::PyValueError; +use crate::instance::Bound; use crate::sync::GILOnceCell; +use crate::types::any::PyAnyMethods; +use crate::types::string::PyStringMethods; use crate::types::PyType; use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; impl FromPyObject<'_> for IpAddr { - fn extract(obj: &PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { match obj.getattr(intern!(obj.py(), "packed")) { Ok(packed) => { if let Ok(packed) = packed.extract::<[u8; 4]>() { @@ -19,7 +22,7 @@ impl FromPyObject<'_> for IpAddr { } Err(_) => { // We don't have a .packed attribute, so we try to construct an IP from str(). - obj.str()?.to_str()?.parse().map_err(PyValueError::new_err) + obj.str()?.to_cow()?.parse().map_err(PyValueError::new_err) } } } diff --git a/src/conversions/std/map.rs b/src/conversions/std/map.rs index a53b0ce9222..1b47c870200 100644 --- a/src/conversions/std/map.rs +++ b/src/conversions/std/map.rs @@ -3,7 +3,9 @@ use std::{cmp, collections, hash}; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - types::{IntoPyDict, PyDict}, + instance::Bound, + types::dict::PyDictMethods, + types::{any::PyAnyMethods, IntoPyDict, PyDict}, FromPyObject, IntoPy, PyAny, PyErr, PyObject, Python, ToPyObject, }; @@ -71,11 +73,11 @@ where V: FromPyObject<'source>, S: hash::BuildHasher + Default, { - fn extract(ob: &'source PyAny) -> Result { - let dict: &PyDict = ob.downcast()?; + fn extract_bound(ob: &Bound<'source, PyAny>) -> Result { + let dict = ob.downcast::()?; let mut ret = collections::HashMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict { - ret.insert(K::extract(k)?, V::extract(v)?); + for (k, v) in dict.iter() { + ret.insert(k.extract()?, v.extract()?); } Ok(ret) } @@ -91,11 +93,11 @@ where K: FromPyObject<'source> + cmp::Ord, V: FromPyObject<'source>, { - fn extract(ob: &'source PyAny) -> Result { - let dict: &PyDict = ob.downcast()?; + fn extract_bound(ob: &Bound<'source, PyAny>) -> Result { + let dict = ob.downcast::()?; let mut ret = collections::BTreeMap::new(); - for (k, v) in dict { - ret.insert(K::extract(k)?, V::extract(v)?); + for (k, v) in dict.iter() { + ret.insert(k.extract()?, v.extract()?); } Ok(ret) } diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index e5c72b98ef0..1ced6cbb958 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -1,7 +1,9 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; +use crate::types::any::PyAnyMethods; use crate::{ - exceptions, ffi, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, + exceptions, ffi, Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, + ToPyObject, }; use std::convert::TryFrom; use std::num::{ @@ -29,8 +31,8 @@ macro_rules! int_fits_larger_int { } } - impl<'source> FromPyObject<'source> for $rust_type { - fn extract(obj: &'source PyAny) -> PyResult { + impl FromPyObject<'_> for $rust_type { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { let val: $larger_type = obj.extract()?; <$rust_type>::try_from(val) .map_err(|e| exceptions::PyOverflowError::new_err(e.to_string())) @@ -94,8 +96,8 @@ macro_rules! int_convert_u64_or_i64 { TypeInfo::builtin("int") } } - impl<'source> FromPyObject<'source> for $rust_type { - fn extract(obj: &'source PyAny) -> PyResult<$rust_type> { + impl FromPyObject<'_> for $rust_type { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult<$rust_type> { extract_int!(obj, !0, $pylong_as_ll_or_ull, $force_index_call) } @@ -126,7 +128,7 @@ macro_rules! int_fits_c_long { } impl<'source> FromPyObject<'source> for $rust_type { - fn extract(obj: &'source PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { let val: c_long = extract_int!(obj, -1, ffi::PyLong_AsLong)?; <$rust_type>::try_from(val) .map_err(|e| exceptions::PyOverflowError::new_err(e.to_string())) @@ -210,8 +212,8 @@ mod fast_128bit_int_conversion { } } - impl<'source> FromPyObject<'source> for $rust_type { - fn extract(ob: &'source PyAny) -> PyResult<$rust_type> { + impl FromPyObject<'_> for $rust_type { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult<$rust_type> { let num = unsafe { PyObject::from_owned_ptr_or_err(ob.py(), ffi::PyNumber_Index(ob.as_ptr()))? }; @@ -279,8 +281,8 @@ mod slow_128bit_int_conversion { } } - impl<'source> FromPyObject<'source> for $rust_type { - fn extract(ob: &'source PyAny) -> PyResult<$rust_type> { + impl FromPyObject<'_> for $rust_type { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult<$rust_type> { let py = ob.py(); unsafe { let lower = err_if_invalid_value( @@ -338,8 +340,8 @@ macro_rules! nonzero_int_impl { } } - impl<'source> FromPyObject<'source> for $nonzero_type { - fn extract(obj: &'source PyAny) -> PyResult { + impl FromPyObject<'_> for $nonzero_type { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { let val: $primitive_type = obj.extract()?; <$nonzero_type>::try_from(val) .map_err(|_| exceptions::PyValueError::new_err("invalid zero value")) diff --git a/src/conversions/std/osstr.rs b/src/conversions/std/osstr.rs index b2c143866ac..be337959b5f 100644 --- a/src/conversions/std/osstr.rs +++ b/src/conversions/std/osstr.rs @@ -1,3 +1,5 @@ +use crate::instance::Bound; +use crate::types::any::PyAnyMethods; use crate::types::PyString; use crate::{ffi, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject}; use std::borrow::Cow; @@ -51,8 +53,8 @@ impl ToPyObject for OsStr { // be impossible to implement on Windows. Hence it's omitted entirely impl FromPyObject<'_> for OsString { - fn extract(ob: &PyAny) -> PyResult { - let pystring: &PyString = ob.downcast()?; + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { + let pystring = ob.downcast::()?; #[cfg(not(windows))] { @@ -79,9 +81,11 @@ impl FromPyObject<'_> for OsString { #[cfg(windows)] { + use crate::types::string::PyStringMethods; + // Take the quick and easy shortcut if UTF-8 - if let Ok(utf8_string) = pystring.to_str() { - return Ok(utf8_string.to_owned().into()); + if let Ok(utf8_string) = pystring.to_cow() { + return Ok(utf8_string.into_owned().into()); } // Get an owned allocated wide char buffer from PyString, which we have to deallocate diff --git a/src/conversions/std/path.rs b/src/conversions/std/path.rs index cc579e493db..5d832e89575 100644 --- a/src/conversions/std/path.rs +++ b/src/conversions/std/path.rs @@ -1,6 +1,7 @@ -use crate::{ - ffi, FromPyObject, FromPyPointer, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject, -}; +use crate::ffi_ptr_ext::FfiPtrExt; +use crate::instance::Bound; +use crate::types::any::PyAnyMethods; +use crate::{ffi, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject}; use std::borrow::Cow; use std::ffi::OsString; use std::path::{Path, PathBuf}; @@ -14,10 +15,10 @@ impl ToPyObject for Path { // See osstr.rs for why there's no FromPyObject impl for &Path impl FromPyObject<'_> for PathBuf { - fn extract(ob: &PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { // We use os.fspath to get the underlying path as bytes or str - let path = unsafe { PyAny::from_owned_ptr_or_err(ob.py(), ffi::PyOS_FSPath(ob.as_ptr())) }?; - Ok(OsString::extract(path)?.into()) + let path = unsafe { ffi::PyOS_FSPath(ob.as_ptr()).assume_owned_or_err(ob.py())? }; + Ok(path.extract::()?.into()) } } diff --git a/src/conversions/std/set.rs b/src/conversions/std/set.rs index 020b2505b11..7d1b91c555a 100644 --- a/src/conversions/std/set.rs +++ b/src/conversions/std/set.rs @@ -3,7 +3,10 @@ use std::{cmp, collections, hash}; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - types::set::new_from_iter, + instance::Bound, + types::any::PyAnyMethods, + types::frozenset::PyFrozenSetMethods, + types::set::{new_from_iter, PySetMethods}, types::{PyFrozenSet, PySet}, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, }; @@ -53,12 +56,12 @@ where K: FromPyObject<'source> + cmp::Eq + hash::Hash, S: hash::BuildHasher + Default, { - fn extract(ob: &'source PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { match ob.downcast::() { - Ok(set) => set.iter().map(K::extract).collect(), + Ok(set) => set.iter().map(|any| any.extract()).collect(), Err(err) => { if let Ok(frozen_set) = ob.downcast::() { - frozen_set.iter().map(K::extract).collect() + frozen_set.iter().map(|any| any.extract()).collect() } else { Err(PyErr::from(err)) } @@ -92,12 +95,12 @@ impl<'source, K> FromPyObject<'source> for collections::BTreeSet where K: FromPyObject<'source> + cmp::Ord, { - fn extract(ob: &'source PyAny) -> PyResult { + fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { match ob.downcast::() { - Ok(set) => set.iter().map(K::extract).collect(), + Ok(set) => set.iter().map(|any| any.extract()).collect(), Err(err) => { if let Ok(frozen_set) = ob.downcast::() { - frozen_set.iter().map(K::extract).collect() + frozen_set.iter().map(|any| any.extract()).collect() } else { Err(PyErr::from(err)) } diff --git a/src/conversions/std/slice.rs b/src/conversions/std/slice.rs index 3d9351f8476..f77cd661834 100644 --- a/src/conversions/std/slice.rs +++ b/src/conversions/std/slice.rs @@ -26,14 +26,13 @@ impl<'a> FromPyObject<'a> for &'a [u8] { #[cfg(test)] mod tests { - use crate::FromPyObject; use crate::Python; #[test] fn test_extract_bytes() { Python::with_gil(|py| { let py_bytes = py.eval("b'Hello Python'", None, None).unwrap(); - let bytes: &[u8] = FromPyObject::extract(py_bytes).unwrap(); + let bytes: &[u8] = py_bytes.extract().unwrap(); assert_eq!(bytes, b"Hello Python"); }); } diff --git a/src/conversions/std/string.rs b/src/conversions/std/string.rs index b43e5422981..2ac08855846 100644 --- a/src/conversions/std/string.rs +++ b/src/conversions/std/string.rs @@ -3,7 +3,9 @@ use std::borrow::Cow; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - types::PyString, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, + instance::Bound, + types::{any::PyAnyMethods, string::PyStringMethods, PyString}, + FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, }; /// Converts a Rust `str` to a Python object. @@ -119,15 +121,15 @@ impl<'source> FromPyObject<'source> for &'source str { #[cfg(feature = "experimental-inspect")] fn type_input() -> TypeInfo { - ::type_input() + ::type_input() } } /// Allows extracting strings from Python objects. /// Accepts Python `str` and `unicode` objects. impl FromPyObject<'_> for String { - fn extract(obj: &PyAny) -> PyResult { - obj.downcast::()?.to_str().map(ToOwned::to_owned) + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { + obj.downcast::()?.to_cow().map(Cow::into_owned) } #[cfg(feature = "experimental-inspect")] @@ -137,8 +139,8 @@ impl FromPyObject<'_> for String { } impl FromPyObject<'_> for char { - fn extract(obj: &PyAny) -> PyResult { - let s = obj.downcast::()?.to_str()?; + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { + let s = obj.downcast::()?.to_cow()?; let mut iter = s.chars(); if let (Some(ch), None) = (iter.next(), iter.next()) { Ok(ch) @@ -158,7 +160,7 @@ impl FromPyObject<'_> for char { #[cfg(test)] mod tests { use crate::Python; - use crate::{FromPyObject, IntoPy, PyObject, ToPyObject}; + use crate::{IntoPy, PyObject, ToPyObject}; use std::borrow::Cow; #[test] @@ -198,7 +200,7 @@ mod tests { let s = "Hello Python"; let py_string = s.to_object(py); - let s2: &str = FromPyObject::extract(py_string.as_ref(py)).unwrap(); + let s2: &str = py_string.as_ref(py).extract().unwrap(); assert_eq!(s, s2); }) } @@ -208,7 +210,7 @@ mod tests { Python::with_gil(|py| { let ch = '😃'; let py_string = ch.to_object(py); - let ch2: char = FromPyObject::extract(py_string.as_ref(py)).unwrap(); + let ch2: char = py_string.as_ref(py).extract().unwrap(); assert_eq!(ch, ch2); }) } @@ -218,7 +220,7 @@ mod tests { Python::with_gil(|py| { let s = "Hello Python"; let py_string = s.to_object(py); - let err: crate::PyResult = FromPyObject::extract(py_string.as_ref(py)); + let err: crate::PyResult = py_string.as_ref(py).extract(); assert!(err .unwrap_err() .to_string() diff --git a/src/conversions/std/time.rs b/src/conversions/std/time.rs index 71cdfa23bdd..bdf938c0bd8 100755 --- a/src/conversions/std/time.rs +++ b/src/conversions/std/time.rs @@ -1,21 +1,24 @@ use crate::exceptions::{PyOverflowError, PyValueError}; use crate::sync::GILOnceCell; +use crate::types::any::PyAnyMethods; #[cfg(Py_LIMITED_API)] use crate::types::PyType; #[cfg(not(Py_LIMITED_API))] use crate::types::{timezone_utc, PyDateTime, PyDelta, PyDeltaAccess}; #[cfg(Py_LIMITED_API)] use crate::Py; -use crate::{intern, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject}; +use crate::{ + intern, Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, +}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; const SECONDS_PER_DAY: u64 = 24 * 60 * 60; impl FromPyObject<'_> for Duration { - fn extract(obj: &PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { #[cfg(not(Py_LIMITED_API))] let (days, seconds, microseconds) = { - let delta: &PyDelta = obj.downcast()?; + let delta = obj.downcast::()?; ( delta.get_days(), delta.get_seconds(), @@ -93,7 +96,7 @@ impl IntoPy for Duration { // TODO: it might be nice to investigate using timestamps anyway, at least when the datetime is a safe range. impl FromPyObject<'_> for SystemTime { - fn extract(obj: &PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { let duration_since_unix_epoch: Duration = obj .call_method1(intern!(obj.py(), "__sub__"), (unix_epoch_py(obj.py()),))? .extract()?; diff --git a/src/coroutine.rs b/src/coroutine.rs index 415bbc88db9..c4b5ddf3185 100644 --- a/src/coroutine.rs +++ b/src/coroutine.rs @@ -118,7 +118,7 @@ impl Coroutine { } // if waker has been waken during future polling, this is roughly equivalent to // `await asyncio.sleep(0)`, so just yield `None`. - Ok(py.None().into()) + Ok(py.None().into_py(py)) } } diff --git a/src/types/float.rs b/src/types/float.rs index 4b4dc93d906..fb4ca9f49e3 100644 --- a/src/types/float.rs +++ b/src/types/float.rs @@ -98,7 +98,7 @@ impl IntoPy for f64 { impl<'source> FromPyObject<'source> for f64 { // PyFloat_AsDouble returns -1.0 upon failure #![allow(clippy::float_cmp)] - fn extract(obj: &'source PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'source, PyAny>) -> PyResult { // On non-limited API, .value() uses PyFloat_AS_DOUBLE which // allows us to have an optimized fast path for the case when // we have exactly a `float` object (it's not worth going through @@ -143,7 +143,7 @@ impl IntoPy for f32 { } impl<'source> FromPyObject<'source> for f32 { - fn extract(obj: &'source PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'source, PyAny>) -> PyResult { Ok(obj.extract::()? as f32) } diff --git a/src/types/mod.rs b/src/types/mod.rs index 5f88d57ce4d..f802348f466 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -224,8 +224,8 @@ macro_rules! pyobject_native_type_extract { ($name:ty $(;$generics:ident)*) => { impl<'py, $($generics,)*> $crate::FromPyObject<'py> for &'py $name { #[inline] - fn extract(obj: &'py $crate::PyAny) -> $crate::PyResult { - obj.downcast().map_err(::std::convert::Into::into) + fn extract_bound(obj: &$crate::Bound<'py, $crate::PyAny>) -> $crate::PyResult { + ::std::clone::Clone::clone(obj).into_gil_ref().downcast().map_err(::std::convert::Into::into) } } } diff --git a/src/types/module.rs b/src/types/module.rs index 7edf164266f..8824dfcf030 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -5,7 +5,7 @@ use crate::pyclass::PyClass; use crate::types::{ any::PyAnyMethods, list::PyListMethods, PyAny, PyCFunction, PyDict, PyList, PyString, }; -use crate::{exceptions, ffi, Bound, IntoPy, Py, PyNativeType, PyObject, Python}; +use crate::{exceptions, ffi, Bound, FromPyObject, IntoPy, Py, PyNativeType, PyObject, Python}; use std::ffi::CString; use std::str; @@ -146,7 +146,7 @@ impl PyModule { return Err(PyErr::fetch(py)); } - <&PyModule as crate::FromPyObject>::extract(py.from_owned_ptr_or_err(mptr)?) + <&PyModule as FromPyObject>::extract(py.from_owned_ptr_or_err(mptr)?) } } diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 64e8f316c74..28a6fd48d63 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -1,4 +1,4 @@ -use crate::err::{self, PyDowncastError, PyErr, PyResult}; +use crate::err::{self, DowncastError, PyDowncastError, PyErr, PyResult}; use crate::exceptions::PyTypeError; use crate::ffi_ptr_ext::FfiPtrExt; #[cfg(feature = "experimental-inspect")] @@ -11,6 +11,8 @@ use crate::type_object::PyTypeInfo; use crate::types::{PyAny, PyList, PyString, PyTuple, PyType}; use crate::{ffi, FromPyObject, Py, PyNativeType, PyTypeCheck, Python, ToPyObject}; +use super::any::PyAnyMethods; + /// Represents a reference to a Python object supporting the sequence protocol. #[repr(transparent)] pub struct PySequence(PyAny); @@ -481,7 +483,7 @@ impl<'a, T> FromPyObject<'a> for Vec where T: FromPyObject<'a>, { - fn extract(obj: &'a PyAny) -> PyResult { + fn extract_bound(obj: &Bound<'a, PyAny>) -> PyResult { if obj.is_instance_of::() { return Err(PyTypeError::new_err("Can't extract `str` to `Vec`")); } @@ -494,17 +496,17 @@ where } } -fn extract_sequence<'s, T>(obj: &'s PyAny) -> PyResult> +fn extract_sequence<'s, T>(obj: &Bound<'s, PyAny>) -> PyResult> where T: FromPyObject<'s>, { // Types that pass `PySequence_Check` usually implement enough of the sequence protocol // to support this function and if not, we will only fail extraction safely. - let seq: &PySequence = unsafe { + let seq = unsafe { if ffi::PySequence_Check(obj.as_ptr()) != 0 { - obj.downcast_unchecked() + obj.downcast_unchecked::() } else { - return Err(PyDowncastError::new(obj, "Sequence").into()); + return Err(DowncastError::new(obj, "Sequence").into()); } }; @@ -602,7 +604,6 @@ mod tests { let v = "London Calling"; let ob = v.to_object(py); - assert!(ob.extract::>(py).is_err()); assert!(ob.extract::>(py).is_err()); assert!(ob.extract::>(py).is_err()); }); diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 38f5f7e94e4..26bd698cf06 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -629,7 +629,7 @@ impl IntoPy> for Bound<'_, PyTuple> { } #[cold] -fn wrong_tuple_length(t: &PyTuple, expected_length: usize) -> PyErr { +fn wrong_tuple_length(t: &Bound<'_, PyTuple>, expected_length: usize) -> PyErr { let msg = format!( "expected tuple of length {}, but got tuple of length {}", expected_length, @@ -667,7 +667,7 @@ fn type_output() -> TypeInfo { } impl<'s, $($T: FromPyObject<'s>),+> FromPyObject<'s> for ($($T,)+) { - fn extract(obj: &'s PyAny) -> PyResult + fn extract_bound(obj: &Bound<'s, PyAny>) -> PyResult { let t = obj.downcast::()?; if t.len() == $length { From 0d4df9c19dee6cb934f8b0003a0c3bd5bd970a88 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 4 Feb 2024 14:12:35 +0000 Subject: [PATCH 092/349] adjust `FromPyObject` implementations to always use `'py` lifetime --- pyo3-macros-backend/src/frompyobject.rs | 4 +-- src/buffer.rs | 2 +- src/conversion.rs | 34 ++++++++++++------------- src/conversions/chrono.rs | 2 +- src/conversions/either.rs | 8 +++--- src/conversions/hashbrown.rs | 14 +++++----- src/conversions/indexmap.rs | 8 +++--- src/conversions/num_bigint.rs | 8 +++--- src/conversions/smallvec.rs | 10 ++++---- src/conversions/std/array.rs | 10 ++++---- src/conversions/std/map.rs | 16 ++++++------ src/conversions/std/num.rs | 2 +- src/conversions/std/set.rs | 12 ++++----- src/conversions/std/slice.rs | 4 +-- src/conversions/std/string.rs | 4 +-- src/instance.rs | 4 +-- src/types/any.rs | 4 +-- src/types/bytes.rs | 4 +-- src/types/float.rs | 8 +++--- src/types/sequence.rs | 10 ++++---- src/types/tuple.rs | 4 +-- 21 files changed, 86 insertions(+), 86 deletions(-) diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 2b527dc8a29..f4774d88c0b 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -568,8 +568,8 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { let lt_param = if let Some(lt) = verify_and_get_lifetime(generics)? { lt.clone() } else { - trait_generics.params.push(parse_quote!('source)); - parse_quote!('source) + trait_generics.params.push(parse_quote!('py)); + parse_quote!('py) }; let mut where_clause: syn::WhereClause = parse_quote!(where); for param in generics.type_params() { diff --git a/src/buffer.rs b/src/buffer.rs index 40bf1a1e99d..d18f05e289d 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -182,7 +182,7 @@ pub unsafe trait Element: Copy { fn is_compatible_format(format: &CStr) -> bool; } -impl<'source, T: Element> FromPyObject<'source> for PyBuffer { +impl<'py, T: Element> FromPyObject<'py> for PyBuffer { fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult> { Self::get(obj.as_gil_ref()) } diff --git a/src/conversion.rs b/src/conversion.rs index 3af038af4d9..d9536aa9445 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -209,7 +209,7 @@ pub trait IntoPy: Sized { /// depend on the lifetime of the `obj` or the `prepared` variable. /// /// For example, when extracting `&str` from a Python byte string, the resulting string slice will -/// point to the existing string data (lifetime: `'source`). +/// point to the existing string data (lifetime: `'py`). /// On the other hand, when extracting `&str` from a Python Unicode string, the preparation step /// will convert the string to UTF-8, and the resulting string slice will have lifetime `'prepared`. /// Since which case applies depends on the runtime type of the Python object, @@ -219,12 +219,12 @@ pub trait IntoPy: Sized { /// has two methods `extract` and `extract_bound` which are defaulted to call each other. To avoid /// infinite recursion, implementors must implement at least one of these methods. The recommendation /// is to implement `extract_bound` and leave `extract` as the default implementation. -pub trait FromPyObject<'source>: Sized { +pub trait FromPyObject<'py>: Sized { /// Extracts `Self` from the source GIL Ref `obj`. /// /// Implementors are encouraged to implement `extract_bound` and leave this method as the /// default implementation, which will forward calls to `extract_bound`. - fn extract(ob: &'source PyAny) -> PyResult { + fn extract(ob: &'py PyAny) -> PyResult { Self::extract_bound(&ob.as_borrowed()) } @@ -232,7 +232,7 @@ pub trait FromPyObject<'source>: Sized { /// /// Implementors are encouraged to implement this method and leave `extract` defaulted, as /// this will be most compatible with PyO3's future API. - fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { Self::extract(ob.clone().into_gil_ref()) } @@ -308,56 +308,56 @@ impl> IntoPy for Cell { } } -impl<'a, T: FromPyObject<'a>> FromPyObject<'a> for Cell { - fn extract(ob: &'a PyAny) -> PyResult { +impl<'py, T: FromPyObject<'py>> FromPyObject<'py> for Cell { + fn extract(ob: &'py PyAny) -> PyResult { T::extract(ob).map(Cell::new) } } -impl<'a, T> FromPyObject<'a> for &'a PyCell +impl<'py, T> FromPyObject<'py> for &'py PyCell where T: PyClass, { - fn extract(obj: &'a PyAny) -> PyResult { + fn extract(obj: &'py PyAny) -> PyResult { obj.downcast().map_err(Into::into) } } -impl<'a, T> FromPyObject<'a> for T +impl FromPyObject<'_> for T where T: PyClass + Clone, { - fn extract(obj: &'a PyAny) -> PyResult { + fn extract(obj: &PyAny) -> PyResult { let cell: &PyCell = obj.downcast()?; Ok(unsafe { cell.try_borrow_unguarded()?.clone() }) } } -impl<'a, T> FromPyObject<'a> for PyRef<'a, T> +impl<'py, T> FromPyObject<'py> for PyRef<'py, T> where T: PyClass, { - fn extract(obj: &'a PyAny) -> PyResult { + fn extract(obj: &'py PyAny) -> PyResult { let cell: &PyCell = obj.downcast()?; cell.try_borrow().map_err(Into::into) } } -impl<'a, T> FromPyObject<'a> for PyRefMut<'a, T> +impl<'py, T> FromPyObject<'py> for PyRefMut<'py, T> where T: PyClass, { - fn extract(obj: &'a PyAny) -> PyResult { + fn extract(obj: &'py PyAny) -> PyResult { let cell: &PyCell = obj.downcast()?; cell.try_borrow_mut().map_err(Into::into) } } -impl<'a, T> FromPyObject<'a> for Option +impl<'py, T> FromPyObject<'py> for Option where - T: FromPyObject<'a>, + T: FromPyObject<'py>, { - fn extract(obj: &'a PyAny) -> PyResult { + fn extract(obj: &'py PyAny) -> PyResult { if obj.as_ptr() == unsafe { ffi::Py_None() } { Ok(None) } else { diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 4a00d8a3975..6e529918ab2 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -275,7 +275,7 @@ impl IntoPy for DateTime { } } -impl FromPyObject<'a>> FromPyObject<'_> for DateTime { +impl FromPyObject<'py>> FromPyObject<'_> for DateTime { fn extract_bound(dt: &Bound<'_, PyAny>) -> PyResult> { #[cfg(not(Py_LIMITED_API))] let dt = dt.downcast::()?; diff --git a/src/conversions/either.rs b/src/conversions/either.rs index 897369e57ce..c763cdf95e5 100644 --- a/src/conversions/either.rs +++ b/src/conversions/either.rs @@ -82,13 +82,13 @@ where } #[cfg_attr(docsrs, doc(cfg(feature = "either")))] -impl<'source, L, R> FromPyObject<'source> for Either +impl<'py, L, R> FromPyObject<'py> for Either where - L: FromPyObject<'source>, - R: FromPyObject<'source>, + L: FromPyObject<'py>, + R: FromPyObject<'py>, { #[inline] - fn extract_bound(obj: &Bound<'source, PyAny>) -> PyResult { + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { if let Ok(l) = obj.extract::() { Ok(Either::Left(l)) } else if let Ok(r) = obj.extract::() { diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index f8260037d3e..d2cbe4ad8c6 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -51,13 +51,13 @@ where } } -impl<'source, K, V, S> FromPyObject<'source> for hashbrown::HashMap +impl<'py, K, V, S> FromPyObject<'py> for hashbrown::HashMap where - K: FromPyObject<'source> + cmp::Eq + hash::Hash, - V: FromPyObject<'source>, + K: FromPyObject<'py> + cmp::Eq + hash::Hash, + V: FromPyObject<'py>, S: hash::BuildHasher + Default, { - fn extract_bound(ob: &Bound<'source, PyAny>) -> Result { + fn extract_bound(ob: &Bound<'py, PyAny>) -> Result { let dict = ob.downcast::()?; let mut ret = hashbrown::HashMap::with_capacity_and_hasher(dict.len(), S::default()); for (k, v) in dict.iter() { @@ -90,12 +90,12 @@ where } } -impl<'source, K, S> FromPyObject<'source> for hashbrown::HashSet +impl<'py, K, S> FromPyObject<'py> for hashbrown::HashSet where - K: FromPyObject<'source> + cmp::Eq + hash::Hash, + K: FromPyObject<'py> + cmp::Eq + hash::Hash, S: hash::BuildHasher + Default, { - fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { match ob.downcast::() { Ok(set) => set.iter().map(|any| any.extract()).collect(), Err(err) => { diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index 1bd638dad25..53f7f9364c3 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -118,13 +118,13 @@ where } } -impl<'source, K, V, S> FromPyObject<'source> for indexmap::IndexMap +impl<'py, K, V, S> FromPyObject<'py> for indexmap::IndexMap where - K: FromPyObject<'source> + cmp::Eq + hash::Hash, - V: FromPyObject<'source>, + K: FromPyObject<'py> + cmp::Eq + hash::Hash, + V: FromPyObject<'py>, S: hash::BuildHasher + Default, { - fn extract_bound(ob: &Bound<'source, PyAny>) -> Result { + fn extract_bound(ob: &Bound<'py, PyAny>) -> Result { let dict = ob.downcast::()?; let mut ret = indexmap::IndexMap::with_capacity_and_hasher(dict.len(), S::default()); for (k, v) in dict.iter() { diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 418458143b7..1d536623a85 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -111,8 +111,8 @@ bigint_conversion!(BigUint, 0, BigUint::to_bytes_le); bigint_conversion!(BigInt, 1, BigInt::to_signed_bytes_le); #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] -impl<'source> FromPyObject<'source> for BigInt { - fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { +impl<'py> FromPyObject<'py> for BigInt { + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { let py = ob.py(); // fast path - checking for subclass of `int` just checks a bit in the type object let num_owned: Py; @@ -159,8 +159,8 @@ impl<'source> FromPyObject<'source> for BigInt { } #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] -impl<'source> FromPyObject<'source> for BigUint { - fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { +impl<'py> FromPyObject<'py> for BigUint { + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { let py = ob.py(); // fast path - checking for subclass of `int` just checks a bit in the type object let num_owned: Py; diff --git a/src/conversions/smallvec.rs b/src/conversions/smallvec.rs index d9ed84194b7..ade64a5106c 100644 --- a/src/conversions/smallvec.rs +++ b/src/conversions/smallvec.rs @@ -54,12 +54,12 @@ where } } -impl<'a, A> FromPyObject<'a> for SmallVec +impl<'py, A> FromPyObject<'py> for SmallVec where A: Array, - A::Item: FromPyObject<'a>, + A::Item: FromPyObject<'py>, { - fn extract_bound(obj: &Bound<'a, PyAny>) -> PyResult { + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { if obj.is_instance_of::() { return Err(PyTypeError::new_err("Can't extract `str` to `SmallVec`")); } @@ -72,10 +72,10 @@ where } } -fn extract_sequence<'s, A>(obj: &Bound<'s, PyAny>) -> PyResult> +fn extract_sequence<'py, A>(obj: &Bound<'py, PyAny>) -> PyResult> where A: Array, - A::Item: FromPyObject<'s>, + A::Item: FromPyObject<'py>, { // Types that pass `PySequence_Check` usually implement enough of the sequence protocol // to support this function and if not, we will only fail extraction safely. diff --git a/src/conversions/std/array.rs b/src/conversions/std/array.rs index e5e1744e74b..0ea167135b6 100644 --- a/src/conversions/std/array.rs +++ b/src/conversions/std/array.rs @@ -45,18 +45,18 @@ where } } -impl<'a, T, const N: usize> FromPyObject<'a> for [T; N] +impl<'py, T, const N: usize> FromPyObject<'py> for [T; N] where - T: FromPyObject<'a>, + T: FromPyObject<'py>, { - fn extract_bound(obj: &Bound<'a, PyAny>) -> PyResult { + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { create_array_from_obj(obj) } } -fn create_array_from_obj<'s, T, const N: usize>(obj: &Bound<'s, PyAny>) -> PyResult<[T; N]> +fn create_array_from_obj<'py, T, const N: usize>(obj: &Bound<'py, PyAny>) -> PyResult<[T; N]> where - T: FromPyObject<'s>, + T: FromPyObject<'py>, { // Types that pass `PySequence_Check` usually implement enough of the sequence protocol // to support this function and if not, we will only fail extraction safely. diff --git a/src/conversions/std/map.rs b/src/conversions/std/map.rs index 1b47c870200..1c3f669eaee 100644 --- a/src/conversions/std/map.rs +++ b/src/conversions/std/map.rs @@ -67,13 +67,13 @@ where } } -impl<'source, K, V, S> FromPyObject<'source> for collections::HashMap +impl<'py, K, V, S> FromPyObject<'py> for collections::HashMap where - K: FromPyObject<'source> + cmp::Eq + hash::Hash, - V: FromPyObject<'source>, + K: FromPyObject<'py> + cmp::Eq + hash::Hash, + V: FromPyObject<'py>, S: hash::BuildHasher + Default, { - fn extract_bound(ob: &Bound<'source, PyAny>) -> Result { + fn extract_bound(ob: &Bound<'py, PyAny>) -> Result { let dict = ob.downcast::()?; let mut ret = collections::HashMap::with_capacity_and_hasher(dict.len(), S::default()); for (k, v) in dict.iter() { @@ -88,12 +88,12 @@ where } } -impl<'source, K, V> FromPyObject<'source> for collections::BTreeMap +impl<'py, K, V> FromPyObject<'py> for collections::BTreeMap where - K: FromPyObject<'source> + cmp::Ord, - V: FromPyObject<'source>, + K: FromPyObject<'py> + cmp::Ord, + V: FromPyObject<'py>, { - fn extract_bound(ob: &Bound<'source, PyAny>) -> Result { + fn extract_bound(ob: &Bound<'py, PyAny>) -> Result { let dict = ob.downcast::()?; let mut ret = collections::BTreeMap::new(); for (k, v) in dict.iter() { diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 1ced6cbb958..82f63016a7d 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -127,7 +127,7 @@ macro_rules! int_fits_c_long { } } - impl<'source> FromPyObject<'source> for $rust_type { + impl<'py> FromPyObject<'py> for $rust_type { fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { let val: c_long = extract_int!(obj, -1, ffi::PyLong_AsLong)?; <$rust_type>::try_from(val) diff --git a/src/conversions/std/set.rs b/src/conversions/std/set.rs index 7d1b91c555a..c955801a916 100644 --- a/src/conversions/std/set.rs +++ b/src/conversions/std/set.rs @@ -51,12 +51,12 @@ where } } -impl<'source, K, S> FromPyObject<'source> for collections::HashSet +impl<'py, K, S> FromPyObject<'py> for collections::HashSet where - K: FromPyObject<'source> + cmp::Eq + hash::Hash, + K: FromPyObject<'py> + cmp::Eq + hash::Hash, S: hash::BuildHasher + Default, { - fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { match ob.downcast::() { Ok(set) => set.iter().map(|any| any.extract()).collect(), Err(err) => { @@ -91,11 +91,11 @@ where } } -impl<'source, K> FromPyObject<'source> for collections::BTreeSet +impl<'py, K> FromPyObject<'py> for collections::BTreeSet where - K: FromPyObject<'source> + cmp::Ord, + K: FromPyObject<'py> + cmp::Ord, { - fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult { + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { match ob.downcast::() { Ok(set) => set.iter().map(|any| any.extract()).collect(), Err(err) => { diff --git a/src/conversions/std/slice.rs b/src/conversions/std/slice.rs index f77cd661834..62809327f57 100644 --- a/src/conversions/std/slice.rs +++ b/src/conversions/std/slice.rs @@ -13,8 +13,8 @@ impl<'a> IntoPy for &'a [u8] { } } -impl<'a> FromPyObject<'a> for &'a [u8] { - fn extract(obj: &'a PyAny) -> PyResult { +impl<'py> FromPyObject<'py> for &'py [u8] { + fn extract(obj: &'py PyAny) -> PyResult { Ok(obj.downcast::()?.as_bytes()) } diff --git a/src/conversions/std/string.rs b/src/conversions/std/string.rs index 2ac08855846..ac29d80491c 100644 --- a/src/conversions/std/string.rs +++ b/src/conversions/std/string.rs @@ -114,8 +114,8 @@ impl<'a> IntoPy for &'a String { /// Allows extracting strings from Python objects. /// Accepts Python `str` and `unicode` objects. -impl<'source> FromPyObject<'source> for &'source str { - fn extract(ob: &'source PyAny) -> PyResult { +impl<'py> FromPyObject<'py> for &'py str { + fn extract(ob: &'py PyAny) -> PyResult { ob.downcast::()?.to_str() } diff --git a/src/instance.rs b/src/instance.rs index 793d0e323fa..92fc01853eb 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -996,9 +996,9 @@ impl Py { /// Extracts some type from the Python object. /// /// This is a wrapper function around `FromPyObject::extract()`. - pub fn extract<'p, D>(&'p self, py: Python<'p>) -> PyResult + pub fn extract<'py, D>(&'py self, py: Python<'py>) -> PyResult where - D: FromPyObject<'p>, + D: FromPyObject<'py>, { FromPyObject::extract(unsafe { py.from_borrowed_ptr(self.as_ptr()) }) } diff --git a/src/types/any.rs b/src/types/any.rs index 61c3c67217a..d79a9227cd7 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -801,9 +801,9 @@ impl PyAny { /// /// This is a wrapper function around [`FromPyObject::extract()`]. #[inline] - pub fn extract<'a, D>(&'a self) -> PyResult + pub fn extract<'py, D>(&'py self) -> PyResult where - D: FromPyObject<'a>, + D: FromPyObject<'py>, { FromPyObject::extract(self) } diff --git a/src/types/bytes.rs b/src/types/bytes.rs index 296925dceff..e0631602542 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -203,8 +203,8 @@ impl> Index for Bound<'_, PyBytes> { /// If the source object is a `bytes` object, the `Cow` will be borrowed and /// pointing into the source object, and no copying or heap allocations will happen. /// If it is a `bytearray`, its contents will be copied to an owned `Cow`. -impl<'source> FromPyObject<'source> for Cow<'source, [u8]> { - fn extract(ob: &'source PyAny) -> PyResult { +impl<'py> FromPyObject<'py> for Cow<'py, [u8]> { + fn extract(ob: &'py PyAny) -> PyResult { if let Ok(bytes) = ob.downcast::() { return Ok(Cow::Borrowed(bytes.as_bytes())); } diff --git a/src/types/float.rs b/src/types/float.rs index fb4ca9f49e3..766c597b2b4 100644 --- a/src/types/float.rs +++ b/src/types/float.rs @@ -95,10 +95,10 @@ impl IntoPy for f64 { } } -impl<'source> FromPyObject<'source> for f64 { +impl<'py> FromPyObject<'py> for f64 { // PyFloat_AsDouble returns -1.0 upon failure #![allow(clippy::float_cmp)] - fn extract_bound(obj: &Bound<'source, PyAny>) -> PyResult { + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { // On non-limited API, .value() uses PyFloat_AS_DOUBLE which // allows us to have an optimized fast path for the case when // we have exactly a `float` object (it's not worth going through @@ -142,8 +142,8 @@ impl IntoPy for f32 { } } -impl<'source> FromPyObject<'source> for f32 { - fn extract_bound(obj: &Bound<'source, PyAny>) -> PyResult { +impl<'py> FromPyObject<'py> for f32 { + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { Ok(obj.extract::()? as f32) } diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 28a6fd48d63..19f6846b4d1 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -479,11 +479,11 @@ fn sequence_slice(seq: &PySequence, start: usize, end: usize) -> &PySequence { index_impls!(PySequence, "sequence", sequence_len, sequence_slice); -impl<'a, T> FromPyObject<'a> for Vec +impl<'py, T> FromPyObject<'py> for Vec where - T: FromPyObject<'a>, + T: FromPyObject<'py>, { - fn extract_bound(obj: &Bound<'a, PyAny>) -> PyResult { + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { if obj.is_instance_of::() { return Err(PyTypeError::new_err("Can't extract `str` to `Vec`")); } @@ -496,9 +496,9 @@ where } } -fn extract_sequence<'s, T>(obj: &Bound<'s, PyAny>) -> PyResult> +fn extract_sequence<'py, T>(obj: &Bound<'py, PyAny>) -> PyResult> where - T: FromPyObject<'s>, + T: FromPyObject<'py>, { // Types that pass `PySequence_Check` usually implement enough of the sequence protocol // to support this function and if not, we will only fail extraction safely. diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 26bd698cf06..a8737c256c4 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -666,8 +666,8 @@ fn type_output() -> TypeInfo { } } - impl<'s, $($T: FromPyObject<'s>),+> FromPyObject<'s> for ($($T,)+) { - fn extract_bound(obj: &Bound<'s, PyAny>) -> PyResult + impl<'py, $($T: FromPyObject<'py>),+> FromPyObject<'py> for ($($T,)+) { + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { let t = obj.downcast::()?; if t.len() == $length { From 304c8e655ad3d56088d490861e393160dc14cb0e Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 31 Jan 2024 21:08:52 +0100 Subject: [PATCH 093/349] convert `marshal` to `Bound` API --- src/marshal.rs | 69 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/src/marshal.rs b/src/marshal.rs index 343b861deff..5e62e6b6289 100644 --- a/src/marshal.rs +++ b/src/marshal.rs @@ -2,14 +2,32 @@ //! Support for the Python `marshal` format. -use crate::ffi; +use crate::ffi_ptr_ext::FfiPtrExt; +use crate::py_result_ext::PyResultExt; use crate::types::{PyAny, PyBytes}; -use crate::{AsPyPointer, FromPyPointer, PyResult, Python}; -use std::os::raw::{c_char, c_int}; +use crate::{ffi, Bound}; +use crate::{AsPyPointer, PyResult, Python}; +use std::os::raw::c_int; /// The current version of the marshal binary format. pub const VERSION: i32 = 4; +/// Deprecated form of [`dumps_bound`] +#[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`dumps` will be replaced by `dumps_bound` in a future PyO3 version" + ) +)] +pub fn dumps<'py>( + py: Python<'py>, + object: &impl AsPyPointer, + version: i32, +) -> PyResult<&'py PyBytes> { + dumps_bound(py, object, version).map(Bound::into_gil_ref) +} + /// Serialize an object to bytes using the Python built-in marshal module. /// /// The built-in marshalling only supports a limited range of objects. @@ -27,33 +45,52 @@ pub const VERSION: i32 = 4; /// dict.set_item("mies", "wim").unwrap(); /// dict.set_item("zus", "jet").unwrap(); /// -/// let bytes = marshal::dumps(py, dict, marshal::VERSION); +/// let bytes = marshal::dumps_bound(py, dict, marshal::VERSION); /// # }); /// ``` -pub fn dumps<'a>(py: Python<'a>, object: &impl AsPyPointer, version: i32) -> PyResult<&'a PyBytes> { +pub fn dumps_bound<'py>( + py: Python<'py>, + object: &impl AsPyPointer, + version: i32, +) -> PyResult> { unsafe { - let bytes = ffi::PyMarshal_WriteObjectToString(object.as_ptr(), version as c_int); - FromPyPointer::from_owned_ptr_or_err(py, bytes) + ffi::PyMarshal_WriteObjectToString(object.as_ptr(), version as c_int) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } +/// Deprecated form of [`loads_bound`] +#[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`loads` will be replaced by `loads_bound` in a future PyO3 version" + ) +)] +pub fn loads<'py, B>(py: Python<'py>, data: &B) -> PyResult<&'py PyAny> +where + B: AsRef<[u8]> + ?Sized, +{ + loads_bound(py, data).map(Bound::into_gil_ref) +} + /// Deserialize an object from bytes using the Python built-in marshal module. -pub fn loads<'a, B>(py: Python<'a>, data: &B) -> PyResult<&'a PyAny> +pub fn loads_bound<'py, B>(py: Python<'py>, data: &B) -> PyResult> where B: AsRef<[u8]> + ?Sized, { let data = data.as_ref(); unsafe { - let c_str = data.as_ptr() as *const c_char; - let object = ffi::PyMarshal_ReadObjectFromString(c_str, data.len() as isize); - FromPyPointer::from_owned_ptr_or_err(py, object) + ffi::PyMarshal_ReadObjectFromString(data.as_ptr().cast(), data.len() as isize) + .assume_owned_or_err(py) } } #[cfg(test)] mod tests { use super::*; - use crate::types::PyDict; + use crate::types::{bytes::PyBytesMethods, PyDict}; #[test] fn marshal_roundtrip() { @@ -63,12 +100,10 @@ mod tests { dict.set_item("mies", "wim").unwrap(); dict.set_item("zus", "jet").unwrap(); - let bytes = dumps(py, dict, VERSION) - .expect("marshalling failed") - .as_bytes(); - let deserialized = loads(py, bytes).expect("unmarshalling failed"); + let pybytes = dumps_bound(py, dict, VERSION).expect("marshalling failed"); + let deserialized = loads_bound(py, pybytes.as_bytes()).expect("unmarshalling failed"); - assert!(equal(py, dict, deserialized)); + assert!(equal(py, dict, &deserialized)); }); } From 8354590ae6cfa254a972e39c4ad270f4bfeac1ef Mon Sep 17 00:00:00 2001 From: Blaz Snuderl Date: Sun, 4 Feb 2024 20:01:15 +0100 Subject: [PATCH 094/349] PyEllipsis get_bound method --- src/marker.rs | 2 +- src/types/ellipsis.rs | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/marker.rs b/src/marker.rs index 74e3dde60cd..f31d109241e 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -706,7 +706,7 @@ impl<'py> Python<'py> { #[allow(non_snake_case)] // the Python keyword starts with uppercase #[inline] pub fn Ellipsis(self) -> PyObject { - PyEllipsis::get(self).into() + PyEllipsis::get_bound(self).into_py(self) } /// Gets the Python builtin value `NotImplemented`. diff --git a/src/types/ellipsis.rs b/src/types/ellipsis.rs index e31f9d4183b..17d51095ce7 100644 --- a/src/types/ellipsis.rs +++ b/src/types/ellipsis.rs @@ -1,4 +1,4 @@ -use crate::{ffi, PyAny, PyTypeInfo, Python}; +use crate::{ffi, ffi_ptr_ext::FfiPtrExt, Borrowed, PyAny, PyTypeInfo, Python}; /// Represents the Python `Ellipsis` object. #[repr(transparent)] @@ -9,10 +9,23 @@ pyobject_native_type_extract!(PyEllipsis); impl PyEllipsis { /// Returns the `Ellipsis` object. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyEllipsis::get` will be replaced by `PyEllipsis::get_bound` in a future PyO3 version" + ) + )] #[inline] pub fn get(py: Python<'_>) -> &PyEllipsis { unsafe { py.from_borrowed_ptr(ffi::Py_Ellipsis()) } } + + /// Returns the `Ellipsis` object. + #[inline] + pub fn get_bound(py: Python<'_>) -> Borrowed<'_, '_, PyEllipsis> { + unsafe { ffi::Py_Ellipsis().assume_borrowed(py).downcast_unchecked() } + } } unsafe impl PyTypeInfo for PyEllipsis { @@ -32,27 +45,28 @@ unsafe impl PyTypeInfo for PyEllipsis { #[inline] fn is_exact_type_of(object: &PyAny) -> bool { - object.is(Self::get(object.py())) + object.is(Self::get_bound(object.py()).as_ref()) } } #[cfg(test)] mod tests { + use crate::types::any::PyAnyMethods; use crate::types::{PyDict, PyEllipsis}; use crate::{PyTypeInfo, Python}; #[test] fn test_ellipsis_is_itself() { Python::with_gil(|py| { - assert!(PyEllipsis::get(py).is_instance_of::()); - assert!(PyEllipsis::get(py).is_exact_instance_of::()); + assert!(PyEllipsis::get_bound(py).is_instance_of::()); + assert!(PyEllipsis::get_bound(py).is_exact_instance_of::()); }) } #[test] fn test_ellipsis_type_object_consistent() { Python::with_gil(|py| { - assert!(PyEllipsis::get(py) + assert!(PyEllipsis::get_bound(py) .get_type() .is(PyEllipsis::type_object(py))); }) From 8388b14369ab8ce12588a3f23511b446cb944b64 Mon Sep 17 00:00:00 2001 From: Blaz Snuderl Date: Sun, 4 Feb 2024 20:04:22 +0100 Subject: [PATCH 095/349] PyNotImplemented get_bound --- src/marker.rs | 2 +- src/types/notimplemented.rs | 28 +++++++++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/marker.rs b/src/marker.rs index f31d109241e..7e5b0ff1044 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -713,7 +713,7 @@ impl<'py> Python<'py> { #[allow(non_snake_case)] // the Python keyword starts with uppercase #[inline] pub fn NotImplemented(self) -> PyObject { - PyNotImplemented::get(self).into() + PyNotImplemented::get_bound(self).into_py(self) } /// Gets the running Python interpreter version as a string. diff --git a/src/types/notimplemented.rs b/src/types/notimplemented.rs index 0c61fd79bee..083eec7f93e 100644 --- a/src/types/notimplemented.rs +++ b/src/types/notimplemented.rs @@ -1,4 +1,4 @@ -use crate::{ffi, PyAny, PyTypeInfo, Python}; +use crate::{ffi, ffi_ptr_ext::FfiPtrExt, Borrowed, PyAny, PyTypeInfo, Python}; /// Represents the Python `NotImplemented` object. #[repr(transparent)] @@ -9,10 +9,27 @@ pyobject_native_type_extract!(PyNotImplemented); impl PyNotImplemented { /// Returns the `NotImplemented` object. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyNotImplemented::get` will be replaced by `PyNotImplemented::get_bound` in a future PyO3 version" + ) + )] #[inline] pub fn get(py: Python<'_>) -> &PyNotImplemented { unsafe { py.from_borrowed_ptr(ffi::Py_NotImplemented()) } } + + /// Returns the `NotImplemented` object. + #[inline] + pub fn get_bound(py: Python<'_>) -> Borrowed<'_, '_, PyNotImplemented> { + unsafe { + ffi::Py_NotImplemented() + .assume_borrowed(py) + .downcast_unchecked() + } + } } unsafe impl PyTypeInfo for PyNotImplemented { @@ -31,27 +48,28 @@ unsafe impl PyTypeInfo for PyNotImplemented { #[inline] fn is_exact_type_of(object: &PyAny) -> bool { - object.is(Self::get(object.py())) + object.is(Self::get_bound(object.py()).as_ref()) } } #[cfg(test)] mod tests { + use crate::types::any::PyAnyMethods; use crate::types::{PyDict, PyNotImplemented}; use crate::{PyTypeInfo, Python}; #[test] fn test_notimplemented_is_itself() { Python::with_gil(|py| { - assert!(PyNotImplemented::get(py).is_instance_of::()); - assert!(PyNotImplemented::get(py).is_exact_instance_of::()); + assert!(PyNotImplemented::get_bound(py).is_instance_of::()); + assert!(PyNotImplemented::get_bound(py).is_exact_instance_of::()); }) } #[test] fn test_notimplemented_type_object_consistent() { Python::with_gil(|py| { - assert!(PyNotImplemented::get(py) + assert!(PyNotImplemented::get_bound(py) .get_type() .is(PyNotImplemented::type_object(py))); }) From 1fd0aa2b19f98c3791c03fe68998d784b5752965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20=C5=A0nuderl?= Date: Mon, 5 Feb 2024 08:03:03 +0100 Subject: [PATCH 096/349] Use new method to implement old --- src/types/ellipsis.rs | 2 +- src/types/notimplemented.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types/ellipsis.rs b/src/types/ellipsis.rs index 17d51095ce7..9151e89df97 100644 --- a/src/types/ellipsis.rs +++ b/src/types/ellipsis.rs @@ -18,7 +18,7 @@ impl PyEllipsis { )] #[inline] pub fn get(py: Python<'_>) -> &PyEllipsis { - unsafe { py.from_borrowed_ptr(ffi::Py_Ellipsis()) } + Self::get_bound(py).into_gil_ref() } /// Returns the `Ellipsis` object. diff --git a/src/types/notimplemented.rs b/src/types/notimplemented.rs index 083eec7f93e..7560dff37d1 100644 --- a/src/types/notimplemented.rs +++ b/src/types/notimplemented.rs @@ -18,7 +18,7 @@ impl PyNotImplemented { )] #[inline] pub fn get(py: Python<'_>) -> &PyNotImplemented { - unsafe { py.from_borrowed_ptr(ffi::Py_NotImplemented()) } + Self::get_bound(py).into_gil_ref() } /// Returns the `NotImplemented` object. From f1384f35829b343c6ee9cc0693bd167eab6e9aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20=C5=A0nuderl?= Date: Mon, 5 Feb 2024 08:06:30 +0100 Subject: [PATCH 097/349] Implement PyNone.get() using PyNone.get_bound() --- src/types/none.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types/none.rs b/src/types/none.rs index 44c483440d2..3c3b171668d 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -15,12 +15,12 @@ impl PyNone { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "`PyNone::get` will be replaced by `PyBool::get_bound` in a future PyO3 version" + note = "`PyNone::get` will be replaced by `PyNone::get_bound` in a future PyO3 version" ) )] #[inline] pub fn get(py: Python<'_>) -> &PyNone { - unsafe { py.from_borrowed_ptr(ffi::Py_None()) } + Self::get_bound(py).into_gil_ref() } /// Returns the `None` object. From de93d15eebfd2045f8cf04dec6ef6ba16f8e3eba Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 5 Feb 2024 07:57:27 +0000 Subject: [PATCH 098/349] ci: fix beta clippy `map_clone` warning --- src/instance.rs | 3 +++ tests/test_proto_methods.rs | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index 92fc01853eb..d21cc307af9 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1443,6 +1443,9 @@ where { /// Extracts `Self` from the source `PyObject`. fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { + // TODO update MSRV past 1.59 and use .cloned() to make + // clippy happy + #[allow(clippy::map_clone)] ob.downcast().map(Clone::clone).map_err(Into::into) } } diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index a328ab9e24d..50dd99ce50d 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -264,13 +264,13 @@ impl Sequence { self.values.len() } - fn __getitem__(&self, index: SequenceIndex<'_>) -> PyResult { + fn __getitem__(&self, index: SequenceIndex<'_>, py: Python<'_>) -> PyResult { match index { SequenceIndex::Integer(index) => { let uindex = self.usize_index(index)?; self.values .get(uindex) - .map(Clone::clone) + .map(|o| o.clone_ref(py)) .ok_or_else(|| PyIndexError::new_err(index)) } // Just to prove that slicing can be implemented From 7281268840931f958ad8662d45506a978b5b9635 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 5 Feb 2024 08:52:07 +0000 Subject: [PATCH 099/349] move `Cow[u8]` conversions into `conversions::std::slice` module --- src/conversions/std/slice.rs | 63 ++++++++++++++++++++++++++++++++++-- src/types/bytes.rs | 61 ++-------------------------------- 2 files changed, 64 insertions(+), 60 deletions(-) diff --git a/src/conversions/std/slice.rs b/src/conversions/std/slice.rs index 62809327f57..378c02c4600 100644 --- a/src/conversions/std/slice.rs +++ b/src/conversions/std/slice.rs @@ -1,6 +1,11 @@ +use std::borrow::Cow; + #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; -use crate::{types::PyBytes, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python}; +use crate::{ + types::{PyByteArray, PyBytes}, + FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, +}; impl<'a> IntoPy for &'a [u8] { fn into_py(self, py: Python<'_>) -> PyObject { @@ -24,9 +29,39 @@ impl<'py> FromPyObject<'py> for &'py [u8] { } } +/// Special-purpose trait impl to efficiently handle both `bytes` and `bytearray` +/// +/// If the source object is a `bytes` object, the `Cow` will be borrowed and +/// pointing into the source object, and no copying or heap allocations will happen. +/// If it is a `bytearray`, its contents will be copied to an owned `Cow`. +impl<'py> FromPyObject<'py> for Cow<'py, [u8]> { + fn extract(ob: &'py PyAny) -> PyResult { + if let Ok(bytes) = ob.downcast::() { + return Ok(Cow::Borrowed(bytes.as_bytes())); + } + + let byte_array = ob.downcast::()?; + Ok(Cow::Owned(byte_array.to_vec())) + } +} + +impl ToPyObject for Cow<'_, [u8]> { + fn to_object(&self, py: Python<'_>) -> Py { + PyBytes::new_bound(py, self.as_ref()).into() + } +} + +impl IntoPy> for Cow<'_, [u8]> { + fn into_py(self, py: Python<'_>) -> Py { + self.to_object(py) + } +} + #[cfg(test)] mod tests { - use crate::Python; + use std::borrow::Cow; + + use crate::{types::PyBytes, Python, ToPyObject}; #[test] fn test_extract_bytes() { @@ -36,4 +71,28 @@ mod tests { assert_eq!(bytes, b"Hello Python"); }); } + + #[test] + fn test_cow_impl() { + Python::with_gil(|py| { + let bytes = py.eval(r#"b"foobar""#, None, None).unwrap(); + let cow = bytes.extract::>().unwrap(); + assert_eq!(cow, Cow::<[u8]>::Borrowed(b"foobar")); + + let byte_array = py.eval(r#"bytearray(b"foobar")"#, None, None).unwrap(); + let cow = byte_array.extract::>().unwrap(); + assert_eq!(cow, Cow::<[u8]>::Owned(b"foobar".to_vec())); + + let something_else_entirely = py.eval("42", None, None).unwrap(); + something_else_entirely + .extract::>() + .unwrap_err(); + + let cow = Cow::<[u8]>::Borrowed(b"foobar").to_object(py); + assert!(cow.as_ref(py).is_instance_of::()); + + let cow = Cow::<[u8]>::Owned(b"foobar".to_vec()).to_object(py); + assert!(cow.as_ref(py).is_instance_of::()); + }); + } } diff --git a/src/types/bytes.rs b/src/types/bytes.rs index e0631602542..0861af630a5 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -1,15 +1,12 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::{Borrowed, Bound}; use crate::types::any::PyAnyMethods; -use crate::{ffi, FromPyObject, IntoPy, Py, PyAny, PyNativeType, PyResult, Python, ToPyObject}; -use std::borrow::Cow; +use crate::{ffi, Py, PyAny, PyNativeType, PyResult, Python}; use std::ops::Index; use std::os::raw::c_char; use std::slice::SliceIndex; use std::str; -use super::bytearray::PyByteArray; - /// Represents a Python `bytes` object. /// /// This type is immutable. @@ -198,34 +195,6 @@ impl> Index for Bound<'_, PyBytes> { } } -/// Special-purpose trait impl to efficiently handle both `bytes` and `bytearray` -/// -/// If the source object is a `bytes` object, the `Cow` will be borrowed and -/// pointing into the source object, and no copying or heap allocations will happen. -/// If it is a `bytearray`, its contents will be copied to an owned `Cow`. -impl<'py> FromPyObject<'py> for Cow<'py, [u8]> { - fn extract(ob: &'py PyAny) -> PyResult { - if let Ok(bytes) = ob.downcast::() { - return Ok(Cow::Borrowed(bytes.as_bytes())); - } - - let byte_array = ob.downcast::()?; - Ok(Cow::Owned(byte_array.to_vec())) - } -} - -impl ToPyObject for Cow<'_, [u8]> { - fn to_object(&self, py: Python<'_>) -> Py { - PyBytes::new_bound(py, self.as_ref()).into() - } -} - -impl IntoPy> for Cow<'_, [u8]> { - fn into_py(self, py: Python<'_>) -> Py { - self.to_object(py) - } -} - #[cfg(test)] #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { @@ -257,7 +226,7 @@ mod tests { b.copy_from_slice(b"Hello Rust"); Ok(()) })?; - let bytes: &[u8] = FromPyObject::extract(py_bytes)?; + let bytes: &[u8] = py_bytes.extract()?; assert_eq!(bytes, b"Hello Rust"); Ok(()) }) @@ -267,7 +236,7 @@ mod tests { fn test_bytes_new_with_zero_initialised() -> super::PyResult<()> { Python::with_gil(|py| -> super::PyResult<()> { let py_bytes = PyBytes::new_with(py, 10, |_b: &mut [u8]| Ok(()))?; - let bytes: &[u8] = FromPyObject::extract(py_bytes)?; + let bytes: &[u8] = py_bytes.extract()?; assert_eq!(bytes, &[0; 10]); Ok(()) }) @@ -287,28 +256,4 @@ mod tests { .is_instance_of::(py)); }); } - - #[test] - fn test_cow_impl() { - Python::with_gil(|py| { - let bytes = py.eval(r#"b"foobar""#, None, None).unwrap(); - let cow = bytes.extract::>().unwrap(); - assert_eq!(cow, Cow::<[u8]>::Borrowed(b"foobar")); - - let byte_array = py.eval(r#"bytearray(b"foobar")"#, None, None).unwrap(); - let cow = byte_array.extract::>().unwrap(); - assert_eq!(cow, Cow::<[u8]>::Owned(b"foobar".to_vec())); - - let something_else_entirely = py.eval("42", None, None).unwrap(); - something_else_entirely - .extract::>() - .unwrap_err(); - - let cow = Cow::<[u8]>::Borrowed(b"foobar").to_object(py); - assert!(cow.as_ref(py).is_instance_of::()); - - let cow = Cow::<[u8]>::Owned(b"foobar".to_vec()).to_object(py); - assert!(cow.as_ref(py).is_instance_of::()); - }); - } } From 42843de47ba918423a2bc0acc9e7302573c60649 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 28 Nov 2023 14:45:45 +0000 Subject: [PATCH 100/349] pyclass methods for `Bound` --- src/instance.rs | 220 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 217 insertions(+), 3 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index 492bef56b32..a42c510af99 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,4 +1,5 @@ use crate::err::{self, PyDowncastError, PyErr, PyResult}; +use crate::ffi_ptr_ext::FfiPtrExt; use crate::pycell::{PyBorrowError, PyBorrowMutError, PyCell}; use crate::pyclass::boolean_struct::{False, True}; use crate::type_object::HasPyGilRef; @@ -66,6 +67,43 @@ pub unsafe trait PyNativeType: Sized { #[repr(transparent)] pub struct Bound<'py, T>(Python<'py>, ManuallyDrop>); +impl<'py, T> Bound<'py, T> +where + T: PyClass, +{ + /// Creates a new instance `Bound` of a `#[pyclass]` on the Python heap. + /// + /// # Examples + /// + /// ```rust + /// use pyo3::prelude::*; + /// + /// #[pyclass] + /// struct Foo {/* fields omitted */} + /// + /// # fn main() -> PyResult<()> { + /// Python::with_gil(|py| -> PyResult> { + /// let foo: Bound<'_, Foo> = Bound::new(py, Foo {})?; + /// Ok(foo.into()) + /// })?; + /// # Ok(()) + /// # } + /// ``` + pub fn new( + py: Python<'py>, + value: impl Into>, + ) -> PyResult> { + let initializer = value.into(); + let obj = initializer.create_cell(py)?; + let ob = unsafe { + obj.cast::() + .assume_owned(py) + .downcast_into_unchecked() + }; + Ok(ob) + } +} + impl<'py> Bound<'py, PyAny> { /// Constructs a new Bound from a pointer. Panics if ptr is null. /// @@ -101,6 +139,149 @@ impl<'py> Bound<'py, PyAny> { } } +impl<'py, T> Bound<'py, T> +where + T: PyClass, +{ + /// Immutably borrows the value `T`. + /// + /// This borrow lasts while the returned [`PyRef`] exists. + /// Multiple immutable borrows can be taken out at the same time. + /// + /// For frozen classes, the simpler [`get`][Self::get] is available. + /// + /// # Examples + /// + /// ```rust + /// # use pyo3::prelude::*; + /// # + /// #[pyclass] + /// struct Foo { + /// inner: u8, + /// } + /// + /// # fn main() -> PyResult<()> { + /// Python::with_gil(|py| -> PyResult<()> { + /// let foo: Bound<'_, Foo> = Bound::new(py, Foo { inner: 73 })?; + /// let inner: &u8 = &foo.borrow().inner; + /// + /// assert_eq!(*inner, 73); + /// Ok(()) + /// })?; + /// # Ok(()) + /// # } + /// ``` + /// + /// # Panics + /// + /// Panics if the value is currently mutably borrowed. For a non-panicking variant, use + /// [`try_borrow`](#method.try_borrow). + pub fn borrow(&'py self) -> PyRef<'py, T> { + self.get_cell().borrow() + } + + /// Mutably borrows the value `T`. + /// + /// This borrow lasts while the returned [`PyRefMut`] exists. + /// + /// # Examples + /// + /// ``` + /// # use pyo3::prelude::*; + /// # + /// #[pyclass] + /// struct Foo { + /// inner: u8, + /// } + /// + /// # fn main() -> PyResult<()> { + /// Python::with_gil(|py| -> PyResult<()> { + /// let foo: Bound<'_, Foo> = Bound::new(py, Foo { inner: 73 })?; + /// foo.borrow_mut().inner = 35; + /// + /// assert_eq!(foo.borrow().inner, 35); + /// Ok(()) + /// })?; + /// # Ok(()) + /// # } + /// ``` + /// + /// # Panics + /// Panics if the value is currently borrowed. For a non-panicking variant, use + /// [`try_borrow_mut`](#method.try_borrow_mut). + pub fn borrow_mut(&'py self) -> PyRefMut<'py, T> + where + T: PyClass, + { + self.get_cell().borrow_mut() + } + + /// Attempts to immutably borrow the value `T`, returning an error if the value is currently mutably borrowed. + /// + /// The borrow lasts while the returned [`PyRef`] exists. + /// + /// This is the non-panicking variant of [`borrow`](#method.borrow). + /// + /// For frozen classes, the simpler [`get`][Self::get] is available. + pub fn try_borrow(&'py self) -> Result, PyBorrowError> { + self.get_cell().try_borrow() + } + + /// Attempts to mutably borrow the value `T`, returning an error if the value is currently borrowed. + /// + /// The borrow lasts while the returned [`PyRefMut`] exists. + /// + /// This is the non-panicking variant of [`borrow_mut`](#method.borrow_mut). + pub fn try_borrow_mut(&'py self) -> Result, PyBorrowMutError> + where + T: PyClass, + { + self.get_cell().try_borrow_mut() + } + + /// Provide an immutable borrow of the value `T` without acquiring the GIL. + /// + /// This is available if the class is [`frozen`][macro@crate::pyclass] and [`Sync`]. + /// + /// # Examples + /// + /// ``` + /// use std::sync::atomic::{AtomicUsize, Ordering}; + /// # use pyo3::prelude::*; + /// + /// #[pyclass(frozen)] + /// struct FrozenCounter { + /// value: AtomicUsize, + /// } + /// + /// Python::with_gil(|py| { + /// let counter = FrozenCounter { value: AtomicUsize::new(0) }; + /// + /// let py_counter = Bound::new(py, counter).unwrap(); + /// + /// py_counter.get().value.fetch_add(1, Ordering::Relaxed); + /// }); + /// ``` + pub fn get(&self) -> &T + where + T: PyClass + Sync, + { + let cell = self.get_cell(); + // SAFETY: The class itself is frozen and `Sync` and we do not access anything but `cell.contents.value`. + unsafe { &*cell.get_ptr() } + } + + fn get_cell(&'py self) -> &'py PyCell { + let cell = self.as_ptr().cast::>(); + // SAFETY: Bound is known to contain an object which is laid out in memory as a + // PyCell. + // + // Strictly speaking for now `&'py PyCell` is part of the "GIL Ref" API, so this + // could use some further refactoring later to avoid going through this reference. + unsafe { &*cell } + } +} + impl<'py, T> std::fmt::Debug for Bound<'py, T> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { let any = self.as_any(); @@ -1757,12 +1938,11 @@ a = A() use super::*; - #[crate::pyclass] - #[pyo3(crate = "crate")] + #[crate::pyclass(crate = "crate")] struct SomeClass(i32); #[test] - fn instance_borrow_methods() { + fn py_borrow_methods() { // More detailed tests of the underlying semantics in pycell.rs Python::with_gil(|py| { let instance = Py::new(py, SomeClass(0)).unwrap(); @@ -1780,6 +1960,40 @@ a = A() }) } + #[test] + fn bound_borrow_methods() { + // More detailed tests of the underlying semantics in pycell.rs + Python::with_gil(|py| { + let instance = Bound::new(py, SomeClass(0)).unwrap(); + assert_eq!(instance.borrow().0, 0); + assert_eq!(instance.try_borrow().unwrap().0, 0); + assert_eq!(instance.borrow_mut().0, 0); + assert_eq!(instance.try_borrow_mut().unwrap().0, 0); + + instance.borrow_mut().0 = 123; + + assert_eq!(instance.borrow().0, 123); + assert_eq!(instance.try_borrow().unwrap().0, 123); + assert_eq!(instance.borrow_mut().0, 123); + assert_eq!(instance.try_borrow_mut().unwrap().0, 123); + }) + } + + #[crate::pyclass(frozen, crate = "crate")] + struct FrozenClass(i32); + + #[test] + fn test_frozen_get() { + Python::with_gil(|py| { + for i in 0..10 { + let instance = Py::new(py, FrozenClass(i)).unwrap(); + assert_eq!(instance.get().0, i); + + assert_eq!(instance.bind(py).get().0, i); + } + }) + } + #[test] #[allow(deprecated)] fn cell_tryfrom() { From 64a6a02bf0b0498ec6d23d3fc13cc260bec5de03 Mon Sep 17 00:00:00 2001 From: Bartosz Telenczuk Date: Fri, 2 Feb 2024 17:16:23 +0100 Subject: [PATCH 101/349] add example for wrapping generic classes --- guide/src/class.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/guide/src/class.md b/guide/src/class.md index 37265aaa2bc..dfe5c8ae4ae 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -82,6 +82,36 @@ When you need to share ownership of data between Python and Rust, instead of usi A Rust `struct Foo` with a generic parameter `T` generates new compiled implementations each time it is used with a different concrete type for `T`. These new implementations are generated by the compiler at each usage site. This is incompatible with wrapping `Foo` in Python, where there needs to be a single compiled implementation of `Foo` which is integrated with the Python interpreter. +Currently, the best alternative is to write a macro which expands to a new #[pyclass] for each instantiation you want: + +```rust +# #![allow(dead_code)] +use pyo3::prelude::*; + +struct GenericClass { + data: T +} + +macro_rules! create_interface { + ($name: ident, $type: ident) => { + #[pyclass] + pub struct $name { + inner: GenericClass<$type>, + } + #[pymethods] + impl $name { + #[new] + pub fn new(data: $type) -> Self { + Self { inner: GenericClass { data: data } } + } + } + }; +} + +create_interface!(IntClass, i64); +create_interface!(FloatClass, String); +``` + #### Must be Send Because Python objects are freely shared between threads by the Python interpreter, there is no guarantee which thread will eventually drop the object. Therefore all types annotated with `#[pyclass]` must implement `Send` (unless annotated with [`#[pyclass(unsendable)]`](#customizing-the-class)). From 662eecfb447830f169eeb2b8b23793f105e194c4 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 5 Feb 2024 12:22:10 +0000 Subject: [PATCH 102/349] add `PyStringMethods::encode_utf8` --- newsfragments/3801.added.md | 1 + src/types/string.rs | 39 ++++++++++++++++++++++++++++++++----- 2 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 newsfragments/3801.added.md diff --git a/newsfragments/3801.added.md b/newsfragments/3801.added.md new file mode 100644 index 00000000000..78f45032ba2 --- /dev/null +++ b/newsfragments/3801.added.md @@ -0,0 +1 @@ +Add `PyStringMethods::encode_utf8`. diff --git a/src/types/string.rs b/src/types/string.rs index b1457950bb8..2b2635715d3 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -305,6 +305,9 @@ pub trait PyStringMethods<'py> { /// replaced with `U+FFFD REPLACEMENT CHARACTER`. fn to_string_lossy(&self) -> Cow<'_, str>; + /// Encodes this string as a Python `bytes` object, using UTF-8 encoding. + fn encode_utf8(&self) -> PyResult>; + /// Obtains the raw data backing the Python string. /// /// If the Python string object was created through legacy APIs, its internal storage format @@ -337,6 +340,14 @@ impl<'py> PyStringMethods<'py> for Bound<'py, PyString> { self.as_borrowed().to_string_lossy() } + fn encode_utf8(&self) -> PyResult> { + unsafe { + ffi::PyUnicode_AsUTF8String(self.as_ptr()) + .assume_owned_or_err(self.py()) + .downcast_into_unchecked::() + } + } + #[cfg(not(Py_LIMITED_API))] unsafe fn data(&self) -> PyResult> { self.as_borrowed().data() @@ -371,11 +382,7 @@ impl<'a> Borrowed<'a, '_, PyString> { #[cfg(not(any(Py_3_10, not(Py_LIMITED_API))))] { - let bytes = unsafe { - ffi::PyUnicode_AsUTF8String(self.as_ptr()) - .assume_owned_or_err(self.py())? - .downcast_into_unchecked::() - }; + let bytes = self.encode_utf8()?; Ok(Cow::Owned( unsafe { str::from_utf8_unchecked(bytes.as_bytes()) }.to_owned(), )) @@ -535,6 +542,28 @@ mod tests { }) } + #[test] + fn test_encode_utf8_unicode() { + Python::with_gil(|py| { + let s = "哈哈🐈"; + let obj = PyString::new_bound(py, s); + assert_eq!(s.as_bytes(), obj.encode_utf8().unwrap().as_bytes()); + }) + } + + #[test] + fn test_encode_utf8_surrogate() { + Python::with_gil(|py| { + let obj: PyObject = py.eval(r"'\ud800'", None, None).unwrap().into(); + assert!(obj + .bind(py) + .downcast::() + .unwrap() + .encode_utf8() + .is_err()); + }) + } + #[test] fn test_to_string_lossy() { Python::with_gil(|py| { From 86f294f6e66ad45789a0814b0f6a1567a6caa8b5 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 16 Jan 2024 21:34:13 +0000 Subject: [PATCH 103/349] expose `Bound::from_owned_ptr` etc --- src/instance.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index 8ae6d80f7e9..e54312a9098 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -105,33 +105,34 @@ where } impl<'py> Bound<'py, PyAny> { - /// Constructs a new Bound from a pointer. Panics if ptr is null. + /// Constructs a new `Bound<'py, PyAny>` from a pointer. Panics if `ptr` is null. /// /// # Safety /// - /// `ptr` must be a valid pointer to a Python object. - pub(crate) unsafe fn from_owned_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { + /// - `ptr` must be a valid pointer to a Python object + /// - `ptr` must be an owned Python reference, as the `Bound<'py, PyAny>` will assume ownership + pub unsafe fn from_owned_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self(py, ManuallyDrop::new(Py::from_owned_ptr(py, ptr))) } - /// Constructs a new Bound from a pointer. Returns None if ptr is null. + /// Constructs a new `Bound<'py, PyAny>` from a pointer. Returns `None`` if `ptr` is null. /// /// # Safety /// - /// `ptr` must be a valid pointer to a Python object, or NULL. - pub(crate) unsafe fn from_owned_ptr_or_opt( - py: Python<'py>, - ptr: *mut ffi::PyObject, - ) -> Option { + /// - `ptr` must be a valid pointer to a Python object, or null + /// - `ptr` must be an owned Python reference, as the `Bound<'py, PyAny>` will assume ownership + pub unsafe fn from_owned_ptr_or_opt(py: Python<'py>, ptr: *mut ffi::PyObject) -> Option { Py::from_owned_ptr_or_opt(py, ptr).map(|obj| Self(py, ManuallyDrop::new(obj))) } - /// Constructs a new Bound from a pointer. Returns error if ptr is null. + /// Constructs a new `Bound<'py, PyAny>` from a pointer. Returns an `Err` by calling `PyErr::fetch` + /// if `ptr` is null. /// /// # Safety /// - /// `ptr` must be a valid pointer to a Python object, or NULL. - pub(crate) unsafe fn from_owned_ptr_or_err( + /// - `ptr` must be a valid pointer to a Python object, or null + /// - `ptr` must be an owned Python reference, as the `Bound<'py, PyAny>` will assume ownership + pub unsafe fn from_owned_ptr_or_err( py: Python<'py>, ptr: *mut ffi::PyObject, ) -> PyResult { From ec0be57c689f1ab3201967940bcae3a8b61be5bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:43:19 +0000 Subject: [PATCH 104/349] Bump codecov/codecov-action from 3 to 4 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79e6b9ee78c..5fefd2b19ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -381,7 +381,7 @@ jobs: if: steps.should-skip.outputs.skip != 'true' - run: nox -s coverage if: steps.should-skip.outputs.skip != 'true' - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 if: steps.should-skip.outputs.skip != 'true' with: file: coverage.json From c85d72bb0e719d50073de22f834c04477f2925ca Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 5 Feb 2024 18:50:18 +0000 Subject: [PATCH 105/349] connect CODECOV_TOKEN to codecov action --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fefd2b19ec..b98495b3f5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -386,6 +386,7 @@ jobs: with: file: coverage.json name: ${{ matrix.os }} + token: ${{ secrets.CODECOV_TOKEN }} emscripten: name: emscripten From dd4df29bada3547e43459fe8fa63d2b1909f320b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 5 Feb 2024 21:50:25 +0000 Subject: [PATCH 106/349] docs: add `chrono` conversions to types table --- guide/src/conversions/tables.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index 6b7c3bfbb80..b0582431156 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -29,12 +29,12 @@ The table below contains the Python type and the corresponding function argument | `type` | - | `&PyType` | | `module` | - | `&PyModule` | | `collections.abc.Buffer` | - | `PyBuffer` | -| `datetime.datetime` | `SystemTime` | `&PyDateTime` | -| `datetime.date` | - | `&PyDate` | -| `datetime.time` | - | `&PyTime` | -| `datetime.tzinfo` | - | `&PyTzInfo` | -| `datetime.timedelta` | `Duration` | `&PyDelta` | -| `decimal.Decimal` | `rust_decimal::Decimal`[^5] | - | +| `datetime.datetime` | `SystemTime`, `chrono::DateTime`[^5], `chrono::NaiveDateTime`[^5] | `&PyDateTime` | +| `datetime.date` | `chrono::NaiveDate`[^5] | `&PyDate` | +| `datetime.time` | `chrono::NaiveTime`[^5] | `&PyTime` | +| `datetime.tzinfo` | `chrono::FixedOffset`[^5], `chrono::Utc`[^5], `chrono_tz::TimeZone`[^6] | `&PyTzInfo` | +| `datetime.timedelta` | `Duration`, `chrono::Duration`[^5] | `&PyDelta` | +| `decimal.Decimal` | `rust_decimal::Decimal`[^7] | - | | `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::IpV4Addr` | - | | `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::IpV6Addr` | - | | `os.PathLike ` | `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | @@ -107,4 +107,8 @@ Finally, the following Rust types are also able to convert to Python as return v [^4]: Requires the `indexmap` optional feature. -[^5]: Requires the `rust_decimal` optional feature. +[^5]: Requires the `chrono` optional feature. + +[^6]: Requires the `chrono-tz` optional feature. + +[^7]: Requires the `rust_decimal` optional feature. From b74d733244eaa6db8fe2c92a16eee22d99914572 Mon Sep 17 00:00:00 2001 From: "Jonatan G. Frausing" Date: Thu, 2 Nov 2023 14:13:06 +0100 Subject: [PATCH 107/349] docs: include section that disables signal in python --- guide/src/python_from_rust.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 99da2e15434..7b364ec5c01 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -487,5 +487,26 @@ class House(object): } ``` +## Handling system signals/interrupts (Ctrl-C) + +The best way to handle system signals when running Rust code is to periodically call `Python::check_signals` to handle any signals captured by Python's signal handler. See also [the FAQ entry](./faq.md#ctrl-c-doesnt-do-anything-while-my-rust-code-is-executing). + +Alternatively, set Python's `signal` module to take the default action for a signal: + +```rust +use pyo3::prelude::*; + +# fn main() -> PyResult<()> { +Python::with_gil(|py| -> PyResult<()> { + let signal = py.import("signal")?; + // Set SIGINT to have the default action + signal + .getattr("signal")? + .call1((signal.getattr("SIGINT")?, signal.getattr("SIG_DFL")?))?; + Ok(()) +}) +# } +``` + [`PyModule::new`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.new From aa3c938b5e4a1267b72c97f7f6d6e76b92455dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20=C5=A0nuderl?= Date: Mon, 5 Feb 2024 08:51:18 +0100 Subject: [PATCH 108/349] PyCFunction bound api --- src/impl_/pyfunction.rs | 2 +- src/types/function.rs | 85 ++++++++++++++++++++++++++------- tests/test_pyfunction.rs | 8 ++-- tests/ui/invalid_closure.rs | 4 +- tests/ui/invalid_closure.stderr | 7 +-- 5 files changed, 81 insertions(+), 25 deletions(-) diff --git a/src/impl_/pyfunction.rs b/src/impl_/pyfunction.rs index 14cdbd48f85..464beeb8ce5 100644 --- a/src/impl_/pyfunction.rs +++ b/src/impl_/pyfunction.rs @@ -6,5 +6,5 @@ pub fn _wrap_pyfunction<'a>( method_def: &PyMethodDef, py_or_module: impl Into>, ) -> PyResult<&'a PyCFunction> { - PyCFunction::internal_new(method_def, py_or_module.into()) + PyCFunction::internal_new(method_def, py_or_module.into()).map(|x| x.into_gil_ref()) } diff --git a/src/types/function.rs b/src/types/function.rs index c6f0f6b0b02..79f91cfe081 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -1,6 +1,8 @@ use crate::derive_utils::PyFunctionArguments; +use crate::ffi_ptr_ext::FfiPtrExt; use crate::methods::PyMethodDefDestructor; use crate::prelude::*; +use crate::py_result_ext::PyResultExt; use crate::types::capsule::PyCapsuleMethods; use crate::{ ffi, @@ -17,13 +19,30 @@ pub struct PyCFunction(PyAny); pyobject_native_type_core!(PyCFunction, pyobject_native_static_type_object!(ffi::PyCFunction_Type), #checkfunction=ffi::PyCFunction_Check); impl PyCFunction { - /// Create a new built-in function with keywords (*args and/or **kwargs). + /// Deprecated form of [`PyCFunction::new_with_keywords_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyCFunction::new_with_keywords` will be replaced by `PyCFunction::new_with_keywords_bound` in a future PyO3 version" + ) + )] pub fn new_with_keywords<'a>( fun: ffi::PyCFunctionWithKeywords, name: &'static str, doc: &'static str, py_or_module: PyFunctionArguments<'a>, ) -> PyResult<&'a Self> { + Self::new_with_keywords_bound(fun, name, doc, py_or_module).map(Bound::into_gil_ref) + } + + /// Create a new built-in function with keywords (*args and/or **kwargs). + pub fn new_with_keywords_bound<'a>( + fun: ffi::PyCFunctionWithKeywords, + name: &'static str, + doc: &'static str, + py_or_module: PyFunctionArguments<'a>, + ) -> PyResult> { Self::internal_new( &PyMethodDef::cfunction_with_keywords( name, @@ -34,19 +53,57 @@ impl PyCFunction { ) } - /// Create a new built-in function which takes no arguments. + /// Deprecated form of [`PyCFunction::new`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyCFunction::new` will be replaced by `PyCFunction::new_bound` in a future PyO3 version" + ) + )] pub fn new<'a>( fun: ffi::PyCFunction, name: &'static str, doc: &'static str, py_or_module: PyFunctionArguments<'a>, ) -> PyResult<&'a Self> { + Self::new_bound(fun, name, doc, py_or_module).map(Bound::into_gil_ref) + } + + /// Create a new built-in function which takes no arguments. + pub fn new_bound<'a>( + fun: ffi::PyCFunction, + name: &'static str, + doc: &'static str, + py_or_module: PyFunctionArguments<'a>, + ) -> PyResult> { Self::internal_new( &PyMethodDef::noargs(name, pymethods::PyCFunction(fun), doc), py_or_module, ) } + /// Deprecated form of [`PyCFunction::new_closure`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyCFunction::new_closure` will be replaced by `PyCFunction::new_closure_bound` in a future PyO3 version" + ) + )] + pub fn new_closure<'a, F, R>( + py: Python<'a>, + name: Option<&'static str>, + doc: Option<&'static str>, + closure: F, + ) -> PyResult<&'a PyCFunction> + where + F: Fn(&PyTuple, Option<&PyDict>) -> R + Send + 'static, + R: crate::callback::IntoPyCallbackOutput<*mut ffi::PyObject>, + { + Self::new_closure_bound(py, name, doc, closure).map(Bound::into_gil_ref) + } + /// Create a new function from a closure. /// /// # Examples @@ -60,16 +117,16 @@ impl PyCFunction { /// let i = args.extract::<(i64,)>()?.0; /// Ok(i+1) /// }; - /// let add_one = PyCFunction::new_closure(py, None, None, add_one).unwrap(); + /// let add_one = PyCFunction::new_closure_bound(py, None, None, add_one).unwrap(); /// py_run!(py, add_one, "assert add_one(42) == 43"); /// }); /// ``` - pub fn new_closure<'a, F, R>( + pub fn new_closure_bound<'a, F, R>( py: Python<'a>, name: Option<&'static str>, doc: Option<&'static str>, closure: F, - ) -> PyResult<&'a PyCFunction> + ) -> PyResult> where F: Fn(&PyTuple, Option<&PyDict>) -> R + Send + 'static, R: crate::callback::IntoPyCallbackOutput<*mut ffi::PyObject>, @@ -95,11 +152,9 @@ impl PyCFunction { let data = unsafe { capsule.reference::>() }; unsafe { - py.from_owned_ptr_or_err::(ffi::PyCFunction_NewEx( - data.def.get(), - capsule.as_ptr(), - std::ptr::null_mut(), - )) + ffi::PyCFunction_NewEx(data.def.get(), capsule.as_ptr(), std::ptr::null_mut()) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } @@ -107,7 +162,7 @@ impl PyCFunction { pub fn internal_new<'py>( method_def: &PyMethodDef, py_or_module: PyFunctionArguments<'py>, - ) -> PyResult<&'py Self> { + ) -> PyResult> { let (py, module) = py_or_module.into_py_and_maybe_module(); let (mod_ptr, module_name): (_, Option>) = if let Some(m) = module { let mod_ptr = m.as_ptr(); @@ -126,11 +181,9 @@ impl PyCFunction { .map_or(std::ptr::null_mut(), Py::as_ptr); unsafe { - py.from_owned_ptr_or_err::(ffi::PyCFunction_NewEx( - def, - mod_ptr, - module_name_ptr, - )) + ffi::PyCFunction_NewEx(def, mod_ptr, module_name_ptr) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } } diff --git a/tests/test_pyfunction.rs b/tests/test_pyfunction.rs index f1116363432..e06f7c2fb9c 100644 --- a/tests/test_pyfunction.rs +++ b/tests/test_pyfunction.rs @@ -323,7 +323,7 @@ fn test_pycfunction_new() { ffi::PyLong_FromLong(4200) } - let py_fn = PyCFunction::new( + let py_fn = PyCFunction::new_bound( c_fn, "py_fn", "py_fn for test (this is the docstring)", @@ -380,7 +380,7 @@ fn test_pycfunction_new_with_keywords() { ffi::PyLong_FromLong(foo * bar) } - let py_fn = PyCFunction::new_with_keywords( + let py_fn = PyCFunction::new_with_keywords_bound( c_fn, "py_fn", "py_fn for test (this is the docstring)", @@ -422,7 +422,7 @@ fn test_closure() { }) }; let closure_py = - PyCFunction::new_closure(py, Some("test_fn"), Some("test_fn doc"), f).unwrap(); + PyCFunction::new_closure_bound(py, Some("test_fn"), Some("test_fn doc"), f).unwrap(); py_assert!(py, closure_py, "closure_py(42) == [43]"); py_assert!(py, closure_py, "closure_py.__name__ == 'test_fn'"); @@ -445,7 +445,7 @@ fn test_closure_counter() { *counter += 1; Ok(*counter) }; - let counter_py = PyCFunction::new_closure(py, None, None, counter_fn).unwrap(); + let counter_py = PyCFunction::new_closure_bound(py, None, None, counter_fn).unwrap(); py_assert!(py, counter_py, "counter_py() == 1"); py_assert!(py, counter_py, "counter_py() == 2"); diff --git a/tests/ui/invalid_closure.rs b/tests/ui/invalid_closure.rs index b724f31b97e..0613b027665 100644 --- a/tests/ui/invalid_closure.rs +++ b/tests/ui/invalid_closure.rs @@ -10,7 +10,9 @@ fn main() { println!("This is five: {:?}", ref_.len()); Ok(()) }; - PyCFunction::new_closure(py, None, None, closure_fn).unwrap().into() + PyCFunction::new_closure_bound(py, None, None, closure_fn) + .unwrap() + .into() }); Python::with_gil(|py| { diff --git a/tests/ui/invalid_closure.stderr b/tests/ui/invalid_closure.stderr index 90240e5db26..7fed8908583 100644 --- a/tests/ui/invalid_closure.stderr +++ b/tests/ui/invalid_closure.stderr @@ -6,7 +6,8 @@ error[E0597]: `local_data` does not live long enough 7 | let ref_: &[u8] = &local_data; | ^^^^^^^^^^^ borrowed value does not live long enough ... -13 | PyCFunction::new_closure(py, None, None, closure_fn).unwrap().into() - | ---------------------------------------------------- argument requires that `local_data` is borrowed for `'static` -14 | }); +13 | PyCFunction::new_closure_bound(py, None, None, closure_fn) + | ---------------------------------------------------------- argument requires that `local_data` is borrowed for `'static` +... +16 | }); | - `local_data` dropped here while still borrowed From 3541506a16cf4ff2cd80de22d9814978dfb320be Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 8 Feb 2024 20:52:03 +0000 Subject: [PATCH 109/349] ci: allow some dead code warnings on nightly --- guide/src/trait_bounds.md | 5 +++++ src/marker.rs | 1 + 2 files changed, 6 insertions(+) diff --git a/guide/src/trait_bounds.md b/guide/src/trait_bounds.md index 7d6e6ea44a4..e0dd988412f 100644 --- a/guide/src/trait_bounds.md +++ b/guide/src/trait_bounds.md @@ -64,6 +64,7 @@ class Model: The following wrapper will call the Python model from Rust, using a struct to hold the model as a `PyAny` object: ```rust +# #![allow(dead_code)] use pyo3::prelude::*; # pub trait Model { @@ -162,6 +163,7 @@ However, we can write a second wrapper around these functions to call them direc This wrapper will also perform the type conversions between Python and Rust. ```rust +# #![allow(dead_code)] # use pyo3::prelude::*; # # pub trait Model { @@ -330,6 +332,7 @@ Let's modify the code performing the type conversion to give a helpful error mes We used in our `get_results` method the following call that performs the type conversion: ```rust +# #![allow(dead_code)] # use pyo3::prelude::*; # # pub trait Model { @@ -382,6 +385,7 @@ impl Model for UserModel { Let's break it down in order to perform better error handling: ```rust +# #![allow(dead_code)] # use pyo3::prelude::*; # # pub trait Model { @@ -460,6 +464,7 @@ Because of this, we can write a function wrapper that takes the `UserModel`--whi It is also required to make the struct public. ```rust +# #![allow(dead_code)] use pyo3::prelude::*; pub trait Model { diff --git a/src/marker.rs b/src/marker.rs index 7e5b0ff1044..1987ffaed13 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -19,6 +19,7 @@ //! defined as the following: //! //! ```rust +//! # #![allow(dead_code)] //! pub unsafe trait Ungil {} //! //! unsafe impl Ungil for T {} From bcb7b88c2327a9d71931be9e876bae1afa9bcaba Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 8 Feb 2024 21:13:58 +0000 Subject: [PATCH 110/349] ci: updates for rust 1.76 --- tests/ui/not_send.stderr | 10 +++++----- tests/ui/not_send2.stderr | 12 ++++++------ tests/ui/traverse.stderr | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/ui/not_send.stderr b/tests/ui/not_send.stderr index 9ac51f36ec0..5d05b3de059 100644 --- a/tests/ui/not_send.stderr +++ b/tests/ui/not_send.stderr @@ -7,23 +7,23 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe | required by a bound introduced by this call | = help: within `pyo3::Python<'_>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` -note: required because it appears within the type `PhantomData<*mut Python<'static>>` +note: required because it appears within the type `PhantomData<*mut pyo3::Python<'static>>` --> $RUST/core/src/marker.rs | | pub struct PhantomData; | ^^^^^^^^^^^ -note: required because it appears within the type `NotSend` +note: required because it appears within the type `impl_::not_send::NotSend` --> src/impl_/not_send.rs | | pub(crate) struct NotSend(PhantomData<*mut Python<'static>>); | ^^^^^^^ - = note: required because it appears within the type `(&GILGuard, NotSend)` -note: required because it appears within the type `PhantomData<(&GILGuard, NotSend)>` + = note: required because it appears within the type `(&pyo3::gil::GILGuard, impl_::not_send::NotSend)` +note: required because it appears within the type `PhantomData<(&pyo3::gil::GILGuard, impl_::not_send::NotSend)>` --> $RUST/core/src/marker.rs | | pub struct PhantomData; | ^^^^^^^^^^^ -note: required because it appears within the type `Python<'_>` +note: required because it appears within the type `pyo3::Python<'_>` --> src/marker.rs | | pub struct Python<'py>(PhantomData<(&'py GILGuard, NotSend)>); diff --git a/tests/ui/not_send2.stderr b/tests/ui/not_send2.stderr index 30189e09b9f..eec2b644e9a 100644 --- a/tests/ui/not_send2.stderr +++ b/tests/ui/not_send2.stderr @@ -10,28 +10,28 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe | |_________^ `*mut pyo3::Python<'static>` cannot be shared between threads safely | = help: within `pyo3::Bound<'_, PyString>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` -note: required because it appears within the type `PhantomData<*mut Python<'static>>` +note: required because it appears within the type `PhantomData<*mut pyo3::Python<'static>>` --> $RUST/core/src/marker.rs | | pub struct PhantomData; | ^^^^^^^^^^^ -note: required because it appears within the type `NotSend` +note: required because it appears within the type `impl_::not_send::NotSend` --> src/impl_/not_send.rs | | pub(crate) struct NotSend(PhantomData<*mut Python<'static>>); | ^^^^^^^ - = note: required because it appears within the type `(&GILGuard, NotSend)` -note: required because it appears within the type `PhantomData<(&GILGuard, NotSend)>` + = note: required because it appears within the type `(&pyo3::gil::GILGuard, impl_::not_send::NotSend)` +note: required because it appears within the type `PhantomData<(&pyo3::gil::GILGuard, impl_::not_send::NotSend)>` --> $RUST/core/src/marker.rs | | pub struct PhantomData; | ^^^^^^^^^^^ -note: required because it appears within the type `Python<'_>` +note: required because it appears within the type `pyo3::Python<'_>` --> src/marker.rs | | pub struct Python<'py>(PhantomData<(&'py GILGuard, NotSend)>); | ^^^^^^ -note: required because it appears within the type `Bound<'_, PyString>` +note: required because it appears within the type `pyo3::Bound<'_, PyString>` --> src/instance.rs | | pub struct Bound<'py, T>(Python<'py>, ManuallyDrop>); diff --git a/tests/ui/traverse.stderr b/tests/ui/traverse.stderr index e2718c76e56..4448e67e13a 100644 --- a/tests/ui/traverse.stderr +++ b/tests/ui/traverse.stderr @@ -15,7 +15,7 @@ error[E0308]: mismatched types | |___________________^ expected fn pointer, found fn item | = note: expected fn pointer `for<'a, 'b> fn(&'a TraverseTriesToTakePyRef, PyVisit<'b>) -> Result<(), PyTraverseError>` - found fn item `for<'a, 'b> fn(pyo3::PyRef<'a, TraverseTriesToTakePyRef>, PyVisit<'b>) {TraverseTriesToTakePyRef::__traverse__}` + found fn item `for<'a, 'b> fn(pyo3::PyRef<'a, TraverseTriesToTakePyRef, >, PyVisit<'b>) {TraverseTriesToTakePyRef::__traverse__}` note: function defined here --> src/impl_/pymethods.rs | From 367eeaeeaba7f9422f3e4af005b759b840b7f7ec Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 30 Jan 2024 10:31:34 +0000 Subject: [PATCH 111/349] add `bound` method variants for `PyTypeInfo` --- guide/src/class/numeric.md | 2 +- guide/src/migration.md | 11 ++++--- src/conversions/chrono_tz.rs | 5 ++- src/conversions/std/ipaddr.rs | 4 +-- src/err/mod.rs | 20 +++++++---- src/marker.rs | 2 +- src/pycell.rs | 4 +-- src/sync.rs | 6 ++-- src/tests/common.rs | 4 +-- src/type_object.rs | 62 ++++++++++++++++++++++++++++++++--- src/types/any.rs | 12 +++---- src/types/ellipsis.rs | 15 +++++---- src/types/iterator.rs | 2 +- src/types/mapping.rs | 10 +++--- src/types/mod.rs | 4 +-- src/types/none.rs | 18 ++++++---- src/types/notimplemented.rs | 15 +++++---- src/types/pysuper.rs | 3 +- src/types/sequence.rs | 16 ++++----- src/types/typeobject.rs | 4 +-- 20 files changed, 144 insertions(+), 75 deletions(-) diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index ee17ea10bd9..6f2e6e18f86 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -388,7 +388,7 @@ fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { # let globals = PyModule::import(py, "__main__")?.dict(); -# globals.set_item("Number", Number::type_object(py))?; +# globals.set_item("Number", Number::type_object_bound(py))?; # # py.run(SCRIPT, Some(globals), None)?; # Ok(()) diff --git a/guide/src/migration.md b/guide/src/migration.md index 6a151b99164..d88a765a16a 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -210,6 +210,7 @@ To minimise breakage of code using the GIL-Refs API, the `Bound` smart pointe For example, the following APIs have gained updated variants: - `PyList::new`, `PyTyple::new` and similar constructors have replacements `PyList::new_bound`, `PyTuple::new_bound` etc. - `FromPyObject::extract` has a new `FromPyObject::extract_bound` (see the section below) +- The `PyTypeInfo` trait has had new `_bound` methods added to accept / return `Bound`. Because the new `Bound` API brings ownership out of the PyO3 framework and into user code, there are a few places where user code is expected to need to adjust while switching to the new API: - Code will need to add the occasional `&` to borrow the new smart pointer as `&Bound` to pass these types around (or use `.clone()` at the very small cost of increasing the Python reference count) @@ -245,6 +246,8 @@ impl<'py> FromPyObject<'py> for MyType { The expectation is that in 0.22 `extract_bound` will have the default implementation removed and in 0.23 `extract` will be removed. + + ## from 0.19.* to 0.20 ### Drop support for older technologies @@ -656,7 +659,7 @@ To migrate, update trait bounds and imports from `PyTypeObject` to `PyTypeInfo`. Before: -```rust,compile_fail +```rust,ignore use pyo3::Python; use pyo3::type_object::PyTypeObject; use pyo3::types::PyType; @@ -668,7 +671,7 @@ fn get_type_object(py: Python<'_>) -> &PyType { After -```rust +```rust,ignore use pyo3::{Python, PyTypeInfo}; use pyo3::types::PyType; @@ -995,13 +998,13 @@ makes it possible to interact with Python exception objects. The new types also have names starting with the "Py" prefix. For example, before: -```rust,compile_fail +```rust,ignore let err: PyErr = TypeError::py_err("error message"); ``` After: -```rust,compile_fail +```rust,ignore # use pyo3::{PyErr, PyResult, Python, type_object::PyTypeObject}; # use pyo3::exceptions::{PyBaseException, PyTypeError}; # Python::with_gil(|py| -> PyResult<()> { diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index 108dae253e3..791be15c96d 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -35,8 +35,7 @@ //! ``` use crate::exceptions::PyValueError; use crate::sync::GILOnceCell; -use crate::types::any::PyAnyMethods; -use crate::types::PyType; +use crate::types::{any::PyAnyMethods, PyType}; use crate::{ intern, Bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, }; @@ -51,7 +50,7 @@ impl ToPyObject for Tz { .unwrap() .call1((self.name(),)) .unwrap() - .into() + .unbind() } } diff --git a/src/conversions/std/ipaddr.rs b/src/conversions/std/ipaddr.rs index 5e313cd7cdd..84aacab43a2 100755 --- a/src/conversions/std/ipaddr.rs +++ b/src/conversions/std/ipaddr.rs @@ -36,7 +36,7 @@ impl ToPyObject for Ipv4Addr { .expect("failed to load ipaddress.IPv4Address") .call1((u32::from_be_bytes(self.octets()),)) .expect("failed to construct ipaddress.IPv4Address") - .to_object(py) + .unbind() } } @@ -48,7 +48,7 @@ impl ToPyObject for Ipv6Addr { .expect("failed to load ipaddress.IPv6Address") .call1((u128::from_be_bytes(self.octets()),)) .expect("failed to construct ipaddress.IPv6Address") - .to_object(py) + .unbind() } } diff --git a/src/err/mod.rs b/src/err/mod.rs index c50e377409e..8568545664b 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -159,7 +159,7 @@ impl PyErr { { PyErr::from_state(PyErrState::Lazy(Box::new(move |py| { PyErrStateLazyFnOutput { - ptype: T::type_object(py).into(), + ptype: T::type_object_bound(py).into(), pvalue: args.arguments(py), } }))) @@ -540,7 +540,7 @@ impl PyErr { where T: PyTypeInfo, { - self.is_instance(py, T::type_object(py)) + self.is_instance(py, T::type_object_bound(py).as_gil_ref()) } /// Writes the error back to the Python interpreter's global state. @@ -1077,18 +1077,24 @@ mod tests { fn test_pyerr_matches() { Python::with_gil(|py| { let err = PyErr::new::("foo"); - assert!(err.matches(py, PyValueError::type_object(py))); + assert!(err.matches(py, PyValueError::type_object_bound(py))); assert!(err.matches( py, - (PyValueError::type_object(py), PyTypeError::type_object(py)) + ( + PyValueError::type_object_bound(py), + PyTypeError::type_object_bound(py) + ) )); - assert!(!err.matches(py, PyTypeError::type_object(py))); + assert!(!err.matches(py, PyTypeError::type_object_bound(py))); // String is not a valid exception class, so we should get a TypeError - let err: PyErr = PyErr::from_type(crate::types::PyString::type_object(py), "foo"); - assert!(err.matches(py, PyTypeError::type_object(py))); + let err: PyErr = PyErr::from_type( + crate::types::PyString::type_object_bound(py).as_gil_ref(), + "foo", + ); + assert!(err.matches(py, PyTypeError::type_object_bound(py))); }) } diff --git a/src/marker.rs b/src/marker.rs index 1987ffaed13..448496de508 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -685,7 +685,7 @@ impl<'py> Python<'py> { where T: PyTypeInfo, { - T::type_object(self) + T::type_object_bound(self).into_gil_ref() } /// Imports the Python module with the specified name. diff --git a/src/pycell.rs b/src/pycell.rs index bde95ad8313..3744c55f67d 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -207,7 +207,7 @@ use crate::{ type_object::get_tp_free, PyTypeInfo, }; -use crate::{ffi, IntoPy, PyErr, PyNativeType, PyObject, PyResult, PyTypeCheck, Python}; +use crate::{ffi, Bound, IntoPy, PyErr, PyNativeType, PyObject, PyResult, PyTypeCheck, Python}; use std::cell::UnsafeCell; use std::fmt; use std::mem::ManuallyDrop; @@ -553,7 +553,7 @@ where { const NAME: &'static str = ::NAME; - fn type_check(object: &PyAny) -> bool { + fn type_check(object: &Bound<'_, PyAny>) -> bool { ::type_check(object) } } diff --git a/src/sync.rs b/src/sync.rs index a88a0f4b485..76c81026884 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,5 +1,5 @@ //! Synchronization mechanisms based on the Python GIL. -use crate::{instance::Bound, types::PyString, types::PyType, Py, PyResult, PyVisit, Python}; +use crate::{types::PyString, types::PyType, Bound, Py, PyResult, PyVisit, Python}; use std::cell::UnsafeCell; /// Value with concurrent access protected by the GIL. @@ -196,9 +196,9 @@ impl GILOnceCell> { py: Python<'py>, module_name: &str, attr_name: &str, - ) -> PyResult<&'py PyType> { + ) -> PyResult<&Bound<'py, PyType>> { self.get_or_try_init(py, || py.import(module_name)?.getattr(attr_name)?.extract()) - .map(|ty| ty.as_ref(py)) + .map(|ty| ty.bind(py)) } } diff --git a/src/tests/common.rs b/src/tests/common.rs index f2082437693..efab3ccb4b3 100644 --- a/src/tests/common.rs +++ b/src/tests/common.rs @@ -138,11 +138,11 @@ mod inner { ($py:expr, $body:expr, [$(($category:ty, $message:literal)),+] $(,)? ) => {{ $crate::tests::common::CatchWarnings::enter($py, |w| { $body; - let expected_warnings = [$((<$category as $crate::type_object::PyTypeInfo>::type_object($py), $message)),+]; + let expected_warnings = [$((<$category as $crate::type_object::PyTypeInfo>::type_object_bound($py), $message)),+]; assert_eq!(w.len(), expected_warnings.len()); for (warning, (category, message)) in w.iter().zip(expected_warnings) { - assert!(warning.getattr("category").unwrap().is(category)); + assert!(warning.getattr("category").unwrap().is(&category)); assert_eq!( warning.getattr("message").unwrap().str().unwrap().to_string_lossy(), message diff --git a/src/type_object.rs b/src/type_object.rs index e7d88f76c18..994781b3fc0 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -1,7 +1,9 @@ //! Python type object information +use crate::ffi_ptr_ext::FfiPtrExt; +use crate::types::any::PyAnyMethods; use crate::types::{PyAny, PyType}; -use crate::{ffi, PyNativeType, Python}; +use crate::{ffi, Bound, PyNativeType, Python}; /// `T: PyLayout` represents that `T` is a concrete representation of `U` in the Python heap. /// E.g., `PyCell` is a concrete representation of all `pyclass`es, and `ffi::PyObject` @@ -64,19 +66,71 @@ pub unsafe trait PyTypeInfo: Sized + HasPyGilRef { /// Returns the safe abstraction over the type object. #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyTypeInfo::type_object` will be replaced by `PyTypeInfo::type_object_bound` in a future PyO3 version" + ) + )] fn type_object(py: Python<'_>) -> &PyType { + // This isn't implemented in terms of `type_object_bound` because this just borrowed the + // object, for legacy reasons. unsafe { py.from_borrowed_ptr(Self::type_object_raw(py) as _) } } + /// Returns the safe abstraction over the type object. + #[inline] + fn type_object_bound(py: Python<'_>) -> Bound<'_, PyType> { + // Making the borrowed object `Bound` is necessary for soundness reasons. It's an extreme + // edge case, but arbitrary Python code _could_ change the __class__ of an object and cause + // the type object to be freed. + // + // By making `Bound` we assume ownership which is then safe against races. + unsafe { + Self::type_object_raw(py) + .cast::() + .assume_borrowed_unchecked(py) + .to_owned() + .downcast_into_unchecked() + } + } + /// Checks if `object` is an instance of this type or a subclass of this type. #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyTypeInfo::is_type_of` will be replaced by `PyTypeInfo::is_type_of_bound` in a future PyO3 version" + ) + )] fn is_type_of(object: &PyAny) -> bool { + Self::is_type_of_bound(&object.as_borrowed()) + } + + /// Checks if `object` is an instance of this type or a subclass of this type. + #[inline] + fn is_type_of_bound(object: &Bound<'_, PyAny>) -> bool { unsafe { ffi::PyObject_TypeCheck(object.as_ptr(), Self::type_object_raw(object.py())) != 0 } } /// Checks if `object` is an instance of this type. #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyTypeInfo::is_exact_type_of` will be replaced by `PyTypeInfo::is_exact_type_of_bound` in a future PyO3 version" + ) + )] fn is_exact_type_of(object: &PyAny) -> bool { + Self::is_exact_type_of_bound(&object.as_borrowed()) + } + + /// Checks if `object` is an instance of this type. + #[inline] + fn is_exact_type_of_bound(object: &Bound<'_, PyAny>) -> bool { unsafe { ffi::Py_TYPE(object.as_ptr()) == Self::type_object_raw(object.py()) } } } @@ -89,7 +143,7 @@ pub trait PyTypeCheck: HasPyGilRef { /// Checks if `object` is an instance of `Self`, which may include a subtype. /// /// This should be equivalent to the Python expression `isinstance(object, Self)`. - fn type_check(object: &PyAny) -> bool; + fn type_check(object: &Bound<'_, PyAny>) -> bool; } impl PyTypeCheck for T @@ -99,8 +153,8 @@ where const NAME: &'static str = ::NAME; #[inline] - fn type_check(object: &PyAny) -> bool { - ::is_type_of(object) + fn type_check(object: &Bound<'_, PyAny>) -> bool { + T::is_type_of_bound(object) } } diff --git a/src/types/any.rs b/src/types/any.rs index d79a9227cd7..a19985ae8bb 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -733,7 +733,7 @@ impl PyAny { where T: PyTypeCheck, { - if T::type_check(self) { + if T::type_check(&self.as_borrowed()) { // Safety: type_check is responsible for ensuring that the type is correct Ok(unsafe { self.downcast_unchecked() }) } else { @@ -776,7 +776,7 @@ impl PyAny { where T: PyTypeInfo, { - if T::is_exact_type_of(self) { + if T::is_exact_type_of_bound(&self.as_borrowed()) { // Safety: type_check is responsible for ensuring that the type is correct Ok(unsafe { self.downcast_unchecked() }) } else { @@ -2100,7 +2100,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { where T: PyTypeCheck, { - if T::type_check(self.as_gil_ref()) { + if T::type_check(self) { // Safety: type_check is responsible for ensuring that the type is correct Ok(unsafe { self.downcast_unchecked() }) } else { @@ -2113,7 +2113,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { where T: PyTypeCheck, { - if T::type_check(self.as_gil_ref()) { + if T::type_check(&self) { // Safety: type_check is responsible for ensuring that the type is correct Ok(unsafe { self.downcast_into_unchecked() }) } else { @@ -2218,12 +2218,12 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { #[inline] fn is_instance_of(&self) -> bool { - T::is_type_of(self.as_gil_ref()) + T::is_type_of_bound(self) } #[inline] fn is_exact_instance_of(&self) -> bool { - T::is_exact_type_of(self.as_gil_ref()) + T::is_exact_type_of_bound(self) } fn contains(&self, value: V) -> PyResult diff --git a/src/types/ellipsis.rs b/src/types/ellipsis.rs index 9151e89df97..b39b74b3832 100644 --- a/src/types/ellipsis.rs +++ b/src/types/ellipsis.rs @@ -1,4 +1,7 @@ -use crate::{ffi, ffi_ptr_ext::FfiPtrExt, Borrowed, PyAny, PyTypeInfo, Python}; +use crate::{ + ffi, ffi_ptr_ext::FfiPtrExt, types::any::PyAnyMethods, Borrowed, Bound, PyAny, PyTypeInfo, + Python, +}; /// Represents the Python `Ellipsis` object. #[repr(transparent)] @@ -38,14 +41,14 @@ unsafe impl PyTypeInfo for PyEllipsis { } #[inline] - fn is_type_of(object: &PyAny) -> bool { + fn is_type_of_bound(object: &Bound<'_, PyAny>) -> bool { // ellipsis is not usable as a base type - Self::is_exact_type_of(object) + Self::is_exact_type_of_bound(object) } #[inline] - fn is_exact_type_of(object: &PyAny) -> bool { - object.is(Self::get_bound(object.py()).as_ref()) + fn is_exact_type_of_bound(object: &Bound<'_, PyAny>) -> bool { + object.is(&**Self::get_bound(object.py())) } } @@ -68,7 +71,7 @@ mod tests { Python::with_gil(|py| { assert!(PyEllipsis::get_bound(py) .get_type() - .is(PyEllipsis::type_object(py))); + .is(&PyEllipsis::type_object_bound(py))); }) } diff --git a/src/types/iterator.rs b/src/types/iterator.rs index e27032867bc..cd5cd4ab9fe 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -114,7 +114,7 @@ impl<'py> Borrowed<'_, 'py, PyIterator> { impl PyTypeCheck for PyIterator { const NAME: &'static str = "Iterator"; - fn type_check(object: &PyAny) -> bool { + fn type_check(object: &Bound<'_, PyAny>) -> bool { unsafe { ffi::PyIter_Check(object.as_ptr()) != 0 } } } diff --git a/src/types/mapping.rs b/src/types/mapping.rs index 4c9677cea8b..81bb2e63912 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -97,7 +97,7 @@ impl PyMapping { /// library). This is equvalent to `collections.abc.Mapping.register(T)` in Python. /// This registration is required for a pyclass to be downcastable from `PyAny` to `PyMapping`. pub fn register(py: Python<'_>) -> PyResult<()> { - let ty = T::type_object(py); + let ty = T::type_object_bound(py); get_mapping_abc(py)?.call_method1("register", (ty,))?; Ok(()) } @@ -232,7 +232,7 @@ impl<'py> PyMappingMethods<'py> for Bound<'py, PyMapping> { } } -fn get_mapping_abc(py: Python<'_>) -> PyResult<&PyType> { +fn get_mapping_abc(py: Python<'_>) -> PyResult<&Bound<'_, PyType>> { static MAPPING_ABC: GILOnceCell> = GILOnceCell::new(); MAPPING_ABC.get_or_try_init_type_ref(py, "collections.abc", "Mapping") @@ -242,10 +242,10 @@ impl PyTypeCheck for PyMapping { const NAME: &'static str = "Mapping"; #[inline] - fn type_check(object: &PyAny) -> bool { + fn type_check(object: &Bound<'_, PyAny>) -> bool { // Using `is_instance` for `collections.abc.Mapping` is slow, so provide // optimized case dict as a well-known mapping - PyDict::is_type_of(object) + PyDict::is_type_of_bound(object) || get_mapping_abc(object.py()) .and_then(|abc| object.is_instance(abc)) .unwrap_or_else(|err| { @@ -263,7 +263,7 @@ impl<'v> crate::PyTryFrom<'v> for PyMapping { fn try_from>(value: V) -> Result<&'v PyMapping, PyDowncastError<'v>> { let value = value.into(); - if PyMapping::type_check(value) { + if PyMapping::type_check(&value.as_borrowed()) { unsafe { return Ok(value.downcast_unchecked()) } } diff --git a/src/types/mod.rs b/src/types/mod.rs index f802348f466..82e025b2d24 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -207,9 +207,9 @@ macro_rules! pyobject_native_type_info( $( #[inline] - fn is_type_of(ptr: &$crate::PyAny) -> bool { + fn is_type_of_bound(obj: &$crate::Bound<'_, $crate::PyAny>) -> bool { #[allow(unused_unsafe)] - unsafe { $checkfunction(ptr.as_ptr()) > 0 } + unsafe { $checkfunction(obj.as_ptr()) > 0 } } )? } diff --git a/src/types/none.rs b/src/types/none.rs index 3c3b171668d..ee650f12f67 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -1,5 +1,8 @@ use crate::ffi_ptr_ext::FfiPtrExt; -use crate::{ffi, Borrowed, IntoPy, PyAny, PyObject, PyTypeInfo, Python, ToPyObject}; +use crate::{ + ffi, types::any::PyAnyMethods, Borrowed, Bound, IntoPy, PyAny, PyObject, PyTypeInfo, Python, + ToPyObject, +}; /// Represents the Python `None` object. #[repr(transparent)] @@ -40,15 +43,14 @@ unsafe impl PyTypeInfo for PyNone { } #[inline] - fn is_type_of(object: &PyAny) -> bool { + fn is_type_of_bound(object: &Bound<'_, PyAny>) -> bool { // NoneType is not usable as a base type - Self::is_exact_type_of(object) + Self::is_exact_type_of_bound(object) } #[inline] - fn is_exact_type_of(object: &PyAny) -> bool { - let none = Self::get_bound(object.py()); - object.is(none.as_ref()) + fn is_exact_type_of_bound(object: &Bound<'_, PyAny>) -> bool { + object.is(&**Self::get_bound(object.py())) } } @@ -82,7 +84,9 @@ mod tests { #[test] fn test_none_type_object_consistent() { Python::with_gil(|py| { - assert!(PyNone::get_bound(py).get_type().is(PyNone::type_object(py))); + assert!(PyNone::get_bound(py) + .get_type() + .is(&PyNone::type_object_bound(py))); }) } diff --git a/src/types/notimplemented.rs b/src/types/notimplemented.rs index 7560dff37d1..5832c6a9803 100644 --- a/src/types/notimplemented.rs +++ b/src/types/notimplemented.rs @@ -1,4 +1,7 @@ -use crate::{ffi, ffi_ptr_ext::FfiPtrExt, Borrowed, PyAny, PyTypeInfo, Python}; +use crate::{ + ffi, ffi_ptr_ext::FfiPtrExt, types::any::PyAnyMethods, Borrowed, Bound, PyAny, PyTypeInfo, + Python, +}; /// Represents the Python `NotImplemented` object. #[repr(transparent)] @@ -41,14 +44,14 @@ unsafe impl PyTypeInfo for PyNotImplemented { } #[inline] - fn is_type_of(object: &PyAny) -> bool { + fn is_type_of_bound(object: &Bound<'_, PyAny>) -> bool { // NotImplementedType is not usable as a base type - Self::is_exact_type_of(object) + Self::is_exact_type_of_bound(object) } #[inline] - fn is_exact_type_of(object: &PyAny) -> bool { - object.is(Self::get_bound(object.py()).as_ref()) + fn is_exact_type_of_bound(object: &Bound<'_, PyAny>) -> bool { + object.is(&**Self::get_bound(object.py())) } } @@ -71,7 +74,7 @@ mod tests { Python::with_gil(|py| { assert!(PyNotImplemented::get_bound(py) .get_type() - .is(PyNotImplemented::type_object(py))); + .is(&PyNotImplemented::type_object_bound(py))); }) } diff --git a/src/types/pysuper.rs b/src/types/pysuper.rs index 25f5e1ceb1c..261a91f289b 100644 --- a/src/types/pysuper.rs +++ b/src/types/pysuper.rs @@ -72,8 +72,7 @@ impl PySuper { ty: &Bound<'py, PyType>, obj: &Bound<'py, PyAny>, ) -> PyResult> { - PySuper::type_object(ty.py()) - .as_borrowed() + PySuper::type_object_bound(ty.py()) .call1((ty, obj)) .map(|any| { // Safety: super() always returns instance of super diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 19f6846b4d1..80306cbf44b 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -8,11 +8,9 @@ use crate::internal_tricks::get_ssize_index; use crate::py_result_ext::PyResultExt; use crate::sync::GILOnceCell; use crate::type_object::PyTypeInfo; -use crate::types::{PyAny, PyList, PyString, PyTuple, PyType}; +use crate::types::{any::PyAnyMethods, PyAny, PyList, PyString, PyTuple, PyType}; use crate::{ffi, FromPyObject, Py, PyNativeType, PyTypeCheck, Python, ToPyObject}; -use super::any::PyAnyMethods; - /// Represents a reference to a Python object supporting the sequence protocol. #[repr(transparent)] pub struct PySequence(PyAny); @@ -182,7 +180,7 @@ impl PySequence { /// library). This is equvalent to `collections.abc.Sequence.register(T)` in Python. /// This registration is required for a pyclass to be downcastable from `PyAny` to `PySequence`. pub fn register(py: Python<'_>) -> PyResult<()> { - let ty = T::type_object(py); + let ty = T::type_object_bound(py); get_sequence_abc(py)?.call_method1("register", (ty,))?; Ok(()) } @@ -517,7 +515,7 @@ where Ok(v) } -fn get_sequence_abc(py: Python<'_>) -> PyResult<&PyType> { +fn get_sequence_abc(py: Python<'_>) -> PyResult<&Bound<'_, PyType>> { static SEQUENCE_ABC: GILOnceCell> = GILOnceCell::new(); SEQUENCE_ABC.get_or_try_init_type_ref(py, "collections.abc", "Sequence") @@ -527,11 +525,11 @@ impl PyTypeCheck for PySequence { const NAME: &'static str = "Sequence"; #[inline] - fn type_check(object: &PyAny) -> bool { + fn type_check(object: &Bound<'_, PyAny>) -> bool { // Using `is_instance` for `collections.abc.Sequence` is slow, so provide // optimized cases for list and tuples as common well-known sequences - PyList::is_type_of(object) - || PyTuple::is_type_of(object) + PyList::is_type_of_bound(object) + || PyTuple::is_type_of_bound(object) || get_sequence_abc(object.py()) .and_then(|abc| object.is_instance(abc)) .unwrap_or_else(|err| { @@ -549,7 +547,7 @@ impl<'v> crate::PyTryFrom<'v> for PySequence { fn try_from>(value: V) -> Result<&'v PySequence, PyDowncastError<'v>> { let value = value.into(); - if PySequence::type_check(value) { + if PySequence::type_check(&value.as_borrowed()) { unsafe { return Ok(value.downcast_unchecked::()) } } diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index a1a053f93a4..50eeaa7153d 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -14,7 +14,7 @@ impl PyType { /// Creates a new type object. #[inline] pub fn new(py: Python<'_>) -> &PyType { - T::type_object(py) + T::type_object_bound(py).into_gil_ref() } /// Retrieves the underlying FFI pointer associated with this Python object. @@ -104,7 +104,7 @@ impl PyType { where T: PyTypeInfo, { - self.is_subclass(T::type_object(self.py())) + self.is_subclass(T::type_object_bound(self.py()).as_gil_ref()) } } From 33dc33ececc49581450867a1993eec6ac14d8a1c Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Mon, 5 Feb 2024 19:13:02 +0100 Subject: [PATCH 112/349] port `Python::eval` to `Bound` API --- README.md | 4 +- guide/src/conversions/traits.md | 2 +- guide/src/memory.md | 22 +++++----- guide/src/migration.md | 6 +-- guide/src/python_from_rust.md | 4 +- src/buffer.rs | 8 ++-- src/conversions/chrono.rs | 5 ++- src/conversions/num_bigint.rs | 6 ++- src/conversions/std/array.rs | 9 +++-- src/conversions/std/num.rs | 14 ++++--- src/conversions/std/slice.rs | 15 ++++--- src/exceptions.rs | 24 ++++++----- src/ffi/tests.rs | 5 ++- src/gil.rs | 5 ++- src/lib.rs | 4 +- src/marker.rs | 72 ++++++++++++++++++++++++--------- src/types/iterator.rs | 4 +- src/types/mod.rs | 4 +- tests/test_datetime.rs | 17 ++++---- tests/test_frompyobject.rs | 16 ++++---- tests/test_inheritance.rs | 2 +- 21 files changed, 149 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 98eeeb80242..606ef926ea9 100644 --- a/README.md +++ b/README.md @@ -152,9 +152,9 @@ fn main() -> PyResult<()> { let sys = py.import("sys")?; let version: String = sys.getattr("version")?.extract()?; - let locals = [("os", py.import("os")?)].into_py_dict(py); + let locals = [("os", py.import("os")?)].into_py_dict(py).as_borrowed(); let code = "os.getenv('USER') or os.getenv('USERNAME') or 'Unknown'"; - let user: String = py.eval(code, None, Some(&locals))?.extract()?; + let user: String = py.eval_bound(code, None, Some(&locals))?.extract()?; println!("Hello {}, I'm Python {}", user, version); Ok(()) diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index 4f3342f2ccc..71e823f8c46 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -155,7 +155,7 @@ struct RustyStruct { # # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let py_dict = py.eval("{'foo': 'foo', 'bar': 'bar', 'foobar': 'foobar'}", None, None)?; +# let py_dict = py.eval_bound("{'foo': 'foo', 'bar': 'bar', 'foobar': 'foobar'}", None, None)?; # let rustystruct: RustyStruct = py_dict.extract()?; # assert_eq!(rustystruct.foo, "foo"); # assert_eq!(rustystruct.bar, "bar"); diff --git a/guide/src/memory.md b/guide/src/memory.md index f9201e3f003..2f5e5d9b0bd 100644 --- a/guide/src/memory.md +++ b/guide/src/memory.md @@ -27,7 +27,7 @@ very simple and easy-to-understand programs like this: # use pyo3::types::PyString; # fn main() -> PyResult<()> { Python::with_gil(|py| -> PyResult<()> { - let hello: &PyString = py.eval("\"Hello World!\"", None, None)?.extract()?; + let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; println!("Python says: {}", hello); Ok(()) })?; @@ -48,7 +48,7 @@ of the time we don't have to think about this, but consider the following: # fn main() -> PyResult<()> { Python::with_gil(|py| -> PyResult<()> { for _ in 0..10 { - let hello: &PyString = py.eval("\"Hello World!\"", None, None)?.extract()?; + let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; println!("Python says: {}", hello); } // There are 10 copies of `hello` on Python's heap here. @@ -76,7 +76,7 @@ is to acquire and release the GIL with each iteration of the loop. # fn main() -> PyResult<()> { for _ in 0..10 { Python::with_gil(|py| -> PyResult<()> { - let hello: &PyString = py.eval("\"Hello World!\"", None, None)?.extract()?; + let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; println!("Python says: {}", hello); Ok(()) })?; // only one copy of `hello` at a time @@ -97,7 +97,7 @@ Python::with_gil(|py| -> PyResult<()> { for _ in 0..10 { let pool = unsafe { py.new_pool() }; let py = pool.python(); - let hello: &PyString = py.eval("\"Hello World!\"", None, None)?.extract()?; + let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; println!("Python says: {}", hello); } Ok(()) @@ -144,8 +144,8 @@ reference count reaches zero? It depends whether or not we are holding the GIL. # use pyo3::types::PyString; # fn main() -> PyResult<()> { Python::with_gil(|py| -> PyResult<()> { - let hello: Py = py.eval("\"Hello World!\"", None, None)?.extract()?; - println!("Python says: {}", hello.as_ref(py)); + let hello: Py = py.eval_bound("\"Hello World!\"", None, None)?.extract()?; + println!("Python says: {}", hello.bind(py)); Ok(()) })?; # Ok(()) @@ -166,7 +166,7 @@ we are *not* holding the GIL? # use pyo3::types::PyString; # fn main() -> PyResult<()> { let hello: Py = Python::with_gil(|py| { - py.eval("\"Hello World!\"", None, None)?.extract() + py.eval_bound("\"Hello World!\"", None, None)?.extract() })?; // Do some stuff... // Now sometime later in the program we want to access `hello`. @@ -197,11 +197,11 @@ We can avoid the delay in releasing memory if we are careful to drop the # use pyo3::types::PyString; # fn main() -> PyResult<()> { let hello: Py = - Python::with_gil(|py| py.eval("\"Hello World!\"", None, None)?.extract())?; + Python::with_gil(|py| py.eval_bound("\"Hello World!\"", None, None)?.extract())?; // Do some stuff... // Now sometime later in the program: Python::with_gil(|py| { - println!("Python says: {}", hello.as_ref(py)); + println!("Python says: {}", hello.bind(py)); drop(hello); // Memory released here. }); # Ok(()) @@ -219,11 +219,11 @@ until the GIL is dropped. # use pyo3::types::PyString; # fn main() -> PyResult<()> { let hello: Py = - Python::with_gil(|py| py.eval("\"Hello World!\"", None, None)?.extract())?; + Python::with_gil(|py| py.eval_bound("\"Hello World!\"", None, None)?.extract())?; // Do some stuff... // Now sometime later in the program: Python::with_gil(|py| { - println!("Python says: {}", hello.into_ref(py)); + println!("Python says: {}", hello.into_bound(py)); // Memory not released yet. // Do more stuff... // Memory released here at end of `with_gil()` closure. diff --git a/guide/src/migration.md b/guide/src/migration.md index 6a151b99164..dce6bddabab 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -1201,7 +1201,7 @@ all you need to do is remove `ObjectProtocol` from your code. Or if you use `ObjectProtocol` by `use pyo3::prelude::*`, you have to do nothing. Before: -```rust,compile_fail +```rust,compile_fail,ignore use pyo3::ObjectProtocol; # pyo3::Python::with_gil(|py| { @@ -1212,7 +1212,7 @@ assert_eq!(hi.len().unwrap(), 5); ``` After: -```rust +```rust,ignore # pyo3::Python::with_gil(|py| { let obj = py.eval("lambda: 'Hi :)'", None, None).unwrap(); let hi: &pyo3::types::PyString = obj.call0().unwrap().downcast().unwrap(); @@ -1351,7 +1351,7 @@ let obj_ref_mut: &mut MyClass = obj.extract().unwrap(); ``` After: -```rust +```rust,ignore # use pyo3::prelude::*; # use pyo3::types::IntoPyDict; # #[pyclass] #[derive(Clone)] struct MyClass {} diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 7b364ec5c01..2f3be53a975 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -157,7 +157,7 @@ use pyo3::prelude::*; # fn main() -> Result<(), ()> { Python::with_gil(|py| { let result = py - .eval("[i * 10 for i in range(5)]", None, None) + .eval_bound("[i * 10 for i in range(5)]", None, None) .map_err(|e| { e.print_and_set_sys_last_vars(py); })?; @@ -466,7 +466,7 @@ class House(object): house.call_method0("__enter__").unwrap(); - let result = py.eval("undefined_variable + 1", None, None); + let result = py.eval_bound("undefined_variable + 1", None, None); // If the eval threw an exception we'll pass it through to the context manager. // Otherwise, __exit__ is called with empty arguments (Python "None"). diff --git a/src/buffer.rs b/src/buffer.rs index d18f05e289d..7360ef67744 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -694,8 +694,8 @@ mod tests { #[test] fn test_debug() { Python::with_gil(|py| { - let bytes = py.eval("b'abcde'", None, None).unwrap(); - let buffer: PyBuffer = PyBuffer::get(bytes).unwrap(); + let bytes = py.eval_bound("b'abcde'", None, None).unwrap(); + let buffer: PyBuffer = PyBuffer::get(bytes.as_gil_ref()).unwrap(); let expected = format!( concat!( "PyBuffer {{ buf: {:?}, obj: {:?}, ", @@ -857,8 +857,8 @@ mod tests { #[test] fn test_bytes_buffer() { Python::with_gil(|py| { - let bytes = py.eval("b'abcde'", None, None).unwrap(); - let buffer = PyBuffer::get(bytes).unwrap(); + let bytes = py.eval_bound("b'abcde'", None, None).unwrap(); + let buffer = PyBuffer::get(bytes.as_gil_ref()).unwrap(); assert_eq!(buffer.dimensions(), 1); assert_eq!(buffer.item_count(), 5); assert_eq!(buffer.format().to_str().unwrap(), "B"); diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 6e529918ab2..83f9830bcdb 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -1095,6 +1095,7 @@ mod tests { mod proptests { use super::*; use crate::tests::common::CatchWarnings; + use crate::types::any::PyAnyMethods; use crate::types::IntoPyDict; use proptest::prelude::*; @@ -1105,9 +1106,9 @@ mod tests { fn test_pyo3_offset_fixed_frompyobject_created_in_python(timestamp in 0..(i32::MAX as i64), timedelta in -86399i32..=86399i32) { Python::with_gil(|py| { - let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict(py); + let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict(py).as_borrowed(); let code = format!("datetime.datetime.fromtimestamp({}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={})))", timestamp, timedelta); - let t = py.eval(&code, Some(globals), None).unwrap(); + let t = py.eval_bound(&code, Some(&globals), None).unwrap(); // Get ISO 8601 string from python let py_iso_str = t.call_method0("isoformat").unwrap(); diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 1d536623a85..ebda6c85348 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -261,6 +261,8 @@ fn int_n_bits(long: &Bound<'_, PyLong>) -> PyResult { #[cfg(test)] mod tests { + use self::{any::PyAnyMethods, dict::PyDictMethods}; + use super::*; use crate::types::{PyDict, PyModule}; use indoc::indoc; @@ -340,9 +342,9 @@ mod tests { fn convert_index_class() { Python::with_gil(|py| { let index = python_index_class(py); - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("index", index).unwrap(); - let ob = py.eval("index.C(10)", None, Some(locals)).unwrap(); + let ob = py.eval_bound("index.C(10)", None, Some(&locals)).unwrap(); let _: BigInt = ob.extract().unwrap(); }); } diff --git a/src/conversions/std/array.rs b/src/conversions/std/array.rs index 0ea167135b6..016ab0a643d 100644 --- a/src/conversions/std/array.rs +++ b/src/conversions/std/array.rs @@ -130,6 +130,7 @@ mod tests { sync::atomic::{AtomicUsize, Ordering}, }; + use crate::types::any::PyAnyMethods; use crate::{types::PyList, IntoPy, PyResult, Python, ToPyObject}; #[test] @@ -157,7 +158,7 @@ mod tests { fn test_extract_bytearray_to_array() { Python::with_gil(|py| { let v: [u8; 33] = py - .eval( + .eval_bound( "bytearray(b'abcabcabcabcabcabcabcabcabcabcabc')", None, None, @@ -173,7 +174,7 @@ mod tests { fn test_extract_small_bytearray_to_array() { Python::with_gil(|py| { let v: [u8; 3] = py - .eval("bytearray(b'abc')", None, None) + .eval_bound("bytearray(b'abc')", None, None) .unwrap() .extract() .unwrap(); @@ -197,7 +198,7 @@ mod tests { fn test_extract_invalid_sequence_length() { Python::with_gil(|py| { let v: PyResult<[u8; 3]> = py - .eval("bytearray(b'abcdefg')", None, None) + .eval_bound("bytearray(b'abcdefg')", None, None) .unwrap() .extract(); assert_eq!( @@ -223,7 +224,7 @@ mod tests { #[test] fn test_extract_non_iterable_to_array() { Python::with_gil(|py| { - let v = py.eval("42", None, None).unwrap(); + let v = py.eval_bound("42", None, None).unwrap(); v.extract::().unwrap(); v.extract::<[i32; 1]>().unwrap_err(); }); diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 82f63016a7d..2d20915d84f 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -371,6 +371,8 @@ nonzero_int_impl!(NonZeroUsize, usize); #[cfg(test)] mod test_128bit_integers { use super::*; + use crate::types::any::PyAnyMethods; + #[cfg(not(target_arch = "wasm32"))] use crate::types::PyDict; @@ -474,7 +476,7 @@ mod test_128bit_integers { #[test] fn test_i128_overflow() { Python::with_gil(|py| { - let obj = py.eval("(1 << 130) * -1", None, None).unwrap(); + let obj = py.eval_bound("(1 << 130) * -1", None, None).unwrap(); let err = obj.extract::().unwrap_err(); assert!(err.is_instance_of::(py)); }) @@ -483,7 +485,7 @@ mod test_128bit_integers { #[test] fn test_u128_overflow() { Python::with_gil(|py| { - let obj = py.eval("1 << 130", None, None).unwrap(); + let obj = py.eval_bound("1 << 130", None, None).unwrap(); let err = obj.extract::().unwrap_err(); assert!(err.is_instance_of::(py)); }) @@ -527,7 +529,7 @@ mod test_128bit_integers { #[test] fn test_nonzero_i128_overflow() { Python::with_gil(|py| { - let obj = py.eval("(1 << 130) * -1", None, None).unwrap(); + let obj = py.eval_bound("(1 << 130) * -1", None, None).unwrap(); let err = obj.extract::().unwrap_err(); assert!(err.is_instance_of::(py)); }) @@ -536,7 +538,7 @@ mod test_128bit_integers { #[test] fn test_nonzero_u128_overflow() { Python::with_gil(|py| { - let obj = py.eval("1 << 130", None, None).unwrap(); + let obj = py.eval_bound("1 << 130", None, None).unwrap(); let err = obj.extract::().unwrap_err(); assert!(err.is_instance_of::(py)); }) @@ -545,7 +547,7 @@ mod test_128bit_integers { #[test] fn test_nonzero_i128_zero_value() { Python::with_gil(|py| { - let obj = py.eval("0", None, None).unwrap(); + let obj = py.eval_bound("0", None, None).unwrap(); let err = obj.extract::().unwrap_err(); assert!(err.is_instance_of::(py)); }) @@ -554,7 +556,7 @@ mod test_128bit_integers { #[test] fn test_nonzero_u128_zero_value() { Python::with_gil(|py| { - let obj = py.eval("0", None, None).unwrap(); + let obj = py.eval_bound("0", None, None).unwrap(); let err = obj.extract::().unwrap_err(); assert!(err.is_instance_of::(py)); }) diff --git a/src/conversions/std/slice.rs b/src/conversions/std/slice.rs index 378c02c4600..2d46abe2953 100644 --- a/src/conversions/std/slice.rs +++ b/src/conversions/std/slice.rs @@ -61,12 +61,15 @@ impl IntoPy> for Cow<'_, [u8]> { mod tests { use std::borrow::Cow; - use crate::{types::PyBytes, Python, ToPyObject}; + use crate::{ + types::{any::PyAnyMethods, PyBytes}, + Python, ToPyObject, + }; #[test] fn test_extract_bytes() { Python::with_gil(|py| { - let py_bytes = py.eval("b'Hello Python'", None, None).unwrap(); + let py_bytes = py.eval_bound("b'Hello Python'", None, None).unwrap(); let bytes: &[u8] = py_bytes.extract().unwrap(); assert_eq!(bytes, b"Hello Python"); }); @@ -75,15 +78,17 @@ mod tests { #[test] fn test_cow_impl() { Python::with_gil(|py| { - let bytes = py.eval(r#"b"foobar""#, None, None).unwrap(); + let bytes = py.eval_bound(r#"b"foobar""#, None, None).unwrap(); let cow = bytes.extract::>().unwrap(); assert_eq!(cow, Cow::<[u8]>::Borrowed(b"foobar")); - let byte_array = py.eval(r#"bytearray(b"foobar")"#, None, None).unwrap(); + let byte_array = py + .eval_bound(r#"bytearray(b"foobar")"#, None, None) + .unwrap(); let cow = byte_array.extract::>().unwrap(); assert_eq!(cow, Cow::<[u8]>::Owned(b"foobar".to_vec())); - let something_else_entirely = py.eval("42", None, None).unwrap(); + let something_else_entirely = py.eval_bound("42", None, None).unwrap(); something_else_entirely .extract::>() .unwrap_err(); diff --git a/src/exceptions.rs b/src/exceptions.rs index 19f59742435..daea55e5abd 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -800,8 +800,9 @@ pub mod socket { #[cfg(test)] mod tests { use super::*; + use crate::types::any::PyAnyMethods; use crate::types::{IntoPyDict, PyDict}; - use crate::{PyErr, Python}; + use crate::{PyErr, PyNativeType, Python}; import_exception!(socket, gaierror); import_exception!(email.errors, MessageError); @@ -863,13 +864,14 @@ mod tests { Python::with_gil(|py| { let error_type = py.get_type::(); - let ctx = [("CustomError", error_type)].into_py_dict(py); + let ctx = [("CustomError", error_type)].into_py_dict(py).as_borrowed(); let type_description: String = py - .eval("str(CustomError)", None, Some(ctx)) + .eval_bound("str(CustomError)", None, Some(&ctx)) .unwrap() .extract() .unwrap(); assert_eq!(type_description, ""); + let ctx = ctx.as_gil_ref(); py.run( "assert CustomError('oops').args == ('oops',)", None, @@ -886,9 +888,9 @@ mod tests { create_exception!(mymodule.exceptions, CustomError, PyException); Python::with_gil(|py| { let error_type = py.get_type::(); - let ctx = [("CustomError", error_type)].into_py_dict(py); + let ctx = [("CustomError", error_type)].into_py_dict(py).as_borrowed(); let type_description: String = py - .eval("str(CustomError)", None, Some(ctx)) + .eval_bound("str(CustomError)", None, Some(&ctx)) .unwrap() .extract() .unwrap(); @@ -905,13 +907,14 @@ mod tests { Python::with_gil(|py| { let error_type = py.get_type::(); - let ctx = [("CustomError", error_type)].into_py_dict(py); + let ctx = [("CustomError", error_type)].into_py_dict(py).as_borrowed(); let type_description: String = py - .eval("str(CustomError)", None, Some(ctx)) + .eval_bound("str(CustomError)", None, Some(&ctx)) .unwrap() .extract() .unwrap(); assert_eq!(type_description, ""); + let ctx = ctx.as_gil_ref(); py.run( "assert CustomError('oops').args == ('oops',)", None, @@ -934,13 +937,14 @@ mod tests { Python::with_gil(|py| { let error_type = py.get_type::(); - let ctx = [("CustomError", error_type)].into_py_dict(py); + let ctx = [("CustomError", error_type)].into_py_dict(py).as_borrowed(); let type_description: String = py - .eval("str(CustomError)", None, Some(ctx)) + .eval_bound("str(CustomError)", None, Some(&ctx)) .unwrap() .extract() .unwrap(); assert_eq!(type_description, ""); + let ctx = ctx.as_gil_ref(); py.run( "assert CustomError('oops').args == ('oops',)", None, @@ -1077,7 +1081,7 @@ mod tests { PyErr::from_value(PyUnicodeDecodeError::new_utf8(py, invalid_utf8, err).unwrap()) }); test_exception!(PyUnicodeEncodeError, |py| py - .eval("chr(40960).encode('ascii')", None, None) + .eval_bound("chr(40960).encode('ascii')", None, None) .unwrap_err()); test_exception!(PyUnicodeTranslateError, |_| { PyUnicodeTranslateError::new_err(("\u{3042}", 0, 1, "ouch")) diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index 0e0d3688e11..fc10bf05f5d 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -1,9 +1,10 @@ use crate::ffi::*; +use crate::types::any::PyAnyMethods; use crate::Python; #[cfg(not(Py_LIMITED_API))] use crate::{ - types::{any::PyAnyMethods, PyDict, PyString}, + types::{PyDict, PyString}, IntoPy, Py, PyAny, }; #[cfg(not(any(Py_3_12, Py_LIMITED_API)))] @@ -293,7 +294,7 @@ fn test_get_tzinfo() { #[test] fn test_inc_dec_ref() { Python::with_gil(|py| { - let obj = py.eval("object()", None, None).unwrap(); + let obj = py.eval_bound("object()", None, None).unwrap(); let ref_count = obj.get_refcnt(); let ptr = obj.as_ptr(); diff --git a/src/gil.rs b/src/gil.rs index d346ad95ea9..61d69ed9576 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -507,6 +507,7 @@ fn decrement_gil_count() { #[cfg(test)] mod tests { use super::{gil_is_acquired, GILPool, GIL_COUNT, OWNED_OBJECTS, POOL}; + use crate::types::any::PyAnyMethods; use crate::{ffi, gil, PyObject, Python, ToPyObject}; #[cfg(not(target_arch = "wasm32"))] use parking_lot::{const_mutex, Condvar, Mutex}; @@ -518,7 +519,7 @@ mod tests { let pool = unsafe { py.new_pool() }; let py = pool.python(); - let obj = py.eval("object()", None, None).unwrap(); + let obj = py.eval_bound("object()", None, None).unwrap(); obj.to_object(py) } @@ -735,7 +736,7 @@ mod tests { fn dropping_gil_does_not_invalidate_references() { // Acquiring GIL for the second time should be safe - see #864 Python::with_gil(|py| { - let obj = Python::with_gil(|_| py.eval("object()", None, None).unwrap()); + let obj = Python::with_gil(|_| py.eval_bound("object()", None, None).unwrap()); // After gil2 drops, obj should still have a reference count of one assert_eq!(obj.get_refcnt(), 1); diff --git a/src/lib.rs b/src/lib.rs index 3f91eb56913..13b027d3e77 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -221,9 +221,9 @@ //! let sys = py.import("sys")?; //! let version: String = sys.getattr("version")?.extract()?; //! -//! let locals = [("os", py.import("os")?)].into_py_dict(py); +//! let locals = [("os", py.import("os")?)].into_py_dict(py).as_borrowed(); //! let code = "os.getenv('USER') or os.getenv('USERNAME') or 'Unknown'"; -//! let user: String = py.eval(code, None, Some(&locals))?.extract()?; +//! let user: String = py.eval_bound(code, None, Some(&locals))?.extract()?; //! //! println!("Hello {}, I'm Python {}", user, version); //! Ok(()) diff --git a/src/marker.rs b/src/marker.rs index 1987ffaed13..5addc33c224 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -117,14 +117,19 @@ //! [`Rc`]: std::rc::Rc //! [`Py`]: crate::Py use crate::err::{self, PyDowncastError, PyErr, PyResult}; +use crate::ffi_ptr_ext::FfiPtrExt; use crate::gil::{GILGuard, GILPool, SuspendGIL}; use crate::impl_::not_send::NotSend; +use crate::py_result_ext::PyResultExt; use crate::type_object::HasPyGilRef; +use crate::types::any::PyAnyMethods; use crate::types::{ PyAny, PyDict, PyEllipsis, PyModule, PyNone, PyNotImplemented, PyString, PyType, }; use crate::version::PythonVersionInfo; -use crate::{ffi, FromPyPointer, IntoPy, Py, PyObject, PyTypeCheck, PyTypeInfo}; +use crate::{ + ffi, Bound, FromPyPointer, IntoPy, Py, PyNativeType, PyObject, PyTypeCheck, PyTypeInfo, +}; use std::ffi::{CStr, CString}; use std::marker::PhantomData; use std::os::raw::c_int; @@ -358,8 +363,8 @@ pub use nightly::Ungil; /// # fn main () -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { /// for _ in 0..10 { -/// let hello: &PyString = py.eval("\"Hello World!\"", None, None)?.extract()?; -/// println!("Python says: {}", hello.to_str()?); +/// let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; +/// println!("Python says: {}", hello.to_cow()?); /// // Normally variables in a loop scope are dropped here, but `hello` is a reference to /// // something owned by the Python interpreter. Dropping this reference does nothing. /// } @@ -417,7 +422,7 @@ impl Python<'_> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let x: i32 = py.eval("5", None, None)?.extract()?; + /// let x: i32 = py.eval_bound("5", None, None)?.extract()?; /// assert_eq!(x, 5); /// Ok(()) /// }) @@ -546,6 +551,28 @@ impl<'py> Python<'py> { f() } + /// Deprecated version of [`Python::eval_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`Python::eval` will be replaced by `Python::eval_bound` in a future PyO3 version" + ) + )] + pub fn eval( + self, + code: &str, + globals: Option<&'py PyDict>, + locals: Option<&'py PyDict>, + ) -> PyResult<&'py PyAny> { + self.eval_bound( + code, + globals.map(PyNativeType::as_borrowed).as_deref(), + locals.map(PyNativeType::as_borrowed).as_deref(), + ) + .map(Bound::into_gil_ref) + } + /// Evaluates a Python expression in the given context and returns the result. /// /// If `globals` is `None`, it defaults to Python module `__main__`. @@ -559,17 +586,17 @@ impl<'py> Python<'py> { /// ``` /// # use pyo3::prelude::*; /// # Python::with_gil(|py| { - /// let result = py.eval("[i * 10 for i in range(5)]", None, None).unwrap(); + /// let result = py.eval_bound("[i * 10 for i in range(5)]", None, None).unwrap(); /// let res: Vec = result.extract().unwrap(); /// assert_eq!(res, vec![0, 10, 20, 30, 40]) /// # }); /// ``` - pub fn eval( + pub fn eval_bound( self, code: &str, - globals: Option<&PyDict>, - locals: Option<&PyDict>, - ) -> PyResult<&'py PyAny> { + globals: Option<&Bound<'py, PyDict>>, + locals: Option<&Bound<'py, PyDict>>, + ) -> PyResult> { self.run_code(code, ffi::Py_eval_input, globals, locals) } @@ -613,7 +640,12 @@ impl<'py> Python<'py> { globals: Option<&PyDict>, locals: Option<&PyDict>, ) -> PyResult<()> { - let res = self.run_code(code, ffi::Py_file_input, globals, locals); + let res = self.run_code( + code, + ffi::Py_file_input, + globals.map(PyNativeType::as_borrowed).as_deref(), + locals.map(PyNativeType::as_borrowed).as_deref(), + ); res.map(|obj| { debug_assert!(obj.is_none()); }) @@ -630,9 +662,9 @@ impl<'py> Python<'py> { self, code: &str, start: c_int, - globals: Option<&PyDict>, - locals: Option<&PyDict>, - ) -> PyResult<&'py PyAny> { + globals: Option<&Bound<'py, PyDict>>, + locals: Option<&Bound<'py, PyDict>>, + ) -> PyResult> { let code = CString::new(code)?; unsafe { let mptr = ffi::PyImport_AddModule("__main__\0".as_ptr() as *const _); @@ -675,7 +707,7 @@ impl<'py> Python<'py> { let res_ptr = ffi::PyEval_EvalCode(code_obj, globals, locals); ffi::Py_DECREF(code_obj); - self.from_owned_ptr_or_err(res_ptr) + res_ptr.assume_owned_or_err(self).downcast_into_unchecked() } } @@ -1077,18 +1109,18 @@ mod tests { Python::with_gil(|py| { // Make sure builtin names are accessible let v: i32 = py - .eval("min(1, 2)", None, None) + .eval_bound("min(1, 2)", None, None) .map_err(|e| e.display(py)) .unwrap() .extract() .unwrap(); assert_eq!(v, 1); - let d = [("foo", 13)].into_py_dict(py); + let d = [("foo", 13)].into_py_dict(py).as_borrowed(); // Inject our own global namespace let v: i32 = py - .eval("foo + 29", Some(d), None) + .eval_bound("foo + 29", Some(&d), None) .unwrap() .extract() .unwrap(); @@ -1096,7 +1128,7 @@ mod tests { // Inject our own local namespace let v: i32 = py - .eval("foo + 29", None, Some(d)) + .eval_bound("foo + 29", None, Some(&d)) .unwrap() .extract() .unwrap(); @@ -1104,7 +1136,7 @@ mod tests { // Make sure builtin names are still accessible when using a local namespace let v: i32 = py - .eval("min(foo, 2)", None, Some(d)) + .eval_bound("min(foo, 2)", None, Some(&d)) .unwrap() .extract() .unwrap(); @@ -1193,7 +1225,7 @@ mod tests { assert_eq!(py.Ellipsis().to_string(), "Ellipsis"); let v = py - .eval("...", None, None) + .eval_bound("...", None, None) .map_err(|e| e.display(py)) .unwrap(); diff --git a/src/types/iterator.rs b/src/types/iterator.rs index e27032867bc..4ea83271eaa 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -14,10 +14,10 @@ use crate::{ /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { -/// let list = py.eval("iter([1, 2, 3, 4])", None, None)?; +/// let list = py.eval_bound("iter([1, 2, 3, 4])", None, None)?; /// let numbers: PyResult> = list /// .iter()? -/// .map(|i| i.and_then(PyAny::extract::)) +/// .map(|i| i.and_then(|i|i.extract::())) /// .collect(); /// let sum: usize = numbers?.iter().sum(); /// assert_eq!(sum, 10); diff --git a/src/types/mod.rs b/src/types/mod.rs index f802348f466..88e78a16d60 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -59,9 +59,9 @@ pub use self::typeobject::PyType; /// /// # pub fn main() -> PyResult<()> { /// Python::with_gil(|py| { -/// let dict: &PyDict = py.eval("{'a':'b', 'c':'d'}", None, None)?.downcast()?; +/// let dict = py.eval_bound("{'a':'b', 'c':'d'}", None, None)?.downcast_into::()?; /// -/// for (key, value) in dict { +/// for (key, value) in dict.iter() { /// println!("key: {}, value: {}", key, value); /// } /// diff --git a/tests/test_datetime.rs b/tests/test_datetime.rs index 32163abe1e0..bfde37d317b 100644 --- a/tests/test_datetime.rs +++ b/tests/test_datetime.rs @@ -4,11 +4,11 @@ use pyo3::prelude::*; use pyo3::types::{timezone_utc, IntoPyDict, PyDate, PyDateTime, PyTime}; use pyo3_ffi::PyDateTime_IMPORT; -fn _get_subclasses<'p>( - py: Python<'p>, +fn _get_subclasses<'py>( + py: Python<'py>, py_type: &str, args: &str, -) -> PyResult<(&'p PyAny, &'p PyAny, &'p PyAny)> { +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>)> { // Import the class from Python and create some subclasses let datetime = py.import("datetime")?; @@ -21,14 +21,15 @@ fn _get_subclasses<'p>( py.run(&make_subclass_py, None, Some(locals))?; py.run(make_sub_subclass_py, None, Some(locals))?; + let locals = locals.as_borrowed(); // Construct an instance of the base class - let obj = py.eval(&format!("{}({})", py_type, args), None, Some(locals))?; + let obj = py.eval_bound(&format!("{}({})", py_type, args), None, Some(&locals))?; // Construct an instance of the subclass - let sub_obj = py.eval(&format!("Subklass({})", args), None, Some(locals))?; + let sub_obj = py.eval_bound(&format!("Subklass({})", args), None, Some(&locals))?; // Construct an instance of the sub-subclass - let sub_sub_obj = py.eval(&format!("SubSubklass({})", args), None, Some(locals))?; + let sub_sub_obj = py.eval_bound(&format!("SubSubklass({})", args), None, Some(&locals))?; Ok((obj, sub_obj, sub_sub_obj)) } @@ -122,10 +123,10 @@ fn test_datetime_utc() { let dt = PyDateTime::new(py, 2018, 1, 1, 0, 0, 0, 0, Some(utc)).unwrap(); - let locals = [("dt", dt)].into_py_dict(py); + let locals = [("dt", dt)].into_py_dict(py).as_borrowed(); let offset: f32 = py - .eval("dt.utcoffset().total_seconds()", None, Some(locals)) + .eval_bound("dt.utcoffset().total_seconds()", None, Some(&locals)) .unwrap() .extract() .unwrap(); diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index c6f6e148943..82e91b84390 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -486,14 +486,14 @@ pub struct Zap { fn test_from_py_with() { Python::with_gil(|py| { let py_zap = py - .eval( + .eval_bound( r#"{"name": "whatever", "my_object": [1, 2, 3]}"#, None, None, ) .expect("failed to create dict"); - let zap = Zap::extract(py_zap).unwrap(); + let zap = Zap::extract_bound(&py_zap).unwrap(); assert_eq!(zap.name, "whatever"); assert_eq!(zap.some_object_length, 3usize); @@ -507,10 +507,10 @@ pub struct ZapTuple(String, #[pyo3(from_py_with = "PyAny::len")] usize); fn test_from_py_with_tuple_struct() { Python::with_gil(|py| { let py_zap = py - .eval(r#"("whatever", [1, 2, 3])"#, None, None) + .eval_bound(r#"("whatever", [1, 2, 3])"#, None, None) .expect("failed to create tuple"); - let zap = ZapTuple::extract(py_zap).unwrap(); + let zap = ZapTuple::extract_bound(&py_zap).unwrap(); assert_eq!(zap.0, "whatever"); assert_eq!(zap.1, 3usize); @@ -521,10 +521,10 @@ fn test_from_py_with_tuple_struct() { fn test_from_py_with_tuple_struct_error() { Python::with_gil(|py| { let py_zap = py - .eval(r#"("whatever", [1, 2, 3], "third")"#, None, None) + .eval_bound(r#"("whatever", [1, 2, 3], "third")"#, None, None) .expect("failed to create tuple"); - let f = ZapTuple::extract(py_zap); + let f = ZapTuple::extract_bound(&py_zap); assert!(f.is_err()); assert_eq!( @@ -544,10 +544,10 @@ pub enum ZapEnum { fn test_from_py_with_enum() { Python::with_gil(|py| { let py_zap = py - .eval(r#"("whatever", [1, 2, 3])"#, None, None) + .eval_bound(r#"("whatever", [1, 2, 3])"#, None, None) .expect("failed to create tuple"); - let zap = ZapEnum::extract(py_zap).unwrap(); + let zap = ZapEnum::extract_bound(&py_zap).unwrap(); let expected_zap = ZapEnum::Zip(2); assert_eq!(zap, expected_zap); diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index d1cfe628ef6..16f87df9d14 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -240,7 +240,7 @@ mod inheriting_native_type { let dict_sub = pyo3::Py::new(py, DictWithName::new()).unwrap(); assert_eq!(dict_sub.get_refcnt(py), 1); - let item = py.eval("object()", None, None).unwrap(); + let item = &py.eval_bound("object()", None, None).unwrap(); assert_eq!(item.get_refcnt(), 1); dict_sub.as_ref(py).set_item("foo", item).unwrap(); From 4d423b0c67e34f3e8820aefc76cb69805f8d1522 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Mon, 5 Feb 2024 19:44:00 +0100 Subject: [PATCH 113/349] port `Python::run` to `Bound` API --- guide/src/class/numeric.md | 4 +-- guide/src/module.md | 4 +-- guide/src/python_from_rust.md | 4 +-- src/conversions/anyhow.rs | 8 ++--- src/conversions/chrono.rs | 9 ++++-- src/conversions/eyre.rs | 8 ++--- src/conversions/rust_decimal.rs | 26 ++++++++-------- src/conversions/std/num.rs | 19 +++++++----- src/err/mod.rs | 8 ++--- src/exceptions.rs | 53 +++++++++++++++++---------------- src/ffi/tests.rs | 18 +++++------ src/gil.rs | 4 +-- src/macros.rs | 5 ++-- src/marker.rs | 48 +++++++++++++++++++---------- src/tests/common.rs | 3 +- src/types/bytearray.rs | 12 ++++---- src/types/traceback.rs | 20 ++++++------- tests/test_anyhow.rs | 7 +++-- tests/test_append_to_inittab.rs | 2 +- tests/test_class_new.rs | 7 +++-- tests/test_coroutine.rs | 18 +++++------ tests/test_datetime.rs | 8 +++-- tests/test_gc.rs | 8 ++--- tests/test_inheritance.rs | 22 +++++++++----- tests/test_proto_methods.rs | 21 +++++++++---- 25 files changed, 198 insertions(+), 148 deletions(-) diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index 6f2e6e18f86..c6b6a65b711 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -387,10 +387,10 @@ fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { # # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let globals = PyModule::import(py, "__main__")?.dict(); +# let globals = PyModule::import(py, "__main__")?.dict().as_borrowed(); # globals.set_item("Number", Number::type_object_bound(py))?; # -# py.run(SCRIPT, Some(globals), None)?; +# py.run_bound(SCRIPT, Some(&globals), None)?; # Ok(()) # }) # } diff --git a/guide/src/module.md b/guide/src/module.md index 9e52ab93e2b..5d224de40ef 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -93,9 +93,9 @@ fn func() -> String { # use pyo3::wrap_pymodule; # use pyo3::types::IntoPyDict; # let parent_module = wrap_pymodule!(parent_module)(py); -# let ctx = [("parent_module", parent_module)].into_py_dict(py); +# let ctx = [("parent_module", parent_module)].into_py_dict(py).as_borrowed(); # -# py.run("assert parent_module.child_module.func() == 'func'", None, Some(&ctx)).unwrap(); +# py.run_bound("assert parent_module.child_module.func() == 'func'", None, Some(&ctx)).unwrap(); # }) ``` diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 2f3be53a975..753ce620011 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -290,7 +290,7 @@ fn foo(_py: Python<'_>, foo_module: &PyModule) -> PyResult<()> { fn main() -> PyResult<()> { pyo3::append_to_inittab!(foo); - Python::with_gil(|py| Python::run(py, "import foo; foo.add_one(6)", None, None)) + Python::with_gil(|py| Python::run_bound(py, "import foo; foo.add_one(6)", None, None)) } ``` @@ -321,7 +321,7 @@ fn main() -> PyResult<()> { py_modules.set_item("foo", foo_module)?; // Now we can import + run our python code - Python::run(py, "import foo; foo.add_one(6)", None, None) + Python::run_bound(py, "import foo; foo.add_one(6)", None, None) }) } ``` diff --git a/src/conversions/anyhow.rs b/src/conversions/anyhow.rs index 3b5aa053e84..809b2aa20d4 100644 --- a/src/conversions/anyhow.rs +++ b/src/conversions/anyhow.rs @@ -147,8 +147,8 @@ mod test_anyhow { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict(py); - let pyerr = py.run("raise err", None, Some(locals)).unwrap_err(); + let locals = [("err", pyerr)].into_py_dict(py).as_borrowed(); + let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } @@ -164,8 +164,8 @@ mod test_anyhow { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict(py); - let pyerr = py.run("raise err", None, Some(locals)).unwrap_err(); + let locals = [("err", pyerr)].into_py_dict(py).as_borrowed(); + let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 83f9830bcdb..069a137fd33 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -574,12 +574,15 @@ mod tests { // tzdata there to make this work. #[cfg(all(Py_3_9, not(target_os = "windows")))] fn test_zoneinfo_is_not_fixed_offset() { + use crate::types::any::PyAnyMethods; + use crate::types::dict::PyDictMethods; + Python::with_gil(|py| { - let locals = crate::types::PyDict::new(py); - py.run( + let locals = crate::types::PyDict::new_bound(py); + py.run_bound( "import zoneinfo; zi = zoneinfo.ZoneInfo('Europe/London')", None, - Some(locals), + Some(&locals), ) .unwrap(); let result: PyResult = locals.get_item("zi").unwrap().unwrap().extract(); diff --git a/src/conversions/eyre.rs b/src/conversions/eyre.rs index b0559ad1469..7f029cf6da0 100644 --- a/src/conversions/eyre.rs +++ b/src/conversions/eyre.rs @@ -152,8 +152,8 @@ mod tests { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict(py); - let pyerr = py.run("raise err", None, Some(locals)).unwrap_err(); + let locals = [("err", pyerr)].into_py_dict(py).as_borrowed(); + let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } @@ -169,8 +169,8 @@ mod tests { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict(py); - let pyerr = py.run("raise err", None, Some(locals)).unwrap_err(); + let locals = [("err", pyerr)].into_py_dict(py).as_borrowed(); + let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) } diff --git a/src/conversions/rust_decimal.rs b/src/conversions/rust_decimal.rs index 2e38e7808e5..e829e88346e 100644 --- a/src/conversions/rust_decimal.rs +++ b/src/conversions/rust_decimal.rs @@ -108,6 +108,8 @@ impl IntoPy for Decimal { mod test_rust_decimal { use super::*; use crate::err::PyErr; + use crate::types::any::PyAnyMethods; + use crate::types::dict::PyDictMethods; use crate::types::PyDict; use rust_decimal::Decimal; @@ -121,16 +123,16 @@ mod test_rust_decimal { Python::with_gil(|py| { let rs_orig = $rs; let rs_dec = rs_orig.into_py(py); - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("rs_dec", &rs_dec).unwrap(); // Checks if Rust Decimal -> Python Decimal conversion is correct - py.run( + py.run_bound( &format!( "import decimal\npy_dec = decimal.Decimal({})\nassert py_dec == rs_dec", $py ), None, - Some(locals), + Some(&locals), ) .unwrap(); // Checks if Python Decimal -> Rust Decimal conversion is correct @@ -163,13 +165,13 @@ mod test_rust_decimal { let num = Decimal::from_parts(lo, mid, high, negative, scale); Python::with_gil(|py| { let rs_dec = num.into_py(py); - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("rs_dec", &rs_dec).unwrap(); - py.run( + py.run_bound( &format!( "import decimal\npy_dec = decimal.Decimal(\"{}\")\nassert py_dec == rs_dec", num), - None, Some(locals)).unwrap(); + None, Some(&locals)).unwrap(); let roundtripped: Decimal = rs_dec.extract(py).unwrap(); assert_eq!(num, roundtripped); }) @@ -189,11 +191,11 @@ mod test_rust_decimal { #[test] fn test_nan() { Python::with_gil(|py| { - let locals = PyDict::new(py); - py.run( + let locals = PyDict::new_bound(py); + py.run_bound( "import decimal\npy_dec = decimal.Decimal(\"NaN\")", None, - Some(locals), + Some(&locals), ) .unwrap(); let py_dec = locals.get_item("py_dec").unwrap().unwrap(); @@ -205,11 +207,11 @@ mod test_rust_decimal { #[test] fn test_infinity() { Python::with_gil(|py| { - let locals = PyDict::new(py); - py.run( + let locals = PyDict::new_bound(py); + py.run_bound( "import decimal\npy_dec = decimal.Decimal(\"Infinity\")", None, - Some(locals), + Some(&locals), ) .unwrap(); let py_dec = locals.get_item("py_dec").unwrap().unwrap(); diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 2d20915d84f..75c2e16b52b 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -376,6 +376,9 @@ mod test_128bit_integers { #[cfg(not(target_arch = "wasm32"))] use crate::types::PyDict; + #[cfg(not(target_arch = "wasm32"))] + use crate::types::dict::PyDictMethods; + #[cfg(not(target_arch = "wasm32"))] use proptest::prelude::*; @@ -385,9 +388,9 @@ mod test_128bit_integers { fn test_i128_roundtrip(x: i128) { Python::with_gil(|py| { let x_py = x.into_py(py); - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("x_py", x_py.clone_ref(py)).unwrap(); - py.run(&format!("assert x_py == {}", x), None, Some(locals)).unwrap(); + py.run_bound(&format!("assert x_py == {}", x), None, Some(&locals)).unwrap(); let roundtripped: i128 = x_py.extract(py).unwrap(); assert_eq!(x, roundtripped); }) @@ -401,9 +404,9 @@ mod test_128bit_integers { ) { Python::with_gil(|py| { let x_py = x.into_py(py); - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("x_py", x_py.clone_ref(py)).unwrap(); - py.run(&format!("assert x_py == {}", x), None, Some(locals)).unwrap(); + py.run_bound(&format!("assert x_py == {}", x), None, Some(&locals)).unwrap(); let roundtripped: NonZeroI128 = x_py.extract(py).unwrap(); assert_eq!(x, roundtripped); }) @@ -416,9 +419,9 @@ mod test_128bit_integers { fn test_u128_roundtrip(x: u128) { Python::with_gil(|py| { let x_py = x.into_py(py); - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("x_py", x_py.clone_ref(py)).unwrap(); - py.run(&format!("assert x_py == {}", x), None, Some(locals)).unwrap(); + py.run_bound(&format!("assert x_py == {}", x), None, Some(&locals)).unwrap(); let roundtripped: u128 = x_py.extract(py).unwrap(); assert_eq!(x, roundtripped); }) @@ -432,9 +435,9 @@ mod test_128bit_integers { ) { Python::with_gil(|py| { let x_py = x.into_py(py); - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("x_py", x_py.clone_ref(py)).unwrap(); - py.run(&format!("assert x_py == {}", x), None, Some(locals)).unwrap(); + py.run_bound(&format!("assert x_py == {}", x), None, Some(&locals)).unwrap(); let roundtripped: NonZeroU128 = x_py.extract(py).unwrap(); assert_eq!(x, roundtripped); }) diff --git a/src/err/mod.rs b/src/err/mod.rs index 8568545664b..e53a9657998 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -1030,7 +1030,7 @@ mod tests { Python::with_gil(|py| { let err = py - .run("raise Exception('banana')", None, None) + .run_bound("raise Exception('banana')", None, None) .expect_err("raising should have given us an error"); let debug_str = format!("{:?}", err); @@ -1055,7 +1055,7 @@ mod tests { fn err_display() { Python::with_gil(|py| { let err = py - .run("raise Exception('banana')", None, None) + .run_bound("raise Exception('banana')", None, None) .expect_err("raising should have given us an error"); assert_eq!(err.to_string(), "Exception: banana"); }); @@ -1102,12 +1102,12 @@ mod tests { fn test_pyerr_cause() { Python::with_gil(|py| { let err = py - .run("raise Exception('banana')", None, None) + .run_bound("raise Exception('banana')", None, None) .expect_err("raising should have given us an error"); assert!(err.cause(py).is_none()); let err = py - .run( + .run_bound( "raise Exception('banana') from Exception('apple')", None, None, diff --git a/src/exceptions.rs b/src/exceptions.rs index daea55e5abd..10121b8a3ad 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -165,18 +165,18 @@ macro_rules! import_exception { /// # fn main() -> PyResult<()> { /// # Python::with_gil(|py| -> PyResult<()> { /// # let fun = wrap_pyfunction!(raise_myerror, py)?; -/// # let locals = pyo3::types::PyDict::new(py); +/// # let locals = pyo3::types::PyDict::new_bound(py); /// # locals.set_item("MyError", py.get_type::())?; /// # locals.set_item("raise_myerror", fun)?; /// # -/// # py.run( +/// # py.run_bound( /// # "try: /// # raise_myerror() /// # except MyError as e: /// # assert e.__doc__ == 'Some description.' /// # assert str(e) == 'Some error happened.'", /// # None, -/// # Some(locals), +/// # Some(&locals), /// # )?; /// # /// # Ok(()) @@ -338,7 +338,7 @@ use pyo3::prelude::*; use pyo3::exceptions::Py", $name, "; Python::with_gil(|py| { - let result: PyResult<()> = py.run(\"raise ", $name, "\", None, None); + let result: PyResult<()> = py.run_bound(\"raise ", $name, "\", None, None); let error_type = match result { Ok(_) => \"Not an error\", @@ -816,7 +816,7 @@ mod tests { .map_err(|e| e.display(py)) .expect("could not import socket"); - let d = PyDict::new(py); + let d = PyDict::new_bound(py); d.set_item("socket", socket) .map_err(|e| e.display(py)) .expect("could not setitem"); @@ -825,7 +825,7 @@ mod tests { .map_err(|e| e.display(py)) .expect("could not setitem"); - py.run("assert isinstance(exc, socket.gaierror)", None, Some(d)) + py.run_bound("assert isinstance(exc, socket.gaierror)", None, Some(&d)) .map_err(|e| e.display(py)) .expect("assertion failed"); }); @@ -840,7 +840,7 @@ mod tests { .map_err(|e| e.display(py)) .expect("could not import email"); - let d = PyDict::new(py); + let d = PyDict::new_bound(py); d.set_item("email", email) .map_err(|e| e.display(py)) .expect("could not setitem"); @@ -848,10 +848,10 @@ mod tests { .map_err(|e| e.display(py)) .expect("could not setitem"); - py.run( + py.run_bound( "assert isinstance(exc, email.errors.MessageError)", None, - Some(d), + Some(&d), ) .map_err(|e| e.display(py)) .expect("assertion failed"); @@ -871,14 +871,13 @@ mod tests { .extract() .unwrap(); assert_eq!(type_description, ""); - let ctx = ctx.as_gil_ref(); - py.run( + py.run_bound( "assert CustomError('oops').args == ('oops',)", None, - Some(ctx), + Some(&ctx), ) .unwrap(); - py.run("assert CustomError.__doc__ is None", None, Some(ctx)) + py.run_bound("assert CustomError.__doc__ is None", None, Some(&ctx)) .unwrap(); }); } @@ -914,15 +913,18 @@ mod tests { .extract() .unwrap(); assert_eq!(type_description, ""); - let ctx = ctx.as_gil_ref(); - py.run( + py.run_bound( "assert CustomError('oops').args == ('oops',)", None, - Some(ctx), + Some(&ctx), + ) + .unwrap(); + py.run_bound( + "assert CustomError.__doc__ == 'Some docs'", + None, + Some(&ctx), ) .unwrap(); - py.run("assert CustomError.__doc__ == 'Some docs'", None, Some(ctx)) - .unwrap(); }); } @@ -944,17 +946,16 @@ mod tests { .extract() .unwrap(); assert_eq!(type_description, ""); - let ctx = ctx.as_gil_ref(); - py.run( + py.run_bound( "assert CustomError('oops').args == ('oops',)", None, - Some(ctx), + Some(&ctx), ) .unwrap(); - py.run( + py.run_bound( "assert CustomError.__doc__ == 'Some more docs'", None, - Some(ctx), + Some(&ctx), ) .unwrap(); }); @@ -964,7 +965,7 @@ mod tests { fn native_exception_debug() { Python::with_gil(|py| { let exc = py - .run("raise Exception('banana')", None, None) + .run_bound("raise Exception('banana')", None, None) .expect_err("raising should have given us an error") .into_value(py) .into_ref(py); @@ -979,7 +980,7 @@ mod tests { fn native_exception_display() { Python::with_gil(|py| { let exc = py - .run("raise Exception('banana')", None, None) + .run_bound("raise Exception('banana')", None, None) .expect_err("raising should have given us an error") .into_value(py) .into_ref(py); @@ -996,7 +997,7 @@ mod tests { Python::with_gil(|py| { let exc = py - .run( + .run_bound( "raise Exception('banana') from TypeError('peach')", None, None, diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index fc10bf05f5d..0735e456950 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -20,12 +20,12 @@ fn test_datetime_fromtimestamp() { PyDateTime_IMPORT(); py.from_owned_ptr(PyDateTime_FromTimestamp(args.as_ptr())) }; - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("dt", dt).unwrap(); - py.run( + py.run_bound( "import datetime; assert dt == datetime.datetime.fromtimestamp(100)", None, - Some(locals), + Some(&locals), ) .unwrap(); }) @@ -41,12 +41,12 @@ fn test_date_fromtimestamp() { PyDateTime_IMPORT(); py.from_owned_ptr(PyDate_FromTimestamp(args.as_ptr())) }; - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("dt", dt).unwrap(); - py.run( + py.run_bound( "import datetime; assert dt == datetime.date.fromtimestamp(100)", None, - Some(locals), + Some(&locals), ) .unwrap(); }) @@ -61,12 +61,12 @@ fn test_utc_timezone() { PyDateTime_IMPORT(); py.from_borrowed_ptr(PyDateTime_TimeZone_UTC()) }; - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("utc_timezone", utc_timezone).unwrap(); - py.run( + py.run_bound( "import datetime; assert utc_timezone is datetime.timezone.utc", None, - Some(locals), + Some(&locals), ) .unwrap(); }) diff --git a/src/gil.rs b/src/gil.rs index 61d69ed9576..379dab4b010 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -75,7 +75,7 @@ fn gil_is_acquired() -> bool { /// /// # fn main() -> PyResult<()> { /// pyo3::prepare_freethreaded_python(); -/// Python::with_gil(|py| py.run("print('Hello World')", None, None)) +/// Python::with_gil(|py| py.run_bound("print('Hello World')", None, None)) /// # } /// ``` #[cfg(not(PyPy))] @@ -118,7 +118,7 @@ pub fn prepare_freethreaded_python() { /// ```rust /// unsafe { /// pyo3::with_embedded_python_interpreter(|py| { -/// if let Err(e) = py.run("print('Hello World')", None, None) { +/// if let Err(e) = py.run_bound("print('Hello World')", None, None) { /// // We must make sure to not return a `PyErr`! /// e.print(py); /// } diff --git a/src/macros.rs b/src/macros.rs index 41de9079c40..79e8ea58cb7 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -104,12 +104,13 @@ macro_rules! py_run_impl { }}; ($py:expr, *$dict:expr, $code:expr) => {{ use ::std::option::Option::*; - if let ::std::result::Result::Err(e) = $py.run($code, None, Some($dict)) { + use $crate::PyNativeType; + if let ::std::result::Result::Err(e) = $py.run_bound($code, None, Some(&$dict.as_borrowed())) { e.print($py); // So when this c api function the last line called printed the error to stderr, // the output is only written into a buffer which is never flushed because we // panic before flushing. This is where this hack comes into place - $py.run("import sys; sys.stderr.flush()", None, None) + $py.run_bound("import sys; sys.stderr.flush()", None, None) .unwrap(); ::std::panic!("{}", $code) } diff --git a/src/marker.rs b/src/marker.rs index 1802d858d10..121cfe6855f 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -600,6 +600,27 @@ impl<'py> Python<'py> { self.run_code(code, ffi::Py_eval_input, globals, locals) } + /// Deprecated version of [`Python::run_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`Python::run` will be replaced by `Python::run_bound` in a future PyO3 version" + ) + )] + pub fn run( + self, + code: &str, + globals: Option<&PyDict>, + locals: Option<&PyDict>, + ) -> PyResult<()> { + self.run_bound( + code, + globals.map(PyNativeType::as_borrowed).as_deref(), + locals.map(PyNativeType::as_borrowed).as_deref(), + ) + } + /// Executes one or more Python statements in the given context. /// /// If `globals` is `None`, it defaults to Python module `__main__`. @@ -615,37 +636,32 @@ impl<'py> Python<'py> { /// types::{PyBytes, PyDict}, /// }; /// Python::with_gil(|py| { - /// let locals = PyDict::new(py); - /// py.run( + /// let locals = PyDict::new_bound(py); + /// py.run_bound( /// r#" /// import base64 /// s = 'Hello Rust!' /// ret = base64.b64encode(s.encode('utf-8')) /// "#, /// None, - /// Some(locals), + /// Some(&locals), /// ) /// .unwrap(); /// let ret = locals.get_item("ret").unwrap().unwrap(); - /// let b64: &PyBytes = ret.downcast().unwrap(); + /// let b64 = ret.downcast::().unwrap(); /// assert_eq!(b64.as_bytes(), b"SGVsbG8gUnVzdCE="); /// }); /// ``` /// /// You can use [`py_run!`](macro.py_run.html) for a handy alternative of `run` /// if you don't need `globals` and unwrapping is OK. - pub fn run( + pub fn run_bound( self, code: &str, - globals: Option<&PyDict>, - locals: Option<&PyDict>, + globals: Option<&Bound<'py, PyDict>>, + locals: Option<&Bound<'py, PyDict>>, ) -> PyResult<()> { - let res = self.run_code( - code, - ffi::Py_file_input, - globals.map(PyNativeType::as_borrowed).as_deref(), - locals.map(PyNativeType::as_borrowed).as_deref(), - ); + let res = self.run_code(code, ffi::Py_file_input, globals, locals); res.map(|obj| { debug_assert!(obj.is_none()); }) @@ -1235,9 +1251,11 @@ mod tests { #[test] fn test_py_run_inserts_globals() { + use crate::types::dict::PyDictMethods; + Python::with_gil(|py| { - let namespace = PyDict::new(py); - py.run("class Foo: pass", Some(namespace), Some(namespace)) + let namespace = PyDict::new_bound(py); + py.run_bound("class Foo: pass", Some(&namespace), Some(&namespace)) .unwrap(); assert!(matches!(namespace.get_item("Foo"), Ok(Some(..)))); assert!(matches!(namespace.get_item("__builtins__"), Ok(Some(..)))); diff --git a/src/tests/common.rs b/src/tests/common.rs index efab3ccb4b3..e424c1ca6fd 100644 --- a/src/tests/common.rs +++ b/src/tests/common.rs @@ -40,7 +40,8 @@ mod inner { }}; // Case2: dict & no err_msg ($py:expr, *$dict:expr, $code:expr, $err:ident) => {{ - let res = $py.run($code, None, Some($dict)); + use pyo3::PyNativeType; + let res = $py.run_bound($code, None, Some(&$dict.as_borrowed())); let err = res.expect_err(&format!("Did not raise {}", stringify!($err))); if !err.matches($py, $py.get_type::()) { panic!("Expected {} but got {:?}", stringify!($err), err) diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 6303a87de3e..7f0fdf9ebbe 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -197,10 +197,10 @@ impl PyByteArray { /// # fn main() -> PyResult<()> { /// # Python::with_gil(|py| -> PyResult<()> { /// # let fun = wrap_pyfunction!(a_valid_function, py)?; - /// # let locals = pyo3::types::PyDict::new(py); + /// # let locals = pyo3::types::PyDict::new_bound(py); /// # locals.set_item("a_valid_function", fun)?; /// # - /// # py.run( + /// # py.run_bound( /// # r#"b = bytearray(b"hello world") /// # a_valid_function(b) /// # @@ -209,7 +209,7 @@ impl PyByteArray { /// # except RuntimeError as e: /// # assert str(e) == 'input is not long enough'"#, /// # None, - /// # Some(locals), + /// # Some(&locals), /// # )?; /// # /// # Ok(()) @@ -359,10 +359,10 @@ pub trait PyByteArrayMethods<'py> { /// # fn main() -> PyResult<()> { /// # Python::with_gil(|py| -> PyResult<()> { /// # let fun = wrap_pyfunction!(a_valid_function, py)?; - /// # let locals = pyo3::types::PyDict::new(py); + /// # let locals = pyo3::types::PyDict::new_bound(py); /// # locals.set_item("a_valid_function", fun)?; /// # - /// # py.run( + /// # py.run_bound( /// # r#"b = bytearray(b"hello world") /// # a_valid_function(b) /// # @@ -371,7 +371,7 @@ pub trait PyByteArrayMethods<'py> { /// # except RuntimeError as e: /// # assert str(e) == 'input is not long enough'"#, /// # None, - /// # Some(locals), + /// # Some(&locals), /// # )?; /// # /// # Ok(()) diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 84ecda747eb..3608772fbd0 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -28,7 +28,7 @@ impl PyTraceback { /// # let result: PyResult<()> = /// Python::with_gil(|py| { /// let err = py - /// .run("raise Exception('banana')", None, None) + /// .run_bound("raise Exception('banana')", None, None) /// .expect_err("raise will create a Python error"); /// /// let traceback = err.traceback_bound(py).expect("raised exception will have a traceback"); @@ -71,7 +71,7 @@ pub trait PyTracebackMethods<'py> { /// # let result: PyResult<()> = /// Python::with_gil(|py| { /// let err = py - /// .run("raise Exception('banana')", None, None) + /// .run_bound("raise Exception('banana')", None, None) /// .expect_err("raise will create a Python error"); /// /// let traceback = err.traceback_bound(py).expect("raised exception will have a traceback"); @@ -121,7 +121,7 @@ mod tests { fn format_traceback() { Python::with_gil(|py| { let err = py - .run("raise Exception('banana')", None, None) + .run_bound("raise Exception('banana')", None, None) .expect_err("raising should have given us an error"); assert_eq!( @@ -134,9 +134,9 @@ mod tests { #[test] fn test_err_from_value() { Python::with_gil(|py| { - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); // Produce an error from python so that it has a traceback - py.run( + py.run_bound( r" try: raise ValueError('raised exception') @@ -144,10 +144,10 @@ except Exception as e: err = e ", None, - Some(locals), + Some(&locals), ) .unwrap(); - let err = PyErr::from_value(locals.get_item("err").unwrap().unwrap()); + let err = PyErr::from_value(locals.get_item("err").unwrap().unwrap().into_gil_ref()); let traceback = err.value(py).getattr("__traceback__").unwrap(); assert!(err.traceback_bound(py).unwrap().is(traceback)); }) @@ -156,15 +156,15 @@ except Exception as e: #[test] fn test_err_into_py() { Python::with_gil(|py| { - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); // Produce an error from python so that it has a traceback - py.run( + py.run_bound( r" def f(): raise ValueError('raised exception') ", None, - Some(locals), + Some(&locals), ) .unwrap(); let f = locals.get_item("f").unwrap().unwrap(); diff --git a/tests/test_anyhow.rs b/tests/test_anyhow.rs index f0df4dcc1a4..1807cfe9708 100644 --- a/tests/test_anyhow.rs +++ b/tests/test_anyhow.rs @@ -25,6 +25,7 @@ fn test_anyhow_py_function_ok_result() { #[test] fn test_anyhow_py_function_err_result() { + use pyo3::prelude::PyDictMethods; use pyo3::{pyfunction, types::PyDict, wrap_pyfunction, Python}; #[pyfunction] @@ -34,15 +35,15 @@ fn test_anyhow_py_function_err_result() { Python::with_gil(|py| { let func = wrap_pyfunction!(produce_err_result)(py).unwrap(); - let locals = PyDict::new(py); + let locals = PyDict::new_bound(py); locals.set_item("func", func).unwrap(); - py.run( + py.run_bound( r#" func() "#, None, - Some(locals), + Some(&locals), ) .unwrap_err(); }); diff --git a/tests/test_append_to_inittab.rs b/tests/test_append_to_inittab.rs index e0a57da1b5c..00cccdbb49e 100644 --- a/tests/test_append_to_inittab.rs +++ b/tests/test_append_to_inittab.rs @@ -18,7 +18,7 @@ fn test_module_append_to_inittab() { use pyo3::append_to_inittab; append_to_inittab!(module_with_functions); Python::with_gil(|py| { - py.run( + py.run_bound( r#" import module_with_functions assert module_with_functions.foo() == 123 diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index 8cb426861db..8084be057cd 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -169,9 +169,12 @@ c = Class() assert c.from_rust is False "# ); - let globals = PyModule::import(py, "__main__").unwrap().dict(); + let globals = PyModule::import(py, "__main__") + .unwrap() + .dict() + .as_borrowed(); globals.set_item("SuperClass", super_cls).unwrap(); - py.run(source, Some(globals), None) + py.run_bound(source, Some(&globals), None) .map_err(|e| e.display(py)) .unwrap(); }); diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index 206c35da93c..d9672cd68f0 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -127,12 +127,12 @@ fn cancelled_coroutine() { await task asyncio.run(main()) "#; - let globals = gil.import("__main__").unwrap().dict(); + let globals = gil.import("__main__").unwrap().dict().as_borrowed(); globals.set_item("sleep", sleep).unwrap(); let err = gil - .run( + .run_bound( &pyo3::unindent::unindent(&handle_windows(test)), - Some(globals), + Some(&globals), None, ) .unwrap_err(); @@ -166,13 +166,13 @@ fn coroutine_cancel_handle() { return await task assert asyncio.run(main()) == 0 "#; - let globals = gil.import("__main__").unwrap().dict(); + let globals = gil.import("__main__").unwrap().dict().as_borrowed(); globals .set_item("cancellable_sleep", cancellable_sleep) .unwrap(); - gil.run( + gil.run_bound( &pyo3::unindent::unindent(&handle_windows(test)), - Some(globals), + Some(&globals), None, ) .unwrap(); @@ -198,11 +198,11 @@ fn coroutine_is_cancelled() { await task asyncio.run(main()) "#; - let globals = gil.import("__main__").unwrap().dict(); + let globals = gil.import("__main__").unwrap().dict().as_borrowed(); globals.set_item("sleep_loop", sleep_loop).unwrap(); - gil.run( + gil.run_bound( &pyo3::unindent::unindent(&handle_windows(test)), - Some(globals), + Some(&globals), None, ) .unwrap(); diff --git a/tests/test_datetime.rs b/tests/test_datetime.rs index bfde37d317b..d1efd1efe2c 100644 --- a/tests/test_datetime.rs +++ b/tests/test_datetime.rs @@ -12,14 +12,16 @@ fn _get_subclasses<'py>( // Import the class from Python and create some subclasses let datetime = py.import("datetime")?; - let locals = [(py_type, datetime.getattr(py_type)?)].into_py_dict(py); + let locals = [(py_type, datetime.getattr(py_type)?)] + .into_py_dict(py) + .as_borrowed(); let make_subclass_py = format!("class Subklass({}):\n pass", py_type); let make_sub_subclass_py = "class SubSubklass(Subklass):\n pass"; - py.run(&make_subclass_py, None, Some(locals))?; - py.run(make_sub_subclass_py, None, Some(locals))?; + py.run_bound(&make_subclass_py, None, Some(&locals))?; + py.run_bound(make_sub_subclass_py, None, Some(&locals))?; let locals = locals.as_borrowed(); // Construct an instance of the base class diff --git a/tests/test_gc.rs b/tests/test_gc.rs index 43cdb1b4811..7ed39436062 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -117,7 +117,7 @@ fn gc_integration() { }); Python::with_gil(|py| { - py.run("import gc; gc.collect()", None, None).unwrap(); + py.run_bound("import gc; gc.collect()", None, None).unwrap(); assert!(drop_called.load(Ordering::Relaxed)); }); } @@ -156,7 +156,7 @@ fn gc_null_traversal() { obj.borrow_mut(py).cycle = Some(obj.clone_ref(py)); // the object doesn't have to be cleaned up, it just needs to be traversed. - py.run("import gc; gc.collect()", None, None).unwrap(); + py.run_bound("import gc; gc.collect()", None, None).unwrap(); }); } @@ -469,7 +469,7 @@ fn drop_during_traversal_with_gil() { // (but not too many) collections to get `inst` actually dropped. for _ in 0..10 { Python::with_gil(|py| { - py.run("import gc; gc.collect()", None, None).unwrap(); + py.run_bound("import gc; gc.collect()", None, None).unwrap(); }); } assert!(drop_called.load(Ordering::Relaxed)); @@ -502,7 +502,7 @@ fn drop_during_traversal_without_gil() { // (but not too many) collections to get `inst` actually dropped. for _ in 0..10 { Python::with_gil(|py| { - py.run("import gc; gc.collect()", None, None).unwrap(); + py.run_bound("import gc; gc.collect()", None, None).unwrap(); }); } assert!(drop_called.load(Ordering::Relaxed)); diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 16f87df9d14..4b024ed75e5 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -20,12 +20,14 @@ struct SubclassAble {} #[test] fn subclass() { Python::with_gil(|py| { - let d = [("SubclassAble", py.get_type::())].into_py_dict(py); + let d = [("SubclassAble", py.get_type::())] + .into_py_dict(py) + .as_borrowed(); - py.run( + py.run_bound( "class A(SubclassAble): pass\nassert issubclass(A, SubclassAble)", None, - Some(d), + Some(&d), ) .map_err(|e| e.display(py)) .unwrap(); @@ -97,9 +99,13 @@ fn call_base_and_sub_methods() { fn mutation_fails() { Python::with_gil(|py| { let obj = PyCell::new(py, SubClass::new()).unwrap(); - let global = Some([("obj", obj)].into_py_dict(py)); + let global = [("obj", obj)].into_py_dict(py).as_borrowed(); let e = py - .run("obj.base_set(lambda: obj.sub_set_and_ret(1))", global, None) + .run_bound( + "obj.base_set(lambda: obj.sub_set_and_ret(1))", + Some(&global), + None, + ) .unwrap_err(); assert_eq!(&e.to_string(), "RuntimeError: Already borrowed"); }); @@ -271,11 +277,11 @@ mod inheriting_native_type { fn custom_exception() { Python::with_gil(|py| { let cls = py.get_type::(); - let dict = [("cls", cls)].into_py_dict(py); - let res = py.run( + let dict = [("cls", cls)].into_py_dict(py).as_borrowed(); + let res = py.run_bound( "e = cls('hello'); assert str(e) == 'hello'; assert e.context == 'Hello :)'; raise e", None, - Some(dict) + Some(&dict) ); let err = res.unwrap_err(); assert!(err.matches(py, cls), "{}", err); diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index 50dd99ce50d..b1fe7548b05 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -687,9 +687,12 @@ if sys.platform == "win32" and sys.version_info >= (3, 8, 0): asyncio.run(main()) "#; - let globals = PyModule::import(py, "__main__").unwrap().dict(); + let globals = PyModule::import(py, "__main__") + .unwrap() + .dict() + .as_borrowed(); globals.set_item("Once", once).unwrap(); - py.run(source, Some(globals), None) + py.run_bound(source, Some(&globals), None) .map_err(|e| e.display(py)) .unwrap(); }); @@ -741,12 +744,15 @@ if sys.platform == "win32" and sys.version_info >= (3, 8, 0): asyncio.run(main()) "#; - let globals = PyModule::import(py, "__main__").unwrap().dict(); + let globals = PyModule::import(py, "__main__") + .unwrap() + .dict() + .as_borrowed(); globals.set_item("Once", once).unwrap(); globals .set_item("AsyncIterator", py.get_type::()) .unwrap(); - py.run(source, Some(globals), None) + py.run_bound(source, Some(&globals), None) .map_err(|e| e.display(py)) .unwrap(); }); @@ -813,9 +819,12 @@ del c.counter assert c.counter.count == 1 "# ); - let globals = PyModule::import(py, "__main__").unwrap().dict(); + let globals = PyModule::import(py, "__main__") + .unwrap() + .dict() + .as_borrowed(); globals.set_item("Counter", counter).unwrap(); - py.run(source, Some(globals), None) + py.run_bound(source, Some(&globals), None) .map_err(|e| e.display(py)) .unwrap(); }); From 559761b2f1255e588043fc4b8afaf89967851f7c Mon Sep 17 00:00:00 2001 From: Juha-Matti Santala Date: Thu, 8 Feb 2024 18:36:10 +0200 Subject: [PATCH 114/349] docs: Clarify the requirement to install nox Installing nox was mentioned in a later section when building the user guide but not at this point earlier in the guide where nox was needed for the first time. --- Contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Contributing.md b/Contributing.md index 3af6a1605db..332645542d7 100644 --- a/Contributing.md +++ b/Contributing.md @@ -48,7 +48,7 @@ There are some specific areas of focus where help is currently needed for the do - Issues requesting documentation improvements are tracked with the [documentation](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) label. - Not all APIs had docs or examples when they were made. The goal is to have documentation on all PyO3 APIs ([#306](https://github.com/PyO3/pyo3/issues/306)). If you see an API lacking a doc, please write one and open a PR! -You can build the docs (including all features) with +To build the docs (including all features), install [`nox`][nox] and then run ```shell nox -s docs -- open From e45fbe493c3dbe5f74f1a4abae5a72cd221b8eb9 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 10 Feb 2024 13:59:55 +0100 Subject: [PATCH 115/349] port `IntoPyDict` to `Bound` API --- README.md | 2 +- guide/src/exception.md | 2 +- guide/src/migration.md | 2 +- guide/src/module.md | 2 +- guide/src/python_from_rust.md | 12 +++---- src/conversions/anyhow.rs | 4 +-- src/conversions/chrono.rs | 2 +- src/conversions/eyre.rs | 4 +-- src/conversions/hashbrown.rs | 6 ++-- src/conversions/indexmap.rs | 10 +++--- src/conversions/std/map.rs | 8 ++--- src/exceptions.rs | 10 +++--- src/impl_/extract_argument.rs | 4 +-- src/instance.rs | 2 +- src/lib.rs | 2 +- src/macros.rs | 5 +-- src/marker.rs | 2 +- src/tests/common.rs | 9 ++--- src/types/any.rs | 4 +-- src/types/dict.rs | 67 ++++++++++++++++++++++------------- tests/test_buffer_protocol.rs | 4 +-- tests/test_class_new.rs | 5 ++- tests/test_coroutine.rs | 4 +-- tests/test_datetime.rs | 7 ++-- tests/test_dict_iter.rs | 2 +- tests/test_getter_setter.rs | 2 +- tests/test_inheritance.rs | 8 ++--- tests/test_macro_docs.rs | 2 +- tests/test_mapping.rs | 4 +-- tests/test_methods.rs | 12 +++---- tests/test_module.rs | 4 +-- tests/test_no_imports.rs | 2 +- tests/test_sequence.rs | 8 ++--- tests/test_static_slots.rs | 4 +-- 34 files changed, 124 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 606ef926ea9..bdb4c3ef196 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ fn main() -> PyResult<()> { let sys = py.import("sys")?; let version: String = sys.getattr("version")?.extract()?; - let locals = [("os", py.import("os")?)].into_py_dict(py).as_borrowed(); + let locals = [("os", py.import("os")?)].into_py_dict_bound(py); let code = "os.getenv('USER') or os.getenv('USERNAME') or 'Unknown'"; let user: String = py.eval_bound(code, None, Some(&locals))?.extract()?; diff --git a/guide/src/exception.md b/guide/src/exception.md index e1ce24980d3..225118576f7 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -24,7 +24,7 @@ use pyo3::exceptions::PyException; create_exception!(mymodule, CustomError, PyException); Python::with_gil(|py| { - let ctx = [("CustomError", py.get_type::())].into_py_dict(py); + let ctx = [("CustomError", py.get_type::())].into_py_dict_bound(py); pyo3::py_run!( py, *ctx, diff --git a/guide/src/migration.md b/guide/src/migration.md index 3c84b4d0925..1c8655be684 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -283,7 +283,7 @@ Python::with_gil(|py| { After: -```rust +```rust,ignore use pyo3::prelude::*; use pyo3::exceptions::PyTypeError; use pyo3::types::{PyDict, IntoPyDict}; diff --git a/guide/src/module.md b/guide/src/module.md index 5d224de40ef..3d984f60d39 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -93,7 +93,7 @@ fn func() -> String { # use pyo3::wrap_pymodule; # use pyo3::types::IntoPyDict; # let parent_module = wrap_pymodule!(parent_module)(py); -# let ctx = [("parent_module", parent_module)].into_py_dict(py).as_borrowed(); +# let ctx = [("parent_module", parent_module)].into_py_dict_bound(py); # # py.run_bound("assert parent_module.child_module.func() == 'func'", None, Some(&ctx)).unwrap(); # }) diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 753ce620011..0c104c2858b 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -94,17 +94,17 @@ fn main() -> PyResult<()> { .into(); // call object with PyDict - let kwargs = [(key1, val1)].into_py_dict(py); - fun.call_bound(py, (), Some(&kwargs.as_borrowed()))?; + let kwargs = [(key1, val1)].into_py_dict_bound(py); + fun.call_bound(py, (), Some(&kwargs))?; // pass arguments as Vec let kwargs = vec![(key1, val1), (key2, val2)]; - fun.call_bound(py, (), Some(&kwargs.into_py_dict(py).as_borrowed()))?; + fun.call_bound(py, (), Some(&kwargs.into_py_dict_bound(py)))?; // pass arguments as HashMap let mut kwargs = HashMap::<&str, i32>::new(); kwargs.insert(key1, 1); - fun.call_bound(py, (), Some(&kwargs.into_py_dict(py).as_borrowed()))?; + fun.call_bound(py, (), Some(&kwargs.into_py_dict_bound(py)))?; Ok(()) }) @@ -250,10 +250,10 @@ def leaky_relu(x, slope=0.01): let relu_result: f64 = activators.getattr("relu")?.call1((-1.0,))?.extract()?; assert_eq!(relu_result, 0.0); - let kwargs = [("slope", 0.2)].into_py_dict(py); + let kwargs = [("slope", 0.2)].into_py_dict_bound(py); let lrelu_result: f64 = activators .getattr("leaky_relu")? - .call((-1.0,), Some(kwargs))? + .call((-1.0,), Some(kwargs.as_gil_ref()))? .extract()?; assert_eq!(lrelu_result, -0.2); # Ok(()) diff --git a/src/conversions/anyhow.rs b/src/conversions/anyhow.rs index 809b2aa20d4..453799c6e8b 100644 --- a/src/conversions/anyhow.rs +++ b/src/conversions/anyhow.rs @@ -147,7 +147,7 @@ mod test_anyhow { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict(py).as_borrowed(); + let locals = [("err", pyerr)].into_py_dict_bound(py); let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) @@ -164,7 +164,7 @@ mod test_anyhow { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict(py).as_borrowed(); + let locals = [("err", pyerr)].into_py_dict_bound(py); let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 069a137fd33..55d49d71d95 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -1109,7 +1109,7 @@ mod tests { fn test_pyo3_offset_fixed_frompyobject_created_in_python(timestamp in 0..(i32::MAX as i64), timedelta in -86399i32..=86399i32) { Python::with_gil(|py| { - let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict(py).as_borrowed(); + let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict_bound(py); let code = format!("datetime.datetime.fromtimestamp({}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={})))", timestamp, timedelta); let t = py.eval_bound(&code, Some(&globals), None).unwrap(); diff --git a/src/conversions/eyre.rs b/src/conversions/eyre.rs index 7f029cf6da0..d25a10af9ee 100644 --- a/src/conversions/eyre.rs +++ b/src/conversions/eyre.rs @@ -152,7 +152,7 @@ mod tests { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict(py).as_borrowed(); + let locals = [("err", pyerr)].into_py_dict_bound(py); let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) @@ -169,7 +169,7 @@ mod tests { let pyerr = PyErr::from(err); Python::with_gil(|py| { - let locals = [("err", pyerr)].into_py_dict(py).as_borrowed(); + let locals = [("err", pyerr)].into_py_dict_bound(py); let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); assert_eq!(pyerr.value(py).to_string(), expected_contents); }) diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index d2cbe4ad8c6..7e57d332d91 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -33,7 +33,7 @@ where H: hash::BuildHasher, { fn to_object(&self, py: Python<'_>) -> PyObject { - IntoPyDict::into_py_dict(self, py).into() + IntoPyDict::into_py_dict_bound(self, py).into() } } @@ -47,7 +47,7 @@ where let iter = self .into_iter() .map(|(k, v)| (k.into_py(py), v.into_py(py))); - IntoPyDict::into_py_dict(iter, py).into() + IntoPyDict::into_py_dict_bound(iter, py).into() } } @@ -164,7 +164,7 @@ mod tests { let mut map = hashbrown::HashMap::::new(); map.insert(1, 1); - let py_map = map.into_py_dict(py); + let py_map = map.into_py_dict_bound(py); assert_eq!(py_map.len(), 1); assert_eq!( diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index 53f7f9364c3..ec7ed5de557 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -100,7 +100,7 @@ where H: hash::BuildHasher, { fn to_object(&self, py: Python<'_>) -> PyObject { - IntoPyDict::into_py_dict(self, py).into() + IntoPyDict::into_py_dict_bound(self, py).into() } } @@ -114,7 +114,7 @@ where let iter = self .into_iter() .map(|(k, v)| (k.into_py(py), v.into_py(py))); - IntoPyDict::into_py_dict(iter, py).into() + IntoPyDict::into_py_dict_bound(iter, py).into() } } @@ -137,6 +137,8 @@ where #[cfg(test)] mod test_indexmap { + use crate::types::any::PyAnyMethods; + use crate::types::dict::PyDictMethods; use crate::types::*; use crate::{IntoPy, PyObject, Python, ToPyObject}; @@ -194,7 +196,7 @@ mod test_indexmap { let mut map = indexmap::IndexMap::::new(); map.insert(1, 1); - let py_map = map.into_py_dict(py); + let py_map = map.into_py_dict_bound(py); assert_eq!(py_map.len(), 1); assert_eq!( @@ -223,7 +225,7 @@ mod test_indexmap { } } - let py_map = map.clone().into_py_dict(py); + let py_map = map.clone().into_py_dict_bound(py); let trip_map = py_map.extract::>().unwrap(); diff --git a/src/conversions/std/map.rs b/src/conversions/std/map.rs index 1c3f669eaee..700617ec17a 100644 --- a/src/conversions/std/map.rs +++ b/src/conversions/std/map.rs @@ -16,7 +16,7 @@ where H: hash::BuildHasher, { fn to_object(&self, py: Python<'_>) -> PyObject { - IntoPyDict::into_py_dict(self, py).into() + IntoPyDict::into_py_dict_bound(self, py).into() } } @@ -26,7 +26,7 @@ where V: ToPyObject, { fn to_object(&self, py: Python<'_>) -> PyObject { - IntoPyDict::into_py_dict(self, py).into() + IntoPyDict::into_py_dict_bound(self, py).into() } } @@ -40,7 +40,7 @@ where let iter = self .into_iter() .map(|(k, v)| (k.into_py(py), v.into_py(py))); - IntoPyDict::into_py_dict(iter, py).into() + IntoPyDict::into_py_dict_bound(iter, py).into() } #[cfg(feature = "experimental-inspect")] @@ -58,7 +58,7 @@ where let iter = self .into_iter() .map(|(k, v)| (k.into_py(py), v.into_py(py))); - IntoPyDict::into_py_dict(iter, py).into() + IntoPyDict::into_py_dict_bound(iter, py).into() } #[cfg(feature = "experimental-inspect")] diff --git a/src/exceptions.rs b/src/exceptions.rs index 10121b8a3ad..7e7dc8eee65 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -71,7 +71,7 @@ macro_rules! impl_exception_boilerplate { /// import_exception!(socket, gaierror); /// /// Python::with_gil(|py| { -/// let ctx = [("gaierror", py.get_type::())].into_py_dict(py); +/// let ctx = [("gaierror", py.get_type::())].into_py_dict_bound(py); /// pyo3::py_run!(py, *ctx, "import socket; assert gaierror is socket.gaierror"); /// }); /// @@ -864,7 +864,7 @@ mod tests { Python::with_gil(|py| { let error_type = py.get_type::(); - let ctx = [("CustomError", error_type)].into_py_dict(py).as_borrowed(); + let ctx = [("CustomError", error_type)].into_py_dict_bound(py); let type_description: String = py .eval_bound("str(CustomError)", None, Some(&ctx)) .unwrap() @@ -887,7 +887,7 @@ mod tests { create_exception!(mymodule.exceptions, CustomError, PyException); Python::with_gil(|py| { let error_type = py.get_type::(); - let ctx = [("CustomError", error_type)].into_py_dict(py).as_borrowed(); + let ctx = [("CustomError", error_type)].into_py_dict_bound(py); let type_description: String = py .eval_bound("str(CustomError)", None, Some(&ctx)) .unwrap() @@ -906,7 +906,7 @@ mod tests { Python::with_gil(|py| { let error_type = py.get_type::(); - let ctx = [("CustomError", error_type)].into_py_dict(py).as_borrowed(); + let ctx = [("CustomError", error_type)].into_py_dict_bound(py); let type_description: String = py .eval_bound("str(CustomError)", None, Some(&ctx)) .unwrap() @@ -939,7 +939,7 @@ mod tests { Python::with_gil(|py| { let error_type = py.get_type::(); - let ctx = [("CustomError", error_type)].into_py_dict(py).as_borrowed(); + let ctx = [("CustomError", error_type)].into_py_dict_bound(py); let type_description: String = py .eval_bound("str(CustomError)", None, Some(&ctx)) .unwrap() diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index cd63fe270b0..ad6b6159819 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -727,7 +727,7 @@ mod tests { Python::with_gil(|py| { let args = PyTuple::new_bound(py, Vec::<&PyAny>::new()); - let kwargs = [("foo", 0u8)].into_py_dict(py); + let kwargs = [("foo", 0u8)].into_py_dict_bound(py); let err = unsafe { function_description .extract_arguments_tuple_dict::( @@ -758,7 +758,7 @@ mod tests { Python::with_gil(|py| { let args = PyTuple::new_bound(py, Vec::<&PyAny>::new()); - let kwargs = [(1u8, 1u8)].into_py_dict(py); + let kwargs = [(1u8, 1u8)].into_py_dict_bound(py); let err = unsafe { function_description .extract_arguments_tuple_dict::( diff --git a/src/instance.rs b/src/instance.rs index e54312a9098..d99fe6f1767 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1765,7 +1765,7 @@ mod tests { "{'x': 1}", ); assert_repr( - obj.call(py, (), Some([('x', 1)].into_py_dict(py))) + obj.call_bound(py, (), Some(&[('x', 1)].into_py_dict_bound(py))) .unwrap() .as_ref(py), "{'x': 1}", diff --git a/src/lib.rs b/src/lib.rs index 13b027d3e77..5ba0d279aa1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -221,7 +221,7 @@ //! let sys = py.import("sys")?; //! let version: String = sys.getattr("version")?.extract()?; //! -//! let locals = [("os", py.import("os")?)].into_py_dict(py).as_borrowed(); +//! let locals = [("os", py.import("os")?)].into_py_dict_bound(py); //! let code = "os.getenv('USER') or os.getenv('USERNAME') or 'Unknown'"; //! let user: String = py.eval_bound(code, None, Some(&locals))?.extract()?; //! diff --git a/src/macros.rs b/src/macros.rs index 79e8ea58cb7..d5fbbdc3dfc 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -73,7 +73,7 @@ /// } /// /// Python::with_gil(|py| { -/// let locals = [("C", py.get_type::())].into_py_dict(py); +/// let locals = [("C", py.get_type::())].into_py_dict_bound(py); /// pyo3::py_run!(py, *locals, "c = C()"); /// }); /// ``` @@ -99,11 +99,12 @@ macro_rules! py_run_impl { ($py:expr, $($val:ident)+, $code:expr) => {{ use $crate::types::IntoPyDict; use $crate::ToPyObject; - let d = [$((stringify!($val), $val.to_object($py)),)+].into_py_dict($py); + let d = [$((stringify!($val), $val.to_object($py)),)+].into_py_dict_bound($py); $crate::py_run_impl!($py, *d, $code) }}; ($py:expr, *$dict:expr, $code:expr) => {{ use ::std::option::Option::*; + #[allow(unused_imports)] use $crate::PyNativeType; if let ::std::result::Result::Err(e) = $py.run_bound($code, None, Some(&$dict.as_borrowed())) { e.print($py); diff --git a/src/marker.rs b/src/marker.rs index 121cfe6855f..642ca783904 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -1132,7 +1132,7 @@ mod tests { .unwrap(); assert_eq!(v, 1); - let d = [("foo", 13)].into_py_dict(py).as_borrowed(); + let d = [("foo", 13)].into_py_dict_bound(py); // Inject our own global namespace let v: i32 = py diff --git a/src/tests/common.rs b/src/tests/common.rs index e424c1ca6fd..bb710d904b4 100644 --- a/src/tests/common.rs +++ b/src/tests/common.rs @@ -35,12 +35,11 @@ mod inner { // Case1: idents & no err_msg ($py:expr, $($val:ident)+, $code:expr, $err:ident) => {{ use pyo3::types::IntoPyDict; - let d = [$((stringify!($val), $val.to_object($py)),)+].into_py_dict($py); + let d = [$((stringify!($val), $val.to_object($py)),)+].into_py_dict_bound($py); py_expect_exception!($py, *d, $code, $err) }}; // Case2: dict & no err_msg ($py:expr, *$dict:expr, $code:expr, $err:ident) => {{ - use pyo3::PyNativeType; let res = $py.run_bound($code, None, Some(&$dict.as_borrowed())); let err = res.expect_err(&format!("Did not raise {}", stringify!($err))); if !err.matches($py, $py.get_type::()) { @@ -117,8 +116,10 @@ mod inner { impl<'py> CatchWarnings<'py> { pub fn enter(py: Python<'py>, f: impl FnOnce(&PyList) -> PyResult) -> PyResult { let warnings = py.import("warnings")?; - let kwargs = [("record", true)].into_py_dict(py); - let catch_warnings = warnings.getattr("catch_warnings")?.call((), Some(kwargs))?; + let kwargs = [("record", true)].into_py_dict_bound(py); + let catch_warnings = warnings + .getattr("catch_warnings")? + .call((), Some(kwargs.as_gil_ref()))?; let list = catch_warnings.call_method0("__enter__")?.extract()?; let _guard = Self { catch_warnings }; f(list) diff --git a/src/types/any.rs b/src/types/any.rs index a19985ae8bb..afc9a83dd41 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -2400,8 +2400,8 @@ class NonHeapNonDescriptorInt: fn test_call_with_kwargs() { Python::with_gil(|py| { let list = vec![3, 6, 5, 4, 7].to_object(py); - let dict = vec![("reverse", true)].into_py_dict(py); - list.call_method(py, "sort", (), Some(dict)).unwrap(); + let dict = vec![("reverse", true)].into_py_dict_bound(py); + list.call_method_bound(py, "sort", (), Some(&dict)).unwrap(); assert_eq!(list.extract::>(py).unwrap(), vec![7, 6, 5, 4, 3]); }); } diff --git a/src/types/dict.rs b/src/types/dict.rs index 3a54e742ed5..34812176da8 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -146,13 +146,13 @@ impl PyDict { /// /// ```rust /// use pyo3::prelude::*; - /// use pyo3::types::{PyDict, IntoPyDict}; + /// use pyo3::types::{IntoPyDict}; /// use pyo3::exceptions::{PyTypeError, PyKeyError}; /// /// # fn main() { /// # let _ = /// Python::with_gil(|py| -> PyResult<()> { - /// let dict: &PyDict = [("a", 1)].into_py_dict(py); + /// let dict = &[("a", 1)].into_py_dict_bound(py); /// // `a` is in the dictionary, with value 1 /// assert!(dict.get_item("a")?.map_or(Ok(false), |x| x.eq(1))?); /// // `b` is not in the dictionary @@ -161,7 +161,7 @@ impl PyDict { /// assert!(dict.get_item(dict).unwrap_err().is_instance_of::(py)); /// /// // `PyAny::get_item("b")` will raise a `KeyError` instead of returning `None` - /// let any: &PyAny = dict.as_ref(); + /// let any = dict.as_any(); /// assert!(any.get_item("b").unwrap_err().is_instance_of::(py)); /// Ok(()) /// }); @@ -650,10 +650,23 @@ impl<'py> IntoIterator for Bound<'py, PyDict> { /// Conversion trait that allows a sequence of tuples to be converted into `PyDict` /// Primary use case for this trait is `call` and `call_method` methods as keywords argument. -pub trait IntoPyDict { +pub trait IntoPyDict: Sized { /// Converts self into a `PyDict` object pointer. Whether pointer owned or borrowed /// depends on implementation. - fn into_py_dict(self, py: Python<'_>) -> &PyDict; + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`IntoPyDict::into_py_dict` will be replaced by `IntoPyDict::into_py_dict_bound` in a future PyO3 version" + ) + )] + fn into_py_dict(self, py: Python<'_>) -> &PyDict { + Self::into_py_dict_bound(self, py).into_gil_ref() + } + + /// Converts self into a `PyDict` object pointer. Whether pointer owned or borrowed + /// depends on implementation. + fn into_py_dict_bound(self, py: Python<'_>) -> Bound<'_, PyDict>; } impl IntoPyDict for I @@ -661,8 +674,8 @@ where T: PyDictItem, I: IntoIterator, { - fn into_py_dict(self, py: Python<'_>) -> &PyDict { - let dict = PyDict::new(py); + fn into_py_dict_bound(self, py: Python<'_>) -> Bound<'_, PyDict> { + let dict = PyDict::new_bound(py); for item in self { dict.set_item(item.key(), item.value()) .expect("Failed to set_item on dict"); @@ -723,7 +736,7 @@ mod tests { #[test] fn test_new() { Python::with_gil(|py| { - let dict = [(7, 32)].into_py_dict(py); + let dict = [(7, 32)].into_py_dict_bound(py); assert_eq!( 32, dict.get_item(7i32) @@ -783,7 +796,7 @@ mod tests { #[test] fn test_copy() { Python::with_gil(|py| { - let dict = [(7, 32)].into_py_dict(py); + let dict = [(7, 32)].into_py_dict_bound(py); let ndict = dict.copy().unwrap(); assert_eq!( @@ -905,7 +918,7 @@ mod tests { { let _pool = unsafe { crate::GILPool::new() }; cnt = obj.get_refcnt(); - let _dict = [(10, obj)].into_py_dict(py); + let _dict = [(10, obj)].into_py_dict_bound(py); } { assert_eq!(cnt, obj.get_refcnt()); @@ -1150,7 +1163,7 @@ mod tests { let mut map = HashMap::::new(); map.insert(1, 1); - let py_map = map.into_py_dict(py); + let py_map = map.into_py_dict_bound(py); assert_eq!(py_map.len(), 1); assert_eq!( @@ -1171,7 +1184,7 @@ mod tests { let mut map = BTreeMap::::new(); map.insert(1, 1); - let py_map = map.into_py_dict(py); + let py_map = map.into_py_dict_bound(py); assert_eq!(py_map.len(), 1); assert_eq!( @@ -1190,7 +1203,7 @@ mod tests { fn test_vec_into_dict() { Python::with_gil(|py| { let vec = vec![("a", 1), ("b", 2), ("c", 3)]; - let py_map = vec.into_py_dict(py); + let py_map = vec.into_py_dict_bound(py); assert_eq!(py_map.len(), 3); assert_eq!( @@ -1209,7 +1222,7 @@ mod tests { fn test_slice_into_dict() { Python::with_gil(|py| { let arr = [("a", 1), ("b", 2), ("c", 3)]; - let py_map = arr.into_py_dict(py); + let py_map = arr.into_py_dict_bound(py); assert_eq!(py_map.len(), 3); assert_eq!( @@ -1230,7 +1243,7 @@ mod tests { let mut map = HashMap::::new(); map.insert(1, 1); - let py_map = map.into_py_dict(py); + let py_map = map.into_py_dict_bound(py); assert_eq!(py_map.as_mapping().len().unwrap(), 1); assert_eq!( @@ -1246,12 +1259,12 @@ mod tests { } #[cfg(not(PyPy))] - fn abc_dict(py: Python<'_>) -> &PyDict { + fn abc_dict(py: Python<'_>) -> Bound<'_, PyDict> { let mut map = HashMap::<&'static str, i32>::new(); map.insert("a", 1); map.insert("b", 2); map.insert("c", 3); - map.into_py_dict(py) + map.into_py_dict_bound(py) } #[test] @@ -1260,7 +1273,9 @@ mod tests { Python::with_gil(|py| { let dict = abc_dict(py); let keys = dict.call_method0("keys").unwrap(); - assert!(keys.is_instance(py.get_type::()).unwrap()); + assert!(keys + .is_instance(&py.get_type::().as_borrowed()) + .unwrap()); }) } @@ -1270,7 +1285,9 @@ mod tests { Python::with_gil(|py| { let dict = abc_dict(py); let values = dict.call_method0("values").unwrap(); - assert!(values.is_instance(py.get_type::()).unwrap()); + assert!(values + .is_instance(&py.get_type::().as_borrowed()) + .unwrap()); }) } @@ -1280,15 +1297,17 @@ mod tests { Python::with_gil(|py| { let dict = abc_dict(py); let items = dict.call_method0("items").unwrap(); - assert!(items.is_instance(py.get_type::()).unwrap()); + assert!(items + .is_instance(&py.get_type::().as_borrowed()) + .unwrap()); }) } #[test] fn dict_update() { Python::with_gil(|py| { - let dict = [("a", 1), ("b", 2), ("c", 3)].into_py_dict(py); - let other = [("b", 4), ("c", 5), ("d", 6)].into_py_dict(py); + let dict = [("a", 1), ("b", 2), ("c", 3)].into_py_dict_bound(py); + let other = [("b", 4), ("c", 5), ("d", 6)].into_py_dict_bound(py); dict.update(other.as_mapping()).unwrap(); assert_eq!(dict.len(), 4); assert_eq!( @@ -1358,8 +1377,8 @@ mod tests { #[test] fn dict_update_if_missing() { Python::with_gil(|py| { - let dict = [("a", 1), ("b", 2), ("c", 3)].into_py_dict(py); - let other = [("b", 4), ("c", 5), ("d", 6)].into_py_dict(py); + let dict = [("a", 1), ("b", 2), ("c", 3)].into_py_dict_bound(py); + let other = [("b", 4), ("c", 5), ("d", 6)].into_py_dict_bound(py); dict.update_if_missing(other.as_mapping()).unwrap(); assert_eq!(dict.len(), 4); assert_eq!( diff --git a/tests/test_buffer_protocol.rs b/tests/test_buffer_protocol.rs index b9f3861c65a..0a3a1758bfa 100644 --- a/tests/test_buffer_protocol.rs +++ b/tests/test_buffer_protocol.rs @@ -57,7 +57,7 @@ fn test_buffer() { }, ) .unwrap(); - let env = [("ob", instance)].into_py_dict(py); + let env = [("ob", instance)].into_py_dict_bound(py); py_assert!(py, *env, "bytes(ob) == b' 23'"); }); @@ -122,7 +122,7 @@ fn test_releasebuffer_unraisable_error() { let capture = UnraisableCapture::install(py); let instance = Py::new(py, ReleaseBufferError {}).unwrap(); - let env = [("ob", instance.clone())].into_py_dict(py); + let env = [("ob", instance.clone())].into_py_dict_bound(py); assert!(capture.borrow(py).capture.is_none()); diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index 8084be057cd..59772cc4cf2 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -29,7 +29,10 @@ fn empty_class_with_new() { // Calling with arbitrary args or kwargs is not ok assert!(typeobj.call(("some", "args"), None).is_err()); assert!(typeobj - .call((), Some([("some", "kwarg")].into_py_dict(py))) + .call( + (), + Some([("some", "kwarg")].into_py_dict_bound(py).as_gil_ref()) + ) .is_err()); }); } diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index d9672cd68f0..fa50fe652e6 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -68,7 +68,7 @@ fn test_coroutine_qualname() { ("my_fn", wrap_pyfunction!(my_fn, gil).unwrap().deref()), ("MyClass", gil.get_type::()), ] - .into_py_dict(gil); + .into_py_dict_bound(gil); py_run!(gil, *locals, &handle_windows(test)); }) } @@ -286,7 +286,7 @@ fn test_async_method_receiver() { assert False assert asyncio.run(coro3) == 1 "#; - let locals = [("Counter", gil.get_type::())].into_py_dict(gil); + let locals = [("Counter", gil.get_type::())].into_py_dict_bound(gil); py_run!(gil, *locals, test); }) } diff --git a/tests/test_datetime.rs b/tests/test_datetime.rs index d1efd1efe2c..5d7b1485260 100644 --- a/tests/test_datetime.rs +++ b/tests/test_datetime.rs @@ -12,9 +12,7 @@ fn _get_subclasses<'py>( // Import the class from Python and create some subclasses let datetime = py.import("datetime")?; - let locals = [(py_type, datetime.getattr(py_type)?)] - .into_py_dict(py) - .as_borrowed(); + let locals = [(py_type, datetime.getattr(py_type)?)].into_py_dict_bound(py); let make_subclass_py = format!("class Subklass({}):\n pass", py_type); @@ -23,7 +21,6 @@ fn _get_subclasses<'py>( py.run_bound(&make_subclass_py, None, Some(&locals))?; py.run_bound(make_sub_subclass_py, None, Some(&locals))?; - let locals = locals.as_borrowed(); // Construct an instance of the base class let obj = py.eval_bound(&format!("{}({})", py_type, args), None, Some(&locals))?; @@ -125,7 +122,7 @@ fn test_datetime_utc() { let dt = PyDateTime::new(py, 2018, 1, 1, 0, 0, 0, 0, Some(utc)).unwrap(); - let locals = [("dt", dt)].into_py_dict(py).as_borrowed(); + let locals = [("dt", dt)].into_py_dict_bound(py); let offset: f32 = py .eval_bound("dt.utcoffset().total_seconds()", None, Some(&locals)) diff --git a/tests/test_dict_iter.rs b/tests/test_dict_iter.rs index dc32eb61fd7..e502a4ca2b6 100644 --- a/tests/test_dict_iter.rs +++ b/tests/test_dict_iter.rs @@ -6,7 +6,7 @@ use pyo3::types::IntoPyDict; fn iter_dict_nosegv() { Python::with_gil(|py| { const LEN: usize = 10_000_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py); + let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); let mut sum = 0; for (k, _v) in dict { let i: u64 = k.extract().unwrap(); diff --git a/tests/test_getter_setter.rs b/tests/test_getter_setter.rs index 148380ecf67..b990a82e134 100644 --- a/tests/test_getter_setter.rs +++ b/tests/test_getter_setter.rs @@ -64,7 +64,7 @@ fn class_with_properties() { py_run!(py, inst, "assert inst.get_num() == inst.unwrapped == 42"); py_run!(py, inst, "assert inst.data_list == [42]"); - let d = [("C", py.get_type::())].into_py_dict(py); + let d = [("C", py.get_type::())].into_py_dict_bound(py); py_assert!(py, *d, "C.DATA.__doc__ == 'a getter for data'"); }); } diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 4b024ed75e5..6c33dec6a9f 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -20,9 +20,7 @@ struct SubclassAble {} #[test] fn subclass() { Python::with_gil(|py| { - let d = [("SubclassAble", py.get_type::())] - .into_py_dict(py) - .as_borrowed(); + let d = [("SubclassAble", py.get_type::())].into_py_dict_bound(py); py.run_bound( "class A(SubclassAble): pass\nassert issubclass(A, SubclassAble)", @@ -99,7 +97,7 @@ fn call_base_and_sub_methods() { fn mutation_fails() { Python::with_gil(|py| { let obj = PyCell::new(py, SubClass::new()).unwrap(); - let global = [("obj", obj)].into_py_dict(py).as_borrowed(); + let global = [("obj", obj)].into_py_dict_bound(py); let e = py .run_bound( "obj.base_set(lambda: obj.sub_set_and_ret(1))", @@ -277,7 +275,7 @@ mod inheriting_native_type { fn custom_exception() { Python::with_gil(|py| { let cls = py.get_type::(); - let dict = [("cls", cls)].into_py_dict(py).as_borrowed(); + let dict = [("cls", cls)].into_py_dict_bound(py); let res = py.run_bound( "e = cls('hello'); assert str(e) == 'hello'; assert e.context == 'Hello :)'; raise e", None, diff --git a/tests/test_macro_docs.rs b/tests/test_macro_docs.rs index 964e762886d..5e240816cd2 100644 --- a/tests/test_macro_docs.rs +++ b/tests/test_macro_docs.rs @@ -23,7 +23,7 @@ impl MacroDocs { #[test] fn meth_doc() { Python::with_gil(|py| { - let d = [("C", py.get_type::())].into_py_dict(py); + let d = [("C", py.get_type::())].into_py_dict_bound(py); py_assert!( py, *d, diff --git a/tests/test_mapping.rs b/tests/test_mapping.rs index c029ce27c50..9c99d56467f 100644 --- a/tests/test_mapping.rs +++ b/tests/test_mapping.rs @@ -68,8 +68,8 @@ impl Mapping { } /// Return a dict with `m = Mapping(['1', '2', '3'])`. -fn map_dict(py: Python<'_>) -> &pyo3::types::PyDict { - let d = [("Mapping", py.get_type::())].into_py_dict(py); +fn map_dict(py: Python<'_>) -> Bound<'_, pyo3::types::PyDict> { + let d = [("Mapping", py.get_type::())].into_py_dict_bound(py); py_run!(py, *d, "m = Mapping(['1', '2', '3'])"); d } diff --git a/tests/test_methods.rs b/tests/test_methods.rs index 50b179fd911..083565334d0 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -89,7 +89,7 @@ impl ClassMethod { #[test] fn class_method() { Python::with_gil(|py| { - let d = [("C", py.get_type::())].into_py_dict(py); + let d = [("C", py.get_type::())].into_py_dict_bound(py); py_assert!(py, *d, "C.method() == 'ClassMethod.method()!'"); py_assert!(py, *d, "C().method() == 'ClassMethod.method()!'"); py_assert!( @@ -116,7 +116,7 @@ impl ClassMethodWithArgs { #[test] fn class_method_with_args() { Python::with_gil(|py| { - let d = [("C", py.get_type::())].into_py_dict(py); + let d = [("C", py.get_type::())].into_py_dict_bound(py); py_assert!( py, *d, @@ -147,7 +147,7 @@ fn static_method() { Python::with_gil(|py| { assert_eq!(StaticMethod::method(py), "StaticMethod.method()!"); - let d = [("C", py.get_type::())].into_py_dict(py); + let d = [("C", py.get_type::())].into_py_dict_bound(py); py_assert!(py, *d, "C.method() == 'StaticMethod.method()!'"); py_assert!(py, *d, "C().method() == 'StaticMethod.method()!'"); py_assert!(py, *d, "C.method.__doc__ == 'Test static method.'"); @@ -171,7 +171,7 @@ fn static_method_with_args() { Python::with_gil(|py| { assert_eq!(StaticMethodWithArgs::method(py, 1234), "0x4d2"); - let d = [("C", py.get_type::())].into_py_dict(py); + let d = [("C", py.get_type::())].into_py_dict_bound(py); py_assert!(py, *d, "C.method(1337) == '0x539'"); }); } @@ -669,7 +669,7 @@ impl MethDocs { #[test] fn meth_doc() { Python::with_gil(|py| { - let d = [("C", py.get_type::())].into_py_dict(py); + let d = [("C", py.get_type::())].into_py_dict_bound(py); py_assert!(py, *d, "C.__doc__ == 'A class with \"documentation\".'"); py_assert!( py, @@ -753,7 +753,7 @@ fn method_with_pyclassarg() { Python::with_gil(|py| { let obj1 = PyCell::new(py, MethodWithPyClassArg { value: 10 }).unwrap(); let obj2 = PyCell::new(py, MethodWithPyClassArg { value: 10 }).unwrap(); - let d = [("obj1", obj1), ("obj2", obj2)].into_py_dict(py); + let d = [("obj1", obj1), ("obj2", obj2)].into_py_dict_bound(py); py_run!(py, *d, "obj = obj1.add(obj2); assert obj.value == 20"); py_run!(py, *d, "obj = obj1.add_pyref(obj2); assert obj.value == 20"); py_run!(py, *d, "obj = obj1.optional_add(); assert obj.value == 20"); diff --git a/tests/test_module.rs b/tests/test_module.rs index 2de23b38324..1bd976e0ae4 100644 --- a/tests/test_module.rs +++ b/tests/test_module.rs @@ -74,7 +74,7 @@ fn test_module_with_functions() { "module_with_functions", wrap_pymodule!(module_with_functions)(py), )] - .into_py_dict(py); + .into_py_dict_bound(py); py_assert!( py, @@ -127,7 +127,7 @@ fn test_module_renaming() { use pyo3::wrap_pymodule; Python::with_gil(|py| { - let d = [("different_name", wrap_pymodule!(some_name)(py))].into_py_dict(py); + let d = [("different_name", wrap_pymodule!(some_name)(py))].into_py_dict_bound(py); py_run!(py, *d, "assert different_name.__name__ == 'other_name'"); }); diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index df73b9a27d8..1ff3dc940aa 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -90,7 +90,7 @@ fn test_basic() { pyo3::Python::with_gil(|py| { let module = pyo3::wrap_pymodule!(basic_module)(py); let cls = py.get_type::(); - let d = pyo3::types::IntoPyDict::into_py_dict( + let d = pyo3::types::IntoPyDict::into_py_dict_bound( [ ("mod", module.as_ref(py).as_ref()), ("cls", cls.as_ref()), diff --git a/tests/test_sequence.rs b/tests/test_sequence.rs index b11e4a6929d..d3c84b76c8c 100644 --- a/tests/test_sequence.rs +++ b/tests/test_sequence.rs @@ -105,8 +105,8 @@ impl ByteSequence { } /// Return a dict with `s = ByteSequence([1, 2, 3])`. -fn seq_dict(py: Python<'_>) -> &pyo3::types::PyDict { - let d = [("ByteSequence", py.get_type::())].into_py_dict(py); +fn seq_dict(py: Python<'_>) -> Bound<'_, pyo3::types::PyDict> { + let d = [("ByteSequence", py.get_type::())].into_py_dict_bound(py); // Though we can construct `s` in Rust, let's test `__new__` works. py_run!(py, *d, "s = ByteSequence([1, 2, 3])"); d @@ -138,7 +138,7 @@ fn test_setitem() { #[test] fn test_delitem() { Python::with_gil(|py| { - let d = [("ByteSequence", py.get_type::())].into_py_dict(py); + let d = [("ByteSequence", py.get_type::())].into_py_dict_bound(py); py_run!( py, @@ -234,7 +234,7 @@ fn test_repeat() { #[test] fn test_inplace_repeat() { Python::with_gil(|py| { - let d = [("ByteSequence", py.get_type::())].into_py_dict(py); + let d = [("ByteSequence", py.get_type::())].into_py_dict_bound(py); py_run!( py, diff --git a/tests/test_static_slots.rs b/tests/test_static_slots.rs index 402128c50df..b01c87cd507 100644 --- a/tests/test_static_slots.rs +++ b/tests/test_static_slots.rs @@ -37,8 +37,8 @@ impl Count5 { } /// Return a dict with `s = Count5()`. -fn test_dict(py: Python<'_>) -> &pyo3::types::PyDict { - let d = [("Count5", py.get_type::())].into_py_dict(py); +fn test_dict(py: Python<'_>) -> Bound<'_, pyo3::types::PyDict> { + let d = [("Count5", py.get_type::())].into_py_dict_bound(py); // Though we can construct `s` in Rust, let's test `__new__` works. py_run!(py, *d, "s = Count5()"); d From 5b1104131f7aba50153bb19470ad4b073b3241fd Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 10 Feb 2024 13:57:20 +0000 Subject: [PATCH 116/349] fix segmentation fault when `datetime` module is invalid --- newsfragments/3818.fixed.md | 1 + src/types/datetime.rs | 56 ++++++++++++++++++----------------- tests/test_datetime_import.rs | 26 ++++++++++++++++ 3 files changed, 56 insertions(+), 27 deletions(-) create mode 100644 newsfragments/3818.fixed.md create mode 100644 tests/test_datetime_import.rs diff --git a/newsfragments/3818.fixed.md b/newsfragments/3818.fixed.md new file mode 100644 index 00000000000..76fe01a545c --- /dev/null +++ b/newsfragments/3818.fixed.md @@ -0,0 +1 @@ +Fix segmentation fault using `datetime` types when an invalid `datetime` module is on sys.path. diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 354414b8b4c..088e37d9536 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -23,21 +23,27 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::PyNativeType; use crate::types::any::PyAnyMethods; use crate::types::PyTuple; -use crate::{Bound, IntoPy, Py, PyAny, Python}; +use crate::{Bound, IntoPy, Py, PyAny, PyErr, Python}; use std::os::raw::c_int; #[cfg(feature = "chrono")] use std::ptr; -fn ensure_datetime_api(_py: Python<'_>) -> &'static PyDateTime_CAPI { - unsafe { - if pyo3_ffi::PyDateTimeAPI().is_null() { - PyDateTime_IMPORT() +fn ensure_datetime_api(py: Python<'_>) -> PyResult<&'static PyDateTime_CAPI> { + if let Some(api) = unsafe { pyo3_ffi::PyDateTimeAPI().as_ref() } { + Ok(api) + } else { + unsafe { + PyDateTime_IMPORT(); + pyo3_ffi::PyDateTimeAPI().as_ref() } - - &*pyo3_ffi::PyDateTimeAPI() + .ok_or_else(|| PyErr::fetch(py)) } } +fn expect_datetime_api(py: Python<'_>) -> &'static PyDateTime_CAPI { + ensure_datetime_api(py).expect("failed to import `datetime` C API") +} + // Type Check macros // // These are bindings around the C API typecheck macros, all of them return @@ -189,7 +195,7 @@ pub struct PyDate(PyAny); pyobject_native_type!( PyDate, crate::ffi::PyDateTime_Date, - |py| ensure_datetime_api(py).DateType, + |py| expect_datetime_api(py).DateType, #module=Some("datetime"), #checkfunction=PyDate_Check ); @@ -197,13 +203,9 @@ pyobject_native_type!( impl PyDate { /// Creates a new `datetime.date`. pub fn new(py: Python<'_>, year: i32, month: u8, day: u8) -> PyResult<&PyDate> { + let api = ensure_datetime_api(py)?; unsafe { - let ptr = (ensure_datetime_api(py).Date_FromDate)( - year, - c_int::from(month), - c_int::from(day), - ensure_datetime_api(py).DateType, - ); + let ptr = (api.Date_FromDate)(year, c_int::from(month), c_int::from(day), api.DateType); py.from_owned_ptr_or_err(ptr) } } @@ -215,7 +217,7 @@ impl PyDate { let time_tuple = PyTuple::new_bound(py, [timestamp]); // safety ensure that the API is loaded - let _api = ensure_datetime_api(py); + let _api = ensure_datetime_api(py)?; unsafe { let ptr = PyDate_FromTimestamp(time_tuple.as_ptr()); @@ -258,7 +260,7 @@ pub struct PyDateTime(PyAny); pyobject_native_type!( PyDateTime, crate::ffi::PyDateTime_DateTime, - |py| ensure_datetime_api(py).DateTimeType, + |py| expect_datetime_api(py).DateTimeType, #module=Some("datetime"), #checkfunction=PyDateTime_Check ); @@ -277,7 +279,7 @@ impl PyDateTime { microsecond: u32, tzinfo: Option<&PyTzInfo>, ) -> PyResult<&'p PyDateTime> { - let api = ensure_datetime_api(py); + let api = ensure_datetime_api(py)?; unsafe { let ptr = (api.DateTime_FromDateAndTime)( year, @@ -314,7 +316,7 @@ impl PyDateTime { tzinfo: Option<&PyTzInfo>, fold: bool, ) -> PyResult<&'p PyDateTime> { - let api = ensure_datetime_api(py); + let api = ensure_datetime_api(py)?; unsafe { let ptr = (api.DateTime_FromDateAndTimeAndFold)( year, @@ -343,7 +345,7 @@ impl PyDateTime { let args: Py = (timestamp, tzinfo).into_py(py); // safety ensure API is loaded - let _api = ensure_datetime_api(py); + let _api = ensure_datetime_api(py)?; unsafe { let ptr = PyDateTime_FromTimestamp(args.as_ptr()); @@ -455,7 +457,7 @@ pub struct PyTime(PyAny); pyobject_native_type!( PyTime, crate::ffi::PyDateTime_Time, - |py| ensure_datetime_api(py).TimeType, + |py| expect_datetime_api(py).TimeType, #module=Some("datetime"), #checkfunction=PyTime_Check ); @@ -470,7 +472,7 @@ impl PyTime { microsecond: u32, tzinfo: Option<&PyTzInfo>, ) -> PyResult<&'p PyTime> { - let api = ensure_datetime_api(py); + let api = ensure_datetime_api(py)?; unsafe { let ptr = (api.Time_FromTime)( c_int::from(hour), @@ -494,7 +496,7 @@ impl PyTime { tzinfo: Option<&PyTzInfo>, fold: bool, ) -> PyResult<&'p PyTime> { - let api = ensure_datetime_api(py); + let api = ensure_datetime_api(py)?; unsafe { let ptr = (api.Time_FromTimeAndFold)( c_int::from(hour), @@ -589,14 +591,14 @@ pub struct PyTzInfo(PyAny); pyobject_native_type!( PyTzInfo, crate::ffi::PyObject, - |py| ensure_datetime_api(py).TZInfoType, + |py| expect_datetime_api(py).TZInfoType, #module=Some("datetime"), #checkfunction=PyTZInfo_Check ); /// Equivalent to `datetime.timezone.utc` pub fn timezone_utc(py: Python<'_>) -> &PyTzInfo { - unsafe { &*(ensure_datetime_api(py).TimeZone_UTC as *const PyTzInfo) } + unsafe { &*(expect_datetime_api(py).TimeZone_UTC as *const PyTzInfo) } } /// Equivalent to `datetime.timezone` constructor @@ -604,7 +606,7 @@ pub fn timezone_utc(py: Python<'_>) -> &PyTzInfo { /// Only used internally #[cfg(feature = "chrono")] pub fn timezone_from_offset<'a>(py: Python<'a>, offset: &PyDelta) -> PyResult<&'a PyTzInfo> { - let api = ensure_datetime_api(py); + let api = ensure_datetime_api(py)?; unsafe { let ptr = (api.TimeZone_FromTimeZone)(offset.as_ptr(), ptr::null_mut()); py.from_owned_ptr_or_err(ptr) @@ -617,7 +619,7 @@ pub struct PyDelta(PyAny); pyobject_native_type!( PyDelta, crate::ffi::PyDateTime_Delta, - |py| ensure_datetime_api(py).DeltaType, + |py| expect_datetime_api(py).DeltaType, #module=Some("datetime"), #checkfunction=PyDelta_Check ); @@ -631,7 +633,7 @@ impl PyDelta { microseconds: i32, normalize: bool, ) -> PyResult<&PyDelta> { - let api = ensure_datetime_api(py); + let api = ensure_datetime_api(py)?; unsafe { let ptr = (api.Delta_FromDelta)( days as c_int, diff --git a/tests/test_datetime_import.rs b/tests/test_datetime_import.rs new file mode 100644 index 00000000000..fee99b07d1a --- /dev/null +++ b/tests/test_datetime_import.rs @@ -0,0 +1,26 @@ +#![cfg(not(Py_LIMITED_API))] + +use pyo3::{types::PyDate, Python}; + +#[test] +#[should_panic(expected = "module 'datetime' has no attribute 'datetime_CAPI'")] +fn test_bad_datetime_module_panic() { + // Create an empty temporary directory + // with an empty "datetime" module which we'll put on the sys.path + let tmpdir = std::env::temp_dir(); + let tmpdir = tmpdir.join("pyo3_test_date_check"); + let _ = std::fs::remove_dir_all(&tmpdir); + std::fs::create_dir(&tmpdir).unwrap(); + std::fs::File::create(tmpdir.join("datetime.py")).unwrap(); + + Python::with_gil(|py: Python<'_>| { + let sys = py.import("sys").unwrap(); + sys.getattr("path") + .unwrap() + .call_method1("insert", (0, tmpdir)) + .unwrap(); + + // This should panic because the "datetime" module is empty + PyDate::new(py, 2018, 1, 1).unwrap(); + }); +} From f721c8c2b72d5b64ec7b0b723fd43569616c4c29 Mon Sep 17 00:00:00 2001 From: Jose <34888496+Jerry-Master@users.noreply.github.com> Date: Sun, 11 Feb 2024 21:52:14 +0100 Subject: [PATCH 117/349] docs: fix link to README on building_and_distribution.md (#3809) * Update building_and_distribution.md Link to README not working. * Update guide/src/building_and_distribution.md Co-authored-by: David Hewitt --------- Co-authored-by: David Hewitt --- guide/src/building_and_distribution.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 8cb675f4303..97e77d24c34 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -2,7 +2,7 @@ This chapter of the guide goes into detail on how to build and distribute projects using PyO3. The way to achieve this is very different depending on whether the project is a Python module implemented in Rust, or a Rust binary embedding Python. For both types of project there are also common problems such as the Python version to build for and the [linker](https://en.wikipedia.org/wiki/Linker_(computing)) arguments to use. -The material in this chapter is intended for users who have already read the PyO3 [README](#index.md). It covers in turn the choices that can be made for Python modules and for Rust binaries. There is also a section at the end about cross-compiling projects using PyO3. +The material in this chapter is intended for users who have already read the PyO3 [README](./index.md). It covers in turn the choices that can be made for Python modules and for Rust binaries. There is also a section at the end about cross-compiling projects using PyO3. There is an additional sub-chapter dedicated to [supporting multiple Python versions](./building_and_distribution/multiple_python_versions.html). From c56cd3dd65980c633bee491194c56fb0b868f660 Mon Sep 17 00:00:00 2001 From: Jose <34888496+Jerry-Master@users.noreply.github.com> Date: Sun, 11 Feb 2024 21:52:49 +0100 Subject: [PATCH 118/349] docs: clarify --pretty option to expand (#3810) * Update debugging.md Added clarification. --pretty no longer works, and it breaks even on nightly at least on cargo 1.78.0-nightly (cdf84b69d 2024-02-02) and rustc 1.78.0-nightly (256b6fb19 2024-02-06). * Update guide/src/debugging.md Co-authored-by: David Hewitt --------- Co-authored-by: David Hewitt --- guide/src/debugging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/debugging.md b/guide/src/debugging.md index e861f454a07..00c22631c3b 100644 --- a/guide/src/debugging.md +++ b/guide/src/debugging.md @@ -16,7 +16,7 @@ You can also debug classic `!`-macros by adding `-Z trace-macros`: cargo rustc --profile=check -- -Z unstable-options --pretty=expanded -Z trace-macros > expanded.rs; rustfmt expanded.rs ``` -See [cargo expand](https://github.com/dtolnay/cargo-expand) for a more elaborate version of those commands. +Note that those commands require using the nightly build of rust and may occasionally have bugs. See [cargo expand](https://github.com/dtolnay/cargo-expand) for a more elaborate and stable version of those commands. ## Running with Valgrind From baf5c8ec0a6598a6a2d1298dea2e7707de1fb08f Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Sun, 11 Feb 2024 21:07:28 +0000 Subject: [PATCH 119/349] Implement PyErr::get_type_bound (#3819) * Implement PyErr::get_type_bound * Update docs for PyErr::get_type_bound * Fix doctest for cloning PyErr * Import the whole prelude in docs example Co-authored-by: David Hewitt * Remove unnecessary self lifetime Co-authored-by: David Hewitt * Remove more unnecessary self lifetimes * Use variables to avoid dangling pointers Co-authored-by: David Hewitt * Avoid using ffi in fn ptype on Py_3_12 Co-authored-by: David Hewitt * Add missing imports to fn ptype --------- Co-authored-by: David Hewitt --- guide/src/python_from_rust.md | 2 +- src/err/err_state.rs | 10 ++++++---- src/err/mod.rs | 28 +++++++++++++++++++++------- src/impl_/extract_argument.rs | 3 ++- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 0c104c2858b..b53b03091e2 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -479,7 +479,7 @@ class House(object): } Err(e) => { house - .call_method1("__exit__", (e.get_type(py), e.value(py), e.traceback_bound(py))) + .call_method1("__exit__", (e.get_type_bound(py), e.value(py), e.traceback_bound(py))) .unwrap(); } } diff --git a/src/err/err_state.rs b/src/err/err_state.rs index 8b31c2b7476..863b276307e 100644 --- a/src/err/err_state.rs +++ b/src/err/err_state.rs @@ -16,13 +16,15 @@ pub(crate) struct PyErrStateNormalized { impl PyErrStateNormalized { #[cfg(not(Py_3_12))] - pub(crate) fn ptype<'py>(&'py self, py: Python<'py>) -> &'py PyType { - self.ptype.as_ref(py) + pub(crate) fn ptype<'py>(&self, py: Python<'py>) -> Bound<'py, PyType> { + self.ptype.bind(py).clone() } #[cfg(Py_3_12)] - pub(crate) fn ptype<'py>(&'py self, py: Python<'py>) -> &'py PyType { - self.pvalue.as_ref(py).get_type() + pub(crate) fn ptype<'py>(&self, py: Python<'py>) -> Bound<'py, PyType> { + use crate::instance::PyNativeType; + use crate::types::any::PyAnyMethods; + self.pvalue.bind(py).get_type().as_borrowed().to_owned() } #[cfg(not(Py_3_12))] diff --git a/src/err/mod.rs b/src/err/mod.rs index e53a9657998..095c89292ce 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -225,18 +225,30 @@ impl PyErr { PyErr::from_state(state) } + /// Deprecated form of [`PyErr::get_type_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyErr::get_type` will be replaced by `PyErr::get_type_bound` in a future PyO3 version" + ) + )] + pub fn get_type<'py>(&'py self, py: Python<'py>) -> &'py PyType { + self.get_type_bound(py).into_gil_ref() + } + /// Returns the type of this exception. /// /// # Examples /// ```rust - /// use pyo3::{exceptions::PyTypeError, types::PyType, PyErr, Python}; + /// use pyo3::{prelude::*, exceptions::PyTypeError, types::PyType}; /// /// Python::with_gil(|py| { /// let err: PyErr = PyTypeError::new_err(("some type error",)); - /// assert!(err.get_type(py).is(PyType::new::(py))); + /// assert!(err.get_type_bound(py).is(PyType::new::(py))); /// }); /// ``` - pub fn get_type<'py>(&'py self, py: Python<'py>) -> &'py PyType { + pub fn get_type_bound<'py>(&self, py: Python<'py>) -> Bound<'py, PyType> { self.normalized(py).ptype(py) } @@ -493,8 +505,9 @@ impl PyErr { // after the argument got evaluated, leading to call with a dangling // pointer. let traceback = self.traceback_bound(py); + let type_bound = self.get_type_bound(py); ffi::PyErr_Display( - self.get_type(py).as_ptr(), + type_bound.as_ptr(), self.value(py).as_ptr(), traceback .as_ref() @@ -531,7 +544,8 @@ impl PyErr { /// Returns true if the current exception is instance of `T`. #[inline] pub fn is_instance(&self, py: Python<'_>, ty: &PyAny) -> bool { - (unsafe { ffi::PyErr_GivenExceptionMatches(self.get_type(py).as_ptr(), ty.as_ptr()) }) != 0 + let type_bound = self.get_type_bound(py); + (unsafe { ffi::PyErr_GivenExceptionMatches(type_bound.as_ptr(), ty.as_ptr()) }) != 0 } /// Returns true if the current exception is instance of `T`. @@ -680,7 +694,7 @@ impl PyErr { /// Python::with_gil(|py| { /// let err: PyErr = PyTypeError::new_err(("some type error",)); /// let err_clone = err.clone_ref(py); - /// assert!(err.get_type(py).is(err_clone.get_type(py))); + /// assert!(err.get_type_bound(py).is(&err_clone.get_type_bound(py))); /// assert!(err.value(py).is(err_clone.value(py))); /// match err.traceback_bound(py) { /// None => assert!(err_clone.traceback_bound(py).is_none()), @@ -763,7 +777,7 @@ impl std::fmt::Debug for PyErr { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { Python::with_gil(|py| { f.debug_struct("PyErr") - .field("type", self.get_type(py)) + .field("type", &self.get_type_bound(py)) .field("value", self.value(py)) .field("traceback", &self.traceback_bound(py)) .finish() diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index ad6b6159819..b9f3fa2757f 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -165,7 +165,8 @@ pub fn from_py_with_with_default<'py, T>( #[doc(hidden)] #[cold] pub fn argument_extraction_error(py: Python<'_>, arg_name: &str, error: PyErr) -> PyErr { - if error.get_type(py).is(py.get_type::()) { + use crate::types::any::PyAnyMethods; + if error.get_type_bound(py).is(py.get_type::()) { let remapped_error = PyTypeError::new_err(format!("argument '{}': {}", arg_name, error.value(py))); remapped_error.set_cause(py, error.cause(py)); From c983dc9773c0d7bd5e338c66d84f08b1e31af721 Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Sun, 11 Feb 2024 23:52:56 +0100 Subject: [PATCH 120/349] Adds johnnycanencrypt project link (#3822) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bdb4c3ef196..64aa3e9ef70 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,7 @@ about this topic. - [html-py-ever](https://github.com/PyO3/setuptools-rust/tree/main/examples/html-py-ever) _Using [html5ever](https://github.com/servo/html5ever) through [kuchiki](https://github.com/kuchiki-rs/kuchiki) to speed up html parsing and css-selecting._ - [hyperjson](https://github.com/mre/hyperjson) _A hyper-fast Python module for reading/writing JSON data using Rust's serde-json._ - [inline-python](https://github.com/fusion-engineering/inline-python) _Inline Python code directly in your Rust code._ +- [johnnycanencrypt](https://github.com/kushaldas/johnnycanencrypt) OpenPGP library with Yubikey support. - [jsonschema-rs](https://github.com/Stranger6667/jsonschema-rs/tree/master/bindings/python) _Fast JSON Schema validation library._ - [mocpy](https://github.com/cds-astro/mocpy) _Astronomical Python library offering data structures for describing any arbitrary coverage regions on the unit sphere._ - [opendal](https://github.com/apache/opendal/tree/main/bindings/python) _A data access layer that allows users to easily and efficiently retrieve data from various storage services in a unified way._ From c359f5ca1d1cf1207cf65a63983d0feb5f78401a Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Mon, 12 Feb 2024 00:55:56 +0100 Subject: [PATCH 121/349] deprecate `PyDict::new` constructor (#3823) * deprecate `PyDict::new` * update benchmarks * convert `test_frompyobject` --- guide/src/conversions/traits.md | 2 +- pyo3-benches/benches/bench_any.rs | 12 +-- pyo3-benches/benches/bench_bigint.rs | 27 ++--- pyo3-benches/benches/bench_decimal.rs | 8 +- pyo3-benches/benches/bench_dict.rs | 26 ++--- pyo3-benches/benches/bench_extract.rs | 25 +++-- src/conversions/num_bigint.rs | 8 +- src/conversions/smallvec.rs | 2 +- src/conversions/std/time.rs | 6 +- src/impl_/extract_argument.rs | 2 +- src/instance.rs | 8 +- src/marshal.rs | 14 +-- src/sync.rs | 14 +-- src/types/any.rs | 24 ++--- src/types/dict.rs | 7 ++ src/types/ellipsis.rs | 2 +- src/types/none.rs | 2 +- src/types/notimplemented.rs | 4 +- tests/test_frompyobject.rs | 129 ++++++++++++++---------- tests/test_proto_methods.rs | 2 +- tests/ui/wrong_aspyref_lifetimes.rs | 6 +- tests/ui/wrong_aspyref_lifetimes.stderr | 12 +-- 22 files changed, 189 insertions(+), 153 deletions(-) diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index 71e823f8c46..b46e5c02f4c 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -86,7 +86,7 @@ struct RustyStruct { # use pyo3::types::PyDict; # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let dict = PyDict::new(py); +# let dict = PyDict::new_bound(py); # dict.set_item("my_string", "test")?; # # let rustystruct: RustyStruct = dict.extract()?; diff --git a/pyo3-benches/benches/bench_any.rs b/pyo3-benches/benches/bench_any.rs index bfd010efd19..b77ab9567a6 100644 --- a/pyo3-benches/benches/bench_any.rs +++ b/pyo3-benches/benches/bench_any.rs @@ -1,11 +1,11 @@ use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ + prelude::*, types::{ PyBool, PyByteArray, PyBytes, PyDict, PyFloat, PyFrozenSet, PyInt, PyList, PyMapping, PySequence, PySet, PyString, PyTuple, }, - PyAny, PyResult, Python, }; #[derive(PartialEq, Eq, Debug)] @@ -27,7 +27,7 @@ enum ObjectType { Unknown, } -fn find_object_type(obj: &PyAny) -> ObjectType { +fn find_object_type(obj: &Bound<'_, PyAny>) -> ObjectType { if obj.is_none() { ObjectType::None } else if obj.is_instance_of::() { @@ -63,17 +63,17 @@ fn find_object_type(obj: &PyAny) -> ObjectType { fn bench_identify_object_type(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let obj = py.eval("object()", None, None).unwrap(); + let obj = py.eval_bound("object()", None, None).unwrap(); - b.iter(|| find_object_type(obj)); + b.iter(|| find_object_type(&obj)); - assert_eq!(find_object_type(obj), ObjectType::Unknown); + assert_eq!(find_object_type(&obj), ObjectType::Unknown); }); } fn bench_collect_generic_iterator(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let collection = py.eval("list(range(1 << 20))", None, None).unwrap(); + let collection = py.eval_bound("list(range(1 << 20))", None, None).unwrap(); b.iter(|| { collection diff --git a/pyo3-benches/benches/bench_bigint.rs b/pyo3-benches/benches/bench_bigint.rs index d3c71629ba4..4a50b437d95 100644 --- a/pyo3-benches/benches/bench_bigint.rs +++ b/pyo3-benches/benches/bench_bigint.rs @@ -1,14 +1,15 @@ use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; -use pyo3::{types::PyDict, PyAny, Python}; +use pyo3::prelude::*; +use pyo3::types::PyDict; use num_bigint::BigInt; fn extract_bigint_extract_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new(py) as &PyAny; + let d = PyDict::new_bound(py).into_any(); - bench.iter(|| match black_box(d).extract::() { + bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), Err(e) => black_box(e), }); @@ -17,10 +18,10 @@ fn extract_bigint_extract_fail(bench: &mut Bencher<'_>) { fn extract_bigint_small(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = py.eval("-42", None, None).unwrap(); + let int = py.eval_bound("-42", None, None).unwrap(); bench.iter(|| { - let v = black_box(int).extract::().unwrap(); + let v = black_box(&int).extract::().unwrap(); black_box(v); }); }); @@ -28,10 +29,10 @@ fn extract_bigint_small(bench: &mut Bencher<'_>) { fn extract_bigint_big_negative(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = py.eval("-10**300", None, None).unwrap(); + let int = py.eval_bound("-10**300", None, None).unwrap(); bench.iter(|| { - let v = black_box(int).extract::().unwrap(); + let v = black_box(&int).extract::().unwrap(); black_box(v); }); }); @@ -39,10 +40,10 @@ fn extract_bigint_big_negative(bench: &mut Bencher<'_>) { fn extract_bigint_big_positive(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = py.eval("10**300", None, None).unwrap(); + let int = py.eval_bound("10**300", None, None).unwrap(); bench.iter(|| { - let v = black_box(int).extract::().unwrap(); + let v = black_box(&int).extract::().unwrap(); black_box(v); }); }); @@ -50,10 +51,10 @@ fn extract_bigint_big_positive(bench: &mut Bencher<'_>) { fn extract_bigint_huge_negative(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = py.eval("-10**3000", None, None).unwrap(); + let int = py.eval_bound("-10**3000", None, None).unwrap(); bench.iter(|| { - let v = black_box(int).extract::().unwrap(); + let v = black_box(&int).extract::().unwrap(); black_box(v); }); }); @@ -61,10 +62,10 @@ fn extract_bigint_huge_negative(bench: &mut Bencher<'_>) { fn extract_bigint_huge_positive(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = py.eval("10**3000", None, None).unwrap(); + let int = py.eval_bound("10**3000", None, None).unwrap(); bench.iter(|| { - let v = black_box(int).extract::().unwrap(); + let v = black_box(&int).extract::().unwrap(); black_box(v); }); }); diff --git a/pyo3-benches/benches/bench_decimal.rs b/pyo3-benches/benches/bench_decimal.rs index 203756b54cc..6db6704bf8e 100644 --- a/pyo3-benches/benches/bench_decimal.rs +++ b/pyo3-benches/benches/bench_decimal.rs @@ -6,20 +6,20 @@ use rust_decimal::Decimal; fn decimal_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let locals = PyDict::new(py); - py.run( + let locals = PyDict::new_bound(py); + py.run_bound( r#" import decimal py_dec = decimal.Decimal("0.0") "#, None, - Some(locals), + Some(&locals), ) .unwrap(); let py_dec = locals.get_item("py_dec").unwrap().unwrap(); b.iter(|| { - let _: Decimal = black_box(py_dec).extract().unwrap(); + let _: Decimal = black_box(&py_dec).extract().unwrap(); }); }) } diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs index 06559519e7e..072dd9408ce 100644 --- a/pyo3-benches/benches/bench_dict.rs +++ b/pyo3-benches/benches/bench_dict.rs @@ -8,10 +8,10 @@ use std::hint::black_box; fn iter_dict(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py); + let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); let mut sum = 0; b.iter(|| { - for (k, _v) in dict { + for (k, _v) in dict.iter() { let i: u64 = k.extract().unwrap(); sum += i; } @@ -22,14 +22,14 @@ fn iter_dict(b: &mut Bencher<'_>) { fn dict_new(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - b.iter(|| (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py)); + b.iter_with_large_drop(|| (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py)); }); } fn dict_get_item(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py); + let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -47,16 +47,16 @@ fn dict_get_item(b: &mut Bencher<'_>) { fn extract_hashmap(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py); - b.iter(|| HashMap::::extract(dict)); + let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + b.iter(|| HashMap::::extract_bound(&dict)); }); } fn extract_btreemap(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py); - b.iter(|| BTreeMap::::extract(dict)); + let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + b.iter(|| BTreeMap::::extract_bound(&dict)); }); } @@ -64,19 +64,15 @@ fn extract_btreemap(b: &mut Bencher<'_>) { fn extract_hashbrown_map(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py); - b.iter(|| hashbrown::HashMap::::extract(dict)); + let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); + b.iter(|| hashbrown::HashMap::::extract_bound(&dict)); }); } fn mapping_from_dict(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let dict = &(0..LEN as u64) - .map(|i| (i, i * 2)) - .into_py_dict(py) - .to_object(py) - .into_bound(py); + let dict = &(0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); b.iter(|| black_box(dict).downcast::().unwrap()); }); } diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs index 479bd0fd547..1c783c3b706 100644 --- a/pyo3-benches/benches/bench_extract.rs +++ b/pyo3-benches/benches/bench_extract.rs @@ -3,7 +3,6 @@ use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Benc use pyo3::{ prelude::*, types::{PyDict, PyFloat, PyInt, PyString}, - IntoPy, PyAny, PyObject, Python, }; fn extract_str_extract_success(bench: &mut Bencher<'_>) { @@ -16,9 +15,9 @@ fn extract_str_extract_success(bench: &mut Bencher<'_>) { fn extract_str_extract_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new(py) as &PyAny; + let d = PyDict::new_bound(py).into_any(); - bench.iter(|| match black_box(d).extract::<&str>() { + bench.iter(|| match black_box(&d).extract::<&str>() { Ok(v) => panic!("should err {}", v), Err(e) => black_box(e), }); @@ -39,9 +38,9 @@ fn extract_str_downcast_success(bench: &mut Bencher<'_>) { fn extract_str_downcast_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new(py) as &PyAny; + let d = PyDict::new_bound(py).into_any(); - bench.iter(|| match black_box(d).downcast::() { + bench.iter(|| match black_box(&d).downcast::() { Ok(v) => panic!("should err {}", v), Err(e) => black_box(e), }); @@ -62,9 +61,9 @@ fn extract_int_extract_success(bench: &mut Bencher<'_>) { fn extract_int_extract_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new(py) as &PyAny; + let d = PyDict::new_bound(py).into_any(); - bench.iter(|| match black_box(d).extract::() { + bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), Err(e) => black_box(e), }); @@ -86,9 +85,9 @@ fn extract_int_downcast_success(bench: &mut Bencher<'_>) { fn extract_int_downcast_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new(py) as &PyAny; + let d = PyDict::new_bound(py).into_any(); - bench.iter(|| match black_box(d).downcast::() { + bench.iter(|| match black_box(&d).downcast::() { Ok(v) => panic!("should err {}", v), Err(e) => black_box(e), }); @@ -109,9 +108,9 @@ fn extract_float_extract_success(bench: &mut Bencher<'_>) { fn extract_float_extract_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new(py) as &PyAny; + let d = PyDict::new_bound(py).into_any(); - bench.iter(|| match black_box(d).extract::() { + bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), Err(e) => black_box(e), }); @@ -133,9 +132,9 @@ fn extract_float_downcast_success(bench: &mut Bencher<'_>) { fn extract_float_downcast_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let d = PyDict::new(py) as &PyAny; + let d = PyDict::new_bound(py).into_any(); - bench.iter(|| match black_box(d).downcast::() { + bench.iter(|| match black_box(&d).downcast::() { Ok(v) => panic!("should err {}", v), Err(e) => black_box(e), }); diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index ebda6c85348..ff5bd0309bd 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -85,14 +85,18 @@ macro_rules! bigint_conversion { let bytes = $to_bytes(self); let bytes_obj = PyBytes::new_bound(py, &bytes); let kwargs = if $is_signed > 0 { - let kwargs = PyDict::new(py); + let kwargs = PyDict::new_bound(py); kwargs.set_item(crate::intern!(py, "signed"), true).unwrap(); Some(kwargs) } else { None }; py.get_type::() - .call_method("from_bytes", (bytes_obj, "little"), kwargs) + .call_method( + "from_bytes", + (bytes_obj, "little"), + kwargs.as_ref().map(crate::Bound::as_gil_ref), + ) .expect("int.from_bytes() failed during to_object()") // FIXME: #1813 or similar .into() } diff --git a/src/conversions/smallvec.rs b/src/conversions/smallvec.rs index ade64a5106c..f51da1de32a 100644 --- a/src/conversions/smallvec.rs +++ b/src/conversions/smallvec.rs @@ -121,7 +121,7 @@ mod tests { #[test] fn test_smallvec_from_py_object_fails() { Python::with_gil(|py| { - let dict = PyDict::new(py); + let dict = PyDict::new_bound(py); let sv: PyResult> = dict.extract(); assert_eq!( sv.unwrap_err().to_string(), diff --git a/src/conversions/std/time.rs b/src/conversions/std/time.rs index bdf938c0bd8..201dd67ceea 100755 --- a/src/conversions/std/time.rs +++ b/src/conversions/std/time.rs @@ -337,9 +337,11 @@ mod tests { fn max_datetime(py: Python<'_>) -> &PyAny { let naive_max = datetime_class(py).getattr("max").unwrap(); - let kargs = PyDict::new(py); + let kargs = PyDict::new_bound(py); kargs.set_item("tzinfo", tz_utc(py)).unwrap(); - naive_max.call_method("replace", (), Some(kargs)).unwrap() + naive_max + .call_method("replace", (), Some(kargs.as_gil_ref())) + .unwrap() } #[test] diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index b9f3fa2757f..4df674e7639 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -681,7 +681,7 @@ impl<'py> VarkeywordsHandler<'py> for DictVarkeywords { _function_description: &FunctionDescription, ) -> PyResult<()> { varkeywords - .get_or_insert_with(|| PyDict::new(name.py())) + .get_or_insert_with(|| PyDict::new_bound(name.py()).into_gil_ref()) .set_item(name, value) } } diff --git a/src/instance.rs b/src/instance.rs index d99fe6f1767..7f92744ccc4 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -634,7 +634,7 @@ impl IntoPy for Borrowed<'_, '_, T> { /// #[new] /// fn __new__() -> Foo { /// Python::with_gil(|py| { -/// let dict: Py = PyDict::new(py).into(); +/// let dict: Py = PyDict::new_bound(py).unbind(); /// Foo { inner: dict } /// }) /// } @@ -706,7 +706,7 @@ impl IntoPy for Borrowed<'_, '_, T> { /// /// # fn main() { /// Python::with_gil(|py| { -/// let first: Py = PyDict::new(py).into(); +/// let first: Py = PyDict::new_bound(py).unbind(); /// /// // All of these are valid syntax /// let second = Py::clone_ref(&first, py); @@ -1130,7 +1130,7 @@ impl Py { /// /// # fn main() { /// Python::with_gil(|py| { - /// let first: Py = PyDict::new(py).into(); + /// let first: Py = PyDict::new_bound(py).unbind(); /// let second = Py::clone_ref(&first, py); /// /// // Both point to the same object @@ -1683,7 +1683,7 @@ impl PyObject { /// use pyo3::types::{PyDict, PyList}; /// /// Python::with_gil(|py| { - /// let any: PyObject = PyDict::new(py).into(); + /// let any: PyObject = PyDict::new_bound(py).into(); /// /// assert!(any.downcast::(py).is_ok()); /// assert!(any.downcast::(py).is_err()); diff --git a/src/marshal.rs b/src/marshal.rs index 5e62e6b6289..9526813976b 100644 --- a/src/marshal.rs +++ b/src/marshal.rs @@ -38,14 +38,14 @@ pub fn dumps<'py>( /// /// # Examples /// ``` -/// # use pyo3::{marshal, types::PyDict}; +/// # use pyo3::{marshal, types::PyDict, prelude::PyDictMethods}; /// # pyo3::Python::with_gil(|py| { -/// let dict = PyDict::new(py); +/// let dict = PyDict::new_bound(py); /// dict.set_item("aap", "noot").unwrap(); /// dict.set_item("mies", "wim").unwrap(); /// dict.set_item("zus", "jet").unwrap(); /// -/// let bytes = marshal::dumps_bound(py, dict, marshal::VERSION); +/// let bytes = marshal::dumps_bound(py, &dict, marshal::VERSION); /// # }); /// ``` pub fn dumps_bound<'py>( @@ -90,20 +90,20 @@ where #[cfg(test)] mod tests { use super::*; - use crate::types::{bytes::PyBytesMethods, PyDict}; + use crate::types::{bytes::PyBytesMethods, dict::PyDictMethods, PyDict}; #[test] fn marshal_roundtrip() { Python::with_gil(|py| { - let dict = PyDict::new(py); + let dict = PyDict::new_bound(py); dict.set_item("aap", "noot").unwrap(); dict.set_item("mies", "wim").unwrap(); dict.set_item("zus", "jet").unwrap(); - let pybytes = dumps_bound(py, dict, VERSION).expect("marshalling failed"); + let pybytes = dumps_bound(py, &dict, VERSION).expect("marshalling failed"); let deserialized = loads_bound(py, pybytes.as_bytes()).expect("unmarshalling failed"); - assert!(equal(py, dict, &deserialized)); + assert!(equal(py, &dict, &deserialized)); }); } diff --git a/src/sync.rs b/src/sync.rs index 76c81026884..04786c8555f 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -210,11 +210,11 @@ impl GILOnceCell> { /// /// ``` /// use pyo3::intern; -/// # use pyo3::{pyfunction, types::PyDict, wrap_pyfunction, PyResult, Python}; +/// # use pyo3::{pyfunction, types::PyDict, wrap_pyfunction, PyResult, Python, prelude::PyDictMethods, Bound}; /// /// #[pyfunction] -/// fn create_dict(py: Python<'_>) -> PyResult<&PyDict> { -/// let dict = PyDict::new(py); +/// fn create_dict(py: Python<'_>) -> PyResult> { +/// let dict = PyDict::new_bound(py); /// // 👇 A new `PyString` is created /// // for every call of this function. /// dict.set_item("foo", 42)?; @@ -222,8 +222,8 @@ impl GILOnceCell> { /// } /// /// #[pyfunction] -/// fn create_dict_faster(py: Python<'_>) -> PyResult<&PyDict> { -/// let dict = PyDict::new(py); +/// fn create_dict_faster(py: Python<'_>) -> PyResult> { +/// let dict = PyDict::new_bound(py); /// // 👇 A `PyString` is created once and reused /// // for the lifetime of the program. /// dict.set_item(intern!(py, "foo"), 42)?; @@ -270,7 +270,7 @@ impl Interned { mod tests { use super::*; - use crate::types::PyDict; + use crate::types::{any::PyAnyMethods, dict::PyDictMethods, PyDict}; #[test] fn test_intern() { @@ -279,7 +279,7 @@ mod tests { let foo2 = intern!(py, "foo"); let foo3 = intern!(py, stringify!(foo)); - let dict = PyDict::new(py); + let dict = PyDict::new_bound(py); dict.set_item(foo1, 42_usize).unwrap(); assert!(dict.contains(foo2).unwrap()); assert_eq!( diff --git a/src/types/any.rs b/src/types/any.rs index afc9a83dd41..73582bbcdc2 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -388,9 +388,9 @@ impl PyAny { /// let module = PyModule::from_code(py, CODE, "", "")?; /// let fun = module.getattr("function")?; /// let args = ("hello",); - /// let kwargs = PyDict::new(py); + /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; - /// let result = fun.call(args, Some(kwargs))?; + /// let result = fun.call(args, Some(kwargs.as_gil_ref()))?; /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); /// Ok(()) /// }) @@ -488,9 +488,9 @@ impl PyAny { /// let module = PyModule::from_code(py, CODE, "", "")?; /// let instance = module.getattr("a")?; /// let args = ("hello",); - /// let kwargs = PyDict::new(py); + /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; - /// let result = instance.call_method("method", args, Some(kwargs))?; + /// let result = instance.call_method("method", args, Some(kwargs.as_gil_ref()))?; /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); /// Ok(()) /// }) @@ -691,9 +691,9 @@ impl PyAny { /// use pyo3::types::{PyDict, PyList}; /// /// Python::with_gil(|py| { - /// let dict = PyDict::new(py); + /// let dict = PyDict::new_bound(py); /// assert!(dict.is_instance_of::()); - /// let any: &PyAny = dict.as_ref(); + /// let any = dict.as_any(); /// /// assert!(any.downcast::().is_ok()); /// assert!(any.downcast::().is_err()); @@ -1268,9 +1268,9 @@ pub trait PyAnyMethods<'py> { /// let module = PyModule::from_code(py, CODE, "", "")?; /// let fun = module.getattr("function")?; /// let args = ("hello",); - /// let kwargs = PyDict::new(py); + /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; - /// let result = fun.call(args, Some(kwargs))?; + /// let result = fun.call(args, Some(kwargs.as_gil_ref()))?; /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); /// Ok(()) /// }) @@ -1360,9 +1360,9 @@ pub trait PyAnyMethods<'py> { /// let module = PyModule::from_code(py, CODE, "", "")?; /// let instance = module.getattr("a")?; /// let args = ("hello",); - /// let kwargs = PyDict::new(py); + /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; - /// let result = instance.call_method("method", args, Some(kwargs))?; + /// let result = instance.call_method("method", args, Some(kwargs.as_gil_ref()))?; /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); /// Ok(()) /// }) @@ -1519,9 +1519,9 @@ pub trait PyAnyMethods<'py> { /// use pyo3::types::{PyDict, PyList}; /// /// Python::with_gil(|py| { - /// let dict = PyDict::new(py); + /// let dict = PyDict::new_bound(py); /// assert!(dict.is_instance_of::()); - /// let any: &PyAny = dict.as_ref(); + /// let any = dict.as_any(); /// /// assert!(any.downcast::().is_ok()); /// assert!(any.downcast::().is_err()); diff --git a/src/types/dict.rs b/src/types/dict.rs index 34812176da8..741d974e264 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -57,6 +57,13 @@ pyobject_native_type_core!( impl PyDict { /// Deprecated form of [`new_bound`][PyDict::new_bound]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyDict::new` will be replaced by `PyDict::new_bound` in a future PyO3 version" + ) + )] #[inline] pub fn new(py: Python<'_>) -> &PyDict { Self::new_bound(py).into_gil_ref() diff --git a/src/types/ellipsis.rs b/src/types/ellipsis.rs index b39b74b3832..4d091c6f9d0 100644 --- a/src/types/ellipsis.rs +++ b/src/types/ellipsis.rs @@ -78,7 +78,7 @@ mod tests { #[test] fn test_dict_is_not_ellipsis() { Python::with_gil(|py| { - assert!(PyDict::new(py).downcast::().is_err()); + assert!(PyDict::new_bound(py).downcast::().is_err()); }) } } diff --git a/src/types/none.rs b/src/types/none.rs index ee650f12f67..6460482e5ff 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -118,7 +118,7 @@ mod tests { #[test] fn test_dict_is_not_none() { Python::with_gil(|py| { - assert!(PyDict::new(py).downcast::().is_err()); + assert!(PyDict::new_bound(py).downcast::().is_err()); }) } } diff --git a/src/types/notimplemented.rs b/src/types/notimplemented.rs index 5832c6a9803..9d31e670e2b 100644 --- a/src/types/notimplemented.rs +++ b/src/types/notimplemented.rs @@ -81,7 +81,9 @@ mod tests { #[test] fn test_dict_is_not_notimplemented() { Python::with_gil(|py| { - assert!(PyDict::new(py).downcast::().is_err()); + assert!(PyDict::new_bound(py) + .downcast::() + .is_err()); }) } } diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index 82e91b84390..c475d8ea81f 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -22,13 +22,13 @@ fn extract_traceback(py: Python<'_>, mut error: PyErr) -> String { } #[derive(Debug, FromPyObject)] -pub struct A<'a> { +pub struct A<'py> { #[pyo3(attribute)] s: String, #[pyo3(item)] - t: &'a PyString, + t: Bound<'py, PyString>, #[pyo3(attribute("foo"))] - p: &'a PyAny, + p: Bound<'py, PyAny>, } #[pyclass] @@ -58,8 +58,9 @@ fn test_named_fields_struct() { foo: None, }; let py_c = Py::new(py, pya).unwrap(); - let a: A<'_> = - FromPyObject::extract(py_c.as_ref(py)).expect("Failed to extract A from PyA"); + let a = py_c + .extract::>(py) + .expect("Failed to extract A from PyA"); assert_eq!(a.s, "foo"); assert_eq!(a.t.to_string_lossy(), "bar"); assert!(a.p.is_none()); @@ -76,10 +77,12 @@ pub struct B { fn test_transparent_named_field_struct() { Python::with_gil(|py| { let test: PyObject = "test".into_py(py); - let b: B = FromPyObject::extract(test.as_ref(py)).expect("Failed to extract B from String"); + let b = test + .extract::(py) + .expect("Failed to extract B from String"); assert_eq!(b.test, "test"); let test: PyObject = 1.into_py(py); - let b = B::extract(test.as_ref(py)); + let b = test.extract::(py); assert!(b.is_err()); }); } @@ -94,12 +97,14 @@ pub struct D { fn test_generic_transparent_named_field_struct() { Python::with_gil(|py| { let test: PyObject = "test".into_py(py); - let d: D = - D::extract(test.as_ref(py)).expect("Failed to extract D from String"); + let d = test + .extract::>(py) + .expect("Failed to extract D from String"); assert_eq!(d.test, "test"); let test = 1usize.into_py(py); - let d: D = - D::extract(test.as_ref(py)).expect("Failed to extract D from String"); + let d = test + .extract::>(py) + .expect("Failed to extract D from String"); assert_eq!(d.test, 1); }); } @@ -128,11 +133,12 @@ fn test_generic_named_fields_struct() { } .into_py(py); - let e: E = - E::extract(pye.as_ref(py)).expect("Failed to extract E from PyE"); + let e = pye + .extract::>(py) + .expect("Failed to extract E from PyE"); assert_eq!(e.test, "test"); assert_eq!(e.test2, 2); - let e = E::::extract(pye.as_ref(py)); + let e = pye.extract::>(py); assert!(e.is_err()); }); } @@ -151,7 +157,7 @@ fn test_named_field_with_ext_fn() { test2: 0, } .into_py(py); - let c = C::extract(pyc.as_ref(py)).expect("Failed to extract C from PyE"); + let c = pyc.extract::(py).expect("Failed to extract C from PyE"); assert_eq!(c.test, "foo"); }); } @@ -163,10 +169,12 @@ pub struct Tuple(String, usize); fn test_tuple_struct() { Python::with_gil(|py| { let tup = PyTuple::new_bound(py, &[1.into_py(py), "test".into_py(py)]); - let tup = Tuple::extract(tup.as_gil_ref()); + let tup = tup.extract::(); assert!(tup.is_err()); let tup = PyTuple::new_bound(py, &["test".into_py(py), 1.into_py(py)]); - let tup = Tuple::extract(tup.as_gil_ref()).expect("Failed to extract Tuple from PyTuple"); + let tup = tup + .extract::() + .expect("Failed to extract Tuple from PyTuple"); assert_eq!(tup.0, "test"); assert_eq!(tup.1, 1); }); @@ -179,10 +187,11 @@ pub struct TransparentTuple(String); fn test_transparent_tuple_struct() { Python::with_gil(|py| { let tup: PyObject = 1.into_py(py); - let tup = TransparentTuple::extract(tup.as_ref(py)); + let tup = tup.extract::(py); assert!(tup.is_err()); let test: PyObject = "test".into_py(py); - let tup = TransparentTuple::extract(test.as_ref(py)) + let tup = test + .extract::(py) .expect("Failed to extract TransparentTuple from PyTuple"); assert_eq!(tup.0, "test"); }); @@ -215,7 +224,7 @@ fn test_struct_nested_type_errors() { } .into_py(py); - let test: PyResult> = FromPyObject::extract(pybaz.as_ref(py)); + let test = pybaz.extract::>(py); assert!(test.is_err()); assert_eq!( extract_traceback(py,test.unwrap_err()), @@ -237,7 +246,7 @@ fn test_struct_nested_type_errors_with_generics() { } .into_py(py); - let test: PyResult> = FromPyObject::extract(pybaz.as_ref(py)); + let test = pybaz.extract::>(py); assert!(test.is_err()); assert_eq!( extract_traceback(py, test.unwrap_err()), @@ -251,7 +260,7 @@ fn test_struct_nested_type_errors_with_generics() { fn test_transparent_struct_error_message() { Python::with_gil(|py| { let tup: PyObject = 1.into_py(py); - let tup = B::extract(tup.as_ref(py)); + let tup = tup.extract::(py); assert!(tup.is_err()); assert_eq!( extract_traceback(py,tup.unwrap_err()), @@ -265,7 +274,7 @@ fn test_transparent_struct_error_message() { fn test_tuple_struct_error_message() { Python::with_gil(|py| { let tup: PyObject = (1, "test").into_py(py); - let tup = Tuple::extract(tup.as_ref(py)); + let tup = tup.extract::(py); assert!(tup.is_err()); assert_eq!( extract_traceback(py, tup.unwrap_err()), @@ -279,7 +288,7 @@ fn test_tuple_struct_error_message() { fn test_transparent_tuple_error_message() { Python::with_gil(|py| { let tup: PyObject = 1.into_py(py); - let tup = TransparentTuple::extract(tup.as_ref(py)); + let tup = tup.extract::(py); assert!(tup.is_err()); assert_eq!( extract_traceback(py, tup.unwrap_err()), @@ -290,10 +299,10 @@ fn test_transparent_tuple_error_message() { } #[derive(Debug, FromPyObject)] -pub enum Foo<'a> { +pub enum Foo<'py> { TupleVar(usize, String), StructVar { - test: &'a PyString, + test: Bound<'py, PyString>, }, #[pyo3(transparent)] TransparentTuple(usize), @@ -325,7 +334,9 @@ pub struct PyBool { fn test_enum() { Python::with_gil(|py| { let tup = PyTuple::new_bound(py, &[1.into_py(py), "test".into_py(py)]); - let f = Foo::extract(tup.as_gil_ref()).expect("Failed to extract Foo from tuple"); + let f = tup + .extract::>() + .expect("Failed to extract Foo from tuple"); match f { Foo::TupleVar(test, test2) => { assert_eq!(test, 1); @@ -339,43 +350,55 @@ fn test_enum() { test2: 0, } .into_py(py); - let f = Foo::extract(pye.as_ref(py)).expect("Failed to extract Foo from PyE"); + let f = pye + .extract::>(py) + .expect("Failed to extract Foo from PyE"); match f { Foo::StructVar { test } => assert_eq!(test.to_string_lossy(), "foo"), _ => panic!("Expected extracting Foo::StructVar, got {:?}", f), } let int: PyObject = 1.into_py(py); - let f = Foo::extract(int.as_ref(py)).expect("Failed to extract Foo from int"); + let f = int + .extract::>(py) + .expect("Failed to extract Foo from int"); match f { Foo::TransparentTuple(test) => assert_eq!(test, 1), _ => panic!("Expected extracting Foo::TransparentTuple, got {:?}", f), } let none = py.None(); - let f = Foo::extract(none.as_ref(py)).expect("Failed to extract Foo from int"); + let f = none + .extract::>(py) + .expect("Failed to extract Foo from int"); match f { Foo::TransparentStructVar { a } => assert!(a.is_none()), _ => panic!("Expected extracting Foo::TransparentStructVar, got {:?}", f), } let pybool = PyBool { bla: true }.into_py(py); - let f = Foo::extract(pybool.as_ref(py)).expect("Failed to extract Foo from PyBool"); + let f = pybool + .extract::>(py) + .expect("Failed to extract Foo from PyBool"); match f { Foo::StructVarGetAttrArg { a } => assert!(a), _ => panic!("Expected extracting Foo::StructVarGetAttrArg, got {:?}", f), } - let dict = PyDict::new(py); + let dict = PyDict::new_bound(py); dict.set_item("a", "test").expect("Failed to set item"); - let f = Foo::extract(dict.as_ref()).expect("Failed to extract Foo from dict"); + let f = dict + .extract::>() + .expect("Failed to extract Foo from dict"); match f { Foo::StructWithGetItem { a } => assert_eq!(a, "test"), _ => panic!("Expected extracting Foo::StructWithGetItem, got {:?}", f), } - let dict = PyDict::new(py); + let dict = PyDict::new_bound(py); dict.set_item("foo", "test").expect("Failed to set item"); - let f = Foo::extract(dict.as_ref()).expect("Failed to extract Foo from dict"); + let f = dict + .extract::>() + .expect("Failed to extract Foo from dict"); match f { Foo::StructWithGetItemArg { a } => assert_eq!(a, "test"), _ => panic!("Expected extracting Foo::StructWithGetItemArg, got {:?}", f), @@ -386,8 +409,8 @@ fn test_enum() { #[test] fn test_enum_error() { Python::with_gil(|py| { - let dict = PyDict::new(py); - let err = Foo::extract(dict.as_ref()).unwrap_err(); + let dict = PyDict::new_bound(py); + let err = dict.extract::>().unwrap_err(); assert_eq!( err.to_string(), "\ @@ -402,7 +425,7 @@ TypeError: failed to extract enum Foo ('TupleVar | StructVar | TransparentTuple ); let tup = PyTuple::empty_bound(py); - let err = Foo::extract(tup.as_gil_ref()).unwrap_err(); + let err = tup.extract::>().unwrap_err(); assert_eq!( err.to_string(), "\ @@ -419,23 +442,24 @@ TypeError: failed to extract enum Foo ('TupleVar | StructVar | TransparentTuple } #[derive(Debug, FromPyObject)] -enum EnumWithCatchAll<'a> { +enum EnumWithCatchAll<'py> { #[allow(dead_code)] #[pyo3(transparent)] - Foo(Foo<'a>), + Foo(Foo<'py>), #[pyo3(transparent)] - CatchAll(&'a PyAny), + CatchAll(Bound<'py, PyAny>), } #[test] fn test_enum_catch_all() { Python::with_gil(|py| { - let dict = PyDict::new(py); - let f = EnumWithCatchAll::extract(dict.as_ref()) + let dict = PyDict::new_bound(py); + let f = dict + .extract::>() .expect("Failed to extract EnumWithCatchAll from dict"); match f { EnumWithCatchAll::CatchAll(any) => { - let d = <&PyDict>::extract(any).expect("Expected pydict"); + let d = any.extract::>().expect("Expected pydict"); assert!(d.is_empty()); } _ => panic!( @@ -459,8 +483,8 @@ pub enum Bar { #[test] fn test_err_rename() { Python::with_gil(|py| { - let dict = PyDict::new(py); - let f = Bar::extract(dict.as_ref()); + let dict = PyDict::new_bound(py); + let f = dict.extract::(); assert!(f.is_err()); assert_eq!( f.unwrap_err().to_string(), @@ -493,7 +517,7 @@ fn test_from_py_with() { ) .expect("failed to create dict"); - let zap = Zap::extract_bound(&py_zap).unwrap(); + let zap = py_zap.extract::().unwrap(); assert_eq!(zap.name, "whatever"); assert_eq!(zap.some_object_length, 3usize); @@ -510,7 +534,7 @@ fn test_from_py_with_tuple_struct() { .eval_bound(r#"("whatever", [1, 2, 3])"#, None, None) .expect("failed to create tuple"); - let zap = ZapTuple::extract_bound(&py_zap).unwrap(); + let zap = py_zap.extract::().unwrap(); assert_eq!(zap.0, "whatever"); assert_eq!(zap.1, 3usize); @@ -524,7 +548,7 @@ fn test_from_py_with_tuple_struct_error() { .eval_bound(r#"("whatever", [1, 2, 3], "third")"#, None, None) .expect("failed to create tuple"); - let f = ZapTuple::extract_bound(&py_zap); + let f = py_zap.extract::(); assert!(f.is_err()); assert_eq!( @@ -547,7 +571,7 @@ fn test_from_py_with_enum() { .eval_bound(r#"("whatever", [1, 2, 3])"#, None, None) .expect("failed to create tuple"); - let zap = ZapEnum::extract_bound(&py_zap).unwrap(); + let zap = py_zap.extract::().unwrap(); let expected_zap = ZapEnum::Zip(2); assert_eq!(zap, expected_zap); @@ -564,8 +588,9 @@ pub struct TransparentFromPyWith { #[test] fn test_transparent_from_py_with() { Python::with_gil(|py| { - let result = - TransparentFromPyWith::extract(PyList::new_bound(py, [1, 2, 3]).as_gil_ref()).unwrap(); + let result = PyList::new_bound(py, [1, 2, 3]) + .extract::() + .unwrap(); let expected = TransparentFromPyWith { len: 3 }; assert_eq!(result, expected); diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index b1fe7548b05..872777bf706 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -216,7 +216,7 @@ fn mapping() { let inst = Py::new( py, Mapping { - values: PyDict::new(py).into(), + values: PyDict::new_bound(py).into(), }, ) .unwrap(); diff --git a/tests/ui/wrong_aspyref_lifetimes.rs b/tests/ui/wrong_aspyref_lifetimes.rs index 5cd8a2d3cb6..755b0cf2a2c 100644 --- a/tests/ui/wrong_aspyref_lifetimes.rs +++ b/tests/ui/wrong_aspyref_lifetimes.rs @@ -1,10 +1,10 @@ -use pyo3::{types::PyDict, Py, Python}; +use pyo3::{types::PyDict, Bound, Py, Python}; fn main() { - let dict: Py = Python::with_gil(|py| PyDict::new(py).into()); + let dict: Py = Python::with_gil(|py| PyDict::new_bound(py).unbind()); // Should not be able to get access to Py contents outside of with_gil. - let dict: &PyDict = Python::with_gil(|py| dict.as_ref(py)); + let dict: &Bound<'_, PyDict> = Python::with_gil(|py| dict.bind(py)); let _py: Python = dict.py(); // Obtain a Python<'p> without GIL. } diff --git a/tests/ui/wrong_aspyref_lifetimes.stderr b/tests/ui/wrong_aspyref_lifetimes.stderr index def6f94c02d..30e63bb8261 100644 --- a/tests/ui/wrong_aspyref_lifetimes.stderr +++ b/tests/ui/wrong_aspyref_lifetimes.stderr @@ -1,8 +1,8 @@ error: lifetime may not live long enough - --> tests/ui/wrong_aspyref_lifetimes.rs:7:47 + --> tests/ui/wrong_aspyref_lifetimes.rs:7:58 | -7 | let dict: &PyDict = Python::with_gil(|py| dict.as_ref(py)); - | --- ^^^^^^^^^^^^^^^ returning this value requires that `'1` must outlive `'2` - | | | - | | return type of closure is &'2 PyDict - | has type `pyo3::Python<'1>` +7 | let dict: &Bound<'_, PyDict> = Python::with_gil(|py| dict.bind(py)); + | --- ^^^^^^^^^^^^^ returning this value requires that `'1` must outlive `'2` + | | | + | | return type of closure is &'2 pyo3::Bound<'_, PyDict> + | has type `pyo3::Python<'1>` From 1279467d2749ad869ce8121a028854299cd7c8e2 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Mon, 12 Feb 2024 08:32:51 +0000 Subject: [PATCH 122/349] Pyerr isinstance (#3826) * Implement PyErr::is_instance_bound * Update is_instance_bound to take a reference Co-authored-by: David Hewitt * Remove spurious clone --------- Co-authored-by: David Hewitt --- src/err/mod.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/err/mod.rs b/src/err/mod.rs index 095c89292ce..12827d7d948 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -538,12 +538,25 @@ impl PyErr { where T: ToPyObject, { - self.is_instance(py, exc.to_object(py).as_ref(py)) + self.is_instance_bound(py, exc.to_object(py).bind(py)) } - /// Returns true if the current exception is instance of `T`. + /// Deprecated form of `PyErr::is_instance_bound`. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyErr::is_instance` will be replaced by `PyErr::is_instance_bound` in a future PyO3 version" + ) + )] #[inline] pub fn is_instance(&self, py: Python<'_>, ty: &PyAny) -> bool { + self.is_instance_bound(py, &ty.as_borrowed()) + } + + /// Returns true if the current exception is instance of `T`. + #[inline] + pub fn is_instance_bound(&self, py: Python<'_>, ty: &Bound<'_, PyAny>) -> bool { let type_bound = self.get_type_bound(py); (unsafe { ffi::PyErr_GivenExceptionMatches(type_bound.as_ptr(), ty.as_ptr()) }) != 0 } @@ -554,7 +567,7 @@ impl PyErr { where T: PyTypeInfo, { - self.is_instance(py, T::type_object_bound(py).as_gil_ref()) + self.is_instance_bound(py, &T::type_object_bound(py)) } /// Writes the error back to the Python interpreter's global state. From 5b9b76fe58a3f593350975723c490fdbc33e1f56 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 12 Feb 2024 20:49:58 +0000 Subject: [PATCH 123/349] add `_bound` constructors for datetime types (#3778) * add `_bound` constructors for datetime types * review: Icxolu feedback * update uses of deprecated timezone_utc --- pytests/src/datetime.rs | 84 +++++---- src/conversions/chrono.rs | 47 +++-- src/conversions/std/time.rs | 17 +- src/ffi/tests.rs | 29 ++- src/types/datetime.rs | 328 ++++++++++++++++++++++++++++------ src/types/mod.rs | 7 +- tests/test_datetime.rs | 12 +- tests/test_datetime_import.rs | 2 +- 8 files changed, 384 insertions(+), 142 deletions(-) diff --git a/pytests/src/datetime.rs b/pytests/src/datetime.rs index 1407da3fe76..9d8f32a93c9 100644 --- a/pytests/src/datetime.rs +++ b/pytests/src/datetime.rs @@ -7,8 +7,8 @@ use pyo3::types::{ }; #[pyfunction] -fn make_date(py: Python<'_>, year: i32, month: u8, day: u8) -> PyResult<&PyDate> { - PyDate::new(py, year, month, day) +fn make_date(py: Python<'_>, year: i32, month: u8, day: u8) -> PyResult> { + PyDate::new_bound(py, year, month, day) } #[pyfunction] @@ -17,34 +17,34 @@ fn get_date_tuple<'p>(py: Python<'p>, d: &PyDate) -> Bound<'p, PyTuple> { } #[pyfunction] -fn date_from_timestamp(py: Python<'_>, timestamp: i64) -> PyResult<&PyDate> { - PyDate::from_timestamp(py, timestamp) +fn date_from_timestamp(py: Python<'_>, timestamp: i64) -> PyResult> { + PyDate::from_timestamp_bound(py, timestamp) } #[pyfunction] -fn make_time<'p>( - py: Python<'p>, +fn make_time<'py>( + py: Python<'py>, hour: u8, minute: u8, second: u8, microsecond: u32, - tzinfo: Option<&PyTzInfo>, -) -> PyResult<&'p PyTime> { - PyTime::new(py, hour, minute, second, microsecond, tzinfo) + tzinfo: Option<&Bound<'py, PyTzInfo>>, +) -> PyResult> { + PyTime::new_bound(py, hour, minute, second, microsecond, tzinfo) } #[pyfunction] #[pyo3(signature = (hour, minute, second, microsecond, tzinfo, fold))] -fn time_with_fold<'p>( - py: Python<'p>, +fn time_with_fold<'py>( + py: Python<'py>, hour: u8, minute: u8, second: u8, microsecond: u32, - tzinfo: Option<&PyTzInfo>, + tzinfo: Option<&Bound<'py, PyTzInfo>>, fold: bool, -) -> PyResult<&'p PyTime> { - PyTime::new_with_fold(py, hour, minute, second, microsecond, tzinfo, fold) +) -> PyResult> { + PyTime::new_bound_with_fold(py, hour, minute, second, microsecond, tzinfo, fold) } #[pyfunction] @@ -75,14 +75,19 @@ fn get_time_tuple_fold<'p>(py: Python<'p>, dt: &PyTime) -> Bound<'p, PyTuple> { } #[pyfunction] -fn make_delta(py: Python<'_>, days: i32, seconds: i32, microseconds: i32) -> PyResult<&PyDelta> { - PyDelta::new(py, days, seconds, microseconds, true) +fn make_delta( + py: Python<'_>, + days: i32, + seconds: i32, + microseconds: i32, +) -> PyResult> { + PyDelta::new_bound(py, days, seconds, microseconds, true) } #[pyfunction] -fn get_delta_tuple<'p>(py: Python<'p>, delta: &PyDelta) -> Bound<'p, PyTuple> { +fn get_delta_tuple<'py>(delta: &Bound<'py, PyDelta>) -> Bound<'py, PyTuple> { PyTuple::new_bound( - py, + delta.py(), [ delta.get_days(), delta.get_seconds(), @@ -93,8 +98,8 @@ fn get_delta_tuple<'p>(py: Python<'p>, delta: &PyDelta) -> Bound<'p, PyTuple> { #[allow(clippy::too_many_arguments)] #[pyfunction] -fn make_datetime<'p>( - py: Python<'p>, +fn make_datetime<'py>( + py: Python<'py>, year: i32, month: u8, day: u8, @@ -102,9 +107,9 @@ fn make_datetime<'p>( minute: u8, second: u8, microsecond: u32, - tzinfo: Option<&PyTzInfo>, -) -> PyResult<&'p PyDateTime> { - PyDateTime::new( + tzinfo: Option<&Bound<'py, PyTzInfo>>, +) -> PyResult> { + PyDateTime::new_bound( py, year, month, @@ -118,7 +123,7 @@ fn make_datetime<'p>( } #[pyfunction] -fn get_datetime_tuple<'p>(py: Python<'p>, dt: &PyDateTime) -> Bound<'p, PyTuple> { +fn get_datetime_tuple<'py>(py: Python<'py>, dt: &Bound<'py, PyDateTime>) -> Bound<'py, PyTuple> { PyTuple::new_bound( py, [ @@ -134,7 +139,10 @@ fn get_datetime_tuple<'p>(py: Python<'p>, dt: &PyDateTime) -> Bound<'p, PyTuple> } #[pyfunction] -fn get_datetime_tuple_fold<'p>(py: Python<'p>, dt: &PyDateTime) -> Bound<'p, PyTuple> { +fn get_datetime_tuple_fold<'py>( + py: Python<'py>, + dt: &Bound<'py, PyDateTime>, +) -> Bound<'py, PyTuple> { PyTuple::new_bound( py, [ @@ -151,21 +159,21 @@ fn get_datetime_tuple_fold<'p>(py: Python<'p>, dt: &PyDateTime) -> Bound<'p, PyT } #[pyfunction] -fn datetime_from_timestamp<'p>( - py: Python<'p>, +fn datetime_from_timestamp<'py>( + py: Python<'py>, ts: f64, - tz: Option<&PyTzInfo>, -) -> PyResult<&'p PyDateTime> { - PyDateTime::from_timestamp(py, ts, tz) + tz: Option<&Bound<'py, PyTzInfo>>, +) -> PyResult> { + PyDateTime::from_timestamp_bound(py, ts, tz) } #[pyfunction] -fn get_datetime_tzinfo(dt: &PyDateTime) -> Option> { +fn get_datetime_tzinfo<'py>(dt: &Bound<'py, PyDateTime>) -> Option> { dt.get_tzinfo_bound() } #[pyfunction] -fn get_time_tzinfo(dt: &PyTime) -> Option> { +fn get_time_tzinfo<'py>(dt: &Bound<'py, PyTime>) -> Option> { dt.get_tzinfo_bound() } @@ -179,15 +187,19 @@ impl TzClass { TzClass {} } - fn utcoffset<'p>(&self, py: Python<'p>, _dt: &PyDateTime) -> PyResult<&'p PyDelta> { - PyDelta::new(py, 0, 3600, 0, true) + fn utcoffset<'py>( + &self, + py: Python<'py>, + _dt: &Bound<'py, PyDateTime>, + ) -> PyResult> { + PyDelta::new_bound(py, 0, 3600, 0, true) } - fn tzname(&self, _py: Python<'_>, _dt: &PyDateTime) -> String { + fn tzname(&self, _dt: &Bound<'_, PyDateTime>) -> String { String::from("+01:00") } - fn dst(&self, _py: Python<'_>, _dt: &PyDateTime) -> Option<&PyDelta> { + fn dst<'py>(&self, _dt: &Bound<'py, PyDateTime>) -> Option> { None } } diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 55d49d71d95..80a83b23915 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -47,14 +47,12 @@ use crate::types::any::PyAnyMethods; use crate::types::datetime::timezone_from_offset; #[cfg(not(Py_LIMITED_API))] use crate::types::{ - timezone_utc, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, PyTimeAccess, - PyTzInfo, PyTzInfoAccess, + timezone_utc_bound, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, + PyTimeAccess, PyTzInfo, PyTzInfoAccess, }; #[cfg(Py_LIMITED_API)] use crate::{intern, DowncastError}; -use crate::{ - Bound, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, ToPyObject, -}; +use crate::{Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject}; use chrono::offset::{FixedOffset, Utc}; use chrono::{ DateTime, Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Offset, TimeZone, Timelike, @@ -81,7 +79,7 @@ impl ToPyObject for Duration { // We pass true as the `normalize` parameter since we'd need to do several checks here to // avoid that, and it shouldn't have a big performance impact. // The seconds and microseconds cast should never overflow since it's at most the number of seconds per day - PyDelta::new( + PyDelta::new_bound( py, days.try_into().unwrap_or(i32::MAX), secs.try_into().unwrap(), @@ -144,7 +142,7 @@ impl ToPyObject for NaiveDate { let DateArgs { year, month, day } = self.into(); #[cfg(not(Py_LIMITED_API))] { - PyDate::new(py, year, month, day) + PyDate::new_bound(py, year, month, day) .expect("failed to construct date") .into() } @@ -189,15 +187,16 @@ impl ToPyObject for NaiveTime { truncated_leap_second, } = self.into(); #[cfg(not(Py_LIMITED_API))] - let time = PyTime::new(py, hour, min, sec, micro, None).expect("Failed to construct time"); + let time = + PyTime::new_bound(py, hour, min, sec, micro, None).expect("Failed to construct time"); #[cfg(Py_LIMITED_API)] let time = DatetimeTypes::get(py) .time - .as_ref(py) + .bind(py) .call1((hour, min, sec, micro)) .expect("failed to construct datetime.time"); if truncated_leap_second { - warn_truncated_leap_second(time); + warn_truncated_leap_second(&time); } time.into() } @@ -264,7 +263,7 @@ impl ToPyObject for DateTime { // FIXME: convert to better timezone representation here than just convert to fixed offset // See https://github.com/PyO3/pyo3/issues/3266 let tz = self.offset().fix().to_object(py); - let tz = tz.downcast(py).unwrap(); + let tz = tz.bind(py).downcast().unwrap(); naive_datetime_to_py_datetime(py, &self.naive_local(), Some(tz)) } } @@ -310,9 +309,9 @@ impl ToPyObject for FixedOffset { #[cfg(not(Py_LIMITED_API))] { - let td = PyDelta::new(py, 0, seconds_offset, 0, true) + let td = PyDelta::new_bound(py, 0, seconds_offset, 0, true) .expect("failed to construct timedelta"); - timezone_from_offset(py, td) + timezone_from_offset(&td) .expect("Failed to construct PyTimezone") .into() } @@ -366,7 +365,7 @@ impl FromPyObject<'_> for FixedOffset { impl ToPyObject for Utc { fn to_object(&self, py: Python<'_>) -> PyObject { - timezone_utc(py).into() + timezone_utc_bound(py).into() } } @@ -378,7 +377,7 @@ impl IntoPy for Utc { impl FromPyObject<'_> for Utc { fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - let py_utc = timezone_utc(ob.py()); + let py_utc = timezone_utc_bound(ob.py()); if ob.eq(py_utc)? { Ok(Utc) } else { @@ -430,8 +429,8 @@ impl From<&NaiveTime> for TimeArgs { fn naive_datetime_to_py_datetime( py: Python<'_>, naive_datetime: &NaiveDateTime, - #[cfg(not(Py_LIMITED_API))] tzinfo: Option<&PyTzInfo>, - #[cfg(Py_LIMITED_API)] tzinfo: Option<&PyAny>, + #[cfg(not(Py_LIMITED_API))] tzinfo: Option<&Bound<'_, PyTzInfo>>, + #[cfg(Py_LIMITED_API)] tzinfo: Option<&Bound<'_, PyAny>>, ) -> PyObject { let DateArgs { year, month, day } = (&naive_datetime.date()).into(); let TimeArgs { @@ -442,21 +441,21 @@ fn naive_datetime_to_py_datetime( truncated_leap_second, } = (&naive_datetime.time()).into(); #[cfg(not(Py_LIMITED_API))] - let datetime = PyDateTime::new(py, year, month, day, hour, min, sec, micro, tzinfo) + let datetime = PyDateTime::new_bound(py, year, month, day, hour, min, sec, micro, tzinfo) .expect("failed to construct datetime"); #[cfg(Py_LIMITED_API)] let datetime = DatetimeTypes::get(py) .datetime - .as_ref(py) + .bind(py) .call1((year, month, day, hour, min, sec, micro, tzinfo)) .expect("failed to construct datetime.datetime"); if truncated_leap_second { - warn_truncated_leap_second(datetime); + warn_truncated_leap_second(&datetime); } datetime.into() } -fn warn_truncated_leap_second(obj: &PyAny) { +fn warn_truncated_leap_second(obj: &Bound<'_, PyAny>) { let py = obj.py(); if let Err(e) = PyErr::warn( py, @@ -558,8 +557,8 @@ impl DatetimeTypes { } #[cfg(Py_LIMITED_API)] -fn timezone_utc(py: Python<'_>) -> &PyAny { - DatetimeTypes::get(py).timezone_utc.as_ref(py) +fn timezone_utc_bound(py: Python<'_>) -> Bound<'_, PyAny> { + DatetimeTypes::get(py).timezone_utc.bind(py).clone() } #[cfg(test)] @@ -913,7 +912,7 @@ mod tests { let minute = 8; let second = 9; let micro = 999_999; - let tz_utc = timezone_utc(py); + let tz_utc = timezone_utc_bound(py); let py_datetime = new_py_datetime_ob( py, "datetime", diff --git a/src/conversions/std/time.rs b/src/conversions/std/time.rs index 201dd67ceea..4e6009e30db 100755 --- a/src/conversions/std/time.rs +++ b/src/conversions/std/time.rs @@ -4,7 +4,7 @@ use crate::types::any::PyAnyMethods; #[cfg(Py_LIMITED_API)] use crate::types::PyType; #[cfg(not(Py_LIMITED_API))] -use crate::types::{timezone_utc, PyDateTime, PyDelta, PyDeltaAccess}; +use crate::types::{timezone_utc_bound, PyDateTime, PyDelta, PyDeltaAccess}; #[cfg(Py_LIMITED_API)] use crate::Py; use crate::{ @@ -59,7 +59,7 @@ impl ToPyObject for Duration { #[cfg(not(Py_LIMITED_API))] { - PyDelta::new( + PyDelta::new_bound( py, days.try_into() .expect("Too large Rust duration for timedelta"), @@ -130,7 +130,18 @@ fn unix_epoch_py(py: Python<'_>) -> &PyObject { #[cfg(not(Py_LIMITED_API))] { Ok::<_, PyErr>( - PyDateTime::new(py, 1970, 1, 1, 0, 0, 0, 0, Some(timezone_utc(py)))?.into(), + PyDateTime::new_bound( + py, + 1970, + 1, + 1, + 0, + 0, + 0, + 0, + Some(&timezone_utc_bound(py)), + )? + .into(), ) } #[cfg(Py_LIMITED_API)] diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index 0735e456950..610edb1c92c 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -77,11 +77,11 @@ fn test_utc_timezone() { #[cfg(feature = "macros")] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_timezone_from_offset() { - use crate::types::PyDelta; + use crate::{ffi_ptr_ext::FfiPtrExt, types::PyDelta}; Python::with_gil(|py| { - let delta = PyDelta::new(py, 0, 100, 0, false).unwrap(); - let tz: &PyAny = unsafe { py.from_borrowed_ptr(PyTimeZone_FromOffset(delta.as_ptr())) }; + let delta = PyDelta::new_bound(py, 0, 100, 0, false).unwrap(); + let tz = unsafe { PyTimeZone_FromOffset(delta.as_ptr()).assume_owned(py) }; crate::py_run!( py, tz, @@ -95,16 +95,13 @@ fn test_timezone_from_offset() { #[cfg(feature = "macros")] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_timezone_from_offset_and_name() { - use crate::types::PyDelta; + use crate::{ffi_ptr_ext::FfiPtrExt, types::PyDelta}; Python::with_gil(|py| { - let delta = PyDelta::new(py, 0, 100, 0, false).unwrap(); + let delta = PyDelta::new_bound(py, 0, 100, 0, false).unwrap(); let tzname = PyString::new_bound(py, "testtz"); - let tz: &PyAny = unsafe { - py.from_borrowed_ptr(PyTimeZone_FromOffsetAndName( - delta.as_ptr(), - tzname.as_ptr(), - )) + let tz = unsafe { + PyTimeZone_FromOffsetAndName(delta.as_ptr(), tzname.as_ptr()).assume_owned(py) }; crate::py_run!( py, @@ -253,36 +250,36 @@ fn ucs4() { #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[cfg(not(PyPy))] fn test_get_tzinfo() { - use crate::types::timezone_utc; + use crate::types::timezone_utc_bound; crate::Python::with_gil(|py| { use crate::types::{PyDateTime, PyTime}; use crate::PyAny; - let utc = timezone_utc(py); + let utc = &timezone_utc_bound(py); - let dt = PyDateTime::new(py, 2018, 1, 1, 0, 0, 0, 0, Some(utc)).unwrap(); + let dt = PyDateTime::new_bound(py, 2018, 1, 1, 0, 0, 0, 0, Some(utc)).unwrap(); assert!( unsafe { py.from_borrowed_ptr::(PyDateTime_DATE_GET_TZINFO(dt.as_ptr())) } .is(utc) ); - let dt = PyDateTime::new(py, 2018, 1, 1, 0, 0, 0, 0, None).unwrap(); + let dt = PyDateTime::new_bound(py, 2018, 1, 1, 0, 0, 0, 0, None).unwrap(); assert!( unsafe { py.from_borrowed_ptr::(PyDateTime_DATE_GET_TZINFO(dt.as_ptr())) } .is_none() ); - let t = PyTime::new(py, 0, 0, 0, 0, Some(utc)).unwrap(); + let t = PyTime::new_bound(py, 0, 0, 0, 0, Some(utc)).unwrap(); assert!( unsafe { py.from_borrowed_ptr::(PyDateTime_TIME_GET_TZINFO(t.as_ptr())) } .is(utc) ); - let t = PyTime::new(py, 0, 0, 0, 0, None).unwrap(); + let t = PyTime::new_bound(py, 0, 0, 0, 0, None).unwrap(); assert!( unsafe { py.from_borrowed_ptr::(PyDateTime_TIME_GET_TZINFO(t.as_ptr())) } diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 088e37d9536..d46a2b77c33 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -21,6 +21,7 @@ use crate::ffi::{ }; use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::PyNativeType; +use crate::py_result_ext::PyResultExt; use crate::types::any::PyAnyMethods; use crate::types::PyTuple; use crate::{Bound, IntoPy, Py, PyAny, PyErr, Python}; @@ -201,27 +202,53 @@ pyobject_native_type!( ); impl PyDate { - /// Creates a new `datetime.date`. + /// Deprecated form of [`PyDate::new_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyDate::new` will be replaced by `PyDate::new_bound` in a future PyO3 version" + ) + )] pub fn new(py: Python<'_>, year: i32, month: u8, day: u8) -> PyResult<&PyDate> { + Self::new_bound(py, year, month, day).map(Bound::into_gil_ref) + } + + /// Creates a new `datetime.date`. + pub fn new_bound(py: Python<'_>, year: i32, month: u8, day: u8) -> PyResult> { let api = ensure_datetime_api(py)?; unsafe { - let ptr = (api.Date_FromDate)(year, c_int::from(month), c_int::from(day), api.DateType); - py.from_owned_ptr_or_err(ptr) + (api.Date_FromDate)(year, c_int::from(month), c_int::from(day), api.DateType) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } + /// Deprecated form of [`PyDate::from_timestamp_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyDate::from_timestamp` will be replaced by `PyDate::from_timestamp_bound` in a future PyO3 version" + ) + )] + pub fn from_timestamp(py: Python<'_>, timestamp: i64) -> PyResult<&PyDate> { + Self::from_timestamp_bound(py, timestamp).map(Bound::into_gil_ref) + } + /// Construct a `datetime.date` from a POSIX timestamp /// /// This is equivalent to `datetime.date.fromtimestamp` - pub fn from_timestamp(py: Python<'_>, timestamp: i64) -> PyResult<&PyDate> { + pub fn from_timestamp_bound(py: Python<'_>, timestamp: i64) -> PyResult> { let time_tuple = PyTuple::new_bound(py, [timestamp]); // safety ensure that the API is loaded let _api = ensure_datetime_api(py)?; unsafe { - let ptr = PyDate_FromTimestamp(time_tuple.as_ptr()); - py.from_owned_ptr_or_err(ptr) + PyDate_FromTimestamp(time_tuple.as_ptr()) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } } @@ -266,10 +293,44 @@ pyobject_native_type!( ); impl PyDateTime { + /// Deprecated form of [`PyDateTime::new_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyDateTime::new` will be replaced by `PyDateTime::new_bound` in a future PyO3 version" + ) + )] + #[allow(clippy::too_many_arguments)] + pub fn new<'py>( + py: Python<'py>, + year: i32, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, + microsecond: u32, + tzinfo: Option<&'py PyTzInfo>, + ) -> PyResult<&'py PyDateTime> { + Self::new_bound( + py, + year, + month, + day, + hour, + minute, + second, + microsecond, + tzinfo.map(PyTzInfo::as_borrowed).as_deref(), + ) + .map(Bound::into_gil_ref) + } + /// Creates a new `datetime.datetime` object. #[allow(clippy::too_many_arguments)] - pub fn new<'p>( - py: Python<'p>, + pub fn new_bound<'py>( + py: Python<'py>, year: i32, month: u8, day: u8, @@ -277,11 +338,11 @@ impl PyDateTime { minute: u8, second: u8, microsecond: u32, - tzinfo: Option<&PyTzInfo>, - ) -> PyResult<&'p PyDateTime> { + tzinfo: Option<&Bound<'py, PyTzInfo>>, + ) -> PyResult> { let api = ensure_datetime_api(py)?; unsafe { - let ptr = (api.DateTime_FromDateAndTime)( + (api.DateTime_FromDateAndTime)( year, c_int::from(month), c_int::from(day), @@ -291,11 +352,48 @@ impl PyDateTime { microsecond as c_int, opt_to_pyobj(tzinfo), api.DateTimeType, - ); - py.from_owned_ptr_or_err(ptr) + ) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } + /// Deprecated form of [`PyDateTime::new_bound_with_fold`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyDateTime::new_with_fold` will be replaced by `PyDateTime::new_bound_with_fold` in a future PyO3 version" + ) + )] + #[allow(clippy::too_many_arguments)] + pub fn new_with_fold<'py>( + py: Python<'py>, + year: i32, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, + microsecond: u32, + tzinfo: Option<&'py PyTzInfo>, + fold: bool, + ) -> PyResult<&'py PyDateTime> { + Self::new_bound_with_fold( + py, + year, + month, + day, + hour, + minute, + second, + microsecond, + tzinfo.map(PyTzInfo::as_borrowed).as_deref(), + fold, + ) + .map(Bound::into_gil_ref) + } + /// Alternate constructor that takes a `fold` parameter. A `true` value for this parameter /// signifies this this datetime is the later of two moments with the same representation, /// during a repeated interval. @@ -304,8 +402,8 @@ impl PyDateTime { /// represented time is ambiguous. /// See [PEP 495](https://www.python.org/dev/peps/pep-0495/) for more detail. #[allow(clippy::too_many_arguments)] - pub fn new_with_fold<'p>( - py: Python<'p>, + pub fn new_bound_with_fold<'py>( + py: Python<'py>, year: i32, month: u8, day: u8, @@ -313,12 +411,12 @@ impl PyDateTime { minute: u8, second: u8, microsecond: u32, - tzinfo: Option<&PyTzInfo>, + tzinfo: Option<&Bound<'py, PyTzInfo>>, fold: bool, - ) -> PyResult<&'p PyDateTime> { + ) -> PyResult> { let api = ensure_datetime_api(py)?; unsafe { - let ptr = (api.DateTime_FromDateAndTimeAndFold)( + (api.DateTime_FromDateAndTimeAndFold)( year, c_int::from(month), c_int::from(day), @@ -329,27 +427,46 @@ impl PyDateTime { opt_to_pyobj(tzinfo), c_int::from(fold), api.DateTimeType, - ); - py.from_owned_ptr_or_err(ptr) + ) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } + /// Deprecated form of [`PyDateTime::from_timestamp_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyDateTime::from_timestamp` will be replaced by `PyDateTime::from_timestamp_bound` in a future PyO3 version" + ) + )] + pub fn from_timestamp<'py>( + py: Python<'py>, + timestamp: f64, + tzinfo: Option<&'py PyTzInfo>, + ) -> PyResult<&'py PyDateTime> { + Self::from_timestamp_bound(py, timestamp, tzinfo.map(PyTzInfo::as_borrowed).as_deref()) + .map(Bound::into_gil_ref) + } + /// Construct a `datetime` object from a POSIX timestamp /// /// This is equivalent to `datetime.datetime.fromtimestamp` - pub fn from_timestamp<'p>( - py: Python<'p>, + pub fn from_timestamp_bound<'py>( + py: Python<'py>, timestamp: f64, - tzinfo: Option<&PyTzInfo>, - ) -> PyResult<&'p PyDateTime> { - let args: Py = (timestamp, tzinfo).into_py(py); + tzinfo: Option<&Bound<'py, PyTzInfo>>, + ) -> PyResult> { + let args = IntoPy::>::into_py((timestamp, tzinfo), py).into_bound(py); // safety ensure API is loaded let _api = ensure_datetime_api(py)?; unsafe { - let ptr = PyDateTime_FromTimestamp(args.as_ptr()); - py.from_owned_ptr_or_err(ptr) + PyDateTime_FromTimestamp(args.as_ptr()) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } } @@ -463,42 +580,99 @@ pyobject_native_type!( ); impl PyTime { + /// Deprecated form of [`PyTime::new_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyTime::new` will be replaced by `PyTime::new_bound` in a future PyO3 version" + ) + )] + pub fn new<'py>( + py: Python<'py>, + hour: u8, + minute: u8, + second: u8, + microsecond: u32, + tzinfo: Option<&'py PyTzInfo>, + ) -> PyResult<&'py PyTime> { + Self::new_bound( + py, + hour, + minute, + second, + microsecond, + tzinfo.map(PyTzInfo::as_borrowed).as_deref(), + ) + .map(Bound::into_gil_ref) + } + /// Creates a new `datetime.time` object. - pub fn new<'p>( - py: Python<'p>, + pub fn new_bound<'py>( + py: Python<'py>, hour: u8, minute: u8, second: u8, microsecond: u32, - tzinfo: Option<&PyTzInfo>, - ) -> PyResult<&'p PyTime> { + tzinfo: Option<&Bound<'py, PyTzInfo>>, + ) -> PyResult> { let api = ensure_datetime_api(py)?; unsafe { - let ptr = (api.Time_FromTime)( + (api.Time_FromTime)( c_int::from(hour), c_int::from(minute), c_int::from(second), microsecond as c_int, opt_to_pyobj(tzinfo), api.TimeType, - ); - py.from_owned_ptr_or_err(ptr) + ) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } + /// Deprecated form of [`PyTime::new_bound_with_fold`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyTime::new_with_fold` will be replaced by `PyTime::new_bound_with_fold` in a future PyO3 version" + ) + )] + pub fn new_with_fold<'py>( + py: Python<'py>, + hour: u8, + minute: u8, + second: u8, + microsecond: u32, + tzinfo: Option<&'py PyTzInfo>, + fold: bool, + ) -> PyResult<&'py PyTime> { + Self::new_bound_with_fold( + py, + hour, + minute, + second, + microsecond, + tzinfo.map(PyTzInfo::as_borrowed).as_deref(), + fold, + ) + .map(Bound::into_gil_ref) + } + /// Alternate constructor that takes a `fold` argument. See [`PyDateTime::new_with_fold`]. - pub fn new_with_fold<'p>( - py: Python<'p>, + pub fn new_bound_with_fold<'py>( + py: Python<'py>, hour: u8, minute: u8, second: u8, microsecond: u32, - tzinfo: Option<&PyTzInfo>, + tzinfo: Option<&Bound<'py, PyTzInfo>>, fold: bool, - ) -> PyResult<&'p PyTime> { + ) -> PyResult> { let api = ensure_datetime_api(py)?; unsafe { - let ptr = (api.Time_FromTimeAndFold)( + (api.Time_FromTimeAndFold)( c_int::from(hour), c_int::from(minute), c_int::from(second), @@ -506,8 +680,9 @@ impl PyTime { opt_to_pyobj(tzinfo), fold as c_int, api.TimeType, - ); - py.from_owned_ptr_or_err(ptr) + ) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } } @@ -596,20 +771,45 @@ pyobject_native_type!( #checkfunction=PyTZInfo_Check ); -/// Equivalent to `datetime.timezone.utc` +/// Deprecated form of [`timezone_utc_bound`]. +#[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`timezone_utc` will be replaced by `timezone_utc_bound` in a future PyO3 version" + ) +)] pub fn timezone_utc(py: Python<'_>) -> &PyTzInfo { - unsafe { &*(expect_datetime_api(py).TimeZone_UTC as *const PyTzInfo) } + timezone_utc_bound(py).into_gil_ref() +} + +/// Equivalent to `datetime.timezone.utc` +pub fn timezone_utc_bound(py: Python<'_>) -> Bound<'_, PyTzInfo> { + // TODO: this _could_ have a borrowed form `timezone_utc_borrowed`, but that seems + // like an edge case optimization and we'd prefer in PyO3 0.21 to use `Bound` as + // much as possible + unsafe { + expect_datetime_api(py) + .TimeZone_UTC + .assume_borrowed(py) + .to_owned() + .downcast_into_unchecked() + } } /// Equivalent to `datetime.timezone` constructor /// /// Only used internally #[cfg(feature = "chrono")] -pub fn timezone_from_offset<'a>(py: Python<'a>, offset: &PyDelta) -> PyResult<&'a PyTzInfo> { +pub(crate) fn timezone_from_offset<'py>( + offset: &Bound<'py, PyDelta>, +) -> PyResult> { + let py = offset.py(); let api = ensure_datetime_api(py)?; unsafe { - let ptr = (api.TimeZone_FromTimeZone)(offset.as_ptr(), ptr::null_mut()); - py.from_owned_ptr_or_err(ptr) + (api.TimeZone_FromTimeZone)(offset.as_ptr(), ptr::null_mut()) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } @@ -625,7 +825,14 @@ pyobject_native_type!( ); impl PyDelta { - /// Creates a new `timedelta`. + /// Deprecated form of [`PyDelta::new_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyDelta::new` will be replaced by `PyDelta::new_bound` in a future PyO3 version" + ) + )] pub fn new( py: Python<'_>, days: i32, @@ -633,16 +840,28 @@ impl PyDelta { microseconds: i32, normalize: bool, ) -> PyResult<&PyDelta> { + Self::new_bound(py, days, seconds, microseconds, normalize).map(Bound::into_gil_ref) + } + + /// Creates a new `timedelta`. + pub fn new_bound( + py: Python<'_>, + days: i32, + seconds: i32, + microseconds: i32, + normalize: bool, + ) -> PyResult> { let api = ensure_datetime_api(py)?; unsafe { - let ptr = (api.Delta_FromDelta)( + (api.Delta_FromDelta)( days as c_int, seconds as c_int, microseconds as c_int, normalize as c_int, api.DeltaType, - ); - py.from_owned_ptr_or_err(ptr) + ) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } } @@ -677,7 +896,7 @@ impl PyDeltaAccess for Bound<'_, PyDelta> { // Utility function which returns a borrowed reference to either // the underlying tzinfo or None. -fn opt_to_pyobj(opt: Option<&PyTzInfo>) -> *mut ffi::PyObject { +fn opt_to_pyobj(opt: Option<&Bound<'_, PyTzInfo>>) -> *mut ffi::PyObject { match opt { Some(tzi) => tzi.as_ptr(), None => unsafe { ffi::Py_None() }, @@ -685,6 +904,7 @@ fn opt_to_pyobj(opt: Option<&PyTzInfo>) -> *mut ffi::PyObject { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; #[cfg(feature = "macros")] @@ -768,7 +988,7 @@ mod tests { fn test_timezone_from_offset() { Python::with_gil(|py| { assert!( - timezone_from_offset(py, PyDelta::new(py, 0, -3600, 0, true).unwrap()) + timezone_from_offset(&PyDelta::new_bound(py, 0, -3600, 0, true).unwrap()) .unwrap() .call_method1("utcoffset", ((),)) .unwrap() @@ -779,7 +999,7 @@ mod tests { ); assert!( - timezone_from_offset(py, PyDelta::new(py, 0, 3600, 0, true).unwrap()) + timezone_from_offset(&PyDelta::new_bound(py, 0, 3600, 0, true).unwrap()) .unwrap() .call_method1("utcoffset", ((),)) .unwrap() @@ -789,7 +1009,7 @@ mod tests { .unwrap() ); - timezone_from_offset(py, PyDelta::new(py, 1, 0, 0, true).unwrap()).unwrap_err(); + timezone_from_offset(&PyDelta::new_bound(py, 1, 0, 0, true).unwrap()).unwrap_err(); }) } } diff --git a/src/types/mod.rs b/src/types/mod.rs index 5162a17ba2d..da22b30729f 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -8,10 +8,13 @@ pub use self::capsule::PyCapsule; #[cfg(not(Py_LIMITED_API))] pub use self::code::PyCode; pub use self::complex::PyComplex; +#[allow(deprecated)] +#[cfg(not(Py_LIMITED_API))] +pub use self::datetime::timezone_utc; #[cfg(not(Py_LIMITED_API))] pub use self::datetime::{ - timezone_utc, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, PyTimeAccess, - PyTzInfo, PyTzInfoAccess, + timezone_utc_bound, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, + PyTimeAccess, PyTzInfo, PyTzInfoAccess, }; pub use self::dict::{IntoPyDict, PyDict}; #[cfg(not(PyPy))] diff --git a/tests/test_datetime.rs b/tests/test_datetime.rs index 5d7b1485260..de8efa9b826 100644 --- a/tests/test_datetime.rs +++ b/tests/test_datetime.rs @@ -1,7 +1,7 @@ #![cfg(not(Py_LIMITED_API))] use pyo3::prelude::*; -use pyo3::types::{timezone_utc, IntoPyDict, PyDate, PyDateTime, PyTime}; +use pyo3::types::{timezone_utc_bound, IntoPyDict, PyDate, PyDateTime, PyTime}; use pyo3_ffi::PyDateTime_IMPORT; fn _get_subclasses<'py>( @@ -118,9 +118,9 @@ fn test_datetime_utc() { use pyo3::types::PyDateTime; Python::with_gil(|py| { - let utc = timezone_utc(py); + let utc = timezone_utc_bound(py); - let dt = PyDateTime::new(py, 2018, 1, 1, 0, 0, 0, 0, Some(utc)).unwrap(); + let dt = PyDateTime::new_bound(py, 2018, 1, 1, 0, 0, 0, 0, Some(&utc)).unwrap(); let locals = [("dt", dt)].into_py_dict_bound(py); @@ -155,7 +155,7 @@ fn test_pydate_out_of_bounds() { Python::with_gil(|py| { for val in INVALID_DATES { let (year, month, day) = val; - let dt = PyDate::new(py, *year, *month, *day); + let dt = PyDate::new_bound(py, *year, *month, *day); dt.unwrap_err(); } }); @@ -168,7 +168,7 @@ fn test_pytime_out_of_bounds() { Python::with_gil(|py| { for val in INVALID_TIMES { let (hour, minute, second, microsecond) = val; - let dt = PyTime::new(py, *hour, *minute, *second, *microsecond, None); + let dt = PyTime::new_bound(py, *hour, *minute, *second, *microsecond, None); dt.unwrap_err(); } }); @@ -192,7 +192,7 @@ fn test_pydatetime_out_of_bounds() { let (date, time) = val; let (year, month, day) = date; let (hour, minute, second, microsecond) = time; - let dt = PyDateTime::new( + let dt = PyDateTime::new_bound( py, *year, *month, diff --git a/tests/test_datetime_import.rs b/tests/test_datetime_import.rs index fee99b07d1a..bf0001b8ea4 100644 --- a/tests/test_datetime_import.rs +++ b/tests/test_datetime_import.rs @@ -21,6 +21,6 @@ fn test_bad_datetime_module_panic() { .unwrap(); // This should panic because the "datetime" module is empty - PyDate::new(py, 2018, 1, 1).unwrap(); + PyDate::new_bound(py, 2018, 1, 1).unwrap(); }); } From 94b7d7e43468db4c32be6c3369780da4b850b83d Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 12 Feb 2024 21:40:05 +0000 Subject: [PATCH 124/349] add `DowncastIntoError::into_inner` (#3829) --- src/err/mod.rs | 8 ++++++++ src/types/any.rs | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/err/mod.rs b/src/err/mod.rs index 12827d7d948..31385f4c24c 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -97,6 +97,14 @@ impl<'py> DowncastIntoError<'py> { to: to.into(), } } + + /// Consumes this `DowncastIntoError` and returns the original object, allowing continued + /// use of it after a failed conversion. + /// + /// See [`downcast_into`][PyAnyMethods::downcast_into] for an example. + pub fn into_inner(self) -> Bound<'py, PyAny> { + self.from + } } impl PyErr { diff --git a/src/types/any.rs b/src/types/any.rs index 73582bbcdc2..90c7f2160d3 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1561,6 +1561,27 @@ pub trait PyAnyMethods<'py> { T: PyTypeCheck; /// Like `downcast` but takes ownership of `self`. + /// + /// In case of an error, it is possible to retrieve `self` again via [`DowncastIntoError::into_inner`]. + /// + /// # Example + /// + /// ```rust + /// use pyo3::prelude::*; + /// use pyo3::types::{PyDict, PyList}; + /// + /// Python::with_gil(|py| { + /// let obj: Bound<'_, PyAny> = PyDict::new_bound(py).into_any(); + /// + /// let obj: Bound<'_, PyAny> = match obj.downcast_into::() { + /// Ok(_) => panic!("obj should not be a list"), + /// Err(err) => err.into_inner(), + /// }; + /// + /// // obj is a dictionary + /// assert!(obj.downcast_into::().is_ok()); + /// }) + /// ``` fn downcast_into(self) -> Result, DowncastIntoError<'py>> where T: PyTypeCheck; From fbfeb2ff034027201e24dc46cd70e50016970f7d Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 13 Feb 2024 01:09:41 +0100 Subject: [PATCH 125/349] update `#[derive(FromPyObject)]` to use `extract_bound` (#3828) * update `#[derive(FromPyObject)]` to use `extract_bound` * type inference for `from_py_with` using function pointers --- pyo3-macros-backend/src/frompyobject.rs | 15 ++++---- src/impl_/frompyobject.rs | 48 +++++++++++++++++++------ tests/test_frompyobject.rs | 16 ++++++--- 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index f4774d88c0b..5e193bf4a24 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -271,7 +271,7 @@ impl<'a> Container<'a> { value: expr_path, .. }) => quote! { Ok(#self_ty { - #ident: _pyo3::impl_::frompyobject::extract_struct_field_with(#expr_path, obj, #struct_name, #field_name)? + #ident: _pyo3::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, #field_name)? }) }, } @@ -283,7 +283,7 @@ impl<'a> Container<'a> { Some(FromPyWithAttribute { value: expr_path, .. }) => quote! ( - _pyo3::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path, obj, #struct_name, 0).map(#self_ty) + _pyo3::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, 0).map(#self_ty) ), } } @@ -298,12 +298,12 @@ impl<'a> Container<'a> { let fields = struct_fields.iter().zip(&field_idents).enumerate().map(|(index, (field, ident))| { match &field.from_py_with { None => quote!( - _pyo3::impl_::frompyobject::extract_tuple_struct_field(#ident, #struct_name, #index)? + _pyo3::impl_::frompyobject::extract_tuple_struct_field(&#ident, #struct_name, #index)? ), Some(FromPyWithAttribute { value: expr_path, .. }) => quote! ( - _pyo3::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path, #ident, #struct_name, #index)? + _pyo3::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, &#ident, #struct_name, #index)? ), } }); @@ -339,12 +339,12 @@ impl<'a> Container<'a> { }; let extractor = match &field.from_py_with { None => { - quote!(_pyo3::impl_::frompyobject::extract_struct_field(obj.#getter?, #struct_name, #field_name)?) + quote!(_pyo3::impl_::frompyobject::extract_struct_field(&obj.#getter?, #struct_name, #field_name)?) } Some(FromPyWithAttribute { value: expr_path, .. }) => { - quote! (_pyo3::impl_::frompyobject::extract_struct_field_with(#expr_path, obj.#getter?, #struct_name, #field_name)?) + quote! (_pyo3::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, &obj.#getter?, #struct_name, #field_name)?) } }; @@ -606,10 +606,11 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { Ok(quote!( const _: () = { use #krate as _pyo3; + use _pyo3::prelude::PyAnyMethods; #[automatically_derived] impl #trait_generics _pyo3::FromPyObject<#lt_param> for #ident #generics #where_clause { - fn extract(obj: &#lt_param _pyo3::PyAny) -> _pyo3::PyResult { + fn extract_bound(obj: &_pyo3::Bound<#lt_param, _pyo3::PyAny>) -> _pyo3::PyResult { #derives } } diff --git a/src/impl_/frompyobject.rs b/src/impl_/frompyobject.rs index a0c7b13df7c..5ab595ca784 100644 --- a/src/impl_/frompyobject.rs +++ b/src/impl_/frompyobject.rs @@ -1,5 +1,33 @@ +use crate::types::any::PyAnyMethods; +use crate::Bound; use crate::{exceptions::PyTypeError, FromPyObject, PyAny, PyErr, PyResult, Python}; +pub enum Extractor<'a, 'py, T> { + Bound(fn(&'a Bound<'py, PyAny>) -> PyResult), + GilRef(fn(&'a PyAny) -> PyResult), +} + +impl<'a, 'py, T> From) -> PyResult> for Extractor<'a, 'py, T> { + fn from(value: fn(&'a Bound<'py, PyAny>) -> PyResult) -> Self { + Self::Bound(value) + } +} + +impl<'a, T> From PyResult> for Extractor<'a, '_, T> { + fn from(value: fn(&'a PyAny) -> PyResult) -> Self { + Self::GilRef(value) + } +} + +impl<'a, 'py, T> Extractor<'a, 'py, T> { + fn call(self, obj: &'a Bound<'py, PyAny>) -> PyResult { + match self { + Extractor::Bound(f) => f(obj), + Extractor::GilRef(f) => f(obj.as_gil_ref()), + } + } +} + #[cold] pub fn failed_to_extract_enum( py: Python<'_>, @@ -41,7 +69,7 @@ fn extract_traceback(py: Python<'_>, mut error: PyErr) -> String { } pub fn extract_struct_field<'py, T>( - obj: &'py PyAny, + obj: &Bound<'py, PyAny>, struct_name: &str, field_name: &str, ) -> PyResult @@ -59,13 +87,13 @@ where } } -pub fn extract_struct_field_with<'py, T>( - extractor: impl FnOnce(&'py PyAny) -> PyResult, - obj: &'py PyAny, +pub fn extract_struct_field_with<'a, 'py, T>( + extractor: impl Into>, + obj: &'a Bound<'py, PyAny>, struct_name: &str, field_name: &str, ) -> PyResult { - match extractor(obj) { + match extractor.into().call(obj) { Ok(value) => Ok(value), Err(err) => Err(failed_to_extract_struct_field( obj.py(), @@ -92,7 +120,7 @@ fn failed_to_extract_struct_field( } pub fn extract_tuple_struct_field<'py, T>( - obj: &'py PyAny, + obj: &Bound<'py, PyAny>, struct_name: &str, index: usize, ) -> PyResult @@ -110,13 +138,13 @@ where } } -pub fn extract_tuple_struct_field_with<'py, T>( - extractor: impl FnOnce(&'py PyAny) -> PyResult, - obj: &'py PyAny, +pub fn extract_tuple_struct_field_with<'a, 'py, T>( + extractor: impl Into>, + obj: &'a Bound<'py, PyAny>, struct_name: &str, index: usize, ) -> PyResult { - match extractor(obj) { + match extractor.into().call(obj) { Ok(value) => Ok(value), Err(err) => Err(failed_to_extract_tuple_struct_field( obj.py(), diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index c475d8ea81f..5c57a954023 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -502,7 +502,7 @@ pub struct Zap { #[pyo3(item)] name: String, - #[pyo3(from_py_with = "PyAny::len", item("my_object"))] + #[pyo3(from_py_with = "Bound::<'_, PyAny>::len", item("my_object"))] some_object_length: usize, } @@ -525,7 +525,10 @@ fn test_from_py_with() { } #[derive(Debug, FromPyObject)] -pub struct ZapTuple(String, #[pyo3(from_py_with = "PyAny::len")] usize); +pub struct ZapTuple( + String, + #[pyo3(from_py_with = "Bound::<'_, PyAny>::len")] usize, +); #[test] fn test_from_py_with_tuple_struct() { @@ -560,8 +563,11 @@ fn test_from_py_with_tuple_struct_error() { #[derive(Debug, FromPyObject, PartialEq, Eq)] pub enum ZapEnum { - Zip(#[pyo3(from_py_with = "PyAny::len")] usize), - Zap(String, #[pyo3(from_py_with = "PyAny::len")] usize), + Zip(#[pyo3(from_py_with = "Bound::<'_, PyAny>::len")] usize), + Zap( + String, + #[pyo3(from_py_with = "Bound::<'_, PyAny>::len")] usize, + ), } #[test] @@ -581,7 +587,7 @@ fn test_from_py_with_enum() { #[derive(Debug, FromPyObject, PartialEq, Eq)] #[pyo3(transparent)] pub struct TransparentFromPyWith { - #[pyo3(from_py_with = "PyAny::len")] + #[pyo3(from_py_with = "Bound::<'_, PyAny>::len")] len: usize, } From e308c8d3ac68c3e8802a5181a3e06504751b0815 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 13 Feb 2024 00:14:55 +0000 Subject: [PATCH 126/349] ci: don't test gevent on pypy (#3830) --- pytests/pyproject.toml | 2 +- pytests/tests/test_misc.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index 920455ca871..126eaf77b09 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ [project.optional-dependencies] dev = [ - "gevent>=22.10.2", + "gevent>=22.10.2; implementation_name == 'cpython'", "hypothesis>=3.55", "pytest-asyncio>=0.21", "pytest-benchmark>=3.4", diff --git a/pytests/tests/test_misc.py b/pytests/tests/test_misc.py index 2f6cee6354e..6645f942f1a 100644 --- a/pytests/tests/test_misc.py +++ b/pytests/tests/test_misc.py @@ -2,7 +2,6 @@ import platform import sys -import gevent import pyo3_pytests.misc import pytest @@ -83,6 +82,8 @@ def __del__(self): def test_gevent(): + gevent = pytest.importorskip("gevent") + def worker(worker_id: int) -> None: for iteration in range(2): d = {"key": ArbitraryClass(worker_id, iteration)} From f5eafe23f29c8f3a9b05b9c9c90726d3fd9c8e7b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 13 Feb 2024 21:52:53 +0000 Subject: [PATCH 127/349] add maximum Python version check (#3821) * add maximum Python version check * restore dependency of `pyo3-macros-backend` on `pyo3-build-config` * fix clippy-all noxfile job --- .github/workflows/ci.yml | 14 +++++ Cargo.toml | 2 +- newsfragments/3821.packaging.md | 1 + noxfile.py | 92 ++++++++++++++++++++++++------- pyo3-build-config/src/impl_.rs | 1 + pyo3-ffi/build.rs | 72 +++++++++++++++++++++--- pyo3-macros-backend/Cargo.toml | 8 +-- pyo3-macros-backend/src/method.rs | 6 +- pyo3-macros-backend/src/utils.rs | 4 ++ pyo3-macros/Cargo.toml | 2 - 10 files changed, 165 insertions(+), 37 deletions(-) create mode 100644 newsfragments/3821.packaging.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b98495b3f5e..22281185caa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -468,6 +468,18 @@ jobs: echo PYO3_CONFIG_FILE=$PYO3_CONFIG_FILE >> $GITHUB_ENV - run: python3 -m nox -s test + test-version-limits: + needs: [fmt] + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + continue-on-error: true + - uses: dtolnay/rust-toolchain@stable + - run: python3 -m pip install --upgrade pip && pip install nox + - run: python3 -m nox -s test-version-limits + conclusion: needs: - fmt @@ -480,6 +492,8 @@ jobs: - docsrs - coverage - emscripten + - test-debug + - test-version-limits if: always() runs-on: ubuntu-latest steps: diff --git a/Cargo.toml b/Cargo.toml index c4ce8b6b033..5386b76f573 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,7 +81,7 @@ multiple-pymethods = ["inventory", "pyo3-macros/multiple-pymethods"] extension-module = ["pyo3-ffi/extension-module"] # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more. -abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3", "pyo3-macros/abi3"] +abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3"] # With abi3, we can manually set the minimum Python version. abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37", "pyo3-ffi/abi3-py37"] diff --git a/newsfragments/3821.packaging.md b/newsfragments/3821.packaging.md new file mode 100644 index 00000000000..4bd89355086 --- /dev/null +++ b/newsfragments/3821.packaging.md @@ -0,0 +1 @@ +Check maximum version of Python at build time and for versions not yet supported require opt-in to the `abi3` stable ABI by the environment variable `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1`. diff --git a/noxfile.py b/noxfile.py index 8288cbb7b1c..3981e62e100 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager import json import os import re @@ -7,9 +8,10 @@ from functools import lru_cache from glob import glob from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple import nox +import nox.command nox.options.sessions = ["test", "clippy", "rustfmt", "ruff", "docs"] @@ -100,7 +102,7 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: "--deny=warnings", env=env, ) - except Exception: + except nox.command.CommandFailed: success = False return success @@ -564,6 +566,33 @@ def ffi_check(session: nox.Session): _run_cargo(session, "run", _FFI_CHECK) +@nox.session(name="test-version-limits") +def test_version_limits(session: nox.Session): + env = os.environ.copy() + with _config_file() as config_file: + env["PYO3_CONFIG_FILE"] = config_file.name + + assert "3.6" not in PY_VERSIONS + config_file.set("CPython", "3.6") + _run_cargo(session, "check", env=env, expect_error=True) + + assert "3.13" not in PY_VERSIONS + config_file.set("CPython", "3.13") + _run_cargo(session, "check", env=env, expect_error=True) + + # 3.13 CPython should build with forward compatibility + env["PYO3_USE_ABI3_FORWARD_COMPATIBILITY"] = "1" + _run_cargo(session, "check", env=env) + + assert "3.6" not in PYPY_VERSIONS + config_file.set("PyPy", "3.6") + _run_cargo(session, "check", env=env, expect_error=True) + + assert "3.11" not in PYPY_VERSIONS + config_file.set("PyPy", "3.11") + _run_cargo(session, "check", env=env, expect_error=True) + + def _build_docs_for_ffi_check(session: nox.Session) -> None: # pyo3-ffi-check needs to scrape docs of pyo3-ffi _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") @@ -652,7 +681,13 @@ def _run(session: nox.Session, *args: str, **kwargs: Any) -> None: print("::endgroup::", file=sys.stderr) -def _run_cargo(session: nox.Session, *args: str, **kwargs: Any) -> None: +def _run_cargo( + session: nox.Session, *args: str, expect_error: bool = False, **kwargs: Any +) -> None: + if expect_error: + if "success_codes" in kwargs: + raise ValueError("expect_error overrides success_codes") + kwargs["success_codes"] = [101] _run(session, "cargo", *args, **kwargs, external=True) @@ -700,24 +735,14 @@ def _get_output(*args: str) -> str: def _for_all_version_configs( session: nox.Session, job: Callable[[Dict[str, str]], None] ) -> None: - with tempfile.NamedTemporaryFile("r+") as config: - env = os.environ.copy() - env["PYO3_CONFIG_FILE"] = config.name - - def _job_with_config(implementation, version) -> bool: - config.seek(0) - config.truncate(0) - config.write( - f"""\ -implementation={implementation} -version={version} -suppress_build_script_link_lines=true -""" - ) - config.flush() + env = os.environ.copy() + with _config_file() as config_file: + env["PYO3_CONFIG_FILE"] = config_file.name + def _job_with_config(implementation, version): session.log(f"{implementation} {version}") - return job(env) + config_file.set(implementation, version) + job(env) for version in PY_VERSIONS: _job_with_config("CPython", version) @@ -726,5 +751,34 @@ def _job_with_config(implementation, version) -> bool: _job_with_config("PyPy", version) +class _ConfigFile: + def __init__(self, config_file) -> None: + self._config_file = config_file + + def set(self, implementation: str, version: str) -> None: + """Set the contents of this config file to the given implementation and version.""" + self._config_file.seek(0) + self._config_file.truncate(0) + self._config_file.write( + f"""\ +implementation={implementation} +version={version} +suppress_build_script_link_lines=true +""" + ) + self._config_file.flush() + + @property + def name(self) -> str: + return self._config_file.name + + +@contextmanager +def _config_file() -> Iterator[_ConfigFile]: + """Creates a temporary config file which can be repeatedly set to different values.""" + with tempfile.NamedTemporaryFile("r+") as config: + yield _ConfigFile(config) + + _BENCHES = "--manifest-path=pyo3-benches/Cargo.toml" _FFI_CHECK = "--manifest-path=pyo3-ffi-check/Cargo.toml" diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index e188767fcd1..fd467b72e0a 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -701,6 +701,7 @@ fn have_python_interpreter() -> bool { /// Must be called from a PyO3 crate build script. fn is_abi3() -> bool { cargo_env_var("CARGO_FEATURE_ABI3").is_some() + || env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").map_or(false, |os_str| os_str == "1") } /// Gets the minimum supported Python version from PyO3 `abi3-py*` features. diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index 940d9808648..286767d8f25 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -4,18 +4,76 @@ use pyo3_build_config::{ cargo_env_var, env_var, errors::Result, is_linking_libpython, resolve_interpreter_config, InterpreterConfig, PythonVersion, }, + PythonImplementation, }; /// Minimum Python version PyO3 supports. -const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; +struct SupportedVersions { + min: PythonVersion, + max: PythonVersion, +} + +const SUPPORTED_VERSIONS_CPYTHON: SupportedVersions = SupportedVersions { + min: PythonVersion { major: 3, minor: 7 }, + max: PythonVersion { + major: 3, + minor: 12, + }, +}; + +const SUPPORTED_VERSIONS_PYPY: SupportedVersions = SupportedVersions { + min: PythonVersion { major: 3, minor: 7 }, + max: PythonVersion { + major: 3, + minor: 10, + }, +}; fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { - ensure!( - interpreter_config.version >= MINIMUM_SUPPORTED_VERSION, - "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})", - interpreter_config.version, - MINIMUM_SUPPORTED_VERSION, - ); + // This is an undocumented env var which is only really intended to be used in CI / for testing + // and development. + if std::env::var("UNSAFE_PYO3_SKIP_VERSION_CHECK").as_deref() == Ok("1") { + return Ok(()); + } + + match interpreter_config.implementation { + PythonImplementation::CPython => { + let versions = SUPPORTED_VERSIONS_CPYTHON; + ensure!( + interpreter_config.version >= versions.min, + "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})", + interpreter_config.version, + versions.min, + ); + ensure!( + interpreter_config.version <= versions.max || env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").map_or(false, |os_str| os_str == "1"), + "the configured Python interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ + = help: please check if an updated version of PyO3 is available. Current version: {}\n\ + = help: set PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 to suppress this check and build anyway using the stable ABI", + interpreter_config.version, + versions.max, + std::env::var("CARGO_PKG_VERSION").unwrap(), + ); + } + PythonImplementation::PyPy => { + let versions = SUPPORTED_VERSIONS_PYPY; + ensure!( + interpreter_config.version >= versions.min, + "the configured PyPy interpreter version ({}) is lower than PyO3's minimum supported version ({})", + interpreter_config.version, + versions.min, + ); + // PyO3 does not support abi3, so we cannot offer forward compatibility + ensure!( + interpreter_config.version <= versions.max, + "the configured PyPy interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ + = help: please check if an updated version of PyO3 is available. Current version: {}", + interpreter_config.version, + versions.max, + std::env::var("CARGO_PKG_VERSION").unwrap() + ); + } + } Ok(()) } diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 3263365dc80..458b280f881 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -14,17 +14,15 @@ edition = "2021" # not to depend on proc-macro itself. # See https://github.com/PyO3/pyo3/pull/810 for more. [dependencies] -quote = { version = "1", default-features = false } -proc-macro2 = { version = "1", default-features = false } heck = "0.4" +proc-macro2 = { version = "1", default-features = false } +pyo3-build-config = { path = "../pyo3-build-config", version = "0.21.0-dev", features = ["resolve-config"] } +quote = { version = "1", default-features = false } [dependencies.syn] version = "2" default-features = false features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"] -[features] -abi3 = [] - [lints] workspace = true diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 7050be23d5c..c158ec9fe05 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -12,7 +12,7 @@ use crate::{ FunctionSignature, PyFunctionArgPyO3Attributes, PyFunctionOptions, SignatureAttribute, }, quotes, - utils::{self, PythonDoc}, + utils::{self, is_abi3, PythonDoc}, }; #[derive(Clone, Debug)] @@ -234,8 +234,8 @@ impl CallingConvention { } else if signature.python_signature.kwargs.is_some() { // for functions that accept **kwargs, always prefer varargs Self::Varargs - } else if cfg!(not(feature = "abi3")) { - // Not available in the Stable ABI as of Python 3.10 + } else if !is_abi3() { + // FIXME: available in the stable ABI since 3.10 Self::Fastcall } else { Self::Varargs diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index 0fc96bd6a1a..9f0f2678476 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -166,3 +166,7 @@ pub fn apply_renaming_rule(rule: RenamingRule, name: &str) -> String { RenamingRule::Uppercase => name.to_uppercase(), } } + +pub(crate) fn is_abi3() -> bool { + pyo3_build_config::get().abi3 +} diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index f34d483dd04..576c94a2bc1 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -16,8 +16,6 @@ proc-macro = true [features] multiple-pymethods = [] -abi3 = ["pyo3-macros-backend/abi3"] - [dependencies] proc-macro2 = { version = "1", default-features = false } quote = "1" From a1e77c5a66d2ce471b63e6ae16582185c6f04e66 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Tue, 13 Feb 2024 23:30:16 +0000 Subject: [PATCH 128/349] Document using as_borrowed in the Bound migration (#3833) --- guide/src/migration.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/guide/src/migration.md b/guide/src/migration.md index 1c8655be684..936784897bd 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -217,6 +217,12 @@ Because the new `Bound` API brings ownership out of the PyO3 framework and in - `Bound` and `Bound` cannot support indexing with `list[0]`, you should use `list.get_item(0)` instead. - `Bound::iter_borrowed` is slightly more efficient than `Bound::iter`. The default iteration of `Bound` cannot return borrowed references because Rust does not (yet) have "lending iterators". Similarly `Bound::get_borrowed_item` is more efficient than `Bound::get_item` for the same reason. - `&Bound` does not implement `FromPyObject` (although it might be possible to do this in the future once the GIL Refs API is completely removed). Use `bound_any.downcast::()` instead of `bound_any.extract::<&Bound>()`. +- To convert between `&PyAny` and `&Bound` you can use the `as_borrowed()` method: + +```rust,ignore +let gil_ref: &PyAny = ...; +let bound: &Bound = &gil_ref.as_borrowed(); +``` #### Migrating `FromPyObject` implementations From 0c12d9137fe73fbe17633d8699bb94964a420e8e Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 14 Feb 2024 01:24:37 +0100 Subject: [PATCH 129/349] port `Python::import` to `Bound` API (#3832) * port `Python::import` to `Bound` API * tidy up imports in tests/test_datetime_import.rs --------- Co-authored-by: David Hewitt --- README.md | 4 +-- guide/src/python_from_rust.md | 4 +-- src/buffer.rs | 7 ++--- src/conversions/chrono.rs | 30 ++++++++++----------- src/conversions/chrono_tz.rs | 13 +++++---- src/conversions/rust_decimal.rs | 2 +- src/conversions/std/time.rs | 43 ++++++++++++++++++------------ src/coroutine/waker.rs | 3 ++- src/err/mod.rs | 3 ++- src/exceptions.rs | 7 ++--- src/gil.rs | 2 +- src/impl_/pymodule.rs | 8 +++--- src/instance.rs | 4 +-- src/lib.rs | 4 +-- src/marker.rs | 21 ++++++++++++++- src/sync.rs | 11 +++++--- src/tests/common.rs | 10 +++---- src/types/any.rs | 32 +++++++++++----------- src/types/traceback.rs | 9 ++++--- tests/test_class_attributes.rs | 16 +++++------ tests/test_coroutine.rs | 6 ++--- tests/test_datetime.rs | 2 +- tests/test_datetime_import.rs | 4 +-- tests/test_various.rs | 2 +- tests/ui/invalid_intern_arg.rs | 2 +- tests/ui/invalid_intern_arg.stderr | 12 ++++----- 26 files changed, 153 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index 64aa3e9ef70..21e09157d8d 100644 --- a/README.md +++ b/README.md @@ -149,10 +149,10 @@ use pyo3::types::IntoPyDict; fn main() -> PyResult<()> { Python::with_gil(|py| { - let sys = py.import("sys")?; + let sys = py.import_bound("sys")?; let version: String = sys.getattr("version")?.extract()?; - let locals = [("os", py.import("os")?)].into_py_dict_bound(py); + let locals = [("os", py.import_bound("os")?)].into_py_dict_bound(py); let code = "os.getenv('USER') or os.getenv('USERNAME') or 'Unknown'"; let user: String = py.eval_bound(code, None, Some(&locals))?.extract()?; diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index b53b03091e2..1ac1a7c217a 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -414,7 +414,7 @@ fn main() -> PyResult<()> { let path = Path::new("/usr/share/python_app"); let py_app = fs::read_to_string(path.join("app.py"))?; let from_python = Python::with_gil(|py| -> PyResult> { - let syspath: &PyList = py.import("sys")?.getattr("path")?.downcast()?; + let syspath = py.import_bound("sys")?.getattr("path")?.downcast_into::()?; syspath.insert(0, &path)?; let app: Py = PyModule::from_code(py, &py_app, "", "")? .getattr("run")? @@ -498,7 +498,7 @@ use pyo3::prelude::*; # fn main() -> PyResult<()> { Python::with_gil(|py| -> PyResult<()> { - let signal = py.import("signal")?; + let signal = py.import_bound("signal")?; // Set SIGINT to have the default action signal .getattr("signal")? diff --git a/src/buffer.rs b/src/buffer.rs index 7360ef67744..2e9afbcae7f 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -689,6 +689,7 @@ impl_element!(f64, Float); mod tests { use super::PyBuffer; use crate::ffi; + use crate::types::any::PyAnyMethods; use crate::Python; #[test] @@ -890,11 +891,11 @@ mod tests { fn test_array_buffer() { Python::with_gil(|py| { let array = py - .import("array") + .import_bound("array") .unwrap() .call_method("array", ("f", (1.0, 1.5, 2.0, 2.5)), None) .unwrap(); - let buffer = PyBuffer::get(array).unwrap(); + let buffer = PyBuffer::get(array.as_gil_ref()).unwrap(); assert_eq!(buffer.dimensions(), 1); assert_eq!(buffer.item_count(), 4); assert_eq!(buffer.format().to_str().unwrap(), "f"); @@ -924,7 +925,7 @@ mod tests { assert_eq!(buffer.to_vec(py).unwrap(), [10.0, 11.0, 12.0, 13.0]); // F-contiguous fns - let buffer = PyBuffer::get(array).unwrap(); + let buffer = PyBuffer::get(array.as_gil_ref()).unwrap(); let slice = buffer.as_fortran_slice(py).unwrap(); assert_eq!(slice.len(), 4); assert_eq!(slice[1].get(), 11.0); diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 80a83b23915..421a7aec83e 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -540,15 +540,15 @@ impl DatetimeTypes { static TYPES: GILOnceCell = GILOnceCell::new(); TYPES .get_or_try_init(py, || { - let datetime = py.import("datetime")?; + let datetime = py.import_bound("datetime")?; let timezone = datetime.getattr("timezone")?; Ok::<_, PyErr>(Self { date: datetime.getattr("date")?.into(), datetime: datetime.getattr("datetime")?.into(), time: datetime.getattr("time")?.into(), timedelta: datetime.getattr("timedelta")?.into(), - timezone: timezone.into(), timezone_utc: timezone.getattr("utc")?.into(), + timezone: timezone.into(), tzinfo: datetime.getattr("tzinfo")?.into(), }) }) @@ -683,7 +683,7 @@ mod tests { let delta = delta.to_object(py); let py_delta = new_py_datetime_ob(py, "timedelta", (py_days, py_seconds, py_ms)); assert!( - delta.as_ref(py).eq(py_delta).unwrap(), + delta.bind(py).eq(&py_delta).unwrap(), "{}: {} != {}", name, delta, @@ -779,7 +779,7 @@ mod tests { .to_object(py); let py_date = new_py_datetime_ob(py, "date", (year, month, day)); assert_eq!( - date.as_ref(py).compare(py_date).unwrap(), + date.bind(py).compare(&py_date).unwrap(), Ordering::Equal, "{}: {} != {}", name, @@ -838,7 +838,7 @@ mod tests { ), ); assert_eq!( - datetime.as_ref(py).compare(py_datetime).unwrap(), + datetime.bind(py).compare(&py_datetime).unwrap(), Ordering::Equal, "{}: {} != {}", name, @@ -880,7 +880,7 @@ mod tests { (year, month, day, hour, minute, second, py_ms, py_tz), ); assert_eq!( - datetime.as_ref(py).compare(py_datetime).unwrap(), + datetime.bind(py).compare(&py_datetime).unwrap(), Ordering::Equal, "{}: {} != {}", name, @@ -1005,7 +1005,7 @@ mod tests { Python::with_gil(|py| { let utc = Utc.to_object(py); let py_utc = python_utc(py); - assert!(utc.as_ref(py).is(py_utc)); + assert!(utc.bind(py).is(&py_utc)); }) } @@ -1036,7 +1036,7 @@ mod tests { .to_object(py); let py_time = new_py_datetime_ob(py, "time", (hour, minute, second, py_ms)); assert!( - time.as_ref(py).eq(py_time).unwrap(), + time.bind(py).eq(&py_time).unwrap(), "{}: {} != {}", name, time, @@ -1071,12 +1071,12 @@ mod tests { }) } - fn new_py_datetime_ob<'a>( - py: Python<'a>, + fn new_py_datetime_ob<'py>( + py: Python<'py>, name: &str, args: impl IntoPy>, - ) -> &'a PyAny { - py.import("datetime") + ) -> Bound<'py, PyAny> { + py.import_bound("datetime") .unwrap() .getattr(name) .unwrap() @@ -1084,8 +1084,8 @@ mod tests { .unwrap() } - fn python_utc(py: Python<'_>) -> &PyAny { - py.import("datetime") + fn python_utc(py: Python<'_>) -> Bound<'_, PyAny> { + py.import_bound("datetime") .unwrap() .getattr("timezone") .unwrap() @@ -1108,7 +1108,7 @@ mod tests { fn test_pyo3_offset_fixed_frompyobject_created_in_python(timestamp in 0..(i32::MAX as i64), timedelta in -86399i32..=86399i32) { Python::with_gil(|py| { - let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict_bound(py); + let globals = [("datetime", py.import_bound("datetime").unwrap())].into_py_dict_bound(py); let code = format!("datetime.datetime.fromtimestamp({}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={})))", timestamp, timedelta); let t = py.eval_bound(&code, Some(&globals), None).unwrap(); diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index 791be15c96d..b4d0e7edf8c 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -89,8 +89,8 @@ mod tests { #[test] fn test_topyobject() { Python::with_gil(|py| { - let assert_eq = |l: PyObject, r: &PyAny| { - assert!(l.as_ref(py).eq(r).unwrap()); + let assert_eq = |l: PyObject, r: Bound<'_, PyAny>| { + assert!(l.bind(py).eq(r).unwrap()); }; assert_eq( @@ -105,11 +105,14 @@ mod tests { }); } - fn new_zoneinfo<'a>(py: Python<'a>, name: &str) -> &'a PyAny { + fn new_zoneinfo<'py>(py: Python<'py>, name: &str) -> Bound<'py, PyAny> { zoneinfo_class(py).call1((name,)).unwrap() } - fn zoneinfo_class(py: Python<'_>) -> &PyAny { - py.import("zoneinfo").unwrap().getattr("ZoneInfo").unwrap() + fn zoneinfo_class(py: Python<'_>) -> Bound<'_, PyAny> { + py.import_bound("zoneinfo") + .unwrap() + .getattr("ZoneInfo") + .unwrap() } } diff --git a/src/conversions/rust_decimal.rs b/src/conversions/rust_decimal.rs index e829e88346e..1d0f64148fa 100644 --- a/src/conversions/rust_decimal.rs +++ b/src/conversions/rust_decimal.rs @@ -77,7 +77,7 @@ static DECIMAL_CLS: GILOnceCell> = GILOnceCell::new(); fn get_decimal_cls(py: Python<'_>) -> PyResult<&PyType> { DECIMAL_CLS .get_or_try_init(py, || { - py.import(intern!(py, "decimal"))? + py.import_bound(intern!(py, "decimal"))? .getattr(intern!(py, "Decimal"))? .extract() }) diff --git a/src/conversions/std/time.rs b/src/conversions/std/time.rs index 4e6009e30db..89d61e696a1 100755 --- a/src/conversions/std/time.rs +++ b/src/conversions/std/time.rs @@ -146,7 +146,7 @@ fn unix_epoch_py(py: Python<'_>) -> &PyObject { } #[cfg(Py_LIMITED_API)] { - let datetime = py.import("datetime")?; + let datetime = py.import_bound("datetime")?; let utc = datetime.getattr("timezone")?.getattr("utc")?; Ok::<_, PyErr>( datetime @@ -216,8 +216,8 @@ mod tests { #[test] fn test_duration_topyobject() { Python::with_gil(|py| { - let assert_eq = |l: PyObject, r: &PyAny| { - assert!(l.as_ref(py).eq(r).unwrap()); + let assert_eq = |l: PyObject, r: Bound<'_, PyAny>| { + assert!(l.bind(py).eq(r).unwrap()); }; assert_eq( @@ -300,8 +300,8 @@ mod tests { #[test] fn test_time_topyobject() { Python::with_gil(|py| { - let assert_eq = |l: PyObject, r: &PyAny| { - assert!(l.as_ref(py).eq(r).unwrap()); + let assert_eq = |l: PyObject, r: Bound<'_, PyAny>| { + assert!(l.bind(py).eq(r).unwrap()); }; assert_eq( @@ -331,7 +331,7 @@ mod tests { minute: u8, second: u8, microsecond: u32, - ) -> &PyAny { + ) -> Bound<'_, PyAny> { datetime_class(py) .call1(( year, @@ -346,13 +346,11 @@ mod tests { .unwrap() } - fn max_datetime(py: Python<'_>) -> &PyAny { + fn max_datetime(py: Python<'_>) -> Bound<'_, PyAny> { let naive_max = datetime_class(py).getattr("max").unwrap(); let kargs = PyDict::new_bound(py); kargs.set_item("tzinfo", tz_utc(py)).unwrap(); - naive_max - .call_method("replace", (), Some(kargs.as_gil_ref())) - .unwrap() + naive_max.call_method("replace", (), Some(&kargs)).unwrap() } #[test] @@ -365,8 +363,8 @@ mod tests { }) } - fn tz_utc(py: Python<'_>) -> &PyAny { - py.import("datetime") + fn tz_utc(py: Python<'_>) -> Bound<'_, PyAny> { + py.import_bound("datetime") .unwrap() .getattr("timezone") .unwrap() @@ -374,17 +372,28 @@ mod tests { .unwrap() } - fn new_timedelta(py: Python<'_>, days: i32, seconds: i32, microseconds: i32) -> &PyAny { + fn new_timedelta( + py: Python<'_>, + days: i32, + seconds: i32, + microseconds: i32, + ) -> Bound<'_, PyAny> { timedelta_class(py) .call1((days, seconds, microseconds)) .unwrap() } - fn datetime_class(py: Python<'_>) -> &PyAny { - py.import("datetime").unwrap().getattr("datetime").unwrap() + fn datetime_class(py: Python<'_>) -> Bound<'_, PyAny> { + py.import_bound("datetime") + .unwrap() + .getattr("datetime") + .unwrap() } - fn timedelta_class(py: Python<'_>) -> &PyAny { - py.import("datetime").unwrap().getattr("timedelta").unwrap() + fn timedelta_class(py: Python<'_>) -> Bound<'_, PyAny> { + py.import_bound("datetime") + .unwrap() + .getattr("timedelta") + .unwrap() } } diff --git a/src/coroutine/waker.rs b/src/coroutine/waker.rs index 8a1166ce3fb..c46654896fc 100644 --- a/src/coroutine/waker.rs +++ b/src/coroutine/waker.rs @@ -1,4 +1,5 @@ use crate::sync::GILOnceCell; +use crate::types::any::PyAnyMethods; use crate::types::PyCFunction; use crate::{intern, wrap_pyfunction, Py, PyAny, PyObject, PyResult, Python}; use pyo3_macros::pyfunction; @@ -56,7 +57,7 @@ impl LoopAndFuture { fn new(py: Python<'_>) -> PyResult { static GET_RUNNING_LOOP: GILOnceCell = GILOnceCell::new(); let import = || -> PyResult<_> { - let module = py.import("asyncio")?; + let module = py.import_bound("asyncio")?; Ok(module.getattr("get_running_loop")?.into()) }; let event_loop = GET_RUNNING_LOOP.get_or_try_init(py, import)?.call0(py)?; diff --git a/src/err/mod.rs b/src/err/mod.rs index 31385f4c24c..5c2631b9301 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -974,6 +974,7 @@ impl_signed_integer!(isize); mod tests { use super::PyErrState; use crate::exceptions::{self, PyTypeError, PyValueError}; + use crate::types::any::PyAnyMethods; use crate::{PyErr, PyTypeInfo, Python}; #[test] @@ -1174,7 +1175,7 @@ mod tests { let cls = py.get_type::(); // Reset warning filter to default state - let warnings = py.import("warnings").unwrap(); + let warnings = py.import_bound("warnings").unwrap(); warnings.call_method0("resetwarnings").unwrap(); // First, test the warning is emitted diff --git a/src/exceptions.rs b/src/exceptions.rs index 7e7dc8eee65..245c3bbdf05 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -101,13 +101,14 @@ macro_rules! import_exception { fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { use $crate::sync::GILOnceCell; use $crate::prelude::PyTracebackMethods; + use $crate::prelude::PyAnyMethods; static TYPE_OBJECT: GILOnceCell<$crate::Py<$crate::types::PyType>> = GILOnceCell::new(); TYPE_OBJECT .get_or_init(py, || { let imp = py - .import(stringify!($module)) + .import_bound(stringify!($module)) .unwrap_or_else(|err| { let traceback = err .traceback_bound(py) @@ -812,7 +813,7 @@ mod tests { Python::with_gil(|py| { let err: PyErr = gaierror::new_err(()); let socket = py - .import("socket") + .import_bound("socket") .map_err(|e| e.display(py)) .expect("could not import socket"); @@ -836,7 +837,7 @@ mod tests { Python::with_gil(|py| { let err: PyErr = MessageError::new_err(()); let email = py - .import("email") + .import_bound("email") .map_err(|e| e.display(py)) .expect("could not import email"); diff --git a/src/gil.rs b/src/gil.rs index 379dab4b010..08b1b3f745b 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -143,7 +143,7 @@ where // Import the threading module - this ensures that it will associate this thread as the "main" // thread, which is important to avoid an `AssertionError` at finalization. - pool.python().import("threading").unwrap(); + pool.python().import_bound("threading").unwrap(); // Execute the closure. let result = f(pool.python()); diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index 0fe5c3846c2..ff72285558a 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -67,13 +67,14 @@ impl ModuleDef { pub fn make_module(&'static self, py: Python<'_>) -> PyResult> { #[cfg(all(PyPy, not(Py_3_8)))] { + use crate::types::any::PyAnyMethods; const PYPY_GOOD_VERSION: [u8; 3] = [7, 3, 8]; let version = py - .import("sys")? + .import_bound("sys")? .getattr("implementation")? .getattr("version")?; if version.lt(crate::types::PyTuple::new_bound(py, PYPY_GOOD_VERSION))? { - let warn = py.import("warnings")?.getattr("warn")?; + let warn = py.import_bound("warnings")?.getattr("warn")?; warn.call1(( "PyPy 3.7 versions older than 7.3.8 are known to have binary \ compatibility issues which may cause segfaults. Please upgrade.", @@ -202,7 +203,8 @@ mod tests { assert_eq!((*module_def.ffi_def.get()).m_doc, DOC.as_ptr() as _); Python::with_gil(|py| { - module_def.initializer.0(py, py.import("builtins").unwrap()).unwrap(); + module_def.initializer.0(py, py.import_bound("builtins").unwrap().into_gil_ref()) + .unwrap(); assert!(INIT_CALLED.load(Ordering::SeqCst)); }) } diff --git a/src/instance.rs b/src/instance.rs index 7f92744ccc4..54a9345241d 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1196,7 +1196,7 @@ impl Py { /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, IntoPy, Py, Python, PyObject, PyResult}; + /// # use pyo3::{prelude::*, intern}; /// # /// #[pyfunction] /// fn version(sys: Py, py: Python<'_>) -> PyResult { @@ -1204,7 +1204,7 @@ impl Py { /// } /// # /// # Python::with_gil(|py| { - /// # let sys = py.import("sys").unwrap().into_py(py); + /// # let sys = py.import_bound("sys").unwrap().unbind(); /// # version(sys, py).unwrap(); /// # }); /// ``` diff --git a/src/lib.rs b/src/lib.rs index 5ba0d279aa1..91a61688e82 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -218,10 +218,10 @@ //! //! fn main() -> PyResult<()> { //! Python::with_gil(|py| { -//! let sys = py.import("sys")?; +//! let sys = py.import_bound("sys")?; //! let version: String = sys.getattr("version")?.extract()?; //! -//! let locals = [("os", py.import("os")?)].into_py_dict_bound(py); +//! let locals = [("os", py.import_bound("os")?)].into_py_dict_bound(py); //! let code = "os.getenv('USER') or os.getenv('USERNAME') or 'Unknown'"; //! let user: String = py.eval_bound(code, None, Some(&locals))?.extract()?; //! diff --git a/src/marker.rs b/src/marker.rs index 642ca783904..fc84ca54c90 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -736,7 +736,14 @@ impl<'py> Python<'py> { T::type_object_bound(self).into_gil_ref() } - /// Imports the Python module with the specified name. + /// Deprecated form of [`Python::import_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`Python::import` will be replaced by `Python::import_bound` in a future PyO3 version" + ) + )] pub fn import(self, name: N) -> PyResult<&'py PyModule> where N: IntoPy>, @@ -744,6 +751,18 @@ impl<'py> Python<'py> { PyModule::import(self, name) } + /// Imports the Python module with the specified name. + pub fn import_bound(self, name: N) -> PyResult> + where + N: IntoPy>, + { + // FIXME: This should be replaced by `PyModule::import_bound` once thats + // implemented. + PyModule::import(self, name) + .map(PyNativeType::as_borrowed) + .map(crate::Borrowed::to_owned) + } + /// Gets the Python builtin value `None`. #[allow(non_snake_case)] // the Python keyword starts with uppercase #[inline] diff --git a/src/sync.rs b/src/sync.rs index 04786c8555f..8fff0e82c0f 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,5 +1,8 @@ //! Synchronization mechanisms based on the Python GIL. -use crate::{types::PyString, types::PyType, Bound, Py, PyResult, PyVisit, Python}; +use crate::{ + types::{any::PyAnyMethods, PyString, PyType}, + Bound, Py, PyResult, PyVisit, Python, +}; use std::cell::UnsafeCell; /// Value with concurrent access protected by the GIL. @@ -197,8 +200,10 @@ impl GILOnceCell> { module_name: &str, attr_name: &str, ) -> PyResult<&Bound<'py, PyType>> { - self.get_or_try_init(py, || py.import(module_name)?.getattr(attr_name)?.extract()) - .map(|ty| ty.bind(py)) + self.get_or_try_init(py, || { + py.import_bound(module_name)?.getattr(attr_name)?.extract() + }) + .map(|ty| ty.bind(py)) } } diff --git a/src/tests/common.rs b/src/tests/common.rs index bb710d904b4..ed3972f16a6 100644 --- a/src/tests/common.rs +++ b/src/tests/common.rs @@ -83,7 +83,7 @@ mod inner { #[cfg(all(feature = "macros", Py_3_8))] impl UnraisableCapture { pub fn install(py: Python<'_>) -> Py { - let sys = py.import("sys").unwrap(); + let sys = py.import_bound("sys").unwrap(); let old_hook = sys.getattr("unraisablehook").unwrap().into(); let capture = Py::new( @@ -104,22 +104,22 @@ mod inner { pub fn uninstall(&mut self, py: Python<'_>) { let old_hook = self.old_hook.take().unwrap(); - let sys = py.import("sys").unwrap(); + let sys = py.import_bound("sys").unwrap(); sys.setattr("unraisablehook", old_hook).unwrap(); } } pub struct CatchWarnings<'py> { - catch_warnings: &'py PyAny, + catch_warnings: Bound<'py, PyAny>, } impl<'py> CatchWarnings<'py> { pub fn enter(py: Python<'py>, f: impl FnOnce(&PyList) -> PyResult) -> PyResult { - let warnings = py.import("warnings")?; + let warnings = py.import_bound("warnings")?; let kwargs = [("record", true)].into_py_dict_bound(py); let catch_warnings = warnings .getattr("catch_warnings")? - .call((), Some(kwargs.as_gil_ref()))?; + .call((), Some(&kwargs))?; let list = catch_warnings.call_method0("__enter__")?.extract()?; let _guard = Self { catch_warnings }; f(list) diff --git a/src/types/any.rs b/src/types/any.rs index 90c7f2160d3..33b3d96405a 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -86,16 +86,16 @@ impl PyAny { /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, Python, PyResult}; + /// # use pyo3::{prelude::*, intern}; /// # /// #[pyfunction] - /// fn has_version(sys: &PyModule) -> PyResult { + /// fn has_version(sys: &Bound<'_, PyModule>) -> PyResult { /// sys.hasattr(intern!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { - /// # let sys = py.import("sys").unwrap(); - /// # has_version(sys).unwrap(); + /// # let sys = py.import_bound("sys").unwrap(); + /// # has_version(&sys).unwrap(); /// # }); /// ``` pub fn hasattr(&self, attr_name: N) -> PyResult @@ -115,16 +115,16 @@ impl PyAny { /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{prelude::*, intern}; /// # /// #[pyfunction] - /// fn version(sys: &PyModule) -> PyResult<&PyAny> { + /// fn version<'py>(sys: &Bound<'py, PyModule>) -> PyResult> { /// sys.getattr(intern!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { - /// # let sys = py.import("sys").unwrap(); - /// # version(sys).unwrap(); + /// # let sys = py.import_bound("sys").unwrap(); + /// # version(&sys).unwrap(); /// # }); /// ``` pub fn getattr(&self, attr_name: N) -> PyResult<&PyAny> @@ -956,16 +956,16 @@ pub trait PyAnyMethods<'py> { /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, Python, PyResult}; + /// # use pyo3::{prelude::*, intern}; /// # /// #[pyfunction] - /// fn has_version(sys: &PyModule) -> PyResult { + /// fn has_version(sys: &Bound<'_, PyModule>) -> PyResult { /// sys.hasattr(intern!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { - /// # let sys = py.import("sys").unwrap(); - /// # has_version(sys).unwrap(); + /// # let sys = py.import_bound("sys").unwrap(); + /// # has_version(&sys).unwrap(); /// # }); /// ``` fn hasattr(&self, attr_name: N) -> PyResult @@ -982,16 +982,16 @@ pub trait PyAnyMethods<'py> { /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{prelude::*, intern}; /// # /// #[pyfunction] - /// fn version(sys: &PyModule) -> PyResult<&PyAny> { + /// fn version<'py>(sys: &Bound<'py, PyModule>) -> PyResult> { /// sys.getattr(intern!(sys.py(), "version")) /// } /// # /// # Python::with_gil(|py| { - /// # let sys = py.import("sys").unwrap(); - /// # version(sys).unwrap(); + /// # let sys = py.import_bound("sys").unwrap(); + /// # version(&sys).unwrap(); /// # }); /// ``` fn getattr(&self, attr_name: N) -> PyResult> diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 3608772fbd0..b97e2ed0093 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -3,6 +3,9 @@ use crate::types::PyString; use crate::{ffi, Bound}; use crate::{PyAny, PyNativeType}; +use super::any::PyAnyMethods; +use super::string::PyStringMethods; + /// Represents a Python traceback. #[repr(transparent)] pub struct PyTraceback(PyAny); @@ -95,7 +98,7 @@ impl<'py> PyTracebackMethods<'py> for Bound<'py, PyTraceback> { fn format(&self) -> PyResult { let py = self.py(); let string_io = py - .import(intern!(py, "io"))? + .import_bound(intern!(py, "io"))? .getattr(intern!(py, "StringIO"))? .call0()?; let result = unsafe { ffi::PyTraceBack_Print(self.as_ptr(), string_io.as_ptr()) }; @@ -104,8 +107,8 @@ impl<'py> PyTracebackMethods<'py> for Bound<'py, PyTraceback> { .getattr(intern!(py, "getvalue"))? .call0()? .downcast::()? - .to_str()? - .to_owned(); + .to_cow()? + .into_owned(); Ok(formatted) } } diff --git a/tests/test_class_attributes.rs b/tests/test_class_attributes.rs index b589dd08809..906a11c8ea1 100644 --- a/tests/test_class_attributes.rs +++ b/tests/test_class_attributes.rs @@ -101,16 +101,16 @@ fn test_fallible_class_attribute() { use pyo3::{exceptions::PyValueError, types::PyString}; struct CaptureStdErr<'py> { - oldstderr: &'py PyAny, - string_io: &'py PyAny, + oldstderr: Bound<'py, PyAny>, + string_io: Bound<'py, PyAny>, } impl<'py> CaptureStdErr<'py> { fn new(py: Python<'py>) -> PyResult { - let sys = py.import("sys")?; + let sys = py.import_bound("sys")?; let oldstderr = sys.getattr("stderr")?; - let string_io = py.import("io")?.getattr("StringIO")?.call0()?; - sys.setattr("stderr", string_io)?; + let string_io = py.import_bound("io")?.getattr("StringIO")?.call0()?; + sys.setattr("stderr", &string_io)?; Ok(Self { oldstderr, string_io, @@ -124,9 +124,9 @@ fn test_fallible_class_attribute() { .getattr("getvalue")? .call0()? .downcast::()? - .to_str()? - .to_owned(); - let sys = py.import("sys")?; + .to_cow()? + .into_owned(); + let sys = py.import_bound("sys")?; sys.setattr("stderr", self.oldstderr)?; Ok(payload) } diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index fa50fe652e6..549d54aa5c4 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -127,7 +127,7 @@ fn cancelled_coroutine() { await task asyncio.run(main()) "#; - let globals = gil.import("__main__").unwrap().dict().as_borrowed(); + let globals = gil.import_bound("__main__").unwrap().dict(); globals.set_item("sleep", sleep).unwrap(); let err = gil .run_bound( @@ -166,7 +166,7 @@ fn coroutine_cancel_handle() { return await task assert asyncio.run(main()) == 0 "#; - let globals = gil.import("__main__").unwrap().dict().as_borrowed(); + let globals = gil.import_bound("__main__").unwrap().dict(); globals .set_item("cancellable_sleep", cancellable_sleep) .unwrap(); @@ -198,7 +198,7 @@ fn coroutine_is_cancelled() { await task asyncio.run(main()) "#; - let globals = gil.import("__main__").unwrap().dict().as_borrowed(); + let globals = gil.import_bound("__main__").unwrap().dict(); globals.set_item("sleep_loop", sleep_loop).unwrap(); gil.run_bound( &pyo3::unindent::unindent(&handle_windows(test)), diff --git a/tests/test_datetime.rs b/tests/test_datetime.rs index de8efa9b826..8a9d190ff7b 100644 --- a/tests/test_datetime.rs +++ b/tests/test_datetime.rs @@ -10,7 +10,7 @@ fn _get_subclasses<'py>( args: &str, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>)> { // Import the class from Python and create some subclasses - let datetime = py.import("datetime")?; + let datetime = py.import_bound("datetime")?; let locals = [(py_type, datetime.getattr(py_type)?)].into_py_dict_bound(py); diff --git a/tests/test_datetime_import.rs b/tests/test_datetime_import.rs index bf0001b8ea4..619df891944 100644 --- a/tests/test_datetime_import.rs +++ b/tests/test_datetime_import.rs @@ -1,6 +1,6 @@ #![cfg(not(Py_LIMITED_API))] -use pyo3::{types::PyDate, Python}; +use pyo3::{prelude::*, types::PyDate}; #[test] #[should_panic(expected = "module 'datetime' has no attribute 'datetime_CAPI'")] @@ -14,7 +14,7 @@ fn test_bad_datetime_module_panic() { std::fs::File::create(tmpdir.join("datetime.py")).unwrap(); Python::with_gil(|py: Python<'_>| { - let sys = py.import("sys").unwrap(); + let sys = py.import_bound("sys").unwrap(); sys.getattr("path") .unwrap() .call_method1("insert", (0, tmpdir)) diff --git a/tests/test_various.rs b/tests/test_various.rs index 6560610f35f..d0df8d4a04e 100644 --- a/tests/test_various.rs +++ b/tests/test_various.rs @@ -134,7 +134,7 @@ impl PickleSupport { } fn add_module(py: Python<'_>, module: &PyModule) -> PyResult<()> { - py.import("sys")? + py.import_bound("sys")? .dict() .get_item("modules") .unwrap() diff --git a/tests/ui/invalid_intern_arg.rs b/tests/ui/invalid_intern_arg.rs index fa9e1e59f0c..3c7bd592175 100644 --- a/tests/ui/invalid_intern_arg.rs +++ b/tests/ui/invalid_intern_arg.rs @@ -2,5 +2,5 @@ use pyo3::Python; fn main() { let foo = if true { "foo" } else { "bar" }; - Python::with_gil(|py| py.import(pyo3::intern!(py, foo)).unwrap()); + Python::with_gil(|py| py.import_bound(pyo3::intern!(py, foo)).unwrap()); } diff --git a/tests/ui/invalid_intern_arg.stderr b/tests/ui/invalid_intern_arg.stderr index bb84d00e15b..dce2d85bf09 100644 --- a/tests/ui/invalid_intern_arg.stderr +++ b/tests/ui/invalid_intern_arg.stderr @@ -1,8 +1,8 @@ error[E0435]: attempt to use a non-constant value in a constant - --> tests/ui/invalid_intern_arg.rs:5:55 + --> tests/ui/invalid_intern_arg.rs:5:61 | -5 | Python::with_gil(|py| py.import(pyo3::intern!(py, foo)).unwrap()); - | ------------------^^^- - | | | - | | non-constant value - | help: consider using `let` instead of `static`: `let INTERNED` +5 | Python::with_gil(|py| py.import_bound(pyo3::intern!(py, foo)).unwrap()); + | ------------------^^^- + | | | + | | non-constant value + | help: consider using `let` instead of `static`: `let INTERNED` From 99026331160952462120afb09ba8b43193b3bc5c Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 14 Feb 2024 23:03:04 +0100 Subject: [PATCH 130/349] allow `from_py_with` on function args to take a `fn(&Bound) -> PyResult` (#3837) --- pyo3-macros-backend/src/params.rs | 8 ++++---- src/impl_/extract_argument.rs | 14 +++++++------- src/impl_/frompyobject.rs | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 1ef31867406..3781b41b765 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -217,18 +217,18 @@ fn impl_arg_param( quote_arg_span! { #[allow(clippy::redundant_closure)] _pyo3::impl_::extract_argument::from_py_with_with_default( - #arg_value, + #arg_value.map(_pyo3::PyNativeType::as_borrowed).as_deref(), #name_str, - #expr_path, + #expr_path as fn(_) -> _, || #default )? } } else { quote_arg_span! { _pyo3::impl_::extract_argument::from_py_with( - _pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value), + &_pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value).as_borrowed(), #name_str, - #expr_path, + #expr_path as fn(_) -> _, )? } } diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index 4df674e7639..ff1c2436e38 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -133,12 +133,12 @@ where /// Alternative to [`extract_argument`] used when the argument has a `#[pyo3(from_py_with)]` annotation. #[doc(hidden)] -pub fn from_py_with<'py, T>( - obj: &'py PyAny, +pub fn from_py_with<'a, 'py, T>( + obj: &'a Bound<'py, PyAny>, arg_name: &str, - extractor: fn(&'py PyAny) -> PyResult, + extractor: impl Into>, ) -> PyResult { - match extractor(obj) { + match extractor.into().call(obj) { Ok(value) => Ok(value), Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)), } @@ -146,10 +146,10 @@ pub fn from_py_with<'py, T>( /// Alternative to [`extract_argument`] used when the argument has a `#[pyo3(from_py_with)]` annotation and also a default value. #[doc(hidden)] -pub fn from_py_with_with_default<'py, T>( - obj: Option<&'py PyAny>, +pub fn from_py_with_with_default<'a, 'py, T>( + obj: Option<&'a Bound<'py, PyAny>>, arg_name: &str, - extractor: fn(&'py PyAny) -> PyResult, + extractor: impl Into>, default: fn() -> T, ) -> PyResult { match obj { diff --git a/src/impl_/frompyobject.rs b/src/impl_/frompyobject.rs index 5ab595ca784..e38ff3c76b2 100644 --- a/src/impl_/frompyobject.rs +++ b/src/impl_/frompyobject.rs @@ -20,7 +20,7 @@ impl<'a, T> From PyResult> for Extractor<'a, '_, T> { } impl<'a, 'py, T> Extractor<'a, 'py, T> { - fn call(self, obj: &'a Bound<'py, PyAny>) -> PyResult { + pub(crate) fn call(self, obj: &'a Bound<'py, PyAny>) -> PyResult { match self { Extractor::Bound(f) => f(obj), Extractor::GilRef(f) => f(obj.as_gil_ref()), From f3ddd023c92bb2089e9716e8b37ed86af5004554 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 14 Feb 2024 23:10:59 +0100 Subject: [PATCH 131/349] convert `PyBuffer` to `Bound` API (#3836) --- pytests/src/buf_and_str.rs | 4 ++-- src/buffer.rs | 26 +++++++++++++++++++------- tests/test_buffer.rs | 12 ++++++------ tests/test_buffer_protocol.rs | 2 +- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/pytests/src/buf_and_str.rs b/pytests/src/buf_and_str.rs index 196126f7cc1..23db9f0625e 100644 --- a/pytests/src/buf_and_str.rs +++ b/pytests/src/buf_and_str.rs @@ -35,8 +35,8 @@ impl BytesExtractor { } #[staticmethod] - pub fn from_buffer(buf: &PyAny) -> PyResult { - let buf = PyBuffer::::get(buf)?; + pub fn from_buffer(buf: &Bound<'_, PyAny>) -> PyResult { + let buf = PyBuffer::::get_bound(buf)?; Ok(buf.item_count()) } } diff --git a/src/buffer.rs b/src/buffer.rs index 2e9afbcae7f..84b08289771 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -18,8 +18,8 @@ // DEALINGS IN THE SOFTWARE. //! `PyBuffer` implementation -use crate::instance::Bound; use crate::{err, exceptions::PyBufferError, ffi, FromPyObject, PyAny, PyResult, Python}; +use crate::{Bound, PyNativeType}; use std::marker::PhantomData; use std::os::raw; use std::pin::Pin; @@ -184,13 +184,25 @@ pub unsafe trait Element: Copy { impl<'py, T: Element> FromPyObject<'py> for PyBuffer { fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult> { - Self::get(obj.as_gil_ref()) + Self::get_bound(obj) } } impl PyBuffer { - /// Gets the underlying buffer from the specified python object. + /// Deprecated form of [`PyBuffer::get_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyBuffer::get` will be replaced by `PyBuffer::get_bound` in a future PyO3 version" + ) + )] pub fn get(obj: &PyAny) -> PyResult> { + Self::get_bound(&obj.as_borrowed()) + } + + /// Gets the underlying buffer from the specified python object. + pub fn get_bound(obj: &Bound<'_, PyAny>) -> PyResult> { // TODO: use nightly API Box::new_uninit() once stable let mut buf = Box::new(mem::MaybeUninit::uninit()); let buf: Box = { @@ -696,7 +708,7 @@ mod tests { fn test_debug() { Python::with_gil(|py| { let bytes = py.eval_bound("b'abcde'", None, None).unwrap(); - let buffer: PyBuffer = PyBuffer::get(bytes.as_gil_ref()).unwrap(); + let buffer: PyBuffer = PyBuffer::get_bound(&bytes).unwrap(); let expected = format!( concat!( "PyBuffer {{ buf: {:?}, obj: {:?}, ", @@ -859,7 +871,7 @@ mod tests { fn test_bytes_buffer() { Python::with_gil(|py| { let bytes = py.eval_bound("b'abcde'", None, None).unwrap(); - let buffer = PyBuffer::get(bytes.as_gil_ref()).unwrap(); + let buffer = PyBuffer::get_bound(&bytes).unwrap(); assert_eq!(buffer.dimensions(), 1); assert_eq!(buffer.item_count(), 5); assert_eq!(buffer.format().to_str().unwrap(), "B"); @@ -895,7 +907,7 @@ mod tests { .unwrap() .call_method("array", ("f", (1.0, 1.5, 2.0, 2.5)), None) .unwrap(); - let buffer = PyBuffer::get(array.as_gil_ref()).unwrap(); + let buffer = PyBuffer::get_bound(&array).unwrap(); assert_eq!(buffer.dimensions(), 1); assert_eq!(buffer.item_count(), 4); assert_eq!(buffer.format().to_str().unwrap(), "f"); @@ -925,7 +937,7 @@ mod tests { assert_eq!(buffer.to_vec(py).unwrap(), [10.0, 11.0, 12.0, 13.0]); // F-contiguous fns - let buffer = PyBuffer::get(array.as_gil_ref()).unwrap(); + let buffer = PyBuffer::get_bound(&array).unwrap(); let slice = buffer.as_fortran_slice(py).unwrap(); assert_eq!(slice.len(), 4); assert_eq!(slice[1].get(), 11.0); diff --git a/tests/test_buffer.rs b/tests/test_buffer.rs index de8638baa7e..72f94b3bcc8 100644 --- a/tests/test_buffer.rs +++ b/tests/test_buffer.rs @@ -96,11 +96,11 @@ fn test_get_buffer_errors() { ) .unwrap(); - assert!(PyBuffer::::get(instance.as_ref(py)).is_ok()); + assert!(PyBuffer::::get_bound(instance.bind(py).as_any()).is_ok()); instance.borrow_mut(py).error = Some(TestGetBufferError::NullShape); assert_eq!( - PyBuffer::::get(instance.as_ref(py)) + PyBuffer::::get_bound(instance.bind(py).as_any()) .unwrap_err() .to_string(), "BufferError: shape is null" @@ -108,7 +108,7 @@ fn test_get_buffer_errors() { instance.borrow_mut(py).error = Some(TestGetBufferError::NullStrides); assert_eq!( - PyBuffer::::get(instance.as_ref(py)) + PyBuffer::::get_bound(instance.bind(py).as_any()) .unwrap_err() .to_string(), "BufferError: strides is null" @@ -116,7 +116,7 @@ fn test_get_buffer_errors() { instance.borrow_mut(py).error = Some(TestGetBufferError::IncorrectItemSize); assert_eq!( - PyBuffer::::get(instance.as_ref(py)) + PyBuffer::::get_bound(instance.bind(py).as_any()) .unwrap_err() .to_string(), "BufferError: buffer contents are not compatible with u32" @@ -124,7 +124,7 @@ fn test_get_buffer_errors() { instance.borrow_mut(py).error = Some(TestGetBufferError::IncorrectFormat); assert_eq!( - PyBuffer::::get(instance.as_ref(py)) + PyBuffer::::get_bound(instance.bind(py).as_any()) .unwrap_err() .to_string(), "BufferError: buffer contents are not compatible with u32" @@ -132,7 +132,7 @@ fn test_get_buffer_errors() { instance.borrow_mut(py).error = Some(TestGetBufferError::IncorrectAlignment); assert_eq!( - PyBuffer::::get(instance.as_ref(py)) + PyBuffer::::get_bound(instance.bind(py).as_any()) .unwrap_err() .to_string(), "BufferError: buffer contents are insufficiently aligned for u32" diff --git a/tests/test_buffer_protocol.rs b/tests/test_buffer_protocol.rs index 0a3a1758bfa..85ff6a4004e 100644 --- a/tests/test_buffer_protocol.rs +++ b/tests/test_buffer_protocol.rs @@ -77,7 +77,7 @@ fn test_buffer_referenced() { } .into_py(py); - let buf = PyBuffer::::get(instance.as_ref(py)).unwrap(); + let buf = PyBuffer::::get_bound(instance.bind(py)).unwrap(); assert_eq!(buf.to_vec(py).unwrap(), input); drop(instance); buf From dc8b94820147206ef3c29b9bcc1a90d35aeae987 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 15 Feb 2024 07:58:20 +0000 Subject: [PATCH 132/349] add `PyBackedStr` and `PyBackedBytes` (#3802) * add `PyBackedStr` and `PyBackedBytes` * review: adamreichold feedback * use `NonNull<[u8]>` * clippy and newsfragment * drop binding unused after refactoring --------- Co-authored-by: Adam Reichold --- newsfragments/3802.added.md | 1 + src/lib.rs | 1 + src/pybacked.rs | 186 ++++++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 newsfragments/3802.added.md create mode 100644 src/pybacked.rs diff --git a/newsfragments/3802.added.md b/newsfragments/3802.added.md new file mode 100644 index 00000000000..86b98e9df97 --- /dev/null +++ b/newsfragments/3802.added.md @@ -0,0 +1 @@ +Add `PyBackedStr` and `PyBackedBytes`, as alternatives to `&str` and `&bytes` where a Python object owns the data. diff --git a/src/lib.rs b/src/lib.rs index 91a61688e82..0a0b4eae684 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -426,6 +426,7 @@ pub mod marshal; pub mod sync; pub mod panic; pub mod prelude; +pub mod pybacked; pub mod pycell; pub mod pyclass; pub mod pyclass_init; diff --git a/src/pybacked.rs b/src/pybacked.rs new file mode 100644 index 00000000000..bd29c830e65 --- /dev/null +++ b/src/pybacked.rs @@ -0,0 +1,186 @@ +//! Contains types for working with Python objects that own the underlying data. + +use std::{ops::Deref, ptr::NonNull}; + +use crate::{ + types::{ + any::PyAnyMethods, bytearray::PyByteArrayMethods, bytes::PyBytesMethods, + string::PyStringMethods, PyByteArray, PyBytes, PyString, + }, + Bound, DowncastError, FromPyObject, Py, PyAny, PyErr, PyResult, +}; + +/// A wrapper around `str` where the storage is owned by a Python `bytes` or `str` object. +/// +/// This type gives access to the underlying data via a `Deref` implementation. +pub struct PyBackedStr { + #[allow(dead_code)] // only held so that the storage is not dropped + storage: Py, + data: NonNull<[u8]>, +} + +impl Deref for PyBackedStr { + type Target = str; + fn deref(&self) -> &str { + // Safety: `data` is known to be immutable utf8 string and owned by self + unsafe { std::str::from_utf8_unchecked(self.data.as_ref()) } + } +} + +impl TryFrom> for PyBackedStr { + type Error = PyErr; + fn try_from(py_string: Bound<'_, PyString>) -> Result { + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + { + let s = py_string.to_str()?; + let data = NonNull::from(s.as_bytes()); + Ok(Self { + storage: py_string.as_any().to_owned().unbind(), + data, + }) + } + #[cfg(not(any(Py_3_10, not(Py_LIMITED_API))))] + { + let bytes = py_string.encode_utf8()?; + let b = bytes.as_bytes(); + let data = NonNull::from(b); + Ok(Self { + storage: bytes.into_any().unbind(), + data, + }) + } + } +} + +impl FromPyObject<'_> for PyBackedStr { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { + let py_string = obj.downcast::()?.to_owned(); + Self::try_from(py_string) + } +} + +/// A wrapper around `[u8]` where the storage is either owned by a Python `bytes` object, or a Rust `Box<[u8]>`. +/// +/// This type gives access to the underlying data via a `Deref` implementation. +pub struct PyBackedBytes { + #[allow(dead_code)] // only held so that the storage is not dropped + storage: PyBackedBytesStorage, + data: NonNull<[u8]>, +} + +#[allow(dead_code)] +enum PyBackedBytesStorage { + Python(Py), + Rust(Box<[u8]>), +} + +impl Deref for PyBackedBytes { + type Target = [u8]; + fn deref(&self) -> &[u8] { + // Safety: `data` is known to be immutable and owned by self + unsafe { self.data.as_ref() } + } +} + +impl From> for PyBackedBytes { + fn from(py_bytes: Bound<'_, PyBytes>) -> Self { + let b = py_bytes.as_bytes(); + let data = NonNull::from(b); + Self { + storage: PyBackedBytesStorage::Python(py_bytes.to_owned().unbind()), + data, + } + } +} + +impl From> for PyBackedBytes { + fn from(py_bytearray: Bound<'_, PyByteArray>) -> Self { + let s = py_bytearray.to_vec().into_boxed_slice(); + let data = NonNull::from(s.as_ref()); + Self { + storage: PyBackedBytesStorage::Rust(s), + data, + } + } +} + +impl FromPyObject<'_> for PyBackedBytes { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { + if let Ok(bytes) = obj.downcast::() { + Ok(Self::from(bytes.to_owned())) + } else if let Ok(bytearray) = obj.downcast::() { + Ok(Self::from(bytearray.to_owned())) + } else { + Err(DowncastError::new(obj, "`bytes` or `bytearray`").into()) + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::Python; + + #[test] + fn py_backed_str_empty() { + Python::with_gil(|py| { + let s = PyString::new_bound(py, ""); + let py_backed_str = s.extract::().unwrap(); + assert_eq!(&*py_backed_str, ""); + }); + } + + #[test] + fn py_backed_str() { + Python::with_gil(|py| { + let s = PyString::new_bound(py, "hello"); + let py_backed_str = s.extract::().unwrap(); + assert_eq!(&*py_backed_str, "hello"); + }); + } + + #[test] + fn py_backed_str_try_from() { + Python::with_gil(|py| { + let s = PyString::new_bound(py, "hello"); + let py_backed_str = PyBackedStr::try_from(s).unwrap(); + assert_eq!(&*py_backed_str, "hello"); + }); + } + + #[test] + fn py_backed_bytes_empty() { + Python::with_gil(|py| { + let b = PyBytes::new_bound(py, &[]); + let py_backed_bytes = b.extract::().unwrap(); + assert_eq!(&*py_backed_bytes, &[]); + }); + } + + #[test] + fn py_backed_bytes() { + Python::with_gil(|py| { + let b = PyBytes::new_bound(py, b"abcde"); + let py_backed_bytes = b.extract::().unwrap(); + assert_eq!(&*py_backed_bytes, b"abcde"); + }); + } + + #[test] + fn py_backed_bytes_from_bytes() { + Python::with_gil(|py| { + let b = PyBytes::new_bound(py, b"abcde"); + let py_backed_bytes = PyBackedBytes::from(b); + assert_eq!(&*py_backed_bytes, b"abcde"); + }); + } + + #[test] + fn py_backed_bytes_from_bytearray() { + Python::with_gil(|py| { + let b = PyByteArray::new_bound(py, b"abcde"); + let py_backed_bytes = PyBackedBytes::from(b); + assert_eq!(&*py_backed_bytes, b"abcde"); + }); + } +} From 05aedc9032e74cde1dd24958c0836426b42e535a Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 16 Feb 2024 01:12:43 +0100 Subject: [PATCH 133/349] port `PyErr::warn` to `Bound` API (#3842) * port `PyErr::new_type` * port `PyErr::warn` and `PyErr::warn_explicit` --- src/conversions/chrono.rs | 8 ++- src/err/mod.rs | 112 ++++++++++++++++++++++++++++++++------ src/exceptions.rs | 5 +- 3 files changed, 102 insertions(+), 23 deletions(-) diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 421a7aec83e..d67818e645c 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -52,7 +52,9 @@ use crate::types::{ }; #[cfg(Py_LIMITED_API)] use crate::{intern, DowncastError}; -use crate::{Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject}; +use crate::{ + Bound, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, ToPyObject, +}; use chrono::offset::{FixedOffset, Utc}; use chrono::{ DateTime, Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Offset, TimeZone, Timelike, @@ -457,9 +459,9 @@ fn naive_datetime_to_py_datetime( fn warn_truncated_leap_second(obj: &Bound<'_, PyAny>) { let py = obj.py(); - if let Err(e) = PyErr::warn( + if let Err(e) = PyErr::warn_bound( py, - py.get_type::(), + &py.get_type::().as_borrowed(), "ignored leap-second, `datetime` does not support leap-seconds", 0, ) { diff --git a/src/err/mod.rs b/src/err/mod.rs index 5c2631b9301..b5e72a71bf9 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -445,6 +445,30 @@ impl PyErr { } } + /// Deprecated form of [`PyErr::new_type_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyErr::new_type` will be replaced by `PyErr::new_type_bound` in a future PyO3 version" + ) + )] + pub fn new_type( + py: Python<'_>, + name: &str, + doc: Option<&str>, + base: Option<&PyType>, + dict: Option, + ) -> PyResult> { + Self::new_type_bound( + py, + name, + doc, + base.map(PyNativeType::as_borrowed).as_deref(), + dict, + ) + } + /// Creates a new exception type with the given name and docstring. /// /// - `base` can be an existing exception type to subclass, or a tuple of classes. @@ -459,11 +483,11 @@ impl PyErr { /// # Panics /// /// This function will panic if `name` or `doc` cannot be converted to [`CString`]s. - pub fn new_type( - py: Python<'_>, + pub fn new_type_bound<'py>( + py: Python<'py>, name: &str, doc: Option<&str>, - base: Option<&PyType>, + base: Option<&Bound<'py, PyType>>, dict: Option, ) -> PyResult> { let base: *mut ffi::PyObject = match base { @@ -635,6 +659,18 @@ impl PyErr { unsafe { ffi::PyErr_WriteUnraisable(obj.map_or(std::ptr::null_mut(), Bound::as_ptr)) } } + /// Deprecated form of [`PyErr::warn_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyErr::warn` will be replaced by `PyErr::warn_bound` in a future PyO3 version" + ) + )] + pub fn warn(py: Python<'_>, category: &PyAny, message: &str, stacklevel: i32) -> PyResult<()> { + Self::warn_bound(py, &category.as_borrowed(), message, stacklevel) + } + /// Issues a warning message. /// /// May return an `Err(PyErr)` if warnings-as-errors is enabled. @@ -650,13 +686,18 @@ impl PyErr { /// # use pyo3::prelude::*; /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let user_warning = py.get_type::(); - /// PyErr::warn(py, user_warning, "I am warning you", 0)?; + /// let user_warning = py.get_type::().as_borrowed(); + /// PyErr::warn_bound(py, &user_warning, "I am warning you", 0)?; /// Ok(()) /// }) /// # } /// ``` - pub fn warn(py: Python<'_>, category: &PyAny, message: &str, stacklevel: i32) -> PyResult<()> { + pub fn warn_bound<'py>( + py: Python<'py>, + category: &Bound<'py, PyAny>, + message: &str, + stacklevel: i32, + ) -> PyResult<()> { let message = CString::new(message)?; error_on_minusone(py, unsafe { ffi::PyErr_WarnEx( @@ -667,6 +708,34 @@ impl PyErr { }) } + /// Deprecated form of [`PyErr::warn_explicit_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyErr::warn_explicit` will be replaced by `PyErr::warn_explicit_bound` in a future PyO3 version" + ) + )] + pub fn warn_explicit( + py: Python<'_>, + category: &PyAny, + message: &str, + filename: &str, + lineno: i32, + module: Option<&str>, + registry: Option<&PyAny>, + ) -> PyResult<()> { + Self::warn_explicit_bound( + py, + &category.as_borrowed(), + message, + filename, + lineno, + module, + registry.map(PyNativeType::as_borrowed).as_deref(), + ) + } + /// Issues a warning message, with more control over the warning attributes. /// /// May return a `PyErr` if warnings-as-errors is enabled. @@ -675,14 +744,14 @@ impl PyErr { /// /// The `category` should be one of the `Warning` classes available in /// [`pyo3::exceptions`](crate::exceptions), or a subclass. - pub fn warn_explicit( - py: Python<'_>, - category: &PyAny, + pub fn warn_explicit_bound<'py>( + py: Python<'py>, + category: &Bound<'py, PyAny>, message: &str, filename: &str, lineno: i32, module: Option<&str>, - registry: Option<&PyAny>, + registry: Option<&Bound<'py, PyAny>>, ) -> PyResult<()> { let message = CString::new(message)?; let filename = CString::new(filename)?; @@ -975,7 +1044,7 @@ mod tests { use super::PyErrState; use crate::exceptions::{self, PyTypeError, PyValueError}; use crate::types::any::PyAnyMethods; - use crate::{PyErr, PyTypeInfo, Python}; + use crate::{PyErr, PyNativeType, PyTypeInfo, Python}; #[test] fn no_error() { @@ -1172,7 +1241,7 @@ mod tests { // GIL locked should prevent effects to be visible to other testing // threads. Python::with_gil(|py| { - let cls = py.get_type::(); + let cls = py.get_type::().as_borrowed(); // Reset warning filter to default state let warnings = py.import_bound("warnings").unwrap(); @@ -1181,7 +1250,7 @@ mod tests { // First, test the warning is emitted assert_warnings!( py, - { PyErr::warn(py, cls, "I am warning you", 0).unwrap() }, + { PyErr::warn_bound(py, &cls, "I am warning you", 0).unwrap() }, [(exceptions::PyUserWarning, "I am warning you")] ); @@ -1189,7 +1258,7 @@ mod tests { warnings .call_method1("simplefilter", ("error", cls)) .unwrap(); - PyErr::warn(py, cls, "I am warning you", 0).unwrap_err(); + PyErr::warn_bound(py, &cls, "I am warning you", 0).unwrap_err(); // Test with error for an explicit module warnings.call_method0("resetwarnings").unwrap(); @@ -1200,13 +1269,20 @@ mod tests { // This has the wrong module and will not raise, just be emitted assert_warnings!( py, - { PyErr::warn(py, cls, "I am warning you", 0).unwrap() }, + { PyErr::warn_bound(py, &cls, "I am warning you", 0).unwrap() }, [(exceptions::PyUserWarning, "I am warning you")] ); - let err = - PyErr::warn_explicit(py, cls, "I am warning you", "pyo3test.py", 427, None, None) - .unwrap_err(); + let err = PyErr::warn_explicit_bound( + py, + &cls, + "I am warning you", + "pyo3test.py", + 427, + None, + None, + ) + .unwrap_err(); assert!(err .value(py) .getattr("args") diff --git a/src/exceptions.rs b/src/exceptions.rs index 245c3bbdf05..bcbe91b90c7 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -240,16 +240,17 @@ macro_rules! create_exception_type_object { impl $name { fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { use $crate::sync::GILOnceCell; + use $crate::PyNativeType; static TYPE_OBJECT: GILOnceCell<$crate::Py<$crate::types::PyType>> = GILOnceCell::new(); TYPE_OBJECT .get_or_init(py, || - $crate::PyErr::new_type( + $crate::PyErr::new_type_bound( py, concat!(stringify!($module), ".", stringify!($name)), $doc, - ::std::option::Option::Some(py.get_type::<$base>()), + ::std::option::Option::Some(&py.get_type::<$base>().as_borrowed()), ::std::option::Option::None, ).expect("Failed to initialize new exception type.") ).as_ptr() as *mut $crate::ffi::PyTypeObject From ec6d587218a9828234ace50917ea9040a7c08e2b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 16 Feb 2024 00:36:11 +0000 Subject: [PATCH 134/349] support `Bound` for `classmethod` and `pass_module` (#3831) * support `Bound` for `classmethod` and `pass_module` * `from_ref_to_ptr` -> `ref_from_ptr` * add detailed docs to `ref_from_ptr` --- guide/src/class.md | 10 +++--- guide/src/function.md | 3 +- pyo3-macros-backend/src/method.rs | 14 ++++++-- pytests/src/pyclasses.rs | 20 +++++++++++ pytests/tests/test_pyclasses.py | 11 ++++++ src/impl_/pymethods.rs | 53 ++++++++++++++++++++++++++++- src/instance.rs | 18 ++++++++++ src/tests/hygiene/pymethods.rs | 4 +-- tests/test_class_basics.rs | 16 +++++++-- tests/test_methods.rs | 20 ++++++++--- tests/test_module.rs | 51 ++++++++++++++++++--------- tests/test_multiple_pymethods.rs | 2 +- tests/test_no_imports.rs | 10 ++++-- tests/test_text_signature.rs | 8 ++--- tests/ui/invalid_pyfunctions.stderr | 6 ++-- tests/ui/invalid_pymethods.stderr | 10 +++--- 16 files changed, 205 insertions(+), 51 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index dfe5c8ae4ae..6ca582ed963 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -691,7 +691,7 @@ This is the equivalent of the Python decorator `@classmethod`. #[pymethods] impl MyClass { #[classmethod] - fn cls_method(cls: &PyType) -> PyResult { + fn cls_method(cls: &Bound<'_, PyType>) -> PyResult { Ok(10) } } @@ -719,10 +719,10 @@ To create a constructor which takes a positional class argument, you can combine impl BaseClass { #[new] #[classmethod] - fn py_new<'p>(cls: &'p PyType, py: Python<'p>) -> PyResult { + fn py_new(cls: &Bound<'_, PyType>) -> PyResult { // Get an abstract attribute (presumably) declared on a subclass of this class. - let subclass_attr = cls.getattr("a_class_attr")?; - Ok(Self(subclass_attr.to_object(py))) + let subclass_attr: Bound<'_, PyAny> = cls.getattr("a_class_attr")?; + Ok(Self(subclass_attr.unbind())) } } ``` @@ -928,7 +928,7 @@ impl MyClass { // similarly for classmethod arguments, use $cls #[classmethod] #[pyo3(text_signature = "($cls, e, f)")] - fn my_class_method(cls: &PyType, e: i32, f: i32) -> i32 { + fn my_class_method(cls: &Bound<'_, PyType>, e: i32, f: i32) -> i32 { e + f } #[staticmethod] diff --git a/guide/src/function.md b/guide/src/function.md index 49ec716af56..f3955ba554b 100644 --- a/guide/src/function.md +++ b/guide/src/function.md @@ -83,10 +83,11 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python ```rust use pyo3::prelude::*; + use pyo3::types::PyString; #[pyfunction] #[pyo3(pass_module)] - fn pyfunction_with_module(module: &PyModule) -> PyResult<&str> { + fn pyfunction_with_module<'py>(module: &Bound<'py, PyModule>) -> PyResult> { module.name() } diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index c158ec9fe05..f492a330c92 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -127,13 +127,21 @@ impl FnType { let slf: Ident = syn::Ident::new("_slf", Span::call_site()); quote_spanned! { *span => #[allow(clippy::useless_conversion)] - ::std::convert::Into::into(_pyo3::types::PyType::from_type_ptr(#py, #slf.cast())), + ::std::convert::Into::into( + _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf.cast()) + .downcast_unchecked::<_pyo3::types::PyType>() + ), } } FnType::FnModule(span) => { + let py = syn::Ident::new("py", Span::call_site()); + let slf: Ident = syn::Ident::new("_slf", Span::call_site()); quote_spanned! { *span => #[allow(clippy::useless_conversion)] - ::std::convert::Into::into(py.from_borrowed_ptr::<_pyo3::types::PyModule>(_slf)), + ::std::convert::Into::into( + _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf.cast()) + .downcast_unchecked::<_pyo3::types::PyModule>() + ), } } } @@ -409,7 +417,7 @@ impl<'a> FnSpec<'a> { // will error on incorrect type. Some(syn::FnArg::Typed(first_arg)) => first_arg.ty.span(), Some(syn::FnArg::Receiver(_)) | None => bail_spanned!( - sig.paren_token.span.join() => "Expected `&PyType` or `Py` as the first argument to `#[classmethod]`" + sig.paren_token.span.join() => "Expected `&Bound` or `Py` as the first argument to `#[classmethod]`" ), }; FnType::FnClass(span) diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index 326893d123d..9c7b2d250da 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -44,6 +44,25 @@ struct AssertingBaseClass; #[pymethods] impl AssertingBaseClass { + #[new] + #[classmethod] + fn new(cls: &Bound<'_, PyType>, expected_type: Bound<'_, PyType>) -> PyResult { + if !cls.is(&expected_type) { + return Err(PyValueError::new_err(format!( + "{:?} != {:?}", + cls, expected_type + ))); + } + Ok(Self) + } +} + +#[pyclass(subclass)] +#[derive(Clone, Debug)] +struct AssertingBaseClassGilRef; + +#[pymethods] +impl AssertingBaseClassGilRef { #[new] #[classmethod] fn new(cls: &PyType, expected_type: &PyType) -> PyResult { @@ -65,6 +84,7 @@ pub fn pyclasses(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; Ok(()) } diff --git a/pytests/tests/test_pyclasses.py b/pytests/tests/test_pyclasses.py index 5482862811a..9a9b44b52e8 100644 --- a/pytests/tests/test_pyclasses.py +++ b/pytests/tests/test_pyclasses.py @@ -41,6 +41,17 @@ def test_new_classmethod(): _ = AssertingSubClass(expected_type=str) +def test_new_classmethod_gil_ref(): + class AssertingSubClass(pyclasses.AssertingBaseClassGilRef): + pass + + # The `AssertingBaseClass` constructor errors if it is not passed the + # relevant subclass. + _ = AssertingSubClass(expected_type=AssertingSubClass) + with pytest.raises(ValueError): + _ = AssertingSubClass(expected_type=str) + + class ClassWithoutConstructorPy: def __new__(cls): raise TypeError("No constructor defined") diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 4fac39fbdc4..eef3b569d08 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -3,8 +3,10 @@ use crate::exceptions::PyStopAsyncIteration; use crate::gil::LockGIL; use crate::impl_::panic::PanicTrap; use crate::internal_tricks::extract_c_string; +use crate::types::{any::PyAnyMethods, PyModule, PyType}; use crate::{ - ffi, PyAny, PyCell, PyClass, PyErr, PyObject, PyResult, PyTraverseError, PyVisit, Python, + ffi, Bound, Py, PyAny, PyCell, PyClass, PyErr, PyObject, PyResult, PyTraverseError, PyVisit, + Python, }; use std::borrow::Cow; use std::ffi::CStr; @@ -466,3 +468,52 @@ pub trait AsyncIterResultOptionKind { } impl AsyncIterResultOptionKind for Result, Error> {} + +/// Used in `#[classmethod]` to pass the class object to the method +/// and also in `#[pyfunction(pass_module)]`. +/// +/// This is a wrapper to avoid implementing `From` for GIL Refs. +/// +/// Once the GIL Ref API is fully removed, it should be possible to simplify +/// this to just `&'a Bound<'py, T>` and `From` implementations. +pub struct BoundRef<'a, 'py, T>(pub &'a Bound<'py, T>); + +impl<'a, 'py> BoundRef<'a, 'py, PyAny> { + pub unsafe fn ref_from_ptr(py: Python<'py>, ptr: &'a *mut ffi::PyObject) -> Self { + BoundRef(Bound::ref_from_ptr(py, ptr)) + } + + pub unsafe fn downcast_unchecked(self) -> BoundRef<'a, 'py, T> { + BoundRef(self.0.downcast_unchecked::()) + } +} + +// GIL Ref implementations for &'a T ran into trouble with orphan rules, +// so explicit implementations are used instead for the two relevant types. +impl<'a> From> for &'a PyType { + #[inline] + fn from(bound: BoundRef<'a, 'a, PyType>) -> Self { + bound.0.as_gil_ref() + } +} + +impl<'a> From> for &'a PyModule { + #[inline] + fn from(bound: BoundRef<'a, 'a, PyModule>) -> Self { + bound.0.as_gil_ref() + } +} + +impl<'a, 'py, T> From> for &'a Bound<'py, T> { + #[inline] + fn from(bound: BoundRef<'a, 'py, T>) -> Self { + bound.0 + } +} + +impl From> for Py { + #[inline] + fn from(bound: BoundRef<'_, '_, T>) -> Self { + bound.0.clone().unbind() + } +} diff --git a/src/instance.rs b/src/instance.rs index 54a9345241d..ff553c44324 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -138,6 +138,24 @@ impl<'py> Bound<'py, PyAny> { ) -> PyResult { Py::from_owned_ptr_or_err(py, ptr).map(|obj| Self(py, ManuallyDrop::new(obj))) } + + /// This slightly strange method is used to obtain `&Bound` from a pointer in macro code + /// where we need to constrain the lifetime `'a` safely. + /// + /// Note that `'py` is required to outlive `'a` implicitly by the nature of the fact that + /// `&'a Bound<'py>` means that `Bound<'py>` exists for at least the lifetime `'a`. + /// + /// # Safety + /// - `ptr` must be a valid pointer to a Python object for the lifetime `'a`. The `ptr` can + /// be either a borrowed reference or an owned reference, it does not matter, as this is + /// just `&Bound` there will never be any ownership transfer. + #[inline] + pub(crate) unsafe fn ref_from_ptr<'a>( + _py: Python<'py>, + ptr: &'a *mut ffi::PyObject, + ) -> &'a Self { + &*(ptr as *const *mut ffi::PyObject).cast::>() + } } impl<'py, T> Bound<'py, T> diff --git a/src/tests/hygiene/pymethods.rs b/src/tests/hygiene/pymethods.rs index 15ea675943c..a00e67d9a85 100644 --- a/src/tests/hygiene/pymethods.rs +++ b/src/tests/hygiene/pymethods.rs @@ -375,7 +375,7 @@ impl Dummy { #[staticmethod] fn staticmethod() {} #[classmethod] - fn clsmethod(_: &crate::types::PyType) {} + fn clsmethod(_: &crate::Bound<'_, crate::types::PyType>) {} #[pyo3(signature = (*_args, **_kwds))] fn __call__( &self, @@ -770,7 +770,7 @@ impl Dummy { #[staticmethod] fn staticmethod() {} #[classmethod] - fn clsmethod(_: &crate::types::PyType) {} + fn clsmethod(_: &crate::Bound<'_, crate::types::PyType>) {} #[pyo3(signature = (*_args, **_kwds))] fn __call__( &self, diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index bbf37a2d66b..ff3555c3107 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -284,7 +284,7 @@ fn panic_unsendable_child() { test_unsendable::().unwrap(); } -fn get_length(obj: &PyAny) -> PyResult { +fn get_length(obj: &Bound<'_, PyAny>) -> PyResult { let length = obj.len()?; Ok(length) @@ -299,7 +299,18 @@ impl ClassWithFromPyWithMethods { argument } #[classmethod] - fn classmethod(_cls: &PyType, #[pyo3(from_py_with = "PyAny::len")] argument: usize) -> usize { + fn classmethod( + _cls: &Bound<'_, PyType>, + #[pyo3(from_py_with = "Bound::<'_, PyAny>::len")] argument: usize, + ) -> usize { + argument + } + + #[classmethod] + fn classmethod_gil_ref( + _cls: &PyType, + #[pyo3(from_py_with = "PyAny::len")] argument: usize, + ) -> usize { argument } @@ -322,6 +333,7 @@ fn test_pymethods_from_py_with() { assert instance.instance_method(arg) == 2 assert instance.classmethod(arg) == 2 + assert instance.classmethod_gil_ref(arg) == 2 assert instance.staticmethod(arg) == 2 "# ); diff --git a/tests/test_methods.rs b/tests/test_methods.rs index 083565334d0..2114ead25c4 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -73,7 +73,13 @@ impl ClassMethod { #[classmethod] /// Test class method. - fn method(cls: &PyType) -> PyResult { + fn method(cls: &Bound<'_, PyType>) -> PyResult { + Ok(format!("{}.method()!", cls.as_gil_ref().qualname()?)) + } + + #[classmethod] + /// Test class method. + fn method_gil_ref(cls: &PyType) -> PyResult { Ok(format!("{}.method()!", cls.qualname()?)) } @@ -108,8 +114,12 @@ struct ClassMethodWithArgs {} #[pymethods] impl ClassMethodWithArgs { #[classmethod] - fn method(cls: &PyType, input: &PyString) -> PyResult { - Ok(format!("{}.method({})", cls.qualname()?, input)) + fn method(cls: &Bound<'_, PyType>, input: &PyString) -> PyResult { + Ok(format!( + "{}.method({})", + cls.as_gil_ref().qualname()?, + input + )) } } @@ -915,7 +925,7 @@ impl r#RawIdents { } #[classmethod] - pub fn r#class_method(_: &PyType, r#type: PyObject) -> PyObject { + pub fn r#class_method(_: &Bound<'_, PyType>, r#type: PyObject) -> PyObject { r#type } @@ -1082,7 +1092,7 @@ issue_1506!( #[classmethod] fn issue_1506_class( - _cls: &PyType, + _cls: &Bound<'_, PyType>, _py: Python<'_>, _arg: &PyAny, _args: &PyTuple, diff --git a/tests/test_module.rs b/tests/test_module.rs index 1bd976e0ae4..9a59770e047 100644 --- a/tests/test_module.rs +++ b/tests/test_module.rs @@ -3,6 +3,7 @@ use pyo3::prelude::*; use pyo3::py_run; +use pyo3::types::PyString; use pyo3::types::{IntoPyDict, PyDict, PyTuple}; #[path = "../src/tests/common.rs"] @@ -344,47 +345,59 @@ fn test_module_with_constant() { #[pyfunction] #[pyo3(pass_module)] -fn pyfunction_with_module(module: &PyModule) -> PyResult<&str> { +fn pyfunction_with_module<'py>(module: &Bound<'py, PyModule>) -> PyResult> { module.name() } #[pyfunction] #[pyo3(pass_module)] -fn pyfunction_with_module_owned(module: Py) -> PyResult { - Python::with_gil(|gil| module.as_ref(gil).name().map(Into::into)) +fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { + module.name() +} + +#[pyfunction] +#[pyo3(pass_module)] +fn pyfunction_with_module_owned( + module: Py, + py: Python<'_>, +) -> PyResult> { + module.bind(py).name() } #[pyfunction] #[pyo3(pass_module)] -fn pyfunction_with_module_and_py<'a>( - module: &'a PyModule, - _python: Python<'a>, -) -> PyResult<&'a str> { +fn pyfunction_with_module_and_py<'py>( + module: &Bound<'py, PyModule>, + _python: Python<'py>, +) -> PyResult> { module.name() } #[pyfunction] #[pyo3(pass_module)] -fn pyfunction_with_module_and_arg(module: &PyModule, string: String) -> PyResult<(&str, String)> { +fn pyfunction_with_module_and_arg<'py>( + module: &Bound<'py, PyModule>, + string: String, +) -> PyResult<(Bound<'py, PyString>, String)> { module.name().map(|s| (s, string)) } #[pyfunction(signature = (string="foo"))] #[pyo3(pass_module)] -fn pyfunction_with_module_and_default_arg<'a>( - module: &'a PyModule, +fn pyfunction_with_module_and_default_arg<'py>( + module: &Bound<'py, PyModule>, string: &str, -) -> PyResult<(&'a str, String)> { +) -> PyResult<(Bound<'py, PyString>, String)> { module.name().map(|s| (s, string.into())) } #[pyfunction(signature = (*args, **kwargs))] #[pyo3(pass_module)] -fn pyfunction_with_module_and_args_kwargs<'a>( - module: &'a PyModule, - args: &PyTuple, - kwargs: Option<&PyDict>, -) -> PyResult<(&'a str, usize, Option)> { +fn pyfunction_with_module_and_args_kwargs<'py>( + module: &Bound<'py, PyModule>, + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, +) -> PyResult<(Bound<'py, PyString>, usize, Option)> { module .name() .map(|s| (s, args.len(), kwargs.map(|d| d.len()))) @@ -399,6 +412,7 @@ fn pyfunction_with_pass_module_in_attribute(module: &PyModule) -> PyResult<&str> #[pymodule] fn module_with_functions_with_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(pyfunction_with_module, m)?)?; + m.add_function(wrap_pyfunction!(pyfunction_with_module_gil_ref, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_owned, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_and_py, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_and_arg, m)?)?; @@ -421,6 +435,11 @@ fn test_module_functions_with_module() { m, "m.pyfunction_with_module() == 'module_with_functions_with_module'" ); + py_assert!( + py, + m, + "m.pyfunction_with_module_gil_ref() == 'module_with_functions_with_module'" + ); py_assert!( py, m, diff --git a/tests/test_multiple_pymethods.rs b/tests/test_multiple_pymethods.rs index f78a9c60dc1..13baeed3815 100644 --- a/tests/test_multiple_pymethods.rs +++ b/tests/test_multiple_pymethods.rs @@ -35,7 +35,7 @@ impl PyClassWithMultiplePyMethods { #[pymethods] impl PyClassWithMultiplePyMethods { #[classmethod] - fn classmethod(_ty: &PyType) -> &'static str { + fn classmethod(_ty: &Bound<'_, PyType>) -> &'static str { "classmethod" } } diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index 1ff3dc940aa..4abdcdf2e5b 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -60,7 +60,9 @@ impl BasicClass { /// Some documentation here #[classmethod] - fn classmethod(cls: &pyo3::types::PyType) -> &pyo3::types::PyType { + fn classmethod<'a, 'py>( + cls: &'a pyo3::Bound<'py, pyo3::types::PyType>, + ) -> &'a pyo3::Bound<'py, pyo3::types::PyType> { cls } @@ -132,8 +134,10 @@ struct NewClassMethod { impl NewClassMethod { #[new] #[classmethod] - fn new(cls: &pyo3::types::PyType) -> Self { - Self { cls: cls.into() } + fn new(cls: &pyo3::Bound<'_, pyo3::types::PyType>) -> Self { + Self { + cls: cls.clone().into_any().unbind(), + } } } diff --git a/tests/test_text_signature.rs b/tests/test_text_signature.rs index 5b0491d9970..9056ca21e14 100644 --- a/tests/test_text_signature.rs +++ b/tests/test_text_signature.rs @@ -115,7 +115,7 @@ fn test_auto_test_signature_function() { } #[pyfunction(pass_module)] - fn my_function_2(module: &PyModule, a: i32, b: i32, c: i32) { + fn my_function_2(module: &Bound<'_, PyModule>, a: i32, b: i32, c: i32) { let _ = (module, a, b, c); } @@ -232,7 +232,7 @@ fn test_auto_test_signature_method() { } #[classmethod] - fn classmethod(cls: &PyType, a: i32, b: i32, c: i32) { + fn classmethod(cls: &Bound<'_, PyType>, a: i32, b: i32, c: i32) { let _ = (cls, a, b, c); } } @@ -311,7 +311,7 @@ fn test_auto_test_signature_opt_out() { #[classmethod] #[pyo3(text_signature = None)] - fn classmethod(cls: &PyType, a: i32, b: i32, c: i32) { + fn classmethod(cls: &Bound<'_, PyType>, a: i32, b: i32, c: i32) { let _ = (cls, a, b, c); } } @@ -372,7 +372,7 @@ fn test_methods() { } #[classmethod] #[pyo3(text_signature = "($cls, c)")] - fn class_method(_cls: &PyType, c: i32) { + fn class_method(_cls: &Bound<'_, PyType>, c: i32) { let _ = c; } #[staticmethod] diff --git a/tests/ui/invalid_pyfunctions.stderr b/tests/ui/invalid_pyfunctions.stderr index a3fd845d580..6576997a3eb 100644 --- a/tests/ui/invalid_pyfunctions.stderr +++ b/tests/ui/invalid_pyfunctions.stderr @@ -35,11 +35,11 @@ error: expected `&PyModule` or `Py` as first argument with `pass_modul 19 | fn pass_module_but_no_arguments<'py>() {} | ^^ -error[E0277]: the trait bound `&str: From<&pyo3::prelude::PyModule>` is not satisfied +error[E0277]: the trait bound `&str: From>` is not satisfied --> tests/ui/invalid_pyfunctions.rs:22:43 | 22 | fn first_argument_not_module<'py>(string: &str, module: &'py PyModule) -> PyResult<&'py str> { - | ^ the trait `From<&pyo3::prelude::PyModule>` is not implemented for `&str` + | ^ the trait `From>` is not implemented for `&str` | = help: the following other types implement trait `From`: > @@ -48,4 +48,4 @@ error[E0277]: the trait bound `&str: From<&pyo3::prelude::PyModule>` is not sati > > > - = note: required for `&pyo3::prelude::PyModule` to implement `Into<&str>` + = note: required for `BoundRef<'_, '_, pyo3::prelude::PyModule>` to implement `Into<&str>` diff --git a/tests/ui/invalid_pymethods.stderr b/tests/ui/invalid_pymethods.stderr index 1a50c4da0c1..6a8d6ecab78 100644 --- a/tests/ui/invalid_pymethods.stderr +++ b/tests/ui/invalid_pymethods.stderr @@ -22,13 +22,13 @@ error: unexpected receiver 26 | fn staticmethod_with_receiver(&self) {} | ^ -error: Expected `&PyType` or `Py` as the first argument to `#[classmethod]` +error: Expected `&Bound` or `Py` as the first argument to `#[classmethod]` --> tests/ui/invalid_pymethods.rs:32:33 | 32 | fn classmethod_with_receiver(&self) {} | ^^^^^^^ -error: Expected `&PyType` or `Py` as the first argument to `#[classmethod]` +error: Expected `&Bound` or `Py` as the first argument to `#[classmethod]` --> tests/ui/invalid_pymethods.rs:38:36 | 38 | fn classmethod_missing_argument() -> Self { @@ -179,11 +179,11 @@ error: macros cannot be used as items in `#[pymethods]` impl blocks 197 | macro_invocation!(); | ^^^^^^^^^^^^^^^^ -error[E0277]: the trait bound `i32: From<&PyType>` is not satisfied +error[E0277]: the trait bound `i32: From>` is not satisfied --> tests/ui/invalid_pymethods.rs:46:45 | 46 | fn classmethod_wrong_first_argument(_x: i32) -> Self { - | ^^^ the trait `From<&PyType>` is not implemented for `i32` + | ^^^ the trait `From>` is not implemented for `i32` | = help: the following other types implement trait `From`: > @@ -192,4 +192,4 @@ error[E0277]: the trait bound `i32: From<&PyType>` is not satisfied > > > - = note: required for `&PyType` to implement `Into` + = note: required for `BoundRef<'_, '_, PyType>` to implement `Into` From c24478e8bc2785e7f032721d5777e339c21afc74 Mon Sep 17 00:00:00 2001 From: alm Date: Fri, 16 Feb 2024 18:53:25 +0200 Subject: [PATCH 135/349] docs: fix link in Contributing.md (#3845) --- Contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Contributing.md b/Contributing.md index 332645542d7..dad543856cc 100644 --- a/Contributing.md +++ b/Contributing.md @@ -35,7 +35,7 @@ Helping others often reveals bugs, documentation weaknesses, and missing APIs. I ### Implement issues ready for development -Issues where the solution is clear and work is not in progress use the [needs-implementer](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-implemeter) label. +Issues where the solution is clear and work is not in progress use the [needs-implementer](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-implementer) label. Don't be afraid if the solution is not clear to you! The core PyO3 contributors will be happy to mentor you through any questions you have to help you write the solution. From 940804fe0d311e087098c4638083258b100093f5 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Sat, 17 Feb 2024 00:27:45 +0000 Subject: [PATCH 136/349] Pyerr value bound (#3820) * Implement PyErr::value_bound * Use PyErr::value_bound in conversions * Implement PyErr::from_value_bound * Remove unnecessary clone * Return a reference from PyErr::value_bound * Avoid clone in PyErr::from_value_bound * Use PyErr::from_value_bound instead of from_value * Remove unnecessary .as_borrowed() calls * Remove unused import * Simplify UnraisableCapture.hook * Use Bound::from_owned_ptr_or_opt in fn cause * Update PyErrState::lazy to take Py This is easier to work with elsewhere than `&PyAny` or `Bound<'py, PyAny>`. * Add Bound PyUnicodeDecodeError constructors * Update PyErr::from_value_bound to take Bound * Simplify PyErr::from_value * Simplify PyErr::value * Remove unnecessary reference * Simplify Pyerr::cause implementation * Simplify PyUnicodeDecodeError::new_bound --- guide/src/python_from_rust.md | 2 +- src/conversions/anyhow.rs | 4 +- src/conversions/chrono.rs | 8 ++-- src/conversions/eyre.rs | 4 +- src/coroutine.rs | 2 +- src/err/err_state.rs | 10 +++-- src/err/impls.rs | 6 ++- src/err/mod.rs | 83 +++++++++++++++++++++++------------ src/exceptions.rs | 67 +++++++++++++++++++++++----- src/impl_/extract_argument.rs | 7 ++- src/tests/common.rs | 4 +- src/types/string.rs | 40 ++++++++++------- src/types/traceback.rs | 6 +-- tests/test_coroutine.rs | 2 +- 14 files changed, 165 insertions(+), 80 deletions(-) diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 1ac1a7c217a..0b412c871c7 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -479,7 +479,7 @@ class House(object): } Err(e) => { house - .call_method1("__exit__", (e.get_type_bound(py), e.value(py), e.traceback_bound(py))) + .call_method1("__exit__", (e.get_type_bound(py), e.value_bound(py), e.traceback_bound(py))) .unwrap(); } } diff --git a/src/conversions/anyhow.rs b/src/conversions/anyhow.rs index 453799c6e8b..a490bd4ce31 100644 --- a/src/conversions/anyhow.rs +++ b/src/conversions/anyhow.rs @@ -149,7 +149,7 @@ mod test_anyhow { Python::with_gil(|py| { let locals = [("err", pyerr)].into_py_dict_bound(py); let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); - assert_eq!(pyerr.value(py).to_string(), expected_contents); + assert_eq!(pyerr.value_bound(py).to_string(), expected_contents); }) } @@ -166,7 +166,7 @@ mod test_anyhow { Python::with_gil(|py| { let locals = [("err", pyerr)].into_py_dict_bound(py); let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); - assert_eq!(pyerr.value(py).to_string(), expected_contents); + assert_eq!(pyerr.value_bound(py).to_string(), expected_contents); }) } diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index d67818e645c..6737c16f8cf 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -590,7 +590,7 @@ mod tests { assert!(result.is_err()); let res = result.err().unwrap(); // Also check the error message is what we expect - let msg = res.value(py).repr().unwrap().to_string(); + let msg = res.value_bound(py).repr().unwrap().to_string(); assert_eq!(msg, "TypeError(\"zoneinfo.ZoneInfo(key='Europe/London') is not a fixed offset timezone\")"); }); } @@ -605,7 +605,7 @@ mod tests { // Now test that converting a PyDateTime with tzinfo to a NaiveDateTime fails let res: PyResult = py_datetime.extract(); assert_eq!( - res.unwrap_err().value(py).repr().unwrap().to_string(), + res.unwrap_err().value_bound(py).repr().unwrap().to_string(), "TypeError('expected a datetime without tzinfo')" ); }); @@ -620,14 +620,14 @@ mod tests { // Now test that converting a PyDateTime with tzinfo to a NaiveDateTime fails let res: PyResult> = py_datetime.extract(); assert_eq!( - res.unwrap_err().value(py).repr().unwrap().to_string(), + res.unwrap_err().value_bound(py).repr().unwrap().to_string(), "TypeError('expected a datetime with non-None tzinfo')" ); // Now test that converting a PyDateTime with tzinfo to a NaiveDateTime fails let res: PyResult> = py_datetime.extract(); assert_eq!( - res.unwrap_err().value(py).repr().unwrap().to_string(), + res.unwrap_err().value_bound(py).repr().unwrap().to_string(), "TypeError('expected a datetime with non-None tzinfo')" ); }); diff --git a/src/conversions/eyre.rs b/src/conversions/eyre.rs index d25a10af9ee..64ed4a0320f 100644 --- a/src/conversions/eyre.rs +++ b/src/conversions/eyre.rs @@ -154,7 +154,7 @@ mod tests { Python::with_gil(|py| { let locals = [("err", pyerr)].into_py_dict_bound(py); let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); - assert_eq!(pyerr.value(py).to_string(), expected_contents); + assert_eq!(pyerr.value_bound(py).to_string(), expected_contents); }) } @@ -171,7 +171,7 @@ mod tests { Python::with_gil(|py| { let locals = [("err", pyerr)].into_py_dict_bound(py); let pyerr = py.run_bound("raise err", None, Some(&locals)).unwrap_err(); - assert_eq!(pyerr.value(py).to_string(), expected_contents); + assert_eq!(pyerr.value_bound(py).to_string(), expected_contents); }) } diff --git a/src/coroutine.rs b/src/coroutine.rs index c4b5ddf3185..3a0df983076 100644 --- a/src/coroutine.rs +++ b/src/coroutine.rs @@ -78,7 +78,7 @@ impl Coroutine { (Some(exc), Some(cb)) => cb.throw(exc.as_ref(py)), (Some(exc), None) => { self.close(); - return Err(PyErr::from_value(exc.as_ref(py))); + return Err(PyErr::from_value_bound(exc.into_bound(py))); } (None, _) => {} } diff --git a/src/err/err_state.rs b/src/err/err_state.rs index 863b276307e..eb03d8b9ff2 100644 --- a/src/err/err_state.rs +++ b/src/err/err_state.rs @@ -101,19 +101,20 @@ where } impl PyErrState { - pub(crate) fn lazy(ptype: &PyAny, args: impl PyErrArguments + 'static) -> Self { - let ptype = ptype.into(); + pub(crate) fn lazy(ptype: Py, args: impl PyErrArguments + 'static) -> Self { PyErrState::Lazy(Box::new(move |py| PyErrStateLazyFnOutput { ptype, pvalue: args.arguments(py), })) } - pub(crate) fn normalized(pvalue: &PyBaseException) -> Self { + pub(crate) fn normalized(pvalue: Bound<'_, PyBaseException>) -> Self { + #[cfg(not(Py_3_12))] + use crate::types::any::PyAnyMethods; + Self::Normalized(PyErrStateNormalized { #[cfg(not(Py_3_12))] ptype: pvalue.get_type().into(), - pvalue: pvalue.into(), #[cfg(not(Py_3_12))] ptraceback: unsafe { Py::from_owned_ptr_or_opt( @@ -121,6 +122,7 @@ impl PyErrState { ffi::PyException_GetTraceback(pvalue.as_ptr()), ) }, + pvalue: pvalue.into(), }) } diff --git a/src/err/impls.rs b/src/err/impls.rs index f49ac3eb1a8..f14a839b471 100644 --- a/src/err/impls.rs +++ b/src/err/impls.rs @@ -124,6 +124,8 @@ mod tests { #[test] fn io_errors() { + use crate::types::any::PyAnyMethods; + let check_err = |kind, expected_ty| { Python::with_gil(|py| { let rust_err = io::Error::new(kind, "some error msg"); @@ -139,8 +141,8 @@ mod tests { let py_err_recovered_from_rust_err: PyErr = rust_err_from_py_err.into(); assert!(py_err_recovered_from_rust_err - .value(py) - .is(py_error_clone.value(py))); // It should be the same exception + .value_bound(py) + .is(py_error_clone.value_bound(py))); // It should be the same exception }) }; diff --git a/src/err/mod.rs b/src/err/mod.rs index b5e72a71bf9..b5cd02c0954 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -187,7 +187,19 @@ impl PyErr { where A: PyErrArguments + Send + Sync + 'static, { - PyErr::from_state(PyErrState::lazy(ty, args)) + PyErr::from_state(PyErrState::lazy(ty.into(), args)) + } + + /// Deprecated form of [`PyErr::from_value_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyErr::from_value` will be replaced by `PyErr::from_value_bound` in a future PyO3 version" + ) + )] + pub fn from_value(obj: &PyAny) -> PyErr { + PyErr::from_value_bound(obj.as_borrowed().to_owned()) } /// Creates a new PyErr. @@ -201,33 +213,38 @@ impl PyErr { /// # Examples /// ```rust /// use pyo3::prelude::*; + /// use pyo3::PyTypeInfo; /// use pyo3::exceptions::PyTypeError; - /// use pyo3::types::{PyType, PyString}; + /// use pyo3::types::PyString; /// /// Python::with_gil(|py| { /// // Case #1: Exception object - /// let err = PyErr::from_value(PyTypeError::new_err("some type error").value(py)); + /// let err = PyErr::from_value_bound(PyTypeError::new_err("some type error") + /// .value_bound(py).clone().into_any()); /// assert_eq!(err.to_string(), "TypeError: some type error"); /// /// // Case #2: Exception type - /// let err = PyErr::from_value(PyType::new::(py)); + /// let err = PyErr::from_value_bound(PyTypeError::type_object_bound(py).into_any()); /// assert_eq!(err.to_string(), "TypeError: "); /// /// // Case #3: Invalid exception value - /// let err = PyErr::from_value(PyString::new_bound(py, "foo").as_gil_ref()); + /// let err = PyErr::from_value_bound(PyString::new_bound(py, "foo").into_any()); /// assert_eq!( /// err.to_string(), /// "TypeError: exceptions must derive from BaseException" /// ); /// }); /// ``` - pub fn from_value(obj: &PyAny) -> PyErr { - let state = if let Ok(obj) = obj.downcast::() { - PyErrState::normalized(obj) - } else { - // Assume obj is Type[Exception]; let later normalization handle if this - // is not the case - PyErrState::lazy(obj, obj.py().None()) + pub fn from_value_bound(obj: Bound<'_, PyAny>) -> PyErr { + let state = match obj.downcast_into::() { + Ok(obj) => PyErrState::normalized(obj), + Err(err) => { + // Assume obj is Type[Exception]; let later normalization handle if this + // is not the case + let obj = err.into_inner(); + let py = obj.py(); + PyErrState::lazy(obj.into_py(py), py.None()) + } }; PyErr::from_state(state) @@ -260,6 +277,18 @@ impl PyErr { self.normalized(py).ptype(py) } + /// Deprecated form of [`PyErr::value_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyErr::value` will be replaced by `PyErr::value_bound` in a future PyO3 version" + ) + )] + pub fn value<'py>(&'py self, py: Python<'py>) -> &'py PyBaseException { + self.value_bound(py).as_gil_ref() + } + /// Returns the value of this exception. /// /// # Examples @@ -270,11 +299,11 @@ impl PyErr { /// Python::with_gil(|py| { /// let err: PyErr = PyTypeError::new_err(("some type error",)); /// assert!(err.is_instance_of::(py)); - /// assert_eq!(err.value(py).to_string(), "some type error"); + /// assert_eq!(err.value_bound(py).to_string(), "some type error"); /// }); /// ``` - pub fn value<'py>(&'py self, py: Python<'py>) -> &'py PyBaseException { - self.normalized(py).pvalue.as_ref(py) + pub fn value_bound<'py>(&self, py: Python<'py>) -> &Bound<'py, PyBaseException> { + self.normalized(py).pvalue.bind(py) } /// Consumes self to take ownership of the exception value contained in this error. @@ -527,7 +556,7 @@ impl PyErr { pub fn display(&self, py: Python<'_>) { #[cfg(Py_3_12)] unsafe { - ffi::PyErr_DisplayException(self.value(py).as_ptr()) + ffi::PyErr_DisplayException(self.value_bound(py).as_ptr()) } #[cfg(not(Py_3_12))] @@ -540,7 +569,7 @@ impl PyErr { let type_bound = self.get_type_bound(py); ffi::PyErr_Display( type_bound.as_ptr(), - self.value(py).as_ptr(), + self.value_bound(py).as_ptr(), traceback .as_ref() .map_or(std::ptr::null_mut(), |traceback| traceback.as_ptr()), @@ -785,7 +814,7 @@ impl PyErr { /// let err: PyErr = PyTypeError::new_err(("some type error",)); /// let err_clone = err.clone_ref(py); /// assert!(err.get_type_bound(py).is(&err_clone.get_type_bound(py))); - /// assert!(err.value(py).is(err_clone.value(py))); + /// assert!(err.value_bound(py).is(err_clone.value_bound(py))); /// match err.traceback_bound(py) { /// None => assert!(err_clone.traceback_bound(py).is_none()), /// Some(tb) => assert!(err_clone.traceback_bound(py).unwrap().is(&tb)), @@ -800,15 +829,14 @@ impl PyErr { /// Return the cause (either an exception instance, or None, set by `raise ... from ...`) /// associated with the exception, as accessible from Python through `__cause__`. pub fn cause(&self, py: Python<'_>) -> Option { - let value = self.value(py); - let obj = - unsafe { py.from_owned_ptr_or_opt::(ffi::PyException_GetCause(value.as_ptr())) }; - obj.map(Self::from_value) + use crate::ffi_ptr_ext::FfiPtrExt; + unsafe { ffi::PyException_GetCause(self.value_bound(py).as_ptr()).assume_owned_or_opt(py) } + .map(Self::from_value_bound) } /// Set the cause associated with the exception, pass `None` to clear it. pub fn set_cause(&self, py: Python<'_>, cause: Option) { - let value = self.value(py); + let value = self.value_bound(py); let cause = cause.map(|err| err.into_value(py)); unsafe { // PyException_SetCause _steals_ a reference to cause, so must use .into_ptr() @@ -868,7 +896,7 @@ impl std::fmt::Debug for PyErr { Python::with_gil(|py| { f.debug_struct("PyErr") .field("type", &self.get_type_bound(py)) - .field("value", self.value(py)) + .field("value", self.value_bound(py)) .field("traceback", &self.traceback_bound(py)) .finish() }) @@ -877,8 +905,9 @@ impl std::fmt::Debug for PyErr { impl std::fmt::Display for PyErr { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use crate::types::string::PyStringMethods; Python::with_gil(|py| { - let value = self.value(py); + let value = self.value_bound(py); let type_name = value.get_type().qualname().map_err(|_| std::fmt::Error)?; write!(f, "{}", type_name)?; if let Ok(s) = value.str() { @@ -1043,7 +1072,6 @@ impl_signed_integer!(isize); mod tests { use super::PyErrState; use crate::exceptions::{self, PyTypeError, PyValueError}; - use crate::types::any::PyAnyMethods; use crate::{PyErr, PyNativeType, PyTypeInfo, Python}; #[test] @@ -1237,6 +1265,7 @@ mod tests { #[test] fn warnings() { + use crate::types::any::PyAnyMethods; // Note: although the warning filter is interpreter global, keeping the // GIL locked should prevent effects to be visible to other testing // threads. @@ -1284,7 +1313,7 @@ mod tests { ) .unwrap_err(); assert!(err - .value(py) + .value_bound(py) .getattr("args") .unwrap() .get_item(0) diff --git a/src/exceptions.rs b/src/exceptions.rs index bcbe91b90c7..e7b6407e721 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -9,7 +9,7 @@ //! yourself to import Python classes that are ultimately derived from //! `BaseException`. -use crate::{ffi, PyResult, Python}; +use crate::{ffi, Bound, PyResult, Python}; use std::ffi::CStr; use std::ops; use std::os::raw::c_char; @@ -22,6 +22,7 @@ macro_rules! impl_exception_boilerplate { impl ::std::convert::From<&$name> for $crate::PyErr { #[inline] fn from(err: &$name) -> $crate::PyErr { + #[allow(deprecated)] $crate::PyErr::from_value(err) } } @@ -613,7 +614,14 @@ impl_windows_native_exception!( ); impl PyUnicodeDecodeError { - /// Creates a Python `UnicodeDecodeError`. + /// Deprecated form of [`PyUnicodeDecodeError::new_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyUnicodeDecodeError::new` will be replaced by `PyUnicodeDecodeError::new_bound` in a future PyO3 version" + ) + )] pub fn new<'p>( py: Python<'p>, encoding: &CStr, @@ -621,16 +629,47 @@ impl PyUnicodeDecodeError { range: ops::Range, reason: &CStr, ) -> PyResult<&'p PyUnicodeDecodeError> { + Ok(PyUnicodeDecodeError::new_bound(py, encoding, input, range, reason)?.into_gil_ref()) + } + + /// Creates a Python `UnicodeDecodeError`. + pub fn new_bound<'p>( + py: Python<'p>, + encoding: &CStr, + input: &[u8], + range: ops::Range, + reason: &CStr, + ) -> PyResult> { + use crate::ffi_ptr_ext::FfiPtrExt; + use crate::py_result_ext::PyResultExt; unsafe { - py.from_owned_ptr_or_err(ffi::PyUnicodeDecodeError_Create( + ffi::PyUnicodeDecodeError_Create( encoding.as_ptr(), input.as_ptr() as *const c_char, input.len() as ffi::Py_ssize_t, range.start as ffi::Py_ssize_t, range.end as ffi::Py_ssize_t, reason.as_ptr(), - )) + ) + .assume_owned_or_err(py) } + .downcast_into() + } + + /// Deprecated form of [`PyUnicodeDecodeError::new_utf8_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyUnicodeDecodeError::new_utf8` will be replaced by `PyUnicodeDecodeError::new_utf8_bound` in a future PyO3 version" + ) + )] + pub fn new_utf8<'p>( + py: Python<'p>, + input: &[u8], + err: std::str::Utf8Error, + ) -> PyResult<&'p PyUnicodeDecodeError> { + Ok(PyUnicodeDecodeError::new_utf8_bound(py, input, err)?.into_gil_ref()) } /// Creates a Python `UnicodeDecodeError` from a Rust UTF-8 decoding error. @@ -646,7 +685,7 @@ impl PyUnicodeDecodeError { /// Python::with_gil(|py| { /// let invalid_utf8 = b"fo\xd8o"; /// let err = std::str::from_utf8(invalid_utf8).expect_err("should be invalid utf8"); - /// let decode_err = PyUnicodeDecodeError::new_utf8(py, invalid_utf8, err)?; + /// let decode_err = PyUnicodeDecodeError::new_utf8_bound(py, invalid_utf8, err)?; /// assert_eq!( /// decode_err.to_string(), /// "'utf-8' codec can't decode byte 0xd8 in position 2: invalid utf-8" @@ -654,13 +693,13 @@ impl PyUnicodeDecodeError { /// Ok(()) /// }) /// # } - pub fn new_utf8<'p>( + pub fn new_utf8_bound<'p>( py: Python<'p>, input: &[u8], err: std::str::Utf8Error, - ) -> PyResult<&'p PyUnicodeDecodeError> { + ) -> PyResult> { let pos = err.valid_up_to(); - PyUnicodeDecodeError::new( + PyUnicodeDecodeError::new_bound( py, CStr::from_bytes_with_nul(b"utf-8\0").unwrap(), input, @@ -745,7 +784,7 @@ macro_rules! test_exception { assert!(err.is_instance_of::<$exc_ty>(py)); - let value: &$exc_ty = err.value(py).downcast().unwrap(); + let value: &$exc_ty = err.value_bound(py).clone().into_gil_ref().downcast().unwrap(); assert!(value.source().is_none()); err.set_cause(py, Some($crate::exceptions::PyValueError::new_err("a cause"))); @@ -1025,14 +1064,14 @@ mod tests { #[cfg_attr(invalid_from_utf8_lint, allow(invalid_from_utf8))] let err = std::str::from_utf8(invalid_utf8).expect_err("should be invalid utf8"); Python::with_gil(|py| { - let decode_err = PyUnicodeDecodeError::new_utf8(py, invalid_utf8, err).unwrap(); + let decode_err = PyUnicodeDecodeError::new_utf8_bound(py, invalid_utf8, err).unwrap(); assert_eq!( format!("{:?}", decode_err), "UnicodeDecodeError('utf-8', b'fo\\xd8o', 2, 3, 'invalid utf-8')" ); // Restoring should preserve the same error - let e: PyErr = decode_err.into(); + let e = PyErr::from_value_bound(decode_err.into_any()); e.restore(py); assert_eq!( @@ -1081,7 +1120,11 @@ mod tests { let invalid_utf8 = b"fo\xd8o"; #[cfg_attr(invalid_from_utf8_lint, allow(invalid_from_utf8))] let err = std::str::from_utf8(invalid_utf8).expect_err("should be invalid utf8"); - PyErr::from_value(PyUnicodeDecodeError::new_utf8(py, invalid_utf8, err).unwrap()) + PyErr::from_value_bound( + PyUnicodeDecodeError::new_utf8_bound(py, invalid_utf8, err) + .unwrap() + .into_any(), + ) }); test_exception!(PyUnicodeEncodeError, |py| py .eval_bound("chr(40960).encode('ascii')", None, None) diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index ff1c2436e38..d9487359cf4 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -167,8 +167,11 @@ pub fn from_py_with_with_default<'a, 'py, T>( pub fn argument_extraction_error(py: Python<'_>, arg_name: &str, error: PyErr) -> PyErr { use crate::types::any::PyAnyMethods; if error.get_type_bound(py).is(py.get_type::()) { - let remapped_error = - PyTypeError::new_err(format!("argument '{}': {}", arg_name, error.value(py))); + let remapped_error = PyTypeError::new_err(format!( + "argument '{}': {}", + arg_name, + error.value_bound(py) + )); remapped_error.set_cause(py, error.cause(py)); remapped_error } else { diff --git a/src/tests/common.rs b/src/tests/common.rs index ed3972f16a6..5eec60949bc 100644 --- a/src/tests/common.rs +++ b/src/tests/common.rs @@ -73,8 +73,8 @@ mod inner { #[cfg(all(feature = "macros", Py_3_8))] #[pymethods(crate = "pyo3")] impl UnraisableCapture { - pub fn hook(&mut self, unraisable: &PyAny) { - let err = PyErr::from_value(unraisable.getattr("exc_value").unwrap()); + pub fn hook(&mut self, unraisable: Bound<'_, PyAny>) { + let err = PyErr::from_value_bound(unraisable.getattr("exc_value").unwrap()); let instance = unraisable.getattr("object").unwrap(); self.capture = Some((err, instance.into())); } diff --git a/src/types/string.rs b/src/types/string.rs index 2b2635715d3..2e17f909df5 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -73,9 +73,9 @@ impl<'a> PyStringData<'a> { match self { Self::Ucs1(data) => match str::from_utf8(data) { Ok(s) => Ok(Cow::Borrowed(s)), - Err(e) => Err(crate::PyErr::from_value(PyUnicodeDecodeError::new_utf8( - py, data, e, - )?)), + Err(e) => Err(crate::PyErr::from_value_bound( + PyUnicodeDecodeError::new_utf8_bound(py, data, e)?.into_any(), + )), }, Self::Ucs2(data) => match String::from_utf16(data) { Ok(s) => Ok(Cow::Owned(s)), @@ -83,24 +83,30 @@ impl<'a> PyStringData<'a> { let mut message = e.to_string().as_bytes().to_vec(); message.push(0); - Err(crate::PyErr::from_value(PyUnicodeDecodeError::new( - py, - CStr::from_bytes_with_nul(b"utf-16\0").unwrap(), - self.as_bytes(), - 0..self.as_bytes().len(), - CStr::from_bytes_with_nul(&message).unwrap(), - )?)) + Err(crate::PyErr::from_value_bound( + PyUnicodeDecodeError::new_bound( + py, + CStr::from_bytes_with_nul(b"utf-16\0").unwrap(), + self.as_bytes(), + 0..self.as_bytes().len(), + CStr::from_bytes_with_nul(&message).unwrap(), + )? + .into_any(), + )) } }, Self::Ucs4(data) => match data.iter().map(|&c| std::char::from_u32(c)).collect() { Some(s) => Ok(Cow::Owned(s)), - None => Err(crate::PyErr::from_value(PyUnicodeDecodeError::new( - py, - CStr::from_bytes_with_nul(b"utf-32\0").unwrap(), - self.as_bytes(), - 0..self.as_bytes().len(), - CStr::from_bytes_with_nul(b"error converting utf-32\0").unwrap(), - )?)), + None => Err(crate::PyErr::from_value_bound( + PyUnicodeDecodeError::new_bound( + py, + CStr::from_bytes_with_nul(b"utf-32\0").unwrap(), + self.as_bytes(), + 0..self.as_bytes().len(), + CStr::from_bytes_with_nul(b"error converting utf-32\0").unwrap(), + )? + .into_any(), + )), }, } } diff --git a/src/types/traceback.rs b/src/types/traceback.rs index b97e2ed0093..19f42d4b64c 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -150,9 +150,9 @@ except Exception as e: Some(&locals), ) .unwrap(); - let err = PyErr::from_value(locals.get_item("err").unwrap().unwrap().into_gil_ref()); - let traceback = err.value(py).getattr("__traceback__").unwrap(); - assert!(err.traceback_bound(py).unwrap().is(traceback)); + let err = PyErr::from_value_bound(locals.get_item("err").unwrap().unwrap()); + let traceback = err.value_bound(py).getattr("__traceback__").unwrap(); + assert!(err.traceback_bound(py).unwrap().is(&traceback)); }) } diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index 549d54aa5c4..5954b9794e7 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -137,7 +137,7 @@ fn cancelled_coroutine() { ) .unwrap_err(); assert_eq!( - err.value(gil).get_type().qualname().unwrap(), + err.value_bound(gil).get_type().qualname().unwrap(), "CancelledError" ); }) From 65cf5808d9b0fb62676cddbb526e46fef6ae77c6 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 17 Feb 2024 10:35:28 +0000 Subject: [PATCH 137/349] docs: add note about mapping to dangling pointer with `Bound` API (#3805) --- guide/src/migration.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/guide/src/migration.md b/guide/src/migration.md index 936784897bd..33b465de8dc 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -224,6 +224,22 @@ let gil_ref: &PyAny = ...; let bound: &Bound = &gil_ref.as_borrowed(); ``` +> Because of the ownership changes, code which uses `.as_ptr()` to convert `&PyAny` and other GIL Refs to a `*mut pyo3_ffi::PyObject` should take care to avoid creating dangling pointers now that `Bound` carries ownership. +> +> For example, the following pattern with `Option<&PyAny>` can easily create a dangling pointer when migrating to the `Bound` smart pointer: +> +> ```rust,ignore +> let opt: Option<&PyAny> = ...; +> let p: *mut ffi::PyObject = opt.map_or(std::ptr::null_mut(), |any| any.as_ptr()); +> ``` +> +> The correct way to migrate this code is to use `.as_ref()` to avoid dropping the `Bound` in the `map_or` closure: +> +> ```rust,ignore +> let opt: Option> = ...; +> let p: *mut ffi::PyObject = opt.as_ref().map_or(std::ptr::null_mut(), Bound::as_ptr); +> ``` + #### Migrating `FromPyObject` implementations `FromPyObject` has had a new method `extract_bound` which takes `&Bound<'py, PyAny>` as an argument instead of `&PyAny`. Both `extract` and `extract_bound` have been given default implementations in terms of the other, to avoid breaking code immediately on update to 0.21. From eb90b81d4481767a49a04242785f1f2f46b007d6 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 17 Feb 2024 10:57:53 +0000 Subject: [PATCH 138/349] always use a Python iterator for sets and frozensets (#3849) * always use a Python iterator for sets and frozensets * add newsfragment --- newsfragments/3849.added.md | 1 + newsfragments/3849.changed.md | 1 + src/types/frozenset.rs | 115 ++++++++---------------------- src/types/set.rs | 129 ++++++++-------------------------- 4 files changed, 62 insertions(+), 184 deletions(-) create mode 100644 newsfragments/3849.added.md create mode 100644 newsfragments/3849.changed.md diff --git a/newsfragments/3849.added.md b/newsfragments/3849.added.md new file mode 100644 index 00000000000..8aa9a55df03 --- /dev/null +++ b/newsfragments/3849.added.md @@ -0,0 +1 @@ +Implement `ExactSizeIterator` for `set` and `frozenset` iterators on `abi3` feature. diff --git a/newsfragments/3849.changed.md b/newsfragments/3849.changed.md new file mode 100644 index 00000000000..ee8dfbf5b2c --- /dev/null +++ b/newsfragments/3849.changed.md @@ -0,0 +1 @@ +`PySet` and `PyFrozenSet` iterators now always iterate the equivalent of `iter(set)`. (A "fast path" with no noticeable performance benefit was removed.) diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index fa56a9c69f7..3be3f0ed0a1 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -1,4 +1,3 @@ -#[cfg(Py_LIMITED_API)] use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, @@ -205,6 +204,13 @@ impl<'py> Iterator for PyFrozenSetIterator<'py> { } } +impl ExactSizeIterator for PyFrozenSetIterator<'_> { + #[inline] + fn len(&self) -> usize { + self.0.len() + } +} + impl<'py> IntoIterator for &'py PyFrozenSet { type Item = &'py PyAny; type IntoIter = PyFrozenSetIterator<'py>; @@ -224,90 +230,42 @@ impl<'py> IntoIterator for Bound<'py, PyFrozenSet> { } } -#[cfg(Py_LIMITED_API)] -mod impl_ { - use super::*; - - /// PyO3 implementation of an iterator for a Python `set` object. - pub struct BoundFrozenSetIterator<'p> { - it: Bound<'p, PyIterator>, - } - - impl<'py> BoundFrozenSetIterator<'py> { - pub(super) fn new(frozenset: Bound<'py, PyFrozenSet>) -> Self { - Self { - it: PyIterator::from_bound_object(&frozenset).unwrap(), - } - } - } - - impl<'py> Iterator for BoundFrozenSetIterator<'py> { - type Item = Bound<'py, super::PyAny>; - - /// Advances the iterator and returns the next value. - #[inline] - fn next(&mut self) -> Option { - self.it.next().map(Result::unwrap) - } - } +/// PyO3 implementation of an iterator for a Python `frozenset` object. +pub struct BoundFrozenSetIterator<'p> { + it: Bound<'p, PyIterator>, + // Remaining elements in the frozenset + remaining: usize, } -#[cfg(not(Py_LIMITED_API))] -mod impl_ { - use super::*; - - /// PyO3 implementation of an iterator for a Python `frozenset` object. - pub struct BoundFrozenSetIterator<'py> { - set: Bound<'py, PyFrozenSet>, - pos: ffi::Py_ssize_t, - } - - impl<'py> BoundFrozenSetIterator<'py> { - pub(super) fn new(set: Bound<'py, PyFrozenSet>) -> Self { - Self { set, pos: 0 } +impl<'py> BoundFrozenSetIterator<'py> { + pub(super) fn new(set: Bound<'py, PyFrozenSet>) -> Self { + Self { + it: PyIterator::from_bound_object(&set).unwrap(), + remaining: set.len(), } } +} - impl<'py> Iterator for BoundFrozenSetIterator<'py> { - type Item = Bound<'py, PyAny>; - - #[inline] - fn next(&mut self) -> Option { - unsafe { - let mut key: *mut ffi::PyObject = std::ptr::null_mut(); - let mut hash: ffi::Py_hash_t = 0; - if ffi::_PySet_NextEntry(self.set.as_ptr(), &mut self.pos, &mut key, &mut hash) != 0 - { - // _PySet_NextEntry returns borrowed object; for safety must make owned (see #890) - Some(key.assume_borrowed(self.set.py()).to_owned()) - } else { - None - } - } - } +impl<'py> Iterator for BoundFrozenSetIterator<'py> { + type Item = Bound<'py, super::PyAny>; - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = self.len(); - (len, Some(len)) - } + /// Advances the iterator and returns the next value. + fn next(&mut self) -> Option { + self.remaining = self.remaining.saturating_sub(1); + self.it.next().map(Result::unwrap) } - impl<'py> ExactSizeIterator for BoundFrozenSetIterator<'py> { - fn len(&self) -> usize { - self.set.len().saturating_sub(self.pos as usize) - } + fn size_hint(&self) -> (usize, Option) { + (self.remaining, Some(self.remaining)) } +} - impl<'py> ExactSizeIterator for PyFrozenSetIterator<'py> { - fn len(&self) -> usize { - self.0.len() - } +impl<'py> ExactSizeIterator for BoundFrozenSetIterator<'py> { + fn len(&self) -> usize { + self.remaining } } -pub use impl_::*; - #[inline] pub(crate) fn new_from_iter( py: Python<'_>, @@ -387,7 +345,6 @@ mod tests { } #[test] - #[cfg(not(Py_LIMITED_API))] fn test_frozenset_iter_size_hint() { Python::with_gil(|py| { let set = PyFrozenSet::new(py, &[1]).unwrap(); @@ -402,18 +359,6 @@ mod tests { }); } - #[test] - #[cfg(Py_LIMITED_API)] - fn test_frozenset_iter_size_hint() { - Python::with_gil(|py| { - let set = PyFrozenSet::new(py, &[1]).unwrap(); - let iter = set.iter(); - - // No known bounds - assert_eq!(iter.size_hint(), (0, None)); - }); - } - #[test] fn test_frozenset_builder() { use super::PyFrozenSetBuilder; diff --git a/src/types/set.rs b/src/types/set.rs index 3204e71f3e5..0c1733527cf 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -1,4 +1,3 @@ -#[cfg(Py_LIMITED_API)] use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, @@ -277,6 +276,12 @@ impl<'py> Iterator for PySetIterator<'py> { } } +impl ExactSizeIterator for PySetIterator<'_> { + fn len(&self) -> usize { + self.0.len() + } +} + impl<'py> IntoIterator for &'py PySet { type Item = &'py PyAny; type IntoIter = PySetIterator<'py>; @@ -304,104 +309,43 @@ impl<'py> IntoIterator for Bound<'py, PySet> { } } -#[cfg(Py_LIMITED_API)] -mod impl_ { - use super::*; - - /// PyO3 implementation of an iterator for a Python `set` object. - pub struct BoundSetIterator<'p> { - it: Bound<'p, PyIterator>, - } - - impl<'py> BoundSetIterator<'py> { - pub(super) fn new(set: Bound<'py, PySet>) -> Self { - Self { - it: PyIterator::from_bound_object(&set).unwrap(), - } - } - } - - impl<'py> Iterator for BoundSetIterator<'py> { - type Item = Bound<'py, super::PyAny>; - - /// Advances the iterator and returns the next value. - /// - /// # Panics - /// - /// If PyO3 detects that the set is mutated during iteration, it will panic. - #[inline] - fn next(&mut self) -> Option { - self.it.next().map(Result::unwrap) - } - } +/// PyO3 implementation of an iterator for a Python `set` object. +pub struct BoundSetIterator<'p> { + it: Bound<'p, PyIterator>, + // Remaining elements in the set. This is fine to store because + // Python will error if the set changes size during iteration. + remaining: usize, } -#[cfg(not(Py_LIMITED_API))] -mod impl_ { - use super::*; - - /// PyO3 implementation of an iterator for a Python `set` object. - pub struct BoundSetIterator<'py> { - set: Bound<'py, super::PySet>, - pos: ffi::Py_ssize_t, - used: ffi::Py_ssize_t, - } - - impl<'py> BoundSetIterator<'py> { - pub(super) fn new(set: Bound<'py, PySet>) -> Self { - let used = unsafe { ffi::PySet_Size(set.as_ptr()) }; - BoundSetIterator { set, pos: 0, used } +impl<'py> BoundSetIterator<'py> { + pub(super) fn new(set: Bound<'py, PySet>) -> Self { + Self { + it: PyIterator::from_bound_object(&set).unwrap(), + remaining: set.len(), } } +} - impl<'py> Iterator for BoundSetIterator<'py> { - type Item = Bound<'py, super::PyAny>; - - /// Advances the iterator and returns the next value. - /// - /// # Panics - /// - /// If PyO3 detects that the set is mutated during iteration, it will panic. - #[inline] - fn next(&mut self) -> Option { - unsafe { - let len = ffi::PySet_Size(self.set.as_ptr()); - assert_eq!(self.used, len, "Set changed size during iteration"); - - let mut key: *mut ffi::PyObject = std::ptr::null_mut(); - let mut hash: ffi::Py_hash_t = 0; - if ffi::_PySet_NextEntry(self.set.as_ptr(), &mut self.pos, &mut key, &mut hash) != 0 - { - // _PySet_NextEntry returns borrowed object; for safety must make owned (see #890) - Some(key.assume_borrowed(self.set.py()).to_owned()) - } else { - None - } - } - } +impl<'py> Iterator for BoundSetIterator<'py> { + type Item = Bound<'py, super::PyAny>; - #[inline] - fn size_hint(&self) -> (usize, Option) { - let len = self.len(); - (len, Some(len)) - } + /// Advances the iterator and returns the next value. + fn next(&mut self) -> Option { + self.remaining = self.remaining.saturating_sub(1); + self.it.next().map(Result::unwrap) } - impl<'py> ExactSizeIterator for BoundSetIterator<'py> { - fn len(&self) -> usize { - self.set.len().saturating_sub(self.pos as usize) - } + fn size_hint(&self) -> (usize, Option) { + (self.remaining, Some(self.remaining)) } +} - impl<'py> ExactSizeIterator for PySetIterator<'py> { - fn len(&self) -> usize { - self.0.len() - } +impl<'py> ExactSizeIterator for BoundSetIterator<'py> { + fn len(&self) -> usize { + self.remaining } } -pub use impl_::*; - #[inline] pub(crate) fn new_from_iter( py: Python<'_>, @@ -571,7 +515,6 @@ mod tests { } #[test] - #[cfg(not(Py_LIMITED_API))] fn test_set_iter_size_hint() { Python::with_gil(|py| { let set = PySet::new(py, &[1]).unwrap(); @@ -585,16 +528,4 @@ mod tests { assert_eq!(iter.size_hint(), (0, Some(0))); }); } - - #[test] - #[cfg(Py_LIMITED_API)] - fn test_set_iter_size_hint() { - Python::with_gil(|py| { - let set = PySet::new(py, &[1]).unwrap(); - let iter = set.iter(); - - // No known bounds - assert_eq!(iter.size_hint(), (0, None)); - }); - } } From 1d8d81db2d3ab99b5ffde5cdac93cc7605ce8d44 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 17 Feb 2024 13:21:41 +0100 Subject: [PATCH 139/349] port `PyFrozenSetBuilder` to `Bound` API (#3850) --- src/types/frozenset.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 3be3f0ed0a1..909e1276a57 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -11,7 +11,7 @@ use std::ptr; /// Allows building a Python `frozenset` one item at a time pub struct PyFrozenSetBuilder<'py> { - py_frozen_set: &'py PyFrozenSet, + py_frozen_set: Bound<'py, PyFrozenSet>, } impl<'py> PyFrozenSetBuilder<'py> { @@ -20,7 +20,7 @@ impl<'py> PyFrozenSetBuilder<'py> { /// panic when running out of memory. pub fn new(py: Python<'py>) -> PyResult> { Ok(PyFrozenSetBuilder { - py_frozen_set: PyFrozenSet::empty(py)?, + py_frozen_set: PyFrozenSet::empty_bound(py)?, }) } @@ -29,17 +29,29 @@ impl<'py> PyFrozenSetBuilder<'py> { where K: ToPyObject, { - fn inner(frozenset: &PyFrozenSet, key: PyObject) -> PyResult<()> { + fn inner(frozenset: &Bound<'_, PyFrozenSet>, key: PyObject) -> PyResult<()> { err::error_on_minusone(frozenset.py(), unsafe { ffi::PySet_Add(frozenset.as_ptr(), key.as_ptr()) }) } - inner(self.py_frozen_set, key.to_object(self.py_frozen_set.py())) + inner(&self.py_frozen_set, key.to_object(self.py_frozen_set.py())) } - /// Finish building the set and take ownership of its current value + /// Deprecated form of [`PyFrozenSetBuilder::finalize_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyFrozenSetBuilder::finalize` will be replaced by `PyFrozenSetBuilder::finalize_bound` in a future PyO3 version" + ) + )] pub fn finalize(self) -> &'py PyFrozenSet { + self.finalize_bound().into_gil_ref() + } + + /// Finish building the set and take ownership of its current value + pub fn finalize_bound(self) -> Bound<'py, PyFrozenSet> { self.py_frozen_set } } @@ -372,7 +384,7 @@ mod tests { builder.add(2).unwrap(); // finalize it - let set = builder.finalize(); + let set = builder.finalize_bound(); assert!(set.contains(1).unwrap()); assert!(set.contains(2).unwrap()); From c33d330b18164bf1afedd0e07f40fa27740eb94a Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 17 Feb 2024 14:40:54 +0100 Subject: [PATCH 140/349] deprecate `PyFrozenSet::empty` (#3851) --- src/types/frozenset.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 909e1276a57..15f892588e5 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -104,6 +104,13 @@ impl PyFrozenSet { } /// Deprecated form of [`PyFrozenSet::empty_bound`]. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyFrozenSet::empty` will be replaced by `PyFrozenSet::empty_bound` in a future PyO3 version" + ) + )] pub fn empty(py: Python<'_>) -> PyResult<&'_ PyFrozenSet> { Self::empty_bound(py).map(Bound::into_gil_ref) } From 5f42c02e4f9ef969f9e71b9c2e456c1bc36d93f1 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Sat, 17 Feb 2024 17:04:35 +0000 Subject: [PATCH 141/349] Remove stray " character from docstring (#3852) --- src/instance.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/instance.rs b/src/instance.rs index ff553c44324..2db63d06a43 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -766,7 +766,7 @@ impl IntoPy for Borrowed<'_, '_, T> { /// # A note on `Send` and `Sync` /// /// Accessing this object is threadsafe, since any access to its API requires a [`Python<'py>`](crate::Python) token. -/// As you can only get this by acquiring the GIL, `Py<...>` "implements [`Send`] and [`Sync`]. +/// As you can only get this by acquiring the GIL, `Py<...>` implements [`Send`] and [`Sync`]. /// /// [`Rc`]: std::rc::Rc /// [`RefCell`]: std::cell::RefCell From 0dd568d397b18c476c25a68157f1805d666f6bdd Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Sun, 18 Feb 2024 00:09:56 +0000 Subject: [PATCH 142/349] Use the new bound API instead of .as_ref(py) (#3853) * Use the new bound API instead of .as_ref(py) * Move import into a nested scope * Use to_cow instead of to_str for compatibility `to_str` is not available before Python 3.10 on the limited api. * Relax &self lifetimes * Use Bound<'py, PyAny> in test Mapping signatures * Use .as_bytes(py) * Simplify ThrowCallback::throw signature * Avoid .as_any call with Py api instead of Bound --- src/conversions/std/ipaddr.rs | 3 ++- src/conversions/std/osstr.rs | 10 ++++------ src/conversions/std/slice.rs | 4 ++-- src/conversions/std/string.rs | 7 ++++--- src/coroutine.rs | 8 ++++---- src/coroutine/cancel.rs | 6 +++--- src/coroutine/waker.rs | 11 +++++++---- src/err/mod.rs | 6 +++--- src/instance.rs | 10 +++++----- src/pycell.rs | 4 ++-- src/types/capsule.rs | 6 +++--- tests/test_class_basics.rs | 2 +- tests/test_class_conversion.rs | 4 ++-- tests/test_field_cfg.rs | 10 ++++++++-- tests/test_inheritance.rs | 2 +- tests/test_mapping.rs | 4 ++-- tests/test_proto_methods.rs | 28 ++++++++++++++-------------- tests/test_pyself.rs | 4 ++-- tests/test_sequence.rs | 2 +- tests/test_various.rs | 2 +- 20 files changed, 71 insertions(+), 62 deletions(-) diff --git a/src/conversions/std/ipaddr.rs b/src/conversions/std/ipaddr.rs index 84aacab43a2..5d030b445d8 100755 --- a/src/conversions/std/ipaddr.rs +++ b/src/conversions/std/ipaddr.rs @@ -87,7 +87,8 @@ mod test_ipaddr { }; let pyobj = ip.into_py(py); - let repr = pyobj.as_ref(py).repr().unwrap().to_string_lossy(); + let repr = pyobj.bind(py).repr().unwrap(); + let repr = repr.to_string_lossy(); assert_eq!(repr, format!("{}('{}')", py_cls, ip)); let ip2: IpAddr = pyobj.extract(py).unwrap(); diff --git a/src/conversions/std/osstr.rs b/src/conversions/std/osstr.rs index be337959b5f..b9382688589 100644 --- a/src/conversions/std/osstr.rs +++ b/src/conversions/std/osstr.rs @@ -68,13 +68,11 @@ impl FromPyObject<'_> for OsString { // Create an OsStr view into the raw bytes from Python #[cfg(target_os = "wasi")] - let os_str: &OsStr = std::os::wasi::ffi::OsStrExt::from_bytes( - fs_encoded_bytes.as_ref(ob.py()).as_bytes(), - ); + let os_str: &OsStr = + std::os::wasi::ffi::OsStrExt::from_bytes(fs_encoded_bytes.as_bytes(ob.py())); #[cfg(not(target_os = "wasi"))] - let os_str: &OsStr = std::os::unix::ffi::OsStrExt::from_bytes( - fs_encoded_bytes.as_ref(ob.py()).as_bytes(), - ); + let os_str: &OsStr = + std::os::unix::ffi::OsStrExt::from_bytes(fs_encoded_bytes.as_bytes(ob.py())); Ok(os_str.to_os_string()) } diff --git a/src/conversions/std/slice.rs b/src/conversions/std/slice.rs index 2d46abe2953..dad27f081cf 100644 --- a/src/conversions/std/slice.rs +++ b/src/conversions/std/slice.rs @@ -94,10 +94,10 @@ mod tests { .unwrap_err(); let cow = Cow::<[u8]>::Borrowed(b"foobar").to_object(py); - assert!(cow.as_ref(py).is_instance_of::()); + assert!(cow.bind(py).is_instance_of::()); let cow = Cow::<[u8]>::Owned(b"foobar".to_vec()).to_object(py); - assert!(cow.as_ref(py).is_instance_of::()); + assert!(cow.bind(py).is_instance_of::()); }); } } diff --git a/src/conversions/std/string.rs b/src/conversions/std/string.rs index ac29d80491c..60e8ff7eb53 100644 --- a/src/conversions/std/string.rs +++ b/src/conversions/std/string.rs @@ -159,6 +159,7 @@ impl FromPyObject<'_> for char { #[cfg(test)] mod tests { + use crate::types::any::PyAnyMethods; use crate::Python; use crate::{IntoPy, PyObject, ToPyObject}; use std::borrow::Cow; @@ -200,7 +201,7 @@ mod tests { let s = "Hello Python"; let py_string = s.to_object(py); - let s2: &str = py_string.as_ref(py).extract().unwrap(); + let s2: &str = py_string.bind(py).extract().unwrap(); assert_eq!(s, s2); }) } @@ -210,7 +211,7 @@ mod tests { Python::with_gil(|py| { let ch = '😃'; let py_string = ch.to_object(py); - let ch2: char = py_string.as_ref(py).extract().unwrap(); + let ch2: char = py_string.bind(py).extract().unwrap(); assert_eq!(ch, ch2); }) } @@ -220,7 +221,7 @@ mod tests { Python::with_gil(|py| { let s = "Hello Python"; let py_string = s.to_object(py); - let err: crate::PyResult = py_string.as_ref(py).extract(); + let err: crate::PyResult = py_string.bind(py).extract(); assert!(err .unwrap_err() .to_string() diff --git a/src/coroutine.rs b/src/coroutine.rs index 3a0df983076..b24197ad593 100644 --- a/src/coroutine.rs +++ b/src/coroutine.rs @@ -14,8 +14,8 @@ use crate::{ coroutine::{cancel::ThrowCallback, waker::AsyncioWaker}, exceptions::{PyAttributeError, PyRuntimeError, PyStopIteration}, panic::PanicException, - types::{PyIterator, PyString}, - IntoPy, Py, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, + types::{string::PyStringMethods, PyIterator, PyString}, + IntoPy, Py, PyAny, PyErr, PyObject, PyResult, Python, }; pub(crate) mod cancel; @@ -75,7 +75,7 @@ impl Coroutine { }; // reraise thrown exception it match (throw, &self.throw_callback) { - (Some(exc), Some(cb)) => cb.throw(exc.as_ref(py)), + (Some(exc), Some(cb)) => cb.throw(exc), (Some(exc), None) => { self.close(); return Err(PyErr::from_value_bound(exc.into_bound(py))); @@ -135,7 +135,7 @@ impl Coroutine { #[getter] fn __qualname__(&self, py: Python<'_>) -> PyResult> { match (&self.name, &self.qualname_prefix) { - (Some(name), Some(prefix)) => Ok(format!("{}.{}", prefix, name.as_ref(py).to_str()?) + (Some(name), Some(prefix)) => Ok(format!("{}.{}", prefix, name.bind(py).to_cow()?) .as_str() .into_py(py)), (Some(name), None) => Ok(name.clone_ref(py)), diff --git a/src/coroutine/cancel.rs b/src/coroutine/cancel.rs index 7828986c11e..2b10fb9a438 100644 --- a/src/coroutine/cancel.rs +++ b/src/coroutine/cancel.rs @@ -1,4 +1,4 @@ -use crate::{PyAny, PyObject}; +use crate::{Py, PyAny, PyObject}; use parking_lot::Mutex; use std::future::Future; use std::pin::Pin; @@ -68,9 +68,9 @@ impl Future for Cancelled<'_> { pub struct ThrowCallback(Arc>); impl ThrowCallback { - pub(super) fn throw(&self, exc: &PyAny) { + pub(super) fn throw(&self, exc: Py) { let mut inner = self.0.lock(); - inner.exception = Some(exc.into()); + inner.exception = Some(exc); if let Some(waker) = inner.waker.take() { waker.wake(); } diff --git a/src/coroutine/waker.rs b/src/coroutine/waker.rs index c46654896fc..096146f8292 100644 --- a/src/coroutine/waker.rs +++ b/src/coroutine/waker.rs @@ -1,7 +1,7 @@ use crate::sync::GILOnceCell; use crate::types::any::PyAnyMethods; use crate::types::PyCFunction; -use crate::{intern, wrap_pyfunction, Py, PyAny, PyObject, PyResult, Python}; +use crate::{intern, wrap_pyfunction, Bound, Py, PyAny, PyObject, PyResult, Python}; use pyo3_macros::pyfunction; use std::sync::Arc; use std::task::Wake; @@ -25,10 +25,13 @@ impl AsyncioWaker { self.0.take(); } - pub(super) fn initialize_future<'a>(&'a self, py: Python<'a>) -> PyResult> { + pub(super) fn initialize_future<'py>( + &self, + py: Python<'py>, + ) -> PyResult>> { let init = || LoopAndFuture::new(py).map(Some); let loop_and_future = self.0.get_or_try_init(py, init)?.as_ref(); - Ok(loop_and_future.map(|LoopAndFuture { future, .. }| future.as_ref(py))) + Ok(loop_and_future.map(|LoopAndFuture { future, .. }| future.bind(py))) } } @@ -74,7 +77,7 @@ impl LoopAndFuture { let call_soon_threadsafe = self.event_loop.call_method1( py, intern!(py, "call_soon_threadsafe"), - (release_waiter, self.future.as_ref(py)), + (release_waiter, self.future.bind(py)), ); if let Err(err) = call_soon_threadsafe { // `call_soon_threadsafe` will raise if the event loop is closed; diff --git a/src/err/mod.rs b/src/err/mod.rs index b5cd02c0954..0c2637a7389 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -2,6 +2,7 @@ use crate::instance::Bound; use crate::panic::PanicException; use crate::type_object::PyTypeInfo; use crate::types::any::PyAnyMethods; +use crate::types::string::PyStringMethods; use crate::types::{PyTraceback, PyType}; use crate::{ exceptions::{self, PyBaseException}, @@ -403,7 +404,7 @@ impl PyErr { if ptype.as_ptr() == PanicException::type_object_raw(py).cast() { let msg = pvalue .as_ref() - .and_then(|obj| obj.as_ref(py).str().ok()) + .and_then(|obj| obj.bind(py).str().ok()) .map(|py_str| py_str.to_string_lossy().into()) .unwrap_or_else(|| String::from("Unwrapped panic from Python code")); @@ -425,7 +426,7 @@ impl PyErr { #[cfg(Py_3_12)] fn _take(py: Python<'_>) -> Option { let state = PyErrStateNormalized::take(py)?; - let pvalue = state.pvalue.as_ref(py); + let pvalue = state.pvalue.bind(py); if pvalue.get_type().as_ptr() == PanicException::type_object_raw(py).cast() { let msg: String = pvalue .str() @@ -905,7 +906,6 @@ impl std::fmt::Debug for PyErr { impl std::fmt::Display for PyErr { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use crate::types::string::PyStringMethods; Python::with_gil(|py| { let value = self.value_bound(py); let type_name = value.get_type().qualname().map_err(|_| std::fmt::Error)?; diff --git a/src/instance.rs b/src/instance.rs index 2db63d06a43..cb803cc6c4d 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -989,7 +989,7 @@ where /// Panics if the value is currently mutably borrowed. For a non-panicking variant, use /// [`try_borrow`](#method.try_borrow). pub fn borrow<'py>(&'py self, py: Python<'py>) -> PyRef<'py, T> { - self.as_ref(py).borrow() + self.bind(py).borrow() } /// Mutably borrows the value `T`. @@ -1028,7 +1028,7 @@ where where T: PyClass, { - self.as_ref(py).borrow_mut() + self.bind(py).borrow_mut() } /// Attempts to immutably borrow the value `T`, returning an error if the value is currently mutably borrowed. @@ -1042,7 +1042,7 @@ where /// Equivalent to `self.as_ref(py).borrow_mut()` - /// see [`PyCell::try_borrow`](crate::pycell::PyCell::try_borrow). pub fn try_borrow<'py>(&'py self, py: Python<'py>) -> Result, PyBorrowError> { - self.as_ref(py).try_borrow() + self.bind(py).try_borrow() } /// Attempts to mutably borrow the value `T`, returning an error if the value is currently borrowed. @@ -1060,7 +1060,7 @@ where where T: PyClass, { - self.as_ref(py).try_borrow_mut() + self.bind(py).try_borrow_mut() } /// Provide an immutable borrow of the value `T` without acquiring the GIL. @@ -1667,7 +1667,7 @@ where T::AsRefTarget: std::fmt::Display, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - Python::with_gil(|py| std::fmt::Display::fmt(self.as_ref(py), f)) + Python::with_gil(|py| std::fmt::Display::fmt(self.bind(py), f)) } } diff --git a/src/pycell.rs b/src/pycell.rs index 3744c55f67d..8409c5cc679 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -96,7 +96,7 @@ //! //! // We borrow the guard and then dereference //! // it to get a mutable reference to Number -//! let mut guard: PyRefMut<'_, Number> = n.as_ref(py).borrow_mut(); +//! let mut guard: PyRefMut<'_, Number> = n.bind(py).borrow_mut(); //! let n_mutable: &mut Number = &mut *guard; //! //! n_mutable.increment(); @@ -105,7 +105,7 @@ //! // `PyRefMut` before borrowing again. //! drop(guard); //! -//! let n_immutable: &Number = &n.as_ref(py).borrow(); +//! let n_immutable: &Number = &n.bind(py).borrow(); //! assert_eq!(n_immutable.inner, 1); //! //! Ok(()) diff --git a/src/types/capsule.rs b/src/types/capsule.rs index 30e77391171..466315ad6c7 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -491,7 +491,7 @@ mod tests { }); Python::with_gil(|py| { - let f = unsafe { cap.as_ref(py).reference:: u32>() }; + let f = unsafe { cap.bind(py).reference:: u32>() }; assert_eq!(f(123), 123); }); } @@ -555,7 +555,7 @@ mod tests { }); Python::with_gil(|py| { - let ctx: &Vec = unsafe { cap.as_ref(py).reference() }; + let ctx: &Vec = unsafe { cap.bind(py).reference() }; assert_eq!(ctx, &[1, 2, 3, 4]); }) } @@ -574,7 +574,7 @@ mod tests { }); Python::with_gil(|py| { - let ctx_ptr: *mut c_void = cap.as_ref(py).context().unwrap(); + let ctx_ptr: *mut c_void = cap.bind(py).context().unwrap(); let ctx = unsafe { *Box::from_raw(ctx_ptr.cast::<&Vec>()) }; assert_eq!(ctx, &vec![1_u8, 2, 3, 4]); }) diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index ff3555c3107..d19d106d206 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -235,7 +235,7 @@ fn test_unsendable() -> PyResult<()> { // Accessing the value inside this thread should not panic let caught_panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> PyResult<_> { - assert_eq!(obj.as_ref(py).getattr("value")?.extract::()?, 5); + assert_eq!(obj.getattr(py, "value")?.extract::(py)?, 5); Ok(()) })) .is_err(); diff --git a/tests/test_class_conversion.rs b/tests/test_class_conversion.rs index 27a8f604b3f..217d2d59d45 100644 --- a/tests/test_class_conversion.rs +++ b/tests/test_class_conversion.rs @@ -89,7 +89,7 @@ fn test_polymorphic_container_stores_sub_class() { .unwrap() .to_object(py); - p.as_ref(py) + p.bind(py) .setattr( "inner", PyCell::new( @@ -116,7 +116,7 @@ fn test_polymorphic_container_does_not_accept_other_types() { .unwrap() .to_object(py); - let setattr = |value: PyObject| p.as_ref(py).setattr("inner", value); + let setattr = |value: PyObject| p.bind(py).setattr("inner", value); assert!(setattr(1i32.into_py(py)).is_err()); assert!(setattr(py.None()).is_err()); diff --git a/tests/test_field_cfg.rs b/tests/test_field_cfg.rs index bd671641e5b..be400f2d749 100644 --- a/tests/test_field_cfg.rs +++ b/tests/test_field_cfg.rs @@ -22,8 +22,14 @@ fn test_cfg() { Python::with_gil(|py| { let cfg = CfgClass { b: 3 }; let py_cfg = Py::new(py, cfg).unwrap(); - assert!(py_cfg.as_ref(py).getattr("a").is_err()); - let b: u32 = py_cfg.as_ref(py).getattr("b").unwrap().extract().unwrap(); + assert!(py_cfg.bind(py).as_any().getattr("a").is_err()); + let b: u32 = py_cfg + .bind(py) + .as_any() + .getattr("b") + .unwrap() + .extract() + .unwrap(); assert_eq!(b, 3); }); } diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 6c33dec6a9f..cd728c77bfe 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -247,7 +247,7 @@ mod inheriting_native_type { let item = &py.eval_bound("object()", None, None).unwrap(); assert_eq!(item.get_refcnt(), 1); - dict_sub.as_ref(py).set_item("foo", item).unwrap(); + dict_sub.bind(py).as_any().set_item("foo", item).unwrap(); assert_eq!(item.get_refcnt(), 2); drop(dict_sub); diff --git a/tests/test_mapping.rs b/tests/test_mapping.rs index 9c99d56467f..9a80ab491a1 100644 --- a/tests/test_mapping.rs +++ b/tests/test_mapping.rs @@ -123,7 +123,7 @@ fn mapping_is_not_sequence() { PyMapping::register::(py).unwrap(); - assert!(m.as_ref(py).downcast::().is_ok()); - assert!(m.as_ref(py).downcast::().is_err()); + assert!(m.bind(py).as_any().downcast::().is_ok()); + assert!(m.bind(py).as_any().downcast::().is_err()); }); } diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index 872777bf706..3151fbe5be7 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -191,20 +191,20 @@ pub struct Mapping { #[pymethods] impl Mapping { fn __len__(&self, py: Python<'_>) -> usize { - self.values.as_ref(py).len() + self.values.bind(py).len() } - fn __getitem__<'a>(&'a self, key: &'a PyAny) -> PyResult<&'a PyAny> { - let any: &PyAny = self.values.as_ref(key.py()).as_ref(); + fn __getitem__<'py>(&self, key: &Bound<'py, PyAny>) -> PyResult> { + let any: &Bound<'py, PyAny> = self.values.bind(key.py()); any.get_item(key) } - fn __setitem__(&self, key: &PyAny, value: &PyAny) -> PyResult<()> { - self.values.as_ref(key.py()).set_item(key, value) + fn __setitem__<'py>(&self, key: &Bound<'py, PyAny>, value: &Bound<'py, PyAny>) -> PyResult<()> { + self.values.bind(key.py()).set_item(key, value) } - fn __delitem__(&self, key: &PyAny) -> PyResult<()> { - self.values.as_ref(key.py()).del_item(key) + fn __delitem__(&self, key: &Bound<'_, PyAny>) -> PyResult<()> { + self.values.bind(key.py()).del_item(key) } } @@ -221,7 +221,7 @@ fn mapping() { ) .unwrap(); - let mapping: &PyMapping = inst.as_ref(py).downcast().unwrap(); + let mapping: &Bound<'_, PyMapping> = inst.bind(py).as_any().downcast().unwrap(); py_assert!(py, inst, "len(inst) == 0"); @@ -323,7 +323,7 @@ fn sequence() { let inst = Py::new(py, Sequence { values: vec![] }).unwrap(); - let sequence: &PySequence = inst.as_ref(py).downcast().unwrap(); + let sequence: &Bound<'_, PySequence> = inst.bind(py).as_any().downcast().unwrap(); py_assert!(py, inst, "len(inst) == 0"); @@ -350,16 +350,16 @@ fn sequence() { // indices. assert!(sequence.len().is_err()); // however regular python len() works thanks to mp_len slot - assert_eq!(inst.as_ref(py).len().unwrap(), 0); + assert_eq!(inst.bind(py).as_any().len().unwrap(), 0); py_run!(py, inst, "inst.append(0)"); sequence.set_item(0, 5).unwrap(); - assert_eq!(inst.as_ref(py).len().unwrap(), 1); + assert_eq!(inst.bind(py).as_any().len().unwrap(), 1); assert_eq!(sequence.get_item(0).unwrap().extract::().unwrap(), 5); sequence.del_item(0).unwrap(); - assert_eq!(inst.as_ref(py).len().unwrap(), 0); + assert_eq!(inst.bind(py).as_any().len().unwrap(), 0); }); } @@ -658,10 +658,10 @@ impl OnceFuture { fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } - fn __next__<'py>(&'py mut self, py: Python<'py>) -> Option<&'py PyAny> { + fn __next__<'py>(&mut self, py: Python<'py>) -> Option<&Bound<'py, PyAny>> { if !self.polled { self.polled = true; - Some(self.future.as_ref(py)) + Some(self.future.bind(py)) } else { None } diff --git a/tests/test_pyself.rs b/tests/test_pyself.rs index 7abec02b0a2..e14d117a639 100644 --- a/tests/test_pyself.rs +++ b/tests/test_pyself.rs @@ -63,12 +63,12 @@ impl Iter { } fn __next__(mut slf: PyRefMut<'_, Self>) -> PyResult> { - let bytes = slf.keys.as_ref(slf.py()).as_bytes(); + let bytes = slf.keys.bind(slf.py()).as_bytes(); match bytes.get(slf.idx) { Some(&b) => { slf.idx += 1; let py = slf.py(); - let reader = slf.reader.as_ref(py); + let reader = slf.reader.bind(py); let reader_ref = reader.try_borrow()?; let res = reader_ref .inner diff --git a/tests/test_sequence.rs b/tests/test_sequence.rs index d3c84b76c8c..f18323dbdeb 100644 --- a/tests/test_sequence.rs +++ b/tests/test_sequence.rs @@ -276,7 +276,7 @@ fn test_generic_list_set() { .items .iter() .zip(&[1u32, 2, 3]) - .all(|(a, b)| a.as_ref(py).eq(&b.into_py(py)).unwrap())); + .all(|(a, b)| a.bind(py).eq(&b.into_py(py)).unwrap())); }); } diff --git a/tests/test_various.rs b/tests/test_various.rs index d0df8d4a04e..a2ca12fbbda 100644 --- a/tests/test_various.rs +++ b/tests/test_various.rs @@ -31,7 +31,7 @@ fn mut_ref_arg() { let inst2 = Py::new(py, MutRefArg { n: 0 }).unwrap(); py_run!(py, inst1 inst2, "inst1.set_other(inst2)"); - let inst2 = inst2.as_ref(py).borrow(); + let inst2 = inst2.bind(py).borrow(); assert_eq!(inst2.n, 100); }); } From 1d295a12a022e7ef977224bb209d3a0c6dbac2fc Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sun, 18 Feb 2024 02:11:43 +0100 Subject: [PATCH 143/349] port `PyObject::downcast` to `Bound` API (#3856) * port `PyObject::downcast` to `Bound` API * relax traits bounds for unchecked variant in `Bound` API * deprecate `Python::(checked_)cast_as` * reword deprecation warning --- src/conversions/hashbrown.rs | 4 +-- src/conversions/indexmap.rs | 4 +-- src/conversions/std/array.rs | 2 +- src/conversions/std/map.rs | 8 ++--- src/err/mod.rs | 9 ++++++ src/instance.rs | 61 +++++++++++++++++++++++++++++------- src/marker.rs | 14 +++++++++ src/types/mapping.rs | 20 ++++++------ src/types/none.rs | 4 +-- src/types/tuple.rs | 4 +-- 10 files changed, 95 insertions(+), 35 deletions(-) diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index 7e57d332d91..c9b6c61c6eb 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -121,7 +121,7 @@ mod tests { map.insert(1, 1); let m = map.to_object(py); - let py_map: &PyDict = m.downcast(py).unwrap(); + let py_map = m.downcast_bound::(py).unwrap(); assert!(py_map.len() == 1); assert!( @@ -143,7 +143,7 @@ mod tests { map.insert(1, 1); let m: PyObject = map.into_py(py); - let py_map: &PyDict = m.downcast(py).unwrap(); + let py_map = m.downcast_bound::(py).unwrap(); assert!(py_map.len() == 1); assert!( diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index ec7ed5de557..5a8cd3d6951 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -149,7 +149,7 @@ mod test_indexmap { map.insert(1, 1); let m = map.to_object(py); - let py_map: &PyDict = m.downcast(py).unwrap(); + let py_map = m.downcast_bound::(py).unwrap(); assert!(py_map.len() == 1); assert!( @@ -175,7 +175,7 @@ mod test_indexmap { map.insert(1, 1); let m: PyObject = map.into_py(py); - let py_map: &PyDict = m.downcast(py).unwrap(); + let py_map = m.downcast_bound::(py).unwrap(); assert!(py_map.len() == 1); assert!( diff --git a/src/conversions/std/array.rs b/src/conversions/std/array.rs index 016ab0a643d..d901ff4e59d 100644 --- a/src/conversions/std/array.rs +++ b/src/conversions/std/array.rs @@ -239,7 +239,7 @@ mod tests { Python::with_gil(|py| { let array: [Foo; 8] = [Foo, Foo, Foo, Foo, Foo, Foo, Foo, Foo]; let pyobject = array.into_py(py); - let list: &PyList = pyobject.downcast(py).unwrap(); + let list = pyobject.downcast_bound::(py).unwrap(); let _cell: &crate::PyCell = list.get_item(4).unwrap().extract().unwrap(); }); } diff --git a/src/conversions/std/map.rs b/src/conversions/std/map.rs index 700617ec17a..8c6835d7215 100644 --- a/src/conversions/std/map.rs +++ b/src/conversions/std/map.rs @@ -121,7 +121,7 @@ mod tests { map.insert(1, 1); let m = map.to_object(py); - let py_map: &PyDict = m.downcast(py).unwrap(); + let py_map = m.downcast_bound::(py).unwrap(); assert!(py_map.len() == 1); assert!( @@ -144,7 +144,7 @@ mod tests { map.insert(1, 1); let m = map.to_object(py); - let py_map: &PyDict = m.downcast(py).unwrap(); + let py_map = m.downcast_bound::(py).unwrap(); assert!(py_map.len() == 1); assert!( @@ -167,7 +167,7 @@ mod tests { map.insert(1, 1); let m: PyObject = map.into_py(py); - let py_map: &PyDict = m.downcast(py).unwrap(); + let py_map = m.downcast_bound::(py).unwrap(); assert!(py_map.len() == 1); assert!( @@ -189,7 +189,7 @@ mod tests { map.insert(1, 1); let m: PyObject = map.into_py(py); - let py_map: &PyDict = m.downcast(py).unwrap(); + let py_map = m.downcast_bound::(py).unwrap(); assert!(py_map.len() == 1); assert!( diff --git a/src/err/mod.rs b/src/err/mod.rs index 0c2637a7389..cf74739db87 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -62,6 +62,15 @@ impl<'a> PyDowncastError<'a> { to: to.into(), } } + + /// Compatibility API to convert the Bound variant `DowncastError` into the + /// gil-ref variant + pub(crate) fn from_downcast_err(DowncastError { from, to }: DowncastError<'a, 'a>) -> Self { + Self { + from: from.as_gil_ref(), + to, + } + } } /// Error that indicates a failure to convert a PyAny to a more specific Python type. diff --git a/src/instance.rs b/src/instance.rs index cb803cc6c4d..821aad81e58 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -7,8 +7,8 @@ use crate::types::any::PyAnyMethods; use crate::types::string::PyStringMethods; use crate::types::{PyDict, PyString, PyTuple}; use crate::{ - ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyClass, PyClassInitializer, PyRef, PyRefMut, - PyTypeInfo, Python, ToPyObject, + ffi, AsPyPointer, DowncastError, FromPyObject, IntoPy, PyAny, PyClass, PyClassInitializer, + PyRef, PyRefMut, PyTypeInfo, Python, ToPyObject, }; use crate::{gil, PyTypeCheck}; use std::marker::PhantomData; @@ -1686,6 +1686,23 @@ impl std::fmt::Debug for Py { pub type PyObject = Py; impl PyObject { + /// Deprecated form of [`PyObject::downcast_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyObject::downcast` will be replaced by `PyObject::downcast_bound` in a future PyO3 version" + ) + )] + #[inline] + pub fn downcast<'py, T>(&'py self, py: Python<'py>) -> Result<&'py T, PyDowncastError<'py>> + where + T: PyTypeCheck, + { + self.downcast_bound::(py) + .map(Bound::as_gil_ref) + .map_err(PyDowncastError::from_downcast_err) + } /// Downcast this `PyObject` to a concrete Python type or pyclass. /// /// Note that you can often avoid downcasting yourself by just specifying @@ -1703,8 +1720,8 @@ impl PyObject { /// Python::with_gil(|py| { /// let any: PyObject = PyDict::new_bound(py).into(); /// - /// assert!(any.downcast::(py).is_ok()); - /// assert!(any.downcast::(py).is_err()); + /// assert!(any.downcast_bound::(py).is_ok()); + /// assert!(any.downcast_bound::(py).is_err()); /// }); /// ``` /// @@ -1725,9 +1742,9 @@ impl PyObject { /// Python::with_gil(|py| { /// let class: PyObject = Py::new(py, Class { i: 0 }).unwrap().into_py(py); /// - /// let class_cell: &PyCell = class.downcast(py)?; + /// let class_bound = class.downcast_bound::(py)?; /// - /// class_cell.borrow_mut().i += 1; + /// class_bound.borrow_mut().i += 1; /// /// // Alternatively you can get a `PyRefMut` directly /// let class_ref: PyRefMut<'_, Class> = class.extract(py)?; @@ -1737,24 +1754,44 @@ impl PyObject { /// # } /// ``` #[inline] - pub fn downcast<'py, T>(&'py self, py: Python<'py>) -> Result<&'py T, PyDowncastError<'py>> + pub fn downcast_bound<'py, T>( + &self, + py: Python<'py>, + ) -> Result<&Bound<'py, T>, DowncastError<'_, 'py>> where - T: PyTypeCheck, + T: PyTypeCheck, { - self.as_ref(py).downcast() + self.bind(py).downcast() } - /// Casts the PyObject to a concrete Python object type without checking validity. + /// Deprecated form of [`PyObject::downcast_bound_unchecked`] /// /// # Safety /// /// Callers must ensure that the type is valid or risk type confusion. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyObject::downcast_unchecked` will be replaced by `PyObject::downcast_bound_unchecked` in a future PyO3 version" + ) + )] #[inline] - pub unsafe fn downcast_unchecked<'p, T>(&'p self, py: Python<'p>) -> &T + pub unsafe fn downcast_unchecked<'py, T>(&'py self, py: Python<'py>) -> &T where T: HasPyGilRef, { - self.as_ref(py).downcast_unchecked() + self.downcast_bound_unchecked::(py).as_gil_ref() + } + + /// Casts the PyObject to a concrete Python object type without checking validity. + /// + /// # Safety + /// + /// Callers must ensure that the type is valid or risk type confusion. + #[inline] + pub unsafe fn downcast_bound_unchecked<'py, T>(&self, py: Python<'py>) -> &Bound<'py, T> { + self.bind(py).downcast_unchecked() } } diff --git a/src/marker.rs b/src/marker.rs index fc84ca54c90..5321eddb2d5 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -826,6 +826,13 @@ impl<'py> Python<'py> { } /// Registers the object in the release pool, and tries to downcast to specific type. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `obj.downcast_bound::(py)` instead of `py.checked_cast_as::(obj)`" + ) + )] pub fn checked_cast_as(self, obj: PyObject) -> Result<&'py T, PyDowncastError<'py>> where T: PyTypeCheck, @@ -839,6 +846,13 @@ impl<'py> Python<'py> { /// # Safety /// /// Callers must ensure that ensure that the cast is valid. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `obj.downcast_bound_unchecked::(py)` instead of `py.cast_as::(obj)`" + ) + )] pub unsafe fn cast_as(self, obj: PyObject) -> &'py T where T: HasPyGilRef, diff --git a/src/types/mapping.rs b/src/types/mapping.rs index 81bb2e63912..86c605f8bed 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -299,13 +299,13 @@ mod tests { Python::with_gil(|py| { let mut v = HashMap::new(); let ob = v.to_object(py); - let mapping: &PyMapping = ob.downcast(py).unwrap(); + let mapping = ob.downcast_bound::(py).unwrap(); assert_eq!(0, mapping.len().unwrap()); assert!(mapping.is_empty().unwrap()); v.insert(7, 32); let ob = v.to_object(py); - let mapping2: &PyMapping = ob.downcast(py).unwrap(); + let mapping2 = ob.downcast_bound::(py).unwrap(); assert_eq!(1, mapping2.len().unwrap()); assert!(!mapping2.is_empty().unwrap()); }); @@ -317,7 +317,7 @@ mod tests { let mut v = HashMap::new(); v.insert("key0", 1234); let ob = v.to_object(py); - let mapping: &PyMapping = ob.downcast(py).unwrap(); + let mapping = ob.downcast_bound::(py).unwrap(); mapping.set_item("key1", "foo").unwrap(); assert!(mapping.contains("key0").unwrap()); @@ -332,7 +332,7 @@ mod tests { let mut v = HashMap::new(); v.insert(7, 32); let ob = v.to_object(py); - let mapping: &PyMapping = ob.downcast(py).unwrap(); + let mapping = ob.downcast_bound::(py).unwrap(); assert_eq!( 32, mapping.get_item(7i32).unwrap().extract::().unwrap() @@ -350,7 +350,7 @@ mod tests { let mut v = HashMap::new(); v.insert(7, 32); let ob = v.to_object(py); - let mapping: &PyMapping = ob.downcast(py).unwrap(); + let mapping = ob.downcast_bound::(py).unwrap(); assert!(mapping.set_item(7i32, 42i32).is_ok()); // change assert!(mapping.set_item(8i32, 123i32).is_ok()); // insert assert_eq!( @@ -370,7 +370,7 @@ mod tests { let mut v = HashMap::new(); v.insert(7, 32); let ob = v.to_object(py); - let mapping: &PyMapping = ob.downcast(py).unwrap(); + let mapping = ob.downcast_bound::(py).unwrap(); assert!(mapping.del_item(7i32).is_ok()); assert_eq!(0, mapping.len().unwrap()); assert!(mapping @@ -388,12 +388,12 @@ mod tests { v.insert(8, 42); v.insert(9, 123); let ob = v.to_object(py); - let mapping: &PyMapping = ob.downcast(py).unwrap(); + let mapping = ob.downcast_bound::(py).unwrap(); // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut key_sum = 0; let mut value_sum = 0; for el in mapping.items().unwrap().iter().unwrap() { - let tuple = el.unwrap().downcast::().unwrap(); + let tuple = el.unwrap().downcast_into::().unwrap(); key_sum += tuple.get_item(0).unwrap().extract::().unwrap(); value_sum += tuple.get_item(1).unwrap().extract::().unwrap(); } @@ -410,7 +410,7 @@ mod tests { v.insert(8, 42); v.insert(9, 123); let ob = v.to_object(py); - let mapping: &PyMapping = ob.downcast(py).unwrap(); + let mapping = ob.downcast_bound::(py).unwrap(); // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut key_sum = 0; for el in mapping.keys().unwrap().iter().unwrap() { @@ -428,7 +428,7 @@ mod tests { v.insert(8, 42); v.insert(9, 123); let ob = v.to_object(py); - let mapping: &PyMapping = ob.downcast(py).unwrap(); + let mapping = ob.downcast_bound::(py).unwrap(); // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut values_sum = 0; for el in mapping.values().unwrap().iter().unwrap() { diff --git a/src/types/none.rs b/src/types/none.rs index 6460482e5ff..a14af044808 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -103,7 +103,7 @@ mod tests { #[test] fn test_unit_to_object_is_none() { Python::with_gil(|py| { - assert!(().to_object(py).downcast::(py).is_ok()); + assert!(().to_object(py).downcast_bound::(py).is_ok()); }) } @@ -111,7 +111,7 @@ mod tests { fn test_unit_into_py_is_none() { Python::with_gil(|py| { let obj: PyObject = ().into_py(py); - assert!(obj.downcast::(py).is_ok()); + assert!(obj.downcast_bound::(py).is_ok()); }) } diff --git a/src/types/tuple.rs b/src/types/tuple.rs index a8737c256c4..e41fcbf0020 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -169,7 +169,7 @@ impl PyTuple { /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { /// let ob = (1, 2, 3).to_object(py); - /// let tuple: &PyTuple = ob.downcast(py).unwrap(); + /// let tuple = ob.downcast_bound::(py).unwrap(); /// let obj = tuple.get_item(0); /// assert_eq!(obj.unwrap().extract::().unwrap(), 1); /// Ok(()) @@ -273,7 +273,7 @@ pub trait PyTupleMethods<'py> { /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { /// let ob = (1, 2, 3).to_object(py); - /// let tuple: &PyTuple = ob.downcast(py).unwrap(); + /// let tuple = ob.downcast_bound::(py).unwrap(); /// let obj = tuple.get_item(0); /// assert_eq!(obj.unwrap().extract::().unwrap(), 1); /// Ok(()) From f04ad56df465170c1e5f5970c2a41cce96454ff3 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 18 Feb 2024 03:07:48 +0000 Subject: [PATCH 144/349] implement `PyTypeMethods` (#3705) * implement `PyTypeMethods` * introduce `PyType` bound constructors * `from_type_ptr_bound` instead of `from_type_ptr_borrowed` * correct conditional code * just make `from_type_ptr_bound` create an owned `Bound` * correct docstrings Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> * Rework as `PyType::from_borrowed_type_ptr` * correct doc link to `from_borrowed_type_ptr` Co-authored-by: Lily Foote * remove unneeded lifetime name --------- Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> Co-authored-by: Lily Foote --- src/err/err_state.rs | 3 +- src/err/mod.rs | 5 +- src/instance.rs | 3 +- src/prelude.rs | 1 + src/types/any.rs | 12 +-- src/types/boolobject.rs | 5 +- src/types/mod.rs | 2 +- src/types/typeobject.rs | 171 ++++++++++++++++++++++++++++++------- tests/test_class_basics.rs | 2 +- 9 files changed, 155 insertions(+), 49 deletions(-) diff --git a/src/err/err_state.rs b/src/err/err_state.rs index eb03d8b9ff2..9f85296f661 100644 --- a/src/err/err_state.rs +++ b/src/err/err_state.rs @@ -22,9 +22,8 @@ impl PyErrStateNormalized { #[cfg(Py_3_12)] pub(crate) fn ptype<'py>(&self, py: Python<'py>) -> Bound<'py, PyType> { - use crate::instance::PyNativeType; use crate::types::any::PyAnyMethods; - self.pvalue.bind(py).get_type().as_borrowed().to_owned() + self.pvalue.bind(py).get_type() } #[cfg(not(Py_3_12))] diff --git a/src/err/mod.rs b/src/err/mod.rs index cf74739db87..d55c8f45afe 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -2,8 +2,7 @@ use crate::instance::Bound; use crate::panic::PanicException; use crate::type_object::PyTypeInfo; use crate::types::any::PyAnyMethods; -use crate::types::string::PyStringMethods; -use crate::types::{PyTraceback, PyType}; +use crate::types::{string::PyStringMethods, typeobject::PyTypeMethods, PyTraceback, PyType}; use crate::{ exceptions::{self, PyBaseException}, ffi, @@ -280,7 +279,7 @@ impl PyErr { /// /// Python::with_gil(|py| { /// let err: PyErr = PyTypeError::new_err(("some type error",)); - /// assert!(err.get_type_bound(py).is(PyType::new::(py))); + /// assert!(err.get_type_bound(py).is(&PyType::new_bound::(py))); /// }); /// ``` pub fn get_type_bound<'py>(&self, py: Python<'py>) -> Bound<'py, PyType> { diff --git a/src/instance.rs b/src/instance.rs index 821aad81e58..3e280c43ffe 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -3,8 +3,7 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::pycell::{PyBorrowError, PyBorrowMutError, PyCell}; use crate::pyclass::boolean_struct::{False, True}; use crate::type_object::HasPyGilRef; -use crate::types::any::PyAnyMethods; -use crate::types::string::PyStringMethods; +use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; use crate::types::{PyDict, PyString, PyTuple}; use crate::{ ffi, AsPyPointer, DowncastError, FromPyObject, IntoPy, PyAny, PyClass, PyClassInitializer, diff --git a/src/prelude.rs b/src/prelude.rs index 47951aedcce..3ac239f49e1 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -41,3 +41,4 @@ pub use crate::types::set::PySetMethods; pub use crate::types::string::PyStringMethods; pub use crate::types::traceback::PyTracebackMethods; pub use crate::types::tuple::PyTupleMethods; +pub use crate::types::typeobject::PyTypeMethods; diff --git a/src/types/any.rs b/src/types/any.rs index 33b3d96405a..1cb6bb779f1 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -667,7 +667,7 @@ impl PyAny { /// Returns the Python type object for this object's type. pub fn get_type(&self) -> &PyType { - self.as_borrowed().get_type() + self.as_borrowed().get_type().into_gil_ref() } /// Returns the Python type pointer for this object. @@ -1499,7 +1499,7 @@ pub trait PyAnyMethods<'py> { fn iter(&self) -> PyResult>; /// Returns the Python type object for this object's type. - fn get_type(&self) -> &'py PyType; + fn get_type(&self) -> Bound<'py, PyType>; /// Returns the Python type pointer for this object. fn get_type_ptr(&self) -> *mut ffi::PyTypeObject; @@ -2107,8 +2107,8 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { PyIterator::from_bound_object(self) } - fn get_type(&self) -> &'py PyType { - unsafe { PyType::from_type_ptr(self.py(), ffi::Py_TYPE(self.as_ptr())) } + fn get_type(&self) -> Bound<'py, PyType> { + unsafe { PyType::from_borrowed_type_ptr(self.py(), ffi::Py_TYPE(self.as_ptr())) } } #[inline] @@ -2265,7 +2265,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { #[cfg(not(PyPy))] fn py_super(&self) -> PyResult> { - PySuper::new_bound(&self.get_type().as_borrowed(), self) + PySuper::new_bound(&self.get_type(), self) } } @@ -2286,7 +2286,7 @@ impl<'py> Bound<'py, PyAny> { N: IntoPy>, { let py = self.py(); - let self_type = self.get_type().as_borrowed(); + let self_type = self.get_type(); let attr = if let Ok(attr) = self_type.getattr(attr_name) { attr } else { diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index c0c4e4ba5c2..01bc14b4431 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -1,8 +1,9 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - exceptions::PyTypeError, ffi, ffi_ptr_ext::FfiPtrExt, instance::Bound, Borrowed, FromPyObject, - IntoPy, PyAny, PyNativeType, PyObject, PyResult, Python, ToPyObject, + exceptions::PyTypeError, ffi, ffi_ptr_ext::FfiPtrExt, instance::Bound, + types::typeobject::PyTypeMethods, Borrowed, FromPyObject, IntoPy, PyAny, PyNativeType, + PyObject, PyResult, Python, ToPyObject, }; use super::any::PyAnyMethods; diff --git a/src/types/mod.rs b/src/types/mod.rs index da22b30729f..46909c880bb 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -309,4 +309,4 @@ mod slice; pub(crate) mod string; pub(crate) mod traceback; pub(crate) mod tuple; -mod typeobject; +pub(crate) mod typeobject; diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 50eeaa7153d..f0e4a8c0a2c 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -1,5 +1,7 @@ use crate::err::{self, PyResult}; -use crate::{ffi, PyAny, PyTypeInfo, Python}; +use crate::instance::Borrowed; +use crate::types::any::PyAnyMethods; +use crate::{ffi, Bound, PyAny, PyNativeType, PyTypeInfo, Python}; use std::borrow::Cow; #[cfg(not(any(Py_LIMITED_API, PyPy)))] use std::ffi::CStr; @@ -11,38 +13,143 @@ pub struct PyType(PyAny); pyobject_native_type_core!(PyType, pyobject_native_static_type_object!(ffi::PyType_Type), #checkfunction=ffi::PyType_Check); impl PyType { - /// Creates a new type object. + /// Deprecated form of [`PyType::new_bound`]. #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyType::new` will be replaced by `PyType::new_bound` in a future PyO3 version" + ) + )] pub fn new(py: Python<'_>) -> &PyType { T::type_object_bound(py).into_gil_ref() } + /// Creates a new type object. + #[inline] + pub fn new_bound(py: Python<'_>) -> Bound<'_, PyType> { + T::type_object_bound(py) + } + /// Retrieves the underlying FFI pointer associated with this Python object. #[inline] pub fn as_type_ptr(&self) -> *mut ffi::PyTypeObject { - self.as_ptr() as *mut ffi::PyTypeObject + self.as_borrowed().as_type_ptr() } - /// Retrieves the `PyType` instance for the given FFI pointer. + /// Deprecated form of [`PyType::from_borrowed_type_ptr`]. /// /// # Safety - /// - The pointer must be non-null. - /// - The pointer must be valid for the entire of the lifetime for which the reference is used. + /// + /// - The pointer must a valid non-null reference to a `PyTypeObject`. #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "Use `PyType::from_borrowed_type_ptr` instead" + ) + )] pub unsafe fn from_type_ptr(py: Python<'_>, p: *mut ffi::PyTypeObject) -> &PyType { - py.from_borrowed_ptr(p as *mut ffi::PyObject) + Self::from_borrowed_type_ptr(py, p).into_gil_ref() + } + + /// Converts the given FFI pointer into `Bound`, to use in safe code. + /// + /// The function creates a new reference from the given pointer, and returns + /// it as a `Bound`. + /// + /// # Safety + /// - The pointer must be a valid non-null reference to a `PyTypeObject` + #[inline] + pub unsafe fn from_borrowed_type_ptr( + py: Python<'_>, + p: *mut ffi::PyTypeObject, + ) -> Bound<'_, PyType> { + Borrowed::from_ptr_unchecked(py, p.cast()) + .downcast_unchecked() + .to_owned() } /// Gets the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name) of the `PyType`. pub fn qualname(&self) -> PyResult { + self.as_borrowed().qualname() + } + + /// Gets the full name, which includes the module, of the `PyType`. + pub fn name(&self) -> PyResult> { + self.as_borrowed().name() + } + + /// Checks whether `self` is a subclass of `other`. + /// + /// Equivalent to the Python expression `issubclass(self, other)`. + pub fn is_subclass(&self, other: &PyAny) -> PyResult { + self.as_borrowed().is_subclass(&other.as_borrowed()) + } + + /// Checks whether `self` is a subclass of type `T`. + /// + /// Equivalent to the Python expression `issubclass(self, T)`, if the type + /// `T` is known at compile time. + pub fn is_subclass_of(&self) -> PyResult + where + T: PyTypeInfo, + { + self.as_borrowed().is_subclass_of::() + } +} + +/// Implementation of functionality for [`PyType`]. +/// +/// These methods are defined for the `Bound<'py, PyType>` smart pointer, so to use method call +/// syntax these methods are separated into a trait, because stable Rust does not yet support +/// `arbitrary_self_types`. +#[doc(alias = "PyType")] +pub trait PyTypeMethods<'py> { + /// Retrieves the underlying FFI pointer associated with this Python object. + fn as_type_ptr(&self) -> *mut ffi::PyTypeObject; + + /// Gets the full name, which includes the module, of the `PyType`. + fn name(&self) -> PyResult>; + + /// Gets the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name) of the `PyType`. + fn qualname(&self) -> PyResult; + + /// Checks whether `self` is a subclass of `other`. + /// + /// Equivalent to the Python expression `issubclass(self, other)`. + fn is_subclass(&self, other: &Bound<'_, PyAny>) -> PyResult; + + /// Checks whether `self` is a subclass of type `T`. + /// + /// Equivalent to the Python expression `issubclass(self, T)`, if the type + /// `T` is known at compile time. + fn is_subclass_of(&self) -> PyResult + where + T: PyTypeInfo; +} + +impl<'py> PyTypeMethods<'py> for Bound<'py, PyType> { + /// Retrieves the underlying FFI pointer associated with this Python object. + #[inline] + fn as_type_ptr(&self) -> *mut ffi::PyTypeObject { + self.as_ptr() as *mut ffi::PyTypeObject + } + + /// Gets the name of the `PyType`. + fn name(&self) -> PyResult> { + Borrowed::from(self).name() + } + + fn qualname(&self) -> PyResult { #[cfg(any(Py_LIMITED_API, PyPy, not(Py_3_11)))] let name = self.getattr(intern!(self.py(), "__qualname__"))?.extract(); #[cfg(not(any(Py_LIMITED_API, PyPy, not(Py_3_11))))] let name = { use crate::ffi_ptr_ext::FfiPtrExt; - use crate::types::any::PyAnyMethods; - let obj = unsafe { ffi::PyType_GetQualName(self.as_type_ptr()).assume_owned_or_err(self.py())? }; @@ -53,8 +160,29 @@ impl PyType { name } - /// Gets the full name, which includes the module, of the `PyType`. - pub fn name(&self) -> PyResult> { + /// Checks whether `self` is a subclass of `other`. + /// + /// Equivalent to the Python expression `issubclass(self, other)`. + fn is_subclass(&self, other: &Bound<'_, PyAny>) -> PyResult { + let result = unsafe { ffi::PyObject_IsSubclass(self.as_ptr(), other.as_ptr()) }; + err::error_on_minusone(self.py(), result)?; + Ok(result == 1) + } + + /// Checks whether `self` is a subclass of type `T`. + /// + /// Equivalent to the Python expression `issubclass(self, T)`, if the type + /// `T` is known at compile time. + fn is_subclass_of(&self) -> PyResult + where + T: PyTypeInfo, + { + self.is_subclass(&T::type_object_bound(self.py())) + } +} + +impl<'a> Borrowed<'a, '_, PyType> { + fn name(self) -> PyResult> { #[cfg(not(any(Py_LIMITED_API, PyPy)))] { let ptr = self.as_type_ptr(); @@ -79,33 +207,12 @@ impl PyType { #[cfg(Py_3_11)] let name = { use crate::ffi_ptr_ext::FfiPtrExt; - unsafe { ffi::PyType_GetName(self.as_type_ptr()).assume_owned_or_err(self.py())? } }; Ok(Cow::Owned(format!("{}.{}", module, name))) } } - - /// Checks whether `self` is a subclass of `other`. - /// - /// Equivalent to the Python expression `issubclass(self, other)`. - pub fn is_subclass(&self, other: &PyAny) -> PyResult { - let result = unsafe { ffi::PyObject_IsSubclass(self.as_ptr(), other.as_ptr()) }; - err::error_on_minusone(self.py(), result)?; - Ok(result == 1) - } - - /// Checks whether `self` is a subclass of type `T`. - /// - /// Equivalent to the Python expression `issubclass(self, T)`, if the type - /// `T` is known at compile time. - pub fn is_subclass_of(&self) -> PyResult - where - T: PyTypeInfo, - { - self.is_subclass(T::type_object_bound(self.py()).as_gil_ref()) - } } #[cfg(test)] diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index d19d106d206..1c2b2a5cce3 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -230,7 +230,7 @@ impl UnsendableChild { fn test_unsendable() -> PyResult<()> { let obj = Python::with_gil(|py| -> PyResult<_> { - let obj: Py = PyType::new::(py).call1((5,))?.extract()?; + let obj: Py = PyType::new_bound::(py).call1((5,))?.extract()?; // Accessing the value inside this thread should not panic let caught_panic = From 4ce9c35983712b76645864ade4ced46f9a5748f7 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sun, 18 Feb 2024 19:27:19 +0100 Subject: [PATCH 145/349] port `Python::get_type` to `Bound` API (#3846) * port `Python::get_type` to `Bound` API * fix `is_subclass_and_is_instance` FIXME --- guide/src/class.md | 22 +++++++++--------- guide/src/exception.md | 4 ++-- pyo3-macros-backend/src/pyclass.rs | 2 +- src/conversions/chrono.rs | 6 ++--- src/conversions/num_bigint.rs | 8 ++----- src/err/mod.rs | 10 ++++---- src/exceptions.rs | 17 +++++++------- src/impl_/extract_argument.rs | 5 +++- src/macros.rs | 2 +- src/marker.rs | 18 ++++++++++++++- src/pyclass_init.rs | 2 +- src/tests/common.rs | 2 +- src/types/typeobject.rs | 12 ++++++---- tests/test_class_attributes.rs | 16 ++++++------- tests/test_class_basics.rs | 14 +++++------ tests/test_class_new.rs | 37 ++++++++++++++---------------- tests/test_coroutine.rs | 11 +++++---- tests/test_enum.rs | 10 ++++---- tests/test_gc.rs | 30 ++++++++++++------------ tests/test_getter_setter.rs | 2 +- tests/test_inheritance.rs | 26 ++++++++++----------- tests/test_macro_docs.rs | 2 +- tests/test_macros.rs | 4 ++-- tests/test_mapping.rs | 2 +- tests/test_methods.rs | 14 +++++------ tests/test_multiple_pymethods.rs | 2 +- tests/test_no_imports.rs | 14 ++++++----- tests/test_proto_methods.rs | 8 +++---- tests/test_sequence.rs | 6 ++--- tests/test_static_slots.rs | 2 +- tests/test_super.rs | 2 +- tests/test_text_signature.rs | 16 ++++++------- tests/test_variable_arguments.rs | 4 ++-- 33 files changed, 175 insertions(+), 157 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index 6ca582ed963..9a2d91c1cb1 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -401,7 +401,7 @@ impl SubSubClass { # pyo3::py_run!(py, subsub, "assert subsub.method3() == 3000"); # let subsub = SubSubClass::factory_method(py, 2).unwrap(); # let subsubsub = SubSubClass::factory_method(py, 3).unwrap(); -# let cls = py.get_type::(); +# let cls = py.get_type_bound::(); # pyo3::py_run!(py, subsub cls, "assert not isinstance(subsub, cls)"); # pyo3::py_run!(py, subsubsub cls, "assert isinstance(subsubsub, cls)"); # }); @@ -497,7 +497,7 @@ impl MyDict { // some custom methods that use `private` here... } # Python::with_gil(|py| { -# let cls = py.get_type::(); +# let cls = py.get_type_bound::(); # pyo3::py_run!(py, cls, "cls(a=1, b=2)") # }); # } @@ -767,7 +767,7 @@ impl MyClass { } Python::with_gil(|py| { - let my_class = py.get_type::(); + let my_class = py.get_type_bound::(); pyo3::py_run!(py, my_class, "assert my_class.my_attribute == 'hello'") }); ``` @@ -1026,7 +1026,7 @@ enum MyEnum { Python::with_gil(|py| { let x = Py::new(py, MyEnum::Variant).unwrap(); let y = Py::new(py, MyEnum::OtherVariant).unwrap(); - let cls = py.get_type::(); + let cls = py.get_type_bound::(); pyo3::py_run!(py, x y cls, r#" assert x == cls.Variant assert y == cls.OtherVariant @@ -1046,7 +1046,7 @@ enum MyEnum { } Python::with_gil(|py| { - let cls = py.get_type::(); + let cls = py.get_type_bound::(); let x = MyEnum::Variant as i32; // The exact value is assigned by the compiler. pyo3::py_run!(py, cls x, r#" assert int(cls.Variant) == x @@ -1068,7 +1068,7 @@ enum MyEnum{ } Python::with_gil(|py| { - let cls = py.get_type::(); + let cls = py.get_type_bound::(); let x = Py::new(py, MyEnum::Variant).unwrap(); pyo3::py_run!(py, cls x, r#" assert repr(x) == 'MyEnum.Variant' @@ -1094,7 +1094,7 @@ impl MyEnum { } Python::with_gil(|py| { - let cls = py.get_type::(); + let cls = py.get_type_bound::(); pyo3::py_run!(py, cls, "assert repr(cls.Answer) == '42'") }) ``` @@ -1111,7 +1111,7 @@ enum MyEnum { Python::with_gil(|py| { let x = Py::new(py, MyEnum::Variant).unwrap(); - let cls = py.get_type::(); + let cls = py.get_type_bound::(); pyo3::py_run!(py, x cls, r#" assert repr(x) == 'RenamedEnum.UPPERCASE' assert x == cls.UPPERCASE @@ -1165,7 +1165,7 @@ enum Shape { Python::with_gil(|py| { let circle = Shape::Circle { radius: 10.0 }.into_py(py); let square = Shape::RegularPolygon { side_count: 4, radius: 10.0 }.into_py(py); - let cls = py.get_type::(); + let cls = py.get_type_bound::(); pyo3::py_run!(py, circle square cls, r#" assert isinstance(circle, cls) assert isinstance(circle, cls.Circle) @@ -1204,7 +1204,7 @@ enum MyEnum { Python::with_gil(|py| { let x = Py::new(py, MyEnum::Variant { i: 42 }).unwrap(); - let cls = py.get_type::(); + let cls = py.get_type_bound::(); pyo3::py_run!(py, x cls, r#" assert isinstance(x, cls) assert not isinstance(x, cls.Variant) @@ -1308,7 +1308,7 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { } # Python::with_gil(|py| { -# let cls = py.get_type::(); +# let cls = py.get_type_bound::(); # pyo3::py_run!(py, cls, "assert cls.__name__ == 'MyClass'") # }); # } diff --git a/guide/src/exception.md b/guide/src/exception.md index 225118576f7..0be44167760 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -24,7 +24,7 @@ use pyo3::exceptions::PyException; create_exception!(mymodule, CustomError, PyException); Python::with_gil(|py| { - let ctx = [("CustomError", py.get_type::())].into_py_dict_bound(py); + let ctx = [("CustomError", py.get_type_bound::())].into_py_dict_bound(py); pyo3::py_run!( py, *ctx, @@ -46,7 +46,7 @@ pyo3::create_exception!(mymodule, CustomError, PyException); #[pymodule] fn mymodule(py: Python<'_>, m: &PyModule) -> PyResult<()> { // ... other elements added to module ... - m.add("CustomError", py.get_type::())?; + m.add("CustomError", py.get_type_bound::())?; Ok(()) } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 02024011366..7d8c868e3d5 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1086,7 +1086,7 @@ pub fn gen_complex_enum_variant_attr( let associated_method = quote! { fn #wrapper_ident(py: _pyo3::Python<'_>) -> _pyo3::PyResult<_pyo3::PyObject> { #deprecations - ::std::result::Result::Ok(py.get_type::<#variant_cls>().into()) + ::std::result::Result::Ok(py.get_type_bound::<#variant_cls>().into_any().unbind()) } }; diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 6737c16f8cf..6878b9d9b1e 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -52,9 +52,7 @@ use crate::types::{ }; #[cfg(Py_LIMITED_API)] use crate::{intern, DowncastError}; -use crate::{ - Bound, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, ToPyObject, -}; +use crate::{Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject}; use chrono::offset::{FixedOffset, Utc}; use chrono::{ DateTime, Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Offset, TimeZone, Timelike, @@ -461,7 +459,7 @@ fn warn_truncated_leap_second(obj: &Bound<'_, PyAny>) { let py = obj.py(); if let Err(e) = PyErr::warn_bound( py, - &py.get_type::().as_borrowed(), + &py.get_type_bound::(), "ignored leap-second, `datetime` does not support leap-seconds", 0, ) { diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index ff5bd0309bd..3c98a90f092 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -91,12 +91,8 @@ macro_rules! bigint_conversion { } else { None }; - py.get_type::() - .call_method( - "from_bytes", - (bytes_obj, "little"), - kwargs.as_ref().map(crate::Bound::as_gil_ref), - ) + py.get_type_bound::() + .call_method("from_bytes", (bytes_obj, "little"), kwargs.as_ref()) .expect("int.from_bytes() failed during to_object()") // FIXME: #1813 or similar .into() } diff --git a/src/err/mod.rs b/src/err/mod.rs index d55c8f45afe..d922baaad21 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -724,7 +724,7 @@ impl PyErr { /// # use pyo3::prelude::*; /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let user_warning = py.get_type::().as_borrowed(); + /// let user_warning = py.get_type_bound::(); /// PyErr::warn_bound(py, &user_warning, "I am warning you", 0)?; /// Ok(()) /// }) @@ -1080,7 +1080,7 @@ impl_signed_integer!(isize); mod tests { use super::PyErrState; use crate::exceptions::{self, PyTypeError, PyValueError}; - use crate::{PyErr, PyNativeType, PyTypeInfo, Python}; + use crate::{PyErr, PyTypeInfo, Python}; #[test] fn no_error() { @@ -1278,7 +1278,7 @@ mod tests { // GIL locked should prevent effects to be visible to other testing // threads. Python::with_gil(|py| { - let cls = py.get_type::().as_borrowed(); + let cls = py.get_type_bound::(); // Reset warning filter to default state let warnings = py.import_bound("warnings").unwrap(); @@ -1293,14 +1293,14 @@ mod tests { // Test with raising warnings - .call_method1("simplefilter", ("error", cls)) + .call_method1("simplefilter", ("error", &cls)) .unwrap(); PyErr::warn_bound(py, &cls, "I am warning you", 0).unwrap_err(); // Test with error for an explicit module warnings.call_method0("resetwarnings").unwrap(); warnings - .call_method1("filterwarnings", ("error", "", cls, "pyo3test")) + .call_method1("filterwarnings", ("error", "", &cls, "pyo3test")) .unwrap(); // This has the wrong module and will not raise, just be emitted diff --git a/src/exceptions.rs b/src/exceptions.rs index e7b6407e721..6ae9febc695 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -72,7 +72,7 @@ macro_rules! impl_exception_boilerplate { /// import_exception!(socket, gaierror); /// /// Python::with_gil(|py| { -/// let ctx = [("gaierror", py.get_type::())].into_py_dict_bound(py); +/// let ctx = [("gaierror", py.get_type_bound::())].into_py_dict_bound(py); /// pyo3::py_run!(py, *ctx, "import socket; assert gaierror is socket.gaierror"); /// }); /// @@ -160,7 +160,7 @@ macro_rules! import_exception { /// /// #[pymodule] /// fn my_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { -/// m.add("MyError", py.get_type::())?; +/// m.add("MyError", py.get_type_bound::())?; /// m.add_function(wrap_pyfunction!(raise_myerror, py)?)?; /// Ok(()) /// } @@ -168,7 +168,7 @@ macro_rules! import_exception { /// # Python::with_gil(|py| -> PyResult<()> { /// # let fun = wrap_pyfunction!(raise_myerror, py)?; /// # let locals = pyo3::types::PyDict::new_bound(py); -/// # locals.set_item("MyError", py.get_type::())?; +/// # locals.set_item("MyError", py.get_type_bound::())?; /// # locals.set_item("raise_myerror", fun)?; /// # /// # py.run_bound( @@ -241,7 +241,6 @@ macro_rules! create_exception_type_object { impl $name { fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { use $crate::sync::GILOnceCell; - use $crate::PyNativeType; static TYPE_OBJECT: GILOnceCell<$crate::Py<$crate::types::PyType>> = GILOnceCell::new(); @@ -251,7 +250,7 @@ macro_rules! create_exception_type_object { py, concat!(stringify!($module), ".", stringify!($name)), $doc, - ::std::option::Option::Some(&py.get_type::<$base>().as_borrowed()), + ::std::option::Option::Some(&py.get_type_bound::<$base>()), ::std::option::Option::None, ).expect("Failed to initialize new exception type.") ).as_ptr() as *mut $crate::ffi::PyTypeObject @@ -904,7 +903,7 @@ mod tests { create_exception!(mymodule, CustomError, PyException); Python::with_gil(|py| { - let error_type = py.get_type::(); + let error_type = py.get_type_bound::(); let ctx = [("CustomError", error_type)].into_py_dict_bound(py); let type_description: String = py .eval_bound("str(CustomError)", None, Some(&ctx)) @@ -927,7 +926,7 @@ mod tests { fn custom_exception_dotted_module() { create_exception!(mymodule.exceptions, CustomError, PyException); Python::with_gil(|py| { - let error_type = py.get_type::(); + let error_type = py.get_type_bound::(); let ctx = [("CustomError", error_type)].into_py_dict_bound(py); let type_description: String = py .eval_bound("str(CustomError)", None, Some(&ctx)) @@ -946,7 +945,7 @@ mod tests { create_exception!(mymodule, CustomError, PyException, "Some docs"); Python::with_gil(|py| { - let error_type = py.get_type::(); + let error_type = py.get_type_bound::(); let ctx = [("CustomError", error_type)].into_py_dict_bound(py); let type_description: String = py .eval_bound("str(CustomError)", None, Some(&ctx)) @@ -979,7 +978,7 @@ mod tests { ); Python::with_gil(|py| { - let error_type = py.get_type::(); + let error_type = py.get_type_bound::(); let ctx = [("CustomError", error_type)].into_py_dict_bound(py); let type_description: String = py .eval_bound("str(CustomError)", None, Some(&ctx)) diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index d9487359cf4..27728c6d46f 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -166,7 +166,10 @@ pub fn from_py_with_with_default<'a, 'py, T>( #[cold] pub fn argument_extraction_error(py: Python<'_>, arg_name: &str, error: PyErr) -> PyErr { use crate::types::any::PyAnyMethods; - if error.get_type_bound(py).is(py.get_type::()) { + if error + .get_type_bound(py) + .is(&py.get_type_bound::()) + { let remapped_error = PyTypeError::new_err(format!( "argument '{}': {}", arg_name, diff --git a/src/macros.rs b/src/macros.rs index d5fbbdc3dfc..0779fb98a4b 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -73,7 +73,7 @@ /// } /// /// Python::with_gil(|py| { -/// let locals = [("C", py.get_type::())].into_py_dict_bound(py); +/// let locals = [("C", py.get_type_bound::())].into_py_dict_bound(py); /// pyo3::py_run!(py, *locals, "c = C()"); /// }); /// ``` diff --git a/src/marker.rs b/src/marker.rs index 5321eddb2d5..4b46d3badbd 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -728,12 +728,28 @@ impl<'py> Python<'py> { } /// Gets the Python type object for type `T`. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`Python::get_type` will be replaced by `Python::get_type_bound` in a future PyO3 version" + ) + )] #[inline] pub fn get_type(self) -> &'py PyType where T: PyTypeInfo, { - T::type_object_bound(self).into_gil_ref() + self.get_type_bound::().into_gil_ref() + } + + /// Gets the Python type object for type `T`. + #[inline] + pub fn get_type_bound(self) -> Bound<'py, PyType> + where + T: PyTypeInfo, + { + T::type_object_bound(self) } /// Deprecated form of [`Python::import_bound`] diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 63761f435bb..ac3aa3ef852 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -122,7 +122,7 @@ impl PyObjectInit for PyNativeTypeInitializer { /// } /// } /// Python::with_gil(|py| { -/// let typeobj = py.get_type::(); +/// let typeobj = py.get_type_bound::(); /// let sub_sub_class = typeobj.call((), None).unwrap(); /// py_run!( /// py, diff --git a/src/tests/common.rs b/src/tests/common.rs index 5eec60949bc..854d73e4d7b 100644 --- a/src/tests/common.rs +++ b/src/tests/common.rs @@ -42,7 +42,7 @@ mod inner { ($py:expr, *$dict:expr, $code:expr, $err:ident) => {{ let res = $py.run_bound($code, None, Some(&$dict.as_borrowed())); let err = res.expect_err(&format!("Did not raise {}", stringify!($err))); - if !err.matches($py, $py.get_type::()) { + if !err.matches($py, $py.get_type_bound::()) { panic!("Expected {} but got {:?}", stringify!($err), err) } err diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index f0e4a8c0a2c..4c1b17a2aa8 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -217,22 +217,26 @@ impl<'a> Borrowed<'a, '_, PyType> { #[cfg(test)] mod tests { + use crate::types::typeobject::PyTypeMethods; use crate::types::{PyBool, PyLong}; use crate::Python; #[test] fn test_type_is_subclass() { Python::with_gil(|py| { - let bool_type = py.get_type::(); - let long_type = py.get_type::(); - assert!(bool_type.is_subclass(long_type).unwrap()); + let bool_type = py.get_type_bound::(); + let long_type = py.get_type_bound::(); + assert!(bool_type.is_subclass(&long_type).unwrap()); }); } #[test] fn test_type_is_subclass_of() { Python::with_gil(|py| { - assert!(py.get_type::().is_subclass_of::().unwrap()); + assert!(py + .get_type_bound::() + .is_subclass_of::() + .unwrap()); }); } } diff --git a/tests/test_class_attributes.rs b/tests/test_class_attributes.rs index 906a11c8ea1..9e544211c3c 100644 --- a/tests/test_class_attributes.rs +++ b/tests/test_class_attributes.rs @@ -56,7 +56,7 @@ impl Foo { #[test] fn class_attributes() { Python::with_gil(|py| { - let foo_obj = py.get_type::(); + let foo_obj = py.get_type_bound::(); py_assert!(py, foo_obj, "foo_obj.MY_CONST == 'foobar'"); py_assert!(py, foo_obj, "foo_obj.RENAMED_CONST == 'foobar_2'"); py_assert!(py, foo_obj, "foo_obj.a == 5"); @@ -72,7 +72,7 @@ fn class_attributes() { #[ignore] fn class_attributes_are_immutable() { Python::with_gil(|py| { - let foo_obj = py.get_type::(); + let foo_obj = py.get_type_bound::(); py_expect_exception!(py, foo_obj, "foo_obj.a = 6", PyTypeError); }); } @@ -88,8 +88,8 @@ impl Bar { #[test] fn recursive_class_attributes() { Python::with_gil(|py| { - let foo_obj = py.get_type::(); - let bar_obj = py.get_type::(); + let foo_obj = py.get_type_bound::(); + let bar_obj = py.get_type_bound::(); py_assert!(py, foo_obj, "foo_obj.a_foo.x == 1"); py_assert!(py, foo_obj, "foo_obj.bar.x == 2"); py_assert!(py, bar_obj, "bar_obj.a_foo.x == 3"); @@ -145,7 +145,7 @@ fn test_fallible_class_attribute() { Python::with_gil(|py| { let stderr = CaptureStdErr::new(py).unwrap(); - assert!(std::panic::catch_unwind(|| py.get_type::()).is_err()); + assert!(std::panic::catch_unwind(|| py.get_type_bound::()).is_err()); assert_eq!( stderr.reset().unwrap().trim(), "\ @@ -187,7 +187,7 @@ fn test_renaming_all_struct_fields() { use pyo3::types::PyBool; Python::with_gil(|py| { - let struct_class = py.get_type::(); + let struct_class = py.get_type_bound::(); let struct_obj = struct_class.call0().unwrap(); assert!(struct_obj .setattr("firstField", PyBool::new_bound(py, false)) @@ -220,11 +220,11 @@ macro_rules! test_case { //use pyo3::types::PyInt; Python::with_gil(|py| { - let struct_class = py.get_type::<$struct_name>(); + let struct_class = py.get_type_bound::<$struct_name>(); let struct_obj = struct_class.call0().unwrap(); assert!(struct_obj.setattr($renamed_field_name, 2).is_ok()); let attr = struct_obj.getattr($renamed_field_name).unwrap(); - assert_eq!(2, PyAny::extract::(attr).unwrap()); + assert_eq!(2, attr.extract().unwrap()); }); } }; diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 1c2b2a5cce3..6d282765c31 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -13,7 +13,7 @@ struct EmptyClass {} #[test] fn empty_class() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); // By default, don't allow creating instances from python. assert!(typeobj.call((), None).is_err()); @@ -27,7 +27,7 @@ struct UnitClass; #[test] fn unit_class() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); // By default, don't allow creating instances from python. assert!(typeobj.call((), None).is_err()); @@ -58,7 +58,7 @@ struct ClassWithDocs { #[test] fn class_with_docstr() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); py_run!( py, typeobj, @@ -104,7 +104,7 @@ impl EmptyClass2 { #[test] fn custom_names() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); py_assert!(py, typeobj, "typeobj.__name__ == 'CustomName'"); py_assert!(py, typeobj, "typeobj.custom_fn.__name__ == 'custom_fn'"); py_assert!( @@ -137,7 +137,7 @@ impl RawIdents { #[test] fn test_raw_idents() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); py_assert!(py, typeobj, "not hasattr(typeobj, 'r#fn')"); py_assert!(py, typeobj, "hasattr(typeobj, 'fn')"); py_assert!(py, typeobj, "hasattr(typeobj, 'type')"); @@ -191,7 +191,7 @@ impl ClassWithObjectField { #[test] fn class_with_object_field() { Python::with_gil(|py| { - let ty = py.get_type::(); + let ty = py.get_type_bound::(); py_assert!(py, ty, "ty(5).value == 5"); py_assert!(py, ty, "ty(None).value == None"); }); @@ -346,7 +346,7 @@ struct TupleClass(#[pyo3(get, set, name = "value")] i32); #[test] fn test_tuple_struct_class() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); assert!(typeobj.call((), None).is_err()); py_assert!(py, typeobj, "typeobj.__name__ == 'TupleClass'"); diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index 59772cc4cf2..b1a0e594799 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -19,7 +19,7 @@ impl EmptyClassWithNew { #[test] fn empty_class_with_new() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); assert!(typeobj .call((), None) .unwrap() @@ -29,10 +29,7 @@ fn empty_class_with_new() { // Calling with arbitrary args or kwargs is not ok assert!(typeobj.call(("some", "args"), None).is_err()); assert!(typeobj - .call( - (), - Some([("some", "kwarg")].into_py_dict_bound(py).as_gil_ref()) - ) + .call((), Some(&[("some", "kwarg")].into_py_dict_bound(py))) .is_err()); }); } @@ -51,7 +48,7 @@ impl UnitClassWithNew { #[test] fn unit_class_with_new() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); assert!(typeobj .call((), None) .unwrap() @@ -74,9 +71,9 @@ impl TupleClassWithNew { #[test] fn tuple_class_with_new() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); let wrp = typeobj.call((42,), None).unwrap(); - let obj = wrp.downcast::>().unwrap(); + let obj = wrp.downcast::().unwrap(); let obj_ref = obj.borrow(); assert_eq!(obj_ref.0, 42); }); @@ -99,9 +96,9 @@ impl NewWithOneArg { #[test] fn new_with_one_arg() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); let wrp = typeobj.call((42,), None).unwrap(); - let obj = wrp.downcast::>().unwrap(); + let obj = wrp.downcast::().unwrap(); let obj_ref = obj.borrow(); assert_eq!(obj_ref.data, 42); }); @@ -127,12 +124,12 @@ impl NewWithTwoArgs { #[test] fn new_with_two_args() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); let wrp = typeobj .call((10, 20), None) .map_err(|e| e.display(py)) .unwrap(); - let obj = wrp.downcast::>().unwrap(); + let obj = wrp.downcast::().unwrap(); let obj_ref = obj.borrow(); assert_eq!(obj_ref.data1, 10); assert_eq!(obj_ref.data2, 20); @@ -158,7 +155,7 @@ impl SuperClass { #[test] fn subclass_new() { Python::with_gil(|py| { - let super_cls = py.get_type::(); + let super_cls = py.get_type_bound::(); let source = pyo3::indoc::indoc!( r#" class Class(SuperClass): @@ -206,7 +203,7 @@ impl NewWithCustomError { #[test] fn new_with_custom_error() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); let err = typeobj.call0().unwrap_err(); assert_eq!(err.to_string(), "ValueError: custom error"); }); @@ -247,7 +244,7 @@ impl NewExisting { #[test] fn test_new_existing() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); let obj1 = typeobj.call1((0,)).unwrap(); let obj2 = typeobj.call1((0,)).unwrap(); @@ -263,10 +260,10 @@ fn test_new_existing() { assert!(obj5.getattr("num").unwrap().extract::().unwrap() == 2); assert!(obj6.getattr("num").unwrap().extract::().unwrap() == 2); - assert!(obj1.is(obj2)); - assert!(obj3.is(obj4)); - assert!(!obj1.is(obj3)); - assert!(!obj1.is(obj5)); - assert!(!obj5.is(obj6)); + assert!(obj1.is(&obj2)); + assert!(obj3.is(&obj4)); + assert!(!obj1.is(&obj3)); + assert!(!obj1.is(&obj5)); + assert!(!obj5.is(&obj6)); }); } diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index 5954b9794e7..b3b8ba7be1d 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -1,6 +1,6 @@ #![cfg(feature = "macros")] #![cfg(not(target_arch = "wasm32"))] -use std::{ops::Deref, task::Poll, thread, time::Duration}; +use std::{task::Poll, thread, time::Duration}; use futures::{channel::oneshot, future::poll_fn, FutureExt}; use pyo3::{ @@ -65,8 +65,11 @@ fn test_coroutine_qualname() { assert coro.__name__ == name and coro.__qualname__ == qualname "#; let locals = [ - ("my_fn", wrap_pyfunction!(my_fn, gil).unwrap().deref()), - ("MyClass", gil.get_type::()), + ( + "my_fn", + wrap_pyfunction!(my_fn, gil).unwrap().as_borrowed().as_any(), + ), + ("MyClass", gil.get_type_bound::().as_any()), ] .into_py_dict_bound(gil); py_run!(gil, *locals, &handle_windows(test)); @@ -286,7 +289,7 @@ fn test_async_method_receiver() { assert False assert asyncio.run(coro3) == 1 "#; - let locals = [("Counter", gil.get_type::())].into_py_dict_bound(gil); + let locals = [("Counter", gil.get_type_bound::())].into_py_dict_bound(gil); py_run!(gil, *locals, test); }) } diff --git a/tests/test_enum.rs b/tests/test_enum.rs index 33e94c241c7..d73316e5512 100644 --- a/tests/test_enum.rs +++ b/tests/test_enum.rs @@ -16,7 +16,7 @@ pub enum MyEnum { #[test] fn test_enum_class_attr() { Python::with_gil(|py| { - let my_enum = py.get_type::(); + let my_enum = py.get_type_bound::(); let var = Py::new(py, MyEnum::Variant).unwrap(); py_assert!(py, my_enum var, "my_enum.Variant == var"); }) @@ -31,7 +31,7 @@ fn return_enum() -> MyEnum { fn test_return_enum() { Python::with_gil(|py| { let f = wrap_pyfunction!(return_enum)(py).unwrap(); - let mynum = py.get_type::(); + let mynum = py.get_type_bound::(); py_run!(py, f mynum, "assert f() == mynum.Variant") }); @@ -46,7 +46,7 @@ fn enum_arg(e: MyEnum) { fn test_enum_arg() { Python::with_gil(|py| { let f = wrap_pyfunction!(enum_arg)(py).unwrap(); - let mynum = py.get_type::(); + let mynum = py.get_type_bound::(); py_run!(py, f mynum, "f(mynum.OtherVariant)") }) @@ -83,7 +83,7 @@ enum CustomDiscriminant { fn test_custom_discriminant() { Python::with_gil(|py| { #[allow(non_snake_case)] - let CustomDiscriminant = py.get_type::(); + let CustomDiscriminant = py.get_type_bound::(); let one = Py::new(py, CustomDiscriminant::One).unwrap(); let two = Py::new(py, CustomDiscriminant::Two).unwrap(); py_run!(py, CustomDiscriminant one two, r#" @@ -204,7 +204,7 @@ enum RenameAllVariantsEnum { #[test] fn test_renaming_all_enum_variants() { Python::with_gil(|py| { - let enum_obj = py.get_type::(); + let enum_obj = py.get_type_bound::(); py_assert!(py, enum_obj, "enum_obj.VARIANT_ONE == enum_obj.VARIANT_ONE"); py_assert!(py, enum_obj, "enum_obj.VARIANT_TWO == enum_obj.VARIANT_TWO"); py_assert!( diff --git a/tests/test_gc.rs b/tests/test_gc.rs index 7ed39436062..9e236a27dbf 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -211,11 +211,11 @@ fn inheritance_with_new_methods_with_drop() { let drop_called2 = Arc::new(AtomicBool::new(false)); Python::with_gil(|py| { - let _typebase = py.get_type::(); - let typeobj = py.get_type::(); + let _typebase = py.get_type_bound::(); + let typeobj = py.get_type_bound::(); let inst = typeobj.call((), None).unwrap(); - let obj: &PyCell = inst.downcast().unwrap(); + let obj = inst.downcast::().unwrap(); let mut obj_ref_mut = obj.borrow_mut(); obj_ref_mut.data = Some(Arc::clone(&drop_called1)); let base: &mut BaseClassWithDrop = obj_ref_mut.as_mut(); @@ -255,8 +255,8 @@ fn gc_during_borrow() { Python::with_gil(|py| { unsafe { // get the traverse function - let ty = py.get_type::().as_type_ptr(); - let traverse = get_type_traverse(ty).unwrap(); + let ty = py.get_type_bound::(); + let traverse = get_type_traverse(ty.as_type_ptr()).unwrap(); // create an object and check that traversing it works normally // when it's not borrowed @@ -303,8 +303,8 @@ impl PartialTraverse { fn traverse_partial() { Python::with_gil(|py| unsafe { // get the traverse function - let ty = py.get_type::().as_type_ptr(); - let traverse = get_type_traverse(ty).unwrap(); + let ty = py.get_type_bound::(); + let traverse = get_type_traverse(ty.as_type_ptr()).unwrap(); // confirm that traversing errors let obj = Py::new(py, PartialTraverse::new(py)).unwrap(); @@ -338,8 +338,8 @@ impl PanickyTraverse { fn traverse_panic() { Python::with_gil(|py| unsafe { // get the traverse function - let ty = py.get_type::().as_type_ptr(); - let traverse = get_type_traverse(ty).unwrap(); + let ty = py.get_type_bound::(); + let traverse = get_type_traverse(ty.as_type_ptr()).unwrap(); // confirm that traversing errors let obj = Py::new(py, PanickyTraverse::new(py)).unwrap(); @@ -361,8 +361,8 @@ impl TriesGILInTraverse { fn tries_gil_in_traverse() { Python::with_gil(|py| unsafe { // get the traverse function - let ty = py.get_type::().as_type_ptr(); - let traverse = get_type_traverse(ty).unwrap(); + let ty = py.get_type_bound::(); + let traverse = get_type_traverse(ty.as_type_ptr()).unwrap(); // confirm that traversing panicks let obj = Py::new(py, TriesGILInTraverse {}).unwrap(); @@ -413,8 +413,8 @@ impl<'a> Traversable for PyRef<'a, HijackedTraverse> { fn traverse_cannot_be_hijacked() { Python::with_gil(|py| unsafe { // get the traverse function - let ty = py.get_type::().as_type_ptr(); - let traverse = get_type_traverse(ty).unwrap(); + let ty = py.get_type_bound::(); + let traverse = get_type_traverse(ty.as_type_ptr()).unwrap(); let cell = PyCell::new(py, HijackedTraverse::new()).unwrap(); let obj = cell.to_object(py); @@ -528,8 +528,8 @@ impl UnsendableTraversal { #[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled fn unsendable_are_not_traversed_on_foreign_thread() { Python::with_gil(|py| unsafe { - let ty = py.get_type::().as_type_ptr(); - let traverse = get_type_traverse(ty).unwrap(); + let ty = py.get_type_bound::(); + let traverse = get_type_traverse(ty.as_type_ptr()).unwrap(); let obj = Py::new( py, diff --git a/tests/test_getter_setter.rs b/tests/test_getter_setter.rs index b990a82e134..1009ce1bd91 100644 --- a/tests/test_getter_setter.rs +++ b/tests/test_getter_setter.rs @@ -64,7 +64,7 @@ fn class_with_properties() { py_run!(py, inst, "assert inst.get_num() == inst.unwrapped == 42"); py_run!(py, inst, "assert inst.data_list == [42]"); - let d = [("C", py.get_type::())].into_py_dict_bound(py); + let d = [("C", py.get_type_bound::())].into_py_dict_bound(py); py_assert!(py, *d, "C.DATA.__doc__ == 'a getter for data'"); }); } diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index cd728c77bfe..af34974c90b 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -20,7 +20,7 @@ struct SubclassAble {} #[test] fn subclass() { Python::with_gil(|py| { - let d = [("SubclassAble", py.get_type::())].into_py_dict_bound(py); + let d = [("SubclassAble", py.get_type_bound::())].into_py_dict_bound(py); py.run_bound( "class A(SubclassAble): pass\nassert issubclass(A, SubclassAble)", @@ -72,7 +72,7 @@ impl SubClass { #[test] fn inheritance_with_new_methods() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); let inst = typeobj.call((), None).unwrap(); py_run!(py, inst, "assert inst.val1 == 10; assert inst.val2 == 5"); }); @@ -112,16 +112,16 @@ fn mutation_fails() { #[test] fn is_subclass_and_is_instance() { Python::with_gil(|py| { - let sub_ty = py.get_type::(); - let base_ty = py.get_type::(); + let sub_ty = py.get_type_bound::(); + let base_ty = py.get_type_bound::(); assert!(sub_ty.is_subclass_of::().unwrap()); - assert!(sub_ty.is_subclass(base_ty).unwrap()); + assert!(sub_ty.is_subclass(&base_ty).unwrap()); - let obj = PyCell::new(py, SubClass::new()).unwrap(); + let obj = Bound::new(py, SubClass::new()).unwrap().into_any(); assert!(obj.is_instance_of::()); assert!(obj.is_instance_of::()); - assert!(obj.is_instance(sub_ty).unwrap()); - assert!(obj.is_instance(base_ty).unwrap()); + assert!(obj.is_instance(&sub_ty).unwrap()); + assert!(obj.is_instance(&base_ty).unwrap()); }); } @@ -155,7 +155,7 @@ impl SubClass2 { #[test] fn handle_result_in_new() { Python::with_gil(|py| { - let subclass = py.get_type::(); + let subclass = py.get_type_bound::(); py_run!( py, subclass, @@ -274,15 +274,15 @@ mod inheriting_native_type { #[test] fn custom_exception() { Python::with_gil(|py| { - let cls = py.get_type::(); - let dict = [("cls", cls)].into_py_dict_bound(py); + let cls = py.get_type_bound::(); + let dict = [("cls", &cls)].into_py_dict_bound(py); let res = py.run_bound( "e = cls('hello'); assert str(e) == 'hello'; assert e.context == 'Hello :)'; raise e", None, Some(&dict) ); let err = res.unwrap_err(); - assert!(err.matches(py, cls), "{}", err); + assert!(err.matches(py, &cls), "{}", err); // catching the exception in Python also works: py_run!( @@ -315,7 +315,7 @@ fn test_subclass_ref_counts() { // regression test for issue #1363 Python::with_gil(|py| { #[allow(non_snake_case)] - let SimpleClass = py.get_type::(); + let SimpleClass = py.get_type_bound::(); py_run!( py, SimpleClass, diff --git a/tests/test_macro_docs.rs b/tests/test_macro_docs.rs index 5e240816cd2..6289626d32e 100644 --- a/tests/test_macro_docs.rs +++ b/tests/test_macro_docs.rs @@ -23,7 +23,7 @@ impl MacroDocs { #[test] fn meth_doc() { Python::with_gil(|py| { - let d = [("C", py.get_type::())].into_py_dict_bound(py); + let d = [("C", py.get_type_bound::())].into_py_dict_bound(py); py_assert!( py, *d, diff --git a/tests/test_macros.rs b/tests/test_macros.rs index c01ba853685..0d2b125b870 100644 --- a/tests/test_macros.rs +++ b/tests/test_macros.rs @@ -73,7 +73,7 @@ property_rename_via_macro!(my_new_property_name); #[test] fn test_macro_rules_interactions() { Python::with_gil(|py| { - let my_base = py.get_type::(); + let my_base = py.get_type_bound::(); py_assert!(py, my_base, "my_base.__name__ == 'MyClass'"); let my_func = wrap_pyfunction!(my_function_in_macro, py).unwrap(); @@ -83,7 +83,7 @@ fn test_macro_rules_interactions() { "my_func.__text_signature__ == '(a, b=None, *, c=42)'" ); - let renamed_prop = py.get_type::(); + let renamed_prop = py.get_type_bound::(); py_assert!( py, renamed_prop, diff --git a/tests/test_mapping.rs b/tests/test_mapping.rs index 9a80ab491a1..06928e74183 100644 --- a/tests/test_mapping.rs +++ b/tests/test_mapping.rs @@ -69,7 +69,7 @@ impl Mapping { /// Return a dict with `m = Mapping(['1', '2', '3'])`. fn map_dict(py: Python<'_>) -> Bound<'_, pyo3::types::PyDict> { - let d = [("Mapping", py.get_type::())].into_py_dict_bound(py); + let d = [("Mapping", py.get_type_bound::())].into_py_dict_bound(py); py_run!(py, *d, "m = Mapping(['1', '2', '3'])"); d } diff --git a/tests/test_methods.rs b/tests/test_methods.rs index 2114ead25c4..96b40174928 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -95,7 +95,7 @@ impl ClassMethod { #[test] fn class_method() { Python::with_gil(|py| { - let d = [("C", py.get_type::())].into_py_dict_bound(py); + let d = [("C", py.get_type_bound::())].into_py_dict_bound(py); py_assert!(py, *d, "C.method() == 'ClassMethod.method()!'"); py_assert!(py, *d, "C().method() == 'ClassMethod.method()!'"); py_assert!( @@ -126,7 +126,7 @@ impl ClassMethodWithArgs { #[test] fn class_method_with_args() { Python::with_gil(|py| { - let d = [("C", py.get_type::())].into_py_dict_bound(py); + let d = [("C", py.get_type_bound::())].into_py_dict_bound(py); py_assert!( py, *d, @@ -157,7 +157,7 @@ fn static_method() { Python::with_gil(|py| { assert_eq!(StaticMethod::method(py), "StaticMethod.method()!"); - let d = [("C", py.get_type::())].into_py_dict_bound(py); + let d = [("C", py.get_type_bound::())].into_py_dict_bound(py); py_assert!(py, *d, "C.method() == 'StaticMethod.method()!'"); py_assert!(py, *d, "C().method() == 'StaticMethod.method()!'"); py_assert!(py, *d, "C.method.__doc__ == 'Test static method.'"); @@ -181,7 +181,7 @@ fn static_method_with_args() { Python::with_gil(|py| { assert_eq!(StaticMethodWithArgs::method(py, 1234), "0x4d2"); - let d = [("C", py.get_type::())].into_py_dict_bound(py); + let d = [("C", py.get_type_bound::())].into_py_dict_bound(py); py_assert!(py, *d, "C.method(1337) == '0x539'"); }); } @@ -679,7 +679,7 @@ impl MethDocs { #[test] fn meth_doc() { Python::with_gil(|py| { - let d = [("C", py.get_type::())].into_py_dict_bound(py); + let d = [("C", py.get_type_bound::())].into_py_dict_bound(py); py_assert!(py, *d, "C.__doc__ == 'A class with \"documentation\".'"); py_assert!( py, @@ -866,7 +866,7 @@ impl FromSequence { #[test] fn test_from_sequence() { Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); py_assert!(py, typeobj, "typeobj(range(0, 4)).numbers == [0, 1, 2, 3]"); }); } @@ -946,7 +946,7 @@ impl r#RawIdents { #[test] fn test_raw_idents() { Python::with_gil(|py| { - let raw_idents_type = py.get_type::(); + let raw_idents_type = py.get_type_bound::(); assert_eq!(raw_idents_type.qualname().unwrap(), "RawIdents"); py_run!( py, diff --git a/tests/test_multiple_pymethods.rs b/tests/test_multiple_pymethods.rs index 13baeed3815..308220e78b2 100644 --- a/tests/test_multiple_pymethods.rs +++ b/tests/test_multiple_pymethods.rs @@ -65,7 +65,7 @@ impl PyClassWithMultiplePyMethods { #[test] fn test_class_with_multiple_pymethods() { Python::with_gil(|py| { - let cls = py.get_type::(); + let cls = py.get_type_bound::(); py_assert!(py, cls, "cls()() == 'call'"); py_assert!(py, cls, "cls().method() == 'method'"); py_assert!(py, cls, "cls.classmethod() == 'classmethod'"); diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index 4abdcdf2e5b..88932ed282a 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -2,6 +2,8 @@ #![cfg(feature = "macros")] +use pyo3::prelude::PyAnyMethods; + #[pyo3::pyfunction] #[pyo3(name = "identity", signature = (x = None))] fn basic_function(py: pyo3::Python<'_>, x: Option) -> pyo3::PyObject { @@ -91,13 +93,13 @@ impl BasicClass { fn test_basic() { pyo3::Python::with_gil(|py| { let module = pyo3::wrap_pymodule!(basic_module)(py); - let cls = py.get_type::(); + let cls = py.get_type_bound::(); let d = pyo3::types::IntoPyDict::into_py_dict_bound( [ - ("mod", module.as_ref(py).as_ref()), - ("cls", cls.as_ref()), - ("a", cls.call1((8,)).unwrap()), - ("b", cls.call1(("foo",)).unwrap()), + ("mod", module.bind(py).as_any()), + ("cls", &cls), + ("a", &cls.call1((8,)).unwrap()), + ("b", &cls.call1(("foo",)).unwrap()), ], py, ); @@ -144,7 +146,7 @@ impl NewClassMethod { #[test] fn test_new_class_method() { pyo3::Python::with_gil(|py| { - let cls = py.get_type::(); + let cls = py.get_type_bound::(); pyo3::py_run!(py, cls, "assert cls().cls is cls"); }); } diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index 3151fbe5be7..9126555df72 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -672,7 +672,7 @@ impl OnceFuture { #[cfg(not(target_arch = "wasm32"))] // Won't work without wasm32 event loop (e.g., Pyodide has WebLoop) fn test_await() { Python::with_gil(|py| { - let once = py.get_type::(); + let once = py.get_type_bound::(); let source = r#" import asyncio import sys @@ -725,7 +725,7 @@ impl AsyncIterator { #[cfg(not(target_arch = "wasm32"))] // Won't work without wasm32 event loop (e.g., Pyodide has WebLoop) fn test_anext_aiter() { Python::with_gil(|py| { - let once = py.get_type::(); + let once = py.get_type_bound::(); let source = r#" import asyncio import sys @@ -750,7 +750,7 @@ asyncio.run(main()) .as_borrowed(); globals.set_item("Once", once).unwrap(); globals - .set_item("AsyncIterator", py.get_type::()) + .set_item("AsyncIterator", py.get_type_bound::()) .unwrap(); py.run_bound(source, Some(&globals), None) .map_err(|e| e.display(py)) @@ -793,7 +793,7 @@ impl DescrCounter { #[test] fn descr_getset() { Python::with_gil(|py| { - let counter = py.get_type::(); + let counter = py.get_type_bound::(); let source = pyo3::indoc::indoc!( r#" class Class: diff --git a/tests/test_sequence.rs b/tests/test_sequence.rs index f18323dbdeb..d73d2d110b4 100644 --- a/tests/test_sequence.rs +++ b/tests/test_sequence.rs @@ -106,7 +106,7 @@ impl ByteSequence { /// Return a dict with `s = ByteSequence([1, 2, 3])`. fn seq_dict(py: Python<'_>) -> Bound<'_, pyo3::types::PyDict> { - let d = [("ByteSequence", py.get_type::())].into_py_dict_bound(py); + let d = [("ByteSequence", py.get_type_bound::())].into_py_dict_bound(py); // Though we can construct `s` in Rust, let's test `__new__` works. py_run!(py, *d, "s = ByteSequence([1, 2, 3])"); d @@ -138,7 +138,7 @@ fn test_setitem() { #[test] fn test_delitem() { Python::with_gil(|py| { - let d = [("ByteSequence", py.get_type::())].into_py_dict_bound(py); + let d = [("ByteSequence", py.get_type_bound::())].into_py_dict_bound(py); py_run!( py, @@ -234,7 +234,7 @@ fn test_repeat() { #[test] fn test_inplace_repeat() { Python::with_gil(|py| { - let d = [("ByteSequence", py.get_type::())].into_py_dict_bound(py); + let d = [("ByteSequence", py.get_type_bound::())].into_py_dict_bound(py); py_run!( py, diff --git a/tests/test_static_slots.rs b/tests/test_static_slots.rs index b01c87cd507..3c270a3154c 100644 --- a/tests/test_static_slots.rs +++ b/tests/test_static_slots.rs @@ -38,7 +38,7 @@ impl Count5 { /// Return a dict with `s = Count5()`. fn test_dict(py: Python<'_>) -> Bound<'_, pyo3::types::PyDict> { - let d = [("Count5", py.get_type::())].into_py_dict_bound(py); + let d = [("Count5", py.get_type_bound::())].into_py_dict_bound(py); // Though we can construct `s` in Rust, let's test `__new__` works. py_run!(py, *d, "s = Count5()"); d diff --git a/tests/test_super.rs b/tests/test_super.rs index 208290df87b..8dcedf808a5 100644 --- a/tests/test_super.rs +++ b/tests/test_super.rs @@ -44,7 +44,7 @@ impl SubClass { #[test] fn test_call_super_method() { Python::with_gil(|py| { - let cls = py.get_type::(); + let cls = py.get_type_bound::(); pyo3::py_run!( py, cls, diff --git a/tests/test_text_signature.rs b/tests/test_text_signature.rs index 9056ca21e14..b72ad2a2efb 100644 --- a/tests/test_text_signature.rs +++ b/tests/test_text_signature.rs @@ -13,7 +13,7 @@ fn class_without_docs_or_signature() { struct MyClass {} Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); py_assert!(py, typeobj, "typeobj.__doc__ is None"); py_assert!(py, typeobj, "typeobj.__text_signature__ is None"); @@ -28,7 +28,7 @@ fn class_with_docs() { struct MyClass {} Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); py_assert!(py, typeobj, "typeobj.__doc__ == 'docs line1\\ndocs line2'"); py_assert!(py, typeobj, "typeobj.__text_signature__ is None"); @@ -52,7 +52,7 @@ fn class_with_signature_no_doc() { } Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); py_assert!(py, typeobj, "typeobj.__doc__ == ''"); py_assert!( py, @@ -81,7 +81,7 @@ fn class_with_docs_and_signature() { } Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); py_assert!(py, typeobj, "typeobj.__doc__ == 'docs line1\\ndocs line2'"); py_assert!( @@ -238,7 +238,7 @@ fn test_auto_test_signature_method() { } Python::with_gil(|py| { - let cls = py.get_type::(); + let cls = py.get_type_bound::(); #[cfg(any(not(Py_LIMITED_API), Py_3_10))] py_assert!(py, cls, "cls.__text_signature__ == '(a, b, c)'"); py_assert!( @@ -323,7 +323,7 @@ fn test_auto_test_signature_opt_out() { let f = wrap_pyfunction!(my_function_2)(py).unwrap(); py_assert!(py, f, "f.__text_signature__ == None"); - let cls = py.get_type::(); + let cls = py.get_type_bound::(); py_assert!(py, cls, "cls.__text_signature__ == None"); py_assert!(py, cls, "cls.method.__text_signature__ == None"); py_assert!(py, cls, "cls.method_2.__text_signature__ == None"); @@ -383,7 +383,7 @@ fn test_methods() { } Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); py_assert!( py, @@ -424,7 +424,7 @@ fn test_raw_identifiers() { } Python::with_gil(|py| { - let typeobj = py.get_type::(); + let typeobj = py.get_type_bound::(); py_assert!(py, typeobj, "typeobj.__text_signature__ == '()'"); diff --git a/tests/test_variable_arguments.rs b/tests/test_variable_arguments.rs index a1f8bb354cf..8d3cd5d3935 100644 --- a/tests/test_variable_arguments.rs +++ b/tests/test_variable_arguments.rs @@ -27,7 +27,7 @@ impl MyClass { #[test] fn variable_args() { Python::with_gil(|py| { - let my_obj = py.get_type::(); + let my_obj = py.get_type_bound::(); py_assert!(py, my_obj, "my_obj.test_args() == ()"); py_assert!(py, my_obj, "my_obj.test_args(1) == (1,)"); py_assert!(py, my_obj, "my_obj.test_args(1, 2) == (1, 2)"); @@ -37,7 +37,7 @@ fn variable_args() { #[test] fn variable_kwargs() { Python::with_gil(|py| { - let my_obj = py.get_type::(); + let my_obj = py.get_type_bound::(); py_assert!(py, my_obj, "my_obj.test_kwargs() == None"); py_assert!(py, my_obj, "my_obj.test_kwargs(test=1) == {'test': 1}"); py_assert!( From b4dc854585064a1df6874c9877cb59eededbd590 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Sun, 18 Feb 2024 22:01:50 +0000 Subject: [PATCH 146/349] Convert LazyTypeObject to use the Bound API (#3855) --- pyo3-macros-backend/src/pyclass.rs | 1 + src/impl_/pyclass/lazy_type_object.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 7d8c868e3d5..806df88eee5 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1283,6 +1283,7 @@ fn impl_pytypeinfo( #[inline] fn type_object_raw(py: _pyo3::Python<'_>) -> *mut _pyo3::ffi::PyTypeObject { + use _pyo3::prelude::PyTypeMethods; #deprecations <#cls as _pyo3::impl_::pyclass::PyClassImpl>::lazy_type_object() diff --git a/src/impl_/pyclass/lazy_type_object.rs b/src/impl_/pyclass/lazy_type_object.rs index af52033a702..1318e1abbbc 100644 --- a/src/impl_/pyclass/lazy_type_object.rs +++ b/src/impl_/pyclass/lazy_type_object.rs @@ -12,7 +12,7 @@ use crate::{ pyclass::{create_type_object, PyClassTypeObject}, sync::{GILOnceCell, GILProtected}, types::PyType, - PyClass, PyErr, PyMethodDefType, PyObject, PyResult, Python, + Bound, PyClass, PyErr, PyMethodDefType, PyObject, PyResult, Python, }; use super::PyClassItemsIter; @@ -46,7 +46,7 @@ impl LazyTypeObject { impl LazyTypeObject { /// Gets the type object contained by this `LazyTypeObject`, initializing it if needed. - pub fn get_or_init<'py>(&'py self, py: Python<'py>) -> &'py PyType { + pub fn get_or_init<'py>(&self, py: Python<'py>) -> &Bound<'py, PyType> { self.get_or_try_init(py).unwrap_or_else(|err| { err.print(py); panic!("failed to create type object for {}", T::NAME) @@ -54,7 +54,7 @@ impl LazyTypeObject { } /// Fallible version of the above. - pub(crate) fn get_or_try_init<'py>(&'py self, py: Python<'py>) -> PyResult<&'py PyType> { + pub(crate) fn get_or_try_init<'py>(&self, py: Python<'py>) -> PyResult<&Bound<'py, PyType>> { self.0 .get_or_try_init(py, create_type_object::, T::NAME, T::items_iter()) } @@ -65,18 +65,18 @@ impl LazyTypeObjectInner { // so that this code is only instantiated once, instead of for every T // like the generic LazyTypeObject methods above. fn get_or_try_init<'py>( - &'py self, + &self, py: Python<'py>, init: fn(Python<'py>) -> PyResult, name: &str, items_iter: PyClassItemsIter, - ) -> PyResult<&'py PyType> { + ) -> PyResult<&Bound<'py, PyType>> { (|| -> PyResult<_> { let type_object = self .value .get_or_try_init(py, || init(py))? .type_object - .as_ref(py); + .bind(py); self.ensure_init(type_object, name, items_iter)?; Ok(type_object) })() @@ -91,7 +91,7 @@ impl LazyTypeObjectInner { fn ensure_init( &self, - type_object: &PyType, + type_object: &Bound<'_, PyType>, name: &str, items_iter: PyClassItemsIter, ) -> PyResult<()> { From a85ed34c454f7074918b0f39455520a358c02416 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 18 Feb 2024 22:03:43 +0000 Subject: [PATCH 147/349] add Bound API constructors from borrowed pointers (#3858) * make `Borrowed` ptr constructors public * introduce `Bound::from_borrowed_ptr` constructors * clippy `assert_eq` -> `assert` * rerrange function order and correct docstrings --- src/instance.rs | 179 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 155 insertions(+), 24 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index 3e280c43ffe..b83b69a20d3 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -114,7 +114,7 @@ impl<'py> Bound<'py, PyAny> { Self(py, ManuallyDrop::new(Py::from_owned_ptr(py, ptr))) } - /// Constructs a new `Bound<'py, PyAny>` from a pointer. Returns `None`` if `ptr` is null. + /// Constructs a new `Bound<'py, PyAny>` from a pointer. Returns `None` if `ptr` is null. /// /// # Safety /// @@ -138,6 +138,42 @@ impl<'py> Bound<'py, PyAny> { Py::from_owned_ptr_or_err(py, ptr).map(|obj| Self(py, ManuallyDrop::new(obj))) } + /// Constructs a new `Bound<'py, PyAny>` from a pointer by creating a new Python reference. + /// Panics if `ptr` is null. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to a Python object + pub unsafe fn from_borrowed_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { + Self(py, ManuallyDrop::new(Py::from_borrowed_ptr(py, ptr))) + } + + /// Constructs a new `Bound<'py, PyAny>` from a pointer by creating a new Python reference. + /// Returns `None` if `ptr` is null. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to a Python object, or null + pub unsafe fn from_borrowed_ptr_or_opt( + py: Python<'py>, + ptr: *mut ffi::PyObject, + ) -> Option { + Py::from_borrowed_ptr_or_opt(py, ptr).map(|obj| Self(py, ManuallyDrop::new(obj))) + } + + /// Constructs a new `Bound<'py, PyAny>` from a pointer by creating a new Python reference. + /// Returns an `Err` by calling `PyErr::fetch` if `ptr` is null. + /// + /// # Safety + /// + /// - `ptr` must be a valid pointer to a Python object, or null + pub unsafe fn from_borrowed_ptr_or_err( + py: Python<'py>, + ptr: *mut ffi::PyObject, + ) -> PyResult { + Py::from_borrowed_ptr_or_err(py, ptr).map(|obj| Self(py, ManuallyDrop::new(obj))) + } + /// This slightly strange method is used to obtain `&Bound` from a pointer in macro code /// where we need to constrain the lifetime `'a` safely. /// @@ -479,37 +515,56 @@ impl<'py, T> Borrowed<'_, 'py, T> { } impl<'a, 'py> Borrowed<'a, 'py, PyAny> { + /// Constructs a new `Borrowed<'a, 'py, PyAny>` from a pointer. Panics if `ptr` is null. + /// + /// Prefer to use [`Bound::from_borrowed_ptr`], as that avoids the major safety risk + /// of needing to precisely define the lifetime `'a` for which the borrow is valid. + /// /// # Safety - /// This is similar to `std::slice::from_raw_parts`, the lifetime `'a` is completely defined by - /// the caller and it's the caller's responsibility to ensure that the reference this is - /// derived from is valid for the lifetime `'a`. - pub(crate) unsafe fn from_ptr_or_err( - py: Python<'py>, - ptr: *mut ffi::PyObject, - ) -> PyResult { - NonNull::new(ptr).map_or_else( - || Err(PyErr::fetch(py)), - |ptr| Ok(Self(ptr, PhantomData, py)), + /// + /// - `ptr` must be a valid pointer to a Python object + /// - similar to `std::slice::from_raw_parts`, the lifetime `'a` is completely defined by + /// the caller and it is the caller's responsibility to ensure that the reference this is + /// derived from is valid for the lifetime `'a`. + pub unsafe fn from_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { + Self( + NonNull::new(ptr).unwrap_or_else(|| crate::err::panic_after_error(py)), + PhantomData, + py, ) } + /// Constructs a new `Borrowed<'a, 'py, PyAny>` from a pointer. Returns `None` if `ptr` is null. + /// + /// Prefer to use [`Bound::from_borrowed_ptr_or_opt`], as that avoids the major safety risk + /// of needing to precisely define the lifetime `'a` for which the borrow is valid. + /// /// # Safety - /// This is similar to `std::slice::from_raw_parts`, the lifetime `'a` is completely defined by - /// the caller and it's the caller's responsibility to ensure that the reference this is - /// derived from is valid for the lifetime `'a`. - pub(crate) unsafe fn from_ptr_or_opt(py: Python<'py>, ptr: *mut ffi::PyObject) -> Option { + /// + /// - `ptr` must be a valid pointer to a Python object, or null + /// - similar to `std::slice::from_raw_parts`, the lifetime `'a` is completely defined by + /// the caller and it is the caller's responsibility to ensure that the reference this is + /// derived from is valid for the lifetime `'a`. + pub unsafe fn from_ptr_or_opt(py: Python<'py>, ptr: *mut ffi::PyObject) -> Option { NonNull::new(ptr).map(|ptr| Self(ptr, PhantomData, py)) } + /// Constructs a new `Borrowed<'a, 'py, PyAny>` from a pointer. Returns an `Err` by calling `PyErr::fetch` + /// if `ptr` is null. + /// + /// Prefer to use [`Bound::from_borrowed_ptr_or_err`], as that avoids the major safety risk + /// of needing to precisely define the lifetime `'a` for which the borrow is valid. + /// /// # Safety - /// This is similar to `std::slice::from_raw_parts`, the lifetime `'a` is completely defined by - /// the caller and it's the caller's responsibility to ensure that the reference this is - /// derived from is valid for the lifetime `'a`. - pub(crate) unsafe fn from_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { - Self( - NonNull::new(ptr).unwrap_or_else(|| crate::err::panic_after_error(py)), - PhantomData, - py, + /// + /// - `ptr` must be a valid pointer to a Python object, or null + /// - similar to `std::slice::from_raw_parts`, the lifetime `'a` is completely defined by + /// the caller and it is the caller's responsibility to ensure that the reference this is + /// derived from is valid for the lifetime `'a`. + pub unsafe fn from_ptr_or_err(py: Python<'py>, ptr: *mut ffi::PyObject) -> PyResult { + NonNull::new(ptr).map_or_else( + || Err(PyErr::fetch(py)), + |ptr| Ok(Self(ptr, PhantomData, py)), ) } @@ -1798,8 +1853,9 @@ impl PyObject { #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::{Bound, Py, PyObject}; + use crate::types::PyCapsule; use crate::types::{dict::IntoPyDict, PyDict, PyString}; - use crate::{PyAny, PyNativeType, PyResult, Python, ToPyObject}; + use crate::{ffi, Borrowed, PyAny, PyNativeType, PyResult, Python, ToPyObject}; #[test] fn test_call() { @@ -1998,6 +2054,81 @@ a = A() }); } + #[test] + fn bound_from_borrowed_ptr_constructors() { + // More detailed tests of the underlying semantics in pycell.rs + Python::with_gil(|py| { + fn check_drop<'py>( + py: Python<'py>, + method: impl FnOnce(*mut ffi::PyObject) -> Bound<'py, PyAny>, + ) { + let mut dropped = false; + let capsule = PyCapsule::new_bound_with_destructor( + py, + (&mut dropped) as *mut _ as usize, + None, + |ptr, _| unsafe { std::ptr::write(ptr as *mut bool, true) }, + ) + .unwrap(); + + let bound = method(capsule.as_ptr()); + assert!(!dropped); + + // creating the bound should have increased the refcount + drop(capsule); + assert!(!dropped); + + // dropping the bound should now also decrease the refcount and free the object + drop(bound); + assert!(dropped); + } + + check_drop(py, |ptr| unsafe { Bound::from_borrowed_ptr(py, ptr) }); + check_drop(py, |ptr| unsafe { + Bound::from_borrowed_ptr_or_opt(py, ptr).unwrap() + }); + check_drop(py, |ptr| unsafe { + Bound::from_borrowed_ptr_or_err(py, ptr).unwrap() + }); + }) + } + + #[test] + fn borrowed_ptr_constructors() { + // More detailed tests of the underlying semantics in pycell.rs + Python::with_gil(|py| { + fn check_drop<'py>( + py: Python<'py>, + method: impl FnOnce(&*mut ffi::PyObject) -> Borrowed<'_, 'py, PyAny>, + ) { + let mut dropped = false; + let capsule = PyCapsule::new_bound_with_destructor( + py, + (&mut dropped) as *mut _ as usize, + None, + |ptr, _| unsafe { std::ptr::write(ptr as *mut bool, true) }, + ) + .unwrap(); + + let ptr = &capsule.as_ptr(); + let _borrowed = method(ptr); + assert!(!dropped); + + // creating the borrow should not have increased the refcount + drop(capsule); + assert!(dropped); + } + + check_drop(py, |&ptr| unsafe { Borrowed::from_ptr(py, ptr) }); + check_drop(py, |&ptr| unsafe { + Borrowed::from_ptr_or_opt(py, ptr).unwrap() + }); + check_drop(py, |&ptr| unsafe { + Borrowed::from_ptr_or_err(py, ptr).unwrap() + }); + }) + } + #[cfg(feature = "macros")] mod using_macros { use crate::PyCell; From 0bb9cab6d312cf308f55204c64381f0fac013493 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Mon, 19 Feb 2024 00:37:02 +0100 Subject: [PATCH 148/349] port `PyComplex::from_complex` to `Bound` API (#3866) * port `PyComplex::from_complex` to `Bound` API * add `PyComplexMethods` to the prelude --- src/conversions/num_complex.rs | 33 +++++++++++++++++++++++++-------- src/prelude.rs | 1 + src/types/mod.rs | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index ba741323611..3b4fe0fc217 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -93,21 +93,37 @@ //! result = get_eigenvalues(m11,m12,m21,m22) //! assert result == [complex(1,-1), complex(-2,0)] //! ``` -#[cfg(any(Py_LIMITED_API, PyPy))] -use crate::types::any::PyAnyMethods; use crate::{ - ffi, types::PyComplex, Bound, FromPyObject, PyAny, PyErr, PyObject, PyResult, Python, - ToPyObject, + ffi, + ffi_ptr_ext::FfiPtrExt, + types::{any::PyAnyMethods, PyComplex}, + Bound, FromPyObject, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, }; use num_complex::Complex; use std::os::raw::c_double; impl PyComplex { - /// Creates a new Python `PyComplex` object from `num_complex`'s [`Complex`]. + /// Deprecated form of [`PyComplex::from_complex_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyComplex::from_complex` will be replaced by `PyComplex::from_complex_bound` in a future PyO3 version" + ) + )] pub fn from_complex>(py: Python<'_>, complex: Complex) -> &PyComplex { + Self::from_complex_bound(py, complex).into_gil_ref() + } + + /// Creates a new Python `PyComplex` object from `num_complex`'s [`Complex`]. + pub fn from_complex_bound>( + py: Python<'_>, + complex: Complex, + ) -> Bound<'_, PyComplex> { unsafe { - let ptr = ffi::PyComplex_FromDoubles(complex.re.into(), complex.im.into()); - py.from_owned_ptr(ptr) + ffi::PyComplex_FromDoubles(complex.re.into(), complex.im.into()) + .assume_owned(py) + .downcast_into_unchecked() } } } @@ -183,13 +199,14 @@ complex_conversion!(f64); #[cfg(test)] mod tests { use super::*; + use crate::types::complex::PyComplexMethods; use crate::types::PyModule; #[test] fn from_complex() { Python::with_gil(|py| { let complex = Complex::new(3.0, 1.2); - let py_c = PyComplex::from_complex(py, complex); + let py_c = PyComplex::from_complex_bound(py, complex); assert_eq!(py_c.real(), 3.0); assert_eq!(py_c.imag(), 1.2); }); diff --git a/src/prelude.rs b/src/prelude.rs index 3ac239f49e1..5a342281eeb 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -30,6 +30,7 @@ pub use crate::types::boolobject::PyBoolMethods; pub use crate::types::bytearray::PyByteArrayMethods; pub use crate::types::bytes::PyBytesMethods; pub use crate::types::capsule::PyCapsuleMethods; +pub use crate::types::complex::PyComplexMethods; pub use crate::types::dict::PyDictMethods; pub use crate::types::float::PyFloatMethods; pub use crate::types::frozenset::PyFrozenSetMethods; diff --git a/src/types/mod.rs b/src/types/mod.rs index 46909c880bb..4ebf050b4b8 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -283,7 +283,7 @@ pub(crate) mod bytes; pub(crate) mod capsule; #[cfg(not(Py_LIMITED_API))] mod code; -mod complex; +pub(crate) mod complex; #[cfg(not(Py_LIMITED_API))] pub(crate) mod datetime; pub(crate) mod dict; From 4efc4b82a381391bdd200063e2774aed4ac3700d Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 19 Feb 2024 22:07:05 +0000 Subject: [PATCH 149/349] ci: fix redundant import warnings on nightly (#3873) --- guide/src/class/numeric.md | 1 - guide/src/python_from_rust.md | 5 ++--- pyo3-build-config/src/impl_.rs | 2 -- src/conversions/chrono.rs | 1 - src/conversions/hashbrown.rs | 1 - src/conversions/num_bigint.rs | 10 ++++------ src/conversions/rust_decimal.rs | 2 -- src/conversions/smallvec.rs | 2 +- src/conversions/std/map.rs | 1 - src/conversions/std/num.rs | 2 -- src/exceptions.rs | 2 +- src/marker.rs | 2 +- src/pyclass/create_type_object.rs | 1 - src/sync.rs | 2 +- src/types/capsule.rs | 2 +- src/types/dict.rs | 4 +--- src/types/function.rs | 2 +- src/types/list.rs | 1 - src/types/mapping.rs | 6 +----- src/types/string.rs | 3 --- src/types/traceback.rs | 4 ++-- src/types/tuple.rs | 1 - tests/test_class_conversion.rs | 1 - tests/test_exceptions.rs | 2 +- tests/test_gc.rs | 3 ++- tests/test_inheritance.rs | 2 +- tests/test_methods.rs | 1 - tests/test_proto_methods.rs | 2 +- tests/test_pyself.rs | 1 - tests/test_text_signature.rs | 2 +- tests/test_various.rs | 2 +- 31 files changed, 23 insertions(+), 50 deletions(-) diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index c6b6a65b711..cbd9db824a0 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -206,7 +206,6 @@ assert hash_djb2('l50_50') == Number(-1152549421) ```rust use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; -use std::convert::TryInto; use pyo3::exceptions::{PyValueError, PyZeroDivisionError}; use pyo3::prelude::*; diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 0b412c871c7..4306795e5b3 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -181,7 +181,7 @@ quickly testing your Python extensions. ```rust use pyo3::prelude::*; -use pyo3::{PyCell, py_run}; +use pyo3::py_run; # fn main() { #[pyclass] @@ -228,7 +228,7 @@ to this function! ```rust use pyo3::{ prelude::*, - types::{IntoPyDict, PyModule}, + types::IntoPyDict, }; # fn main() -> PyResult<()> { @@ -437,7 +437,6 @@ Use context managers by directly invoking `__enter__` and `__exit__`. ```rust use pyo3::prelude::*; -use pyo3::types::PyModule; fn main() { Python::with_gil(|py| { diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index fd467b72e0a..698c58a18ec 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -8,7 +8,6 @@ mod import_lib; use std::{ collections::{HashMap, HashSet}, - convert::AsRef, env, ffi::{OsStr, OsString}, fmt::Display, @@ -1785,7 +1784,6 @@ fn unescape(escaped: &str) -> Vec { #[cfg(test)] mod tests { - use std::iter::FromIterator; use target_lexicon::triple; use super::*; diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 6878b9d9b1e..c4fa106887a 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -1097,7 +1097,6 @@ mod tests { mod proptests { use super::*; use crate::tests::common::CatchWarnings; - use crate::types::any::PyAnyMethods; use crate::types::IntoPyDict; use proptest::prelude::*; diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index c9b6c61c6eb..8ac95d87f76 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -112,7 +112,6 @@ where #[cfg(test)] mod tests { use super::*; - use crate::types::any::PyAnyMethods; #[test] fn test_hashbrown_hashmap_to_python() { diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 3c98a90f092..652df154258 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -48,11 +48,11 @@ //! ``` #[cfg(Py_LIMITED_API)] -use crate::types::bytes::PyBytesMethods; +use crate::types::{bytes::PyBytesMethods, PyBytes}; use crate::{ ffi, instance::Bound, - types::{any::PyAnyMethods, *}, + types::{any::PyAnyMethods, PyLong}, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, }; @@ -85,7 +85,7 @@ macro_rules! bigint_conversion { let bytes = $to_bytes(self); let bytes_obj = PyBytes::new_bound(py, &bytes); let kwargs = if $is_signed > 0 { - let kwargs = PyDict::new_bound(py); + let kwargs = crate::types::PyDict::new_bound(py); kwargs.set_item(crate::intern!(py, "signed"), true).unwrap(); Some(kwargs) } else { @@ -224,7 +224,7 @@ fn int_to_py_bytes<'py>( use crate::intern; let py = long.py(); let kwargs = if is_signed { - let kwargs = PyDict::new_bound(py); + let kwargs = crate::types::PyDict::new_bound(py); kwargs.set_item(intern!(py, "signed"), true)?; Some(kwargs) } else { @@ -261,8 +261,6 @@ fn int_n_bits(long: &Bound<'_, PyLong>) -> PyResult { #[cfg(test)] mod tests { - use self::{any::PyAnyMethods, dict::PyDictMethods}; - use super::*; use crate::types::{PyDict, PyModule}; use indoc::indoc; diff --git a/src/conversions/rust_decimal.rs b/src/conversions/rust_decimal.rs index 1d0f64148fa..2e3151720e6 100644 --- a/src/conversions/rust_decimal.rs +++ b/src/conversions/rust_decimal.rs @@ -108,10 +108,8 @@ impl IntoPy for Decimal { mod test_rust_decimal { use super::*; use crate::err::PyErr; - use crate::types::any::PyAnyMethods; use crate::types::dict::PyDictMethods; use crate::types::PyDict; - use rust_decimal::Decimal; #[cfg(not(target_arch = "wasm32"))] use proptest::prelude::*; diff --git a/src/conversions/smallvec.rs b/src/conversions/smallvec.rs index f51da1de32a..96dbfad14b7 100644 --- a/src/conversions/smallvec.rs +++ b/src/conversions/smallvec.rs @@ -97,7 +97,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::types::{any::PyAnyMethods, PyDict, PyList}; + use crate::types::{PyDict, PyList}; #[test] fn test_smallvec_into_py() { diff --git a/src/conversions/std/map.rs b/src/conversions/std/map.rs index 8c6835d7215..c17caa8b252 100644 --- a/src/conversions/std/map.rs +++ b/src/conversions/std/map.rs @@ -111,7 +111,6 @@ where #[cfg(test)] mod tests { use super::*; - use crate::{IntoPy, PyObject, Python, ToPyObject}; use std::collections::{BTreeMap, HashMap}; #[test] diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 75c2e16b52b..027982461e9 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -5,7 +5,6 @@ use crate::{ exceptions, ffi, Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, }; -use std::convert::TryFrom; use std::num::{ NonZeroI128, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroIsize, NonZeroU128, NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, NonZeroUsize, @@ -371,7 +370,6 @@ nonzero_int_impl!(NonZeroUsize, usize); #[cfg(test)] mod test_128bit_integers { use super::*; - use crate::types::any::PyAnyMethods; #[cfg(not(target_arch = "wasm32"))] use crate::types::PyDict; diff --git a/src/exceptions.rs b/src/exceptions.rs index 6ae9febc695..cf471053264 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -842,7 +842,7 @@ mod tests { use super::*; use crate::types::any::PyAnyMethods; use crate::types::{IntoPyDict, PyDict}; - use crate::{PyErr, PyNativeType, Python}; + use crate::{PyErr, PyNativeType}; import_exception!(socket, gaierror); import_exception!(email.errors, MessageError); diff --git a/src/marker.rs b/src/marker.rs index 4b46d3badbd..8a52bbaf416 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -1166,7 +1166,7 @@ impl<'unbound> Python<'unbound> { #[cfg(test)] mod tests { use super::*; - use crate::types::{any::PyAnyMethods, IntoPyDict, PyDict, PyList}; + use crate::types::{IntoPyDict, PyList}; use std::sync::Arc; #[test] diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index a7196f30288..65d16e59511 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -17,7 +17,6 @@ use crate::{ use std::{ borrow::Cow, collections::HashMap, - convert::TryInto, ffi::{CStr, CString}, os::raw::{c_char, c_int, c_ulong, c_void}, ptr, diff --git a/src/sync.rs b/src/sync.rs index 8fff0e82c0f..f4ffe0409a7 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -275,7 +275,7 @@ impl Interned { mod tests { use super::*; - use crate::types::{any::PyAnyMethods, dict::PyDictMethods, PyDict}; + use crate::types::{dict::PyDictMethods, PyDict}; #[test] fn test_intern() { diff --git a/src/types/capsule.rs b/src/types/capsule.rs index 466315ad6c7..342ee9b44bc 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -1,8 +1,8 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::py_result_ext::PyResultExt; use crate::{ffi, PyAny, PyNativeType}; -use crate::{pyobject_native_type_core, PyErr, PyResult}; use crate::{Bound, Python}; +use crate::{PyErr, PyResult}; use std::ffi::{CStr, CString}; use std::os::raw::{c_char, c_int, c_void}; diff --git a/src/types/dict.rs b/src/types/dict.rs index 741d974e264..39a0c45e35e 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -735,9 +735,7 @@ mod tests { use super::*; #[cfg(not(PyPy))] use crate::exceptions; - #[cfg(not(PyPy))] - use crate::types::PyList; - use crate::{types::PyTuple, Python, ToPyObject}; + use crate::types::PyTuple; use std::collections::{BTreeMap, HashMap}; #[test] diff --git a/src/types/function.rs b/src/types/function.rs index 79f91cfe081..6c7db09aed3 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -1,7 +1,6 @@ use crate::derive_utils::PyFunctionArguments; use crate::ffi_ptr_ext::FfiPtrExt; use crate::methods::PyMethodDefDestructor; -use crate::prelude::*; use crate::py_result_ext::PyResultExt; use crate::types::capsule::PyCapsuleMethods; use crate::{ @@ -9,6 +8,7 @@ use crate::{ impl_::pymethods::{self, PyMethodDef}, types::{PyCapsule, PyDict, PyString, PyTuple}, }; +use crate::{Bound, IntoPy, Py, PyAny, PyResult, Python}; use std::cell::UnsafeCell; use std::ffi::CStr; diff --git a/src/types/list.rs b/src/types/list.rs index 1389d25ed29..3e3942bcc13 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -1,4 +1,3 @@ -use std::convert::TryInto; use std::iter::FusedIterator; use crate::err::{self, PyResult}; diff --git a/src/types/mapping.rs b/src/types/mapping.rs index 86c605f8bed..afbdae688b4 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -286,11 +286,7 @@ impl<'v> crate::PyTryFrom<'v> for PyMapping { mod tests { use std::collections::HashMap; - use crate::{ - exceptions::PyKeyError, - types::{PyDict, PyTuple}, - Python, - }; + use crate::{exceptions::PyKeyError, types::PyTuple}; use super::*; diff --git a/src/types/string.rs b/src/types/string.rs index 2e17f909df5..bc4419944bc 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -514,10 +514,7 @@ impl IntoPy> for &'_ Py { #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; - use crate::Python; use crate::{PyObject, ToPyObject}; - #[cfg(not(Py_LIMITED_API))] - use std::borrow::Cow; #[test] fn test_to_str_utf8() { diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 19f42d4b64c..33a1ec7c749 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -116,8 +116,8 @@ impl<'py> PyTracebackMethods<'py> for Bound<'py, PyTraceback> { #[cfg(test)] mod tests { use crate::{ - prelude::*, - types::{traceback::PyTracebackMethods, PyDict}, + types::{any::PyAnyMethods, dict::PyDictMethods, traceback::PyTracebackMethods, PyDict}, + IntoPy, PyErr, Python, }; #[test] diff --git a/src/types/tuple.rs b/src/types/tuple.rs index e41fcbf0020..caad7d9c0e9 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -1,4 +1,3 @@ -use std::convert::TryInto; use std::iter::FusedIterator; use crate::ffi::{self, Py_ssize_t}; diff --git a/tests/test_class_conversion.rs b/tests/test_class_conversion.rs index 217d2d59d45..29723b4ad97 100644 --- a/tests/test_class_conversion.rs +++ b/tests/test_class_conversion.rs @@ -1,7 +1,6 @@ #![cfg(feature = "macros")] use pyo3::prelude::*; -use pyo3::ToPyObject; #[macro_use] #[path = "../src/tests/common.rs"] diff --git a/tests/test_exceptions.rs b/tests/test_exceptions.rs index d6d6b46fcad..3603586033e 100644 --- a/tests/test_exceptions.rs +++ b/tests/test_exceptions.rs @@ -1,7 +1,7 @@ #![cfg(feature = "macros")] use pyo3::prelude::*; -use pyo3::{exceptions, py_run, PyErr, PyResult}; +use pyo3::{exceptions, py_run}; use std::error::Error; use std::fmt; #[cfg(not(target_os = "windows"))] diff --git a/tests/test_gc.rs b/tests/test_gc.rs index 9e236a27dbf..637596fd0c8 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -3,7 +3,7 @@ use pyo3::class::PyTraverseError; use pyo3::class::PyVisit; use pyo3::prelude::*; -use pyo3::{py_run, PyCell}; +use pyo3::py_run; use std::cell::Cell; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -398,6 +398,7 @@ impl HijackedTraverse { } } +#[allow(dead_code)] trait Traversable { fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>; } diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index af34974c90b..64c4eeffe89 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -177,7 +177,7 @@ except Exception as e: mod inheriting_native_type { use super::*; use pyo3::exceptions::PyException; - use pyo3::types::{IntoPyDict, PyDict}; + use pyo3::types::PyDict; #[cfg(not(PyPy))] #[test] diff --git a/tests/test_methods.rs b/tests/test_methods.rs index 96b40174928..e3ec3c76722 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -3,7 +3,6 @@ use pyo3::prelude::*; use pyo3::py_run; use pyo3::types::{IntoPyDict, PyDict, PyList, PySet, PyString, PyTuple, PyType}; -use pyo3::PyCell; #[path = "../src/tests/common.rs"] mod common; diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index 9126555df72..15c8a0e019c 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -2,7 +2,7 @@ use pyo3::exceptions::{PyAttributeError, PyIndexError, PyValueError}; use pyo3::types::{PyDict, PyList, PyMapping, PySequence, PySlice, PyType}; -use pyo3::{prelude::*, py_run, PyCell}; +use pyo3::{prelude::*, py_run}; use std::{isize, iter}; #[path = "../src/tests/common.rs"] diff --git a/tests/test_pyself.rs b/tests/test_pyself.rs index e14d117a639..5019360236e 100644 --- a/tests/test_pyself.rs +++ b/tests/test_pyself.rs @@ -3,7 +3,6 @@ //! Test slf: PyRef/PyMutRef(especially, slf.into::) works use pyo3::prelude::*; use pyo3::types::{PyBytes, PyString}; -use pyo3::PyCell; use std::collections::HashMap; #[path = "../src/tests/common.rs"] diff --git a/tests/test_text_signature.rs b/tests/test_text_signature.rs index b72ad2a2efb..cb2cd85e99b 100644 --- a/tests/test_text_signature.rs +++ b/tests/test_text_signature.rs @@ -2,7 +2,7 @@ use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{types::PyType, wrap_pymodule, PyCell}; +use pyo3::{types::PyType, wrap_pymodule}; #[path = "../src/tests/common.rs"] mod common; diff --git a/tests/test_various.rs b/tests/test_various.rs index a2ca12fbbda..a1c53fd28db 100644 --- a/tests/test_various.rs +++ b/tests/test_various.rs @@ -1,8 +1,8 @@ #![cfg(feature = "macros")] use pyo3::prelude::*; +use pyo3::py_run; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{py_run, PyCell}; use std::fmt; From ececa86a7d7742db5538e4f739dd5455212a34f1 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 19 Feb 2024 22:14:00 +0000 Subject: [PATCH 150/349] ci: cancel in-progress benches job on push (#3874) --- .github/workflows/benches.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 01ab5f802dd..6c4d31e074a 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -9,6 +9,10 @@ on: # performance analysis in order to generate initial data. workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}-benches + cancel-in-progress: true + jobs: benchmarks: runs-on: ubuntu-latest From 96b8c9facffbd6c4ad90353310a2ad88c561f28b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 19 Feb 2024 22:14:26 +0000 Subject: [PATCH 151/349] migrate some final `FromPyObject` implementations to the `Bound` API (#3869) * update `Py::extract` to use `extract_bound` * update docstring of `FromPyObject` * move `Option` conversions to new module & update * move `Cell` conversions to new module & update --- src/conversion.rs | 121 ++++++---------------------------- src/conversions/std/cell.rs | 24 +++++++ src/conversions/std/mod.rs | 2 + src/conversions/std/option.rs | 73 ++++++++++++++++++++ src/instance.rs | 4 +- 5 files changed, 120 insertions(+), 104 deletions(-) create mode 100644 src/conversions/std/cell.rs create mode 100644 src/conversions/std/option.rs diff --git a/src/conversion.rs b/src/conversion.rs index d9536aa9445..d9f79ffa104 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -8,7 +8,6 @@ use crate::types::PyTuple; use crate::{ ffi, gil, Bound, Py, PyAny, PyCell, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, }; -use std::cell::Cell; use std::ptr::NonNull; /// Returns a borrowed pointer to a Python object. @@ -63,18 +62,6 @@ pub unsafe trait AsPyPointer { fn as_ptr(&self) -> *mut ffi::PyObject; } -/// Convert `None` into a null pointer. -unsafe impl AsPyPointer for Option -where - T: AsPyPointer, -{ - #[inline] - fn as_ptr(&self) -> *mut ffi::PyObject { - self.as_ref() - .map_or_else(std::ptr::null_mut, |t| t.as_ptr()) - } -} - /// Conversion trait that allows various objects to be converted into `PyObject`. pub trait ToPyObject { /// Converts self into a Python object. @@ -180,7 +167,7 @@ pub trait IntoPy: Sized { /// Extract a type from a Python object. /// /// -/// Normal usage is through the `extract` methods on [`Py`] and [`PyAny`], which forward to this trait. +/// Normal usage is through the `extract` methods on [`Bound`] and [`Py`], which forward to this trait. /// /// # Examples /// @@ -190,30 +177,32 @@ pub trait IntoPy: Sized { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { -/// let obj: Py = PyString::new_bound(py, "blah").into(); -/// -/// // Straight from an owned reference -/// let s: &str = obj.extract(py)?; +/// // Calling `.extract()` on a `Bound` smart pointer +/// let obj: Bound<'_, PyString> = PyString::new_bound(py, "blah"); +/// let s: &str = obj.extract()?; /// # assert_eq!(s, "blah"); /// -/// // Or from a borrowed reference -/// let obj: &PyString = obj.as_ref(py); -/// let s: &str = obj.extract()?; +/// // Calling `.extract(py)` on a `Py` smart pointer +/// let obj: Py = obj.unbind(); +/// let s: &str = obj.extract(py)?; /// # assert_eq!(s, "blah"); /// # Ok(()) /// }) /// # } /// ``` /// -/// Note: depending on the implementation, the lifetime of the extracted result may -/// depend on the lifetime of the `obj` or the `prepared` variable. -/// -/// For example, when extracting `&str` from a Python byte string, the resulting string slice will -/// point to the existing string data (lifetime: `'py`). -/// On the other hand, when extracting `&str` from a Python Unicode string, the preparation step -/// will convert the string to UTF-8, and the resulting string slice will have lifetime `'prepared`. -/// Since which case applies depends on the runtime type of the Python object, -/// both the `obj` and `prepared` variables must outlive the resulting string slice. +// /// FIXME: until `FromPyObject` can pick up a second lifetime, the below commentary is no longer +// /// true. Update and restore this documentation at that time. +// /// +// /// Note: depending on the implementation, the lifetime of the extracted result may +// /// depend on the lifetime of the `obj` or the `prepared` variable. +// /// +// /// For example, when extracting `&str` from a Python byte string, the resulting string slice will +// /// point to the existing string data (lifetime: `'py`). +// /// On the other hand, when extracting `&str` from a Python Unicode string, the preparation step +// /// will convert the string to UTF-8, and the resulting string slice will have lifetime `'prepared`. +// /// Since which case applies depends on the runtime type of the Python object, +// /// both the `obj` and `prepared` variables must outlive the resulting string slice. /// /// During the migration of PyO3 from the "GIL Refs" API to the `Bound` smart pointer, this trait /// has two methods `extract` and `extract_bound` which are defaulted to call each other. To avoid @@ -258,27 +247,6 @@ impl ToPyObject for &'_ T { } } -/// `Option::Some` is converted like `T`. -/// `Option::None` is converted to Python `None`. -impl ToPyObject for Option -where - T: ToPyObject, -{ - fn to_object(&self, py: Python<'_>) -> PyObject { - self.as_ref() - .map_or_else(|| py.None(), |val| val.to_object(py)) - } -} - -impl IntoPy for Option -where - T: IntoPy, -{ - fn into_py(self, py: Python<'_>) -> PyObject { - self.map_or_else(|| py.None(), |val| val.into_py(py)) - } -} - impl IntoPy for &'_ PyAny { #[inline] fn into_py(self, py: Python<'_>) -> PyObject { @@ -296,24 +264,6 @@ where } } -impl ToPyObject for Cell { - fn to_object(&self, py: Python<'_>) -> PyObject { - self.get().to_object(py) - } -} - -impl> IntoPy for Cell { - fn into_py(self, py: Python<'_>) -> PyObject { - self.get().into_py(py) - } -} - -impl<'py, T: FromPyObject<'py>> FromPyObject<'py> for Cell { - fn extract(ob: &'py PyAny) -> PyResult { - T::extract(ob).map(Cell::new) - } -} - impl<'py, T> FromPyObject<'py> for &'py PyCell where T: PyClass, @@ -353,19 +303,6 @@ where } } -impl<'py, T> FromPyObject<'py> for Option -where - T: FromPyObject<'py>, -{ - fn extract(obj: &'py PyAny) -> PyResult { - if obj.as_ptr() == unsafe { ffi::Py_None() } { - Ok(None) - } else { - T::extract(obj).map(Some) - } - } -} - /// Trait implemented by Python object types that allow a checked downcast. /// If `T` implements `PyTryFrom`, we can convert `&PyAny` to `&T`. /// @@ -593,8 +530,6 @@ mod test_no_clone {} #[cfg(test)] mod tests { - use crate::{PyObject, Python}; - #[allow(deprecated)] mod deprecated { use super::super::PyTryFrom; @@ -638,22 +573,4 @@ mod tests { }); } } - - #[test] - fn test_option_as_ptr() { - Python::with_gil(|py| { - use crate::AsPyPointer; - let mut option: Option = None; - assert_eq!(option.as_ptr(), std::ptr::null_mut()); - - let none = py.None(); - option = Some(none.clone()); - - let ref_cnt = none.get_refcnt(py); - assert_eq!(option.as_ptr(), none.as_ptr()); - - // Ensure ref count not changed by as_ptr call - assert_eq!(none.get_refcnt(py), ref_cnt); - }); - } } diff --git a/src/conversions/std/cell.rs b/src/conversions/std/cell.rs new file mode 100644 index 00000000000..69d990910d3 --- /dev/null +++ b/src/conversions/std/cell.rs @@ -0,0 +1,24 @@ +use std::cell::Cell; + +use crate::{ + types::any::PyAnyMethods, Bound, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, + ToPyObject, +}; + +impl ToPyObject for Cell { + fn to_object(&self, py: Python<'_>) -> PyObject { + self.get().to_object(py) + } +} + +impl> IntoPy for Cell { + fn into_py(self, py: Python<'_>) -> PyObject { + self.get().into_py(py) + } +} + +impl<'py, T: FromPyObject<'py>> FromPyObject<'py> for Cell { + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { + ob.extract().map(Cell::new) + } +} diff --git a/src/conversions/std/mod.rs b/src/conversions/std/mod.rs index 9b10b59fd3f..305344b1284 100644 --- a/src/conversions/std/mod.rs +++ b/src/conversions/std/mod.rs @@ -1,7 +1,9 @@ mod array; +mod cell; mod ipaddr; mod map; mod num; +mod option; mod osstr; mod path; mod set; diff --git a/src/conversions/std/option.rs b/src/conversions/std/option.rs new file mode 100644 index 00000000000..2fa082ba16a --- /dev/null +++ b/src/conversions/std/option.rs @@ -0,0 +1,73 @@ +use crate::{ + ffi, types::any::PyAnyMethods, AsPyPointer, Bound, FromPyObject, IntoPy, PyAny, PyObject, + PyResult, Python, ToPyObject, +}; + +/// `Option::Some` is converted like `T`. +/// `Option::None` is converted to Python `None`. +impl ToPyObject for Option +where + T: ToPyObject, +{ + fn to_object(&self, py: Python<'_>) -> PyObject { + self.as_ref() + .map_or_else(|| py.None(), |val| val.to_object(py)) + } +} + +impl IntoPy for Option +where + T: IntoPy, +{ + fn into_py(self, py: Python<'_>) -> PyObject { + self.map_or_else(|| py.None(), |val| val.into_py(py)) + } +} + +impl<'py, T> FromPyObject<'py> for Option +where + T: FromPyObject<'py>, +{ + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { + if obj.is_none() { + Ok(None) + } else { + obj.extract().map(Some) + } + } +} + +/// Convert `None` into a null pointer. +unsafe impl AsPyPointer for Option +where + T: AsPyPointer, +{ + #[inline] + fn as_ptr(&self) -> *mut ffi::PyObject { + self.as_ref() + .map_or_else(std::ptr::null_mut, |t| t.as_ptr()) + } +} + +#[cfg(test)] +mod tests { + use crate::{PyObject, Python}; + + #[test] + fn test_option_as_ptr() { + Python::with_gil(|py| { + use crate::AsPyPointer; + let mut option: Option = None; + assert_eq!(option.as_ptr(), std::ptr::null_mut()); + + let none = py.None(); + option = Some(none.clone()); + + let ref_cnt = none.get_refcnt(py); + assert_eq!(option.as_ptr(), none.as_ptr()); + + // Ensure ref count not changed by as_ptr call + assert_eq!(none.get_refcnt(py), ref_cnt); + }); + } +} diff --git a/src/instance.rs b/src/instance.rs index b83b69a20d3..a280f179f05 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1250,11 +1250,11 @@ impl Py { /// Extracts some type from the Python object. /// /// This is a wrapper function around `FromPyObject::extract()`. - pub fn extract<'py, D>(&'py self, py: Python<'py>) -> PyResult + pub fn extract<'py, D>(&self, py: Python<'py>) -> PyResult where D: FromPyObject<'py>, { - FromPyObject::extract(unsafe { py.from_borrowed_ptr(self.as_ptr()) }) + self.bind(py).as_any().extract() } /// Retrieves an attribute value. From 9a36b5078989a7c07a5e880aea3c6da205585ee3 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 19 Feb 2024 22:15:36 +0000 Subject: [PATCH 152/349] update `py.import` -> `py.import_bound` in benches (#3868) --- pyo3-benches/benches/bench_intern.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyo3-benches/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs index d8dd1b8fd30..806c4e95a9d 100644 --- a/pyo3-benches/benches/bench_intern.rs +++ b/pyo3-benches/benches/bench_intern.rs @@ -6,7 +6,7 @@ use pyo3::intern; fn getattr_direct(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let sys = py.import("sys").unwrap(); + let sys = py.import_bound("sys").unwrap(); b.iter(|| sys.getattr("version").unwrap()); }); @@ -14,7 +14,7 @@ fn getattr_direct(b: &mut Bencher<'_>) { fn getattr_intern(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let sys = py.import("sys").unwrap(); + let sys = py.import_bound("sys").unwrap(); b.iter(|| sys.getattr(intern!(py, "version")).unwrap()); }); From 76dabd4e601b6200141ac2cac92d7069c23a23f9 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Mon, 19 Feb 2024 22:39:54 +0000 Subject: [PATCH 153/349] Replace as_ref(py) with Bound APIs (#3863) --- src/err/mod.rs | 2 +- src/pyclass/create_type_object.rs | 3 ++- tests/test_methods.rs | 8 +++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/err/mod.rs b/src/err/mod.rs index d922baaad21..46f482a6879 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -957,7 +957,7 @@ impl PyErrArguments for PyDowncastErrorArguments { format!( "'{}' object cannot be converted to '{}'", self.from - .as_ref(py) + .bind(py) .qualname() .as_deref() .unwrap_or(""), diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index 65d16e59511..d0c271cf31a 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -11,6 +11,7 @@ use crate::{ pymethods::{get_doc, get_name, Getter, Setter}, trampoline::trampoline, }, + types::typeobject::PyTypeMethods, types::PyType, Py, PyCell, PyClass, PyGetterDef, PyMethodDefType, PyResult, PySetterDef, PyTypeInfo, Python, }; @@ -434,7 +435,7 @@ impl PyTypeBuilder { bpo_45315_workaround(py, class_name); for cleanup in std::mem::take(&mut self.cleanup) { - cleanup(&self, type_object.as_ref(py).as_type_ptr()); + cleanup(&self, type_object.bind(py).as_type_ptr()); } Ok(PyClassTypeObject { diff --git a/tests/test_methods.rs b/tests/test_methods.rs index e3ec3c76722..7ca75016aec 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -73,7 +73,7 @@ impl ClassMethod { #[classmethod] /// Test class method. fn method(cls: &Bound<'_, PyType>) -> PyResult { - Ok(format!("{}.method()!", cls.as_gil_ref().qualname()?)) + Ok(format!("{}.method()!", cls.qualname()?)) } #[classmethod] @@ -84,10 +84,8 @@ impl ClassMethod { #[classmethod] fn method_owned(cls: Py) -> PyResult { - Ok(format!( - "{}.method_owned()!", - Python::with_gil(|gil| cls.as_ref(gil).qualname())? - )) + let qualname = Python::with_gil(|gil| cls.bind(gil).qualname())?; + Ok(format!("{}.method_owned()!", qualname)) } } From 7a03a6fe6def65b445c5af424e1f44d8574462eb Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 19 Feb 2024 22:40:08 +0000 Subject: [PATCH 154/349] ci: disable some benchmarks volatile on codspeed (#3861) --- pyo3-benches/benches/bench_dict.rs | 2 ++ pyo3-benches/benches/bench_extract.rs | 9 +++++++++ pyo3-benches/benches/bench_frompyobject.rs | 8 ++++++++ pyo3-benches/benches/bench_list.rs | 2 ++ pyo3-benches/benches/bench_tuple.rs | 2 ++ 5 files changed, 23 insertions(+) diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs index 072dd9408ce..b63e010c1ff 100644 --- a/pyo3-benches/benches/bench_dict.rs +++ b/pyo3-benches/benches/bench_dict.rs @@ -69,6 +69,7 @@ fn extract_hashbrown_map(b: &mut Bencher<'_>) { }); } +#[cfg(not(codspeed))] fn mapping_from_dict(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; @@ -87,6 +88,7 @@ fn criterion_benchmark(c: &mut Criterion) { #[cfg(feature = "hashbrown")] c.bench_function("extract_hashbrown_map", extract_hashbrown_map); + #[cfg(not(codspeed))] c.bench_function("mapping_from_dict", mapping_from_dict); } diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs index 1c783c3b706..434d8eb5b33 100644 --- a/pyo3-benches/benches/bench_extract.rs +++ b/pyo3-benches/benches/bench_extract.rs @@ -70,6 +70,7 @@ fn extract_int_extract_fail(bench: &mut Bencher<'_>) { }); } +#[cfg(not(codspeed))] fn extract_int_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let int_obj: PyObject = 123.into_py(py); @@ -83,6 +84,7 @@ fn extract_int_downcast_success(bench: &mut Bencher<'_>) { }); } +#[cfg(not(codspeed))] fn extract_int_downcast_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let d = PyDict::new_bound(py).into_any(); @@ -94,6 +96,7 @@ fn extract_int_downcast_fail(bench: &mut Bencher<'_>) { }); } +#[cfg(not(codspeed))] fn extract_float_extract_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let float_obj: PyObject = 23.42.into_py(py); @@ -117,6 +120,7 @@ fn extract_float_extract_fail(bench: &mut Bencher<'_>) { }); } +#[cfg(not(codspeed))] fn extract_float_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let float_obj: PyObject = 23.42.into_py(py); @@ -147,15 +151,20 @@ fn criterion_benchmark(c: &mut Criterion) { #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] c.bench_function("extract_str_downcast_success", extract_str_downcast_success); c.bench_function("extract_str_downcast_fail", extract_str_downcast_fail); + #[cfg(not(codspeed))] c.bench_function("extract_int_extract_success", extract_int_extract_success); c.bench_function("extract_int_extract_fail", extract_int_extract_fail); + #[cfg(not(codspeed))] c.bench_function("extract_int_downcast_success", extract_int_downcast_success); + #[cfg(not(codspeed))] c.bench_function("extract_int_downcast_fail", extract_int_downcast_fail); + #[cfg(not(codspeed))] c.bench_function( "extract_float_extract_success", extract_float_extract_success, ); c.bench_function("extract_float_extract_fail", extract_float_extract_fail); + #[cfg(not(codspeed))] c.bench_function( "extract_float_downcast_success", extract_float_downcast_success, diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs index 8114ee5a802..e78c48fcaa1 100644 --- a/pyo3-benches/benches/bench_frompyobject.rs +++ b/pyo3-benches/benches/bench_frompyobject.rs @@ -20,6 +20,7 @@ fn enum_from_pyobject(b: &mut Bencher<'_>) { }) } +#[cfg(not(codspeed))] fn list_via_downcast(b: &mut Bencher<'_>) { Python::with_gil(|py| { let any: &Bound<'_, PyAny> = &PyList::empty_bound(py); @@ -28,6 +29,7 @@ fn list_via_downcast(b: &mut Bencher<'_>) { }) } +#[cfg(not(codspeed))] fn list_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { let any: &Bound<'_, PyAny> = &PyList::empty_bound(py); @@ -36,6 +38,7 @@ fn list_via_extract(b: &mut Bencher<'_>) { }) } +#[cfg(not(codspeed))] fn not_a_list_via_downcast(b: &mut Bencher<'_>) { Python::with_gil(|py| { let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "foobar"); @@ -70,6 +73,7 @@ fn not_a_list_via_extract_enum(b: &mut Bencher<'_>) { }) } +#[cfg(not(codspeed))] fn f64_from_pyobject(b: &mut Bencher<'_>) { Python::with_gil(|py| { let obj = &PyFloat::new_bound(py, 1.234); @@ -79,11 +83,15 @@ fn f64_from_pyobject(b: &mut Bencher<'_>) { fn criterion_benchmark(c: &mut Criterion) { c.bench_function("enum_from_pyobject", enum_from_pyobject); + #[cfg(not(codspeed))] c.bench_function("list_via_downcast", list_via_downcast); + #[cfg(not(codspeed))] c.bench_function("list_via_extract", list_via_extract); + #[cfg(not(codspeed))] c.bench_function("not_a_list_via_downcast", not_a_list_via_downcast); c.bench_function("not_a_list_via_extract", not_a_list_via_extract); c.bench_function("not_a_list_via_extract_enum", not_a_list_via_extract_enum); + #[cfg(not(codspeed))] c.bench_function("f64_from_pyobject", f64_from_pyobject); } diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs index e0f238c5599..dddd04d81ee 100644 --- a/pyo3-benches/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -55,6 +55,7 @@ fn list_get_item_unchecked(b: &mut Bencher<'_>) { }); } +#[cfg(not(codspeed))] fn sequence_from_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; @@ -69,6 +70,7 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("list_get_item", list_get_item); #[cfg(not(Py_LIMITED_API))] c.bench_function("list_get_item_unchecked", list_get_item_unchecked); + #[cfg(not(codspeed))] c.bench_function("sequence_from_list", sequence_from_list); } diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs index 24f32fac364..22bf7588ed1 100644 --- a/pyo3-benches/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -89,6 +89,7 @@ fn tuple_get_borrowed_item_unchecked(b: &mut Bencher<'_>) { }); } +#[cfg(not(codspeed))] fn sequence_from_tuple(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; @@ -131,6 +132,7 @@ fn criterion_benchmark(c: &mut Criterion) { "tuple_get_borrowed_item_unchecked", tuple_get_borrowed_item_unchecked, ); + #[cfg(not(codspeed))] c.bench_function("sequence_from_tuple", sequence_from_tuple); c.bench_function("tuple_new_list", tuple_new_list); c.bench_function("tuple_to_list", tuple_to_list); From 8ac7834f986d087293756b7d6b471399f3099051 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 20 Feb 2024 07:08:49 +0000 Subject: [PATCH 155/349] docs: update example for storing Py in structs (#3876) --- src/instance.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index a280f179f05..5049a66bb32 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -656,9 +656,14 @@ impl IntoPy for Borrowed<'_, '_, T> { /// These require passing in the [`Python<'py>`](crate::Python) token but are otherwise similar to the corresponding /// methods on [`PyAny`]. /// -/// # Example: Storing Python objects in structs +/// # Example: Storing Python objects in `#[pyclass]` structs +/// +/// Usually `Bound<'py, T>` is recommended for interacting with Python objects as its lifetime `'py` +/// is an association to the GIL and that enables many operations to be done as efficiently as possible. +/// +/// However, `#[pyclass]` structs cannot carry a lifetime, so `Py` is the only way to store +/// a Python object in a `#[pyclass]` struct. /// -/// As all the native Python objects only appear as references, storing them in structs doesn't work well. /// For example, this won't compile: /// /// ```compile_fail @@ -667,7 +672,7 @@ impl IntoPy for Borrowed<'_, '_, T> { /// # /// #[pyclass] /// struct Foo<'py> { -/// inner: &'py PyDict, +/// inner: Bound<'py, PyDict>, /// } /// /// impl Foo { @@ -675,9 +680,9 @@ impl IntoPy for Borrowed<'_, '_, T> { /// let foo = Python::with_gil(|py| { /// // `py` will only last for this scope. /// -/// // `&PyDict` derives its lifetime from `py` and +/// // `Bound<'py, PyDict>` inherits the GIL lifetime from `py` and /// // so won't be able to outlive this closure. -/// let dict: &PyDict = PyDict::new(py); +/// let dict: Bound<'_, PyDict> = PyDict::new_bound(py); /// /// // because `Foo` contains `dict` its lifetime /// // is now also tied to `py`. From a93900686ecd6292cecbd7d55db1a7ae691d7bcd Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Tue, 20 Feb 2024 07:10:45 +0000 Subject: [PATCH 156/349] Deprecate Py::into_ref (#3867) * Migrate into_ref calls to Bound api * Mark Py::into_ref as deprecated --- guide/src/types.md | 3 +++ src/conversions/chrono.rs | 2 +- src/exceptions.rs | 5 +++-- src/impl_/pymodule.rs | 4 ++-- src/instance.rs | 8 ++++++++ src/marker.rs | 2 ++ src/types/any.rs | 20 ++++++++++---------- src/types/traceback.rs | 2 +- tests/test_proto_methods.rs | 32 +++++++++++++++++++++----------- 9 files changed, 51 insertions(+), 27 deletions(-) diff --git a/guide/src/types.md b/guide/src/types.md index 882baddb33c..1a926f43dab 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -92,6 +92,7 @@ For a `&PyAny` object reference `any` where the underlying object is a `#[pyclas # use pyo3::prelude::*; # #[pyclass] #[derive(Clone)] struct MyClass { } # Python::with_gil(|py| -> PyResult<()> { +# #[allow(deprecated)] let obj: &PyAny = Py::new(py, MyClass {})?.into_ref(py); // To &PyCell with PyAny::downcast @@ -182,6 +183,7 @@ let _: &PyList = list.as_ref(py); # let list_clone = list.clone(); // Because `.into_ref()` will consume `list`. // To &PyList with Py::into_ref() (moves the pointer into PyO3's object storage) +# #[allow(deprecated)] let _: &PyList = list.into_ref(py); # let list = list_clone; @@ -204,6 +206,7 @@ let _: &PyCell = my_class.as_ref(py); # let my_class_clone = my_class.clone(); // Because `.into_ref()` will consume `my_class`. // To &PyCell with Py::into_ref() (moves the pointer into PyO3's object storage) +# #[allow(deprecated)] let _: &PyCell = my_class.into_ref(py); # let my_class = my_class_clone.clone(); diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index c4fa106887a..3aa6dbeb4ca 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -636,7 +636,7 @@ mod tests { // Test that if a user tries to convert a python's timezone aware datetime into a naive // one, the conversion fails. Python::with_gil(|py| { - let none = py.None().into_ref(py); + let none = py.None().into_bound(py); assert_eq!( none.extract::().unwrap_err().to_string(), "TypeError: 'NoneType' object cannot be converted to 'PyDelta'" diff --git a/src/exceptions.rs b/src/exceptions.rs index cf471053264..70809448bff 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -1008,7 +1008,7 @@ mod tests { .run_bound("raise Exception('banana')", None, None) .expect_err("raising should have given us an error") .into_value(py) - .into_ref(py); + .into_bound(py); assert_eq!( format!("{:?}", exc), exc.repr().unwrap().extract::().unwrap() @@ -1023,7 +1023,7 @@ mod tests { .run_bound("raise Exception('banana')", None, None) .expect_err("raising should have given us an error") .into_value(py) - .into_ref(py); + .into_bound(py); assert_eq!( exc.to_string(), exc.str().unwrap().extract::().unwrap() @@ -1036,6 +1036,7 @@ mod tests { use std::error::Error; Python::with_gil(|py| { + #[allow(deprecated)] let exc = py .run_bound( "raise Exception('banana') from TypeError('peach')", diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index ff72285558a..7103f8b3938 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -137,7 +137,7 @@ impl ModuleDef { mod tests { use std::sync::atomic::{AtomicBool, Ordering}; - use crate::{types::PyModule, PyResult, Python}; + use crate::{types::any::PyAnyMethods, types::PyModule, PyResult, Python}; use super::{ModuleDef, ModuleInitializer}; @@ -154,7 +154,7 @@ mod tests { ) }; Python::with_gil(|py| { - let module = MODULE_DEF.make_module(py).unwrap().into_ref(py); + let module = MODULE_DEF.make_module(py).unwrap().into_bound(py); assert_eq!( module .getattr("__name__") diff --git a/src/instance.rs b/src/instance.rs index 5049a66bb32..3f19637e3b3 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -958,9 +958,17 @@ where /// // This reference's lifetime is determined by `py`'s lifetime. /// // Because that originates from outside this function, /// // this return value is allowed. + /// # #[allow(deprecated)] /// obj.into_ref(py) /// } /// ``` + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `obj.into_bound(py)` instead of `obj.into_ref(py)`" + ) + )] pub fn into_ref(self, py: Python<'_>) -> &T::AsRefTarget { unsafe { py.from_owned_ptr(self.into_ptr()) } } diff --git a/src/marker.rs b/src/marker.rs index 8a52bbaf416..29583e8e080 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -853,6 +853,7 @@ impl<'py> Python<'py> { where T: PyTypeCheck, { + #[allow(deprecated)] obj.into_ref(self).downcast() } @@ -873,6 +874,7 @@ impl<'py> Python<'py> { where T: HasPyGilRef, { + #[allow(deprecated)] obj.into_ref(self).downcast_unchecked() } diff --git a/src/types/any.rs b/src/types/any.rs index 1cb6bb779f1..4d371764d29 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -259,8 +259,8 @@ impl PyAny { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let a: &PyInt = 0_u8.into_py(py).into_ref(py).downcast()?; - /// let b: &PyInt = 42_u8.into_py(py).into_ref(py).downcast()?; + /// let a: Bound<'_, PyInt> = 0_u8.into_py(py).into_bound(py).downcast_into()?; + /// let b: Bound<'_, PyInt> = 42_u8.into_py(py).into_bound(py).downcast_into()?; /// assert!(a.rich_compare(b, CompareOp::Le)?.is_truthy()?); /// Ok(()) /// })?; @@ -715,11 +715,11 @@ impl PyAny { /// } /// /// Python::with_gil(|py| { - /// let class: &PyAny = Py::new(py, Class { i: 0 }).unwrap().into_ref(py); + /// let class = Py::new(py, Class { i: 0 }).unwrap().into_bound(py).into_any(); /// - /// let class_cell: &PyCell = class.downcast()?; + /// let class_bound: &Bound<'_, Class> = class.downcast()?; /// - /// class_cell.borrow_mut().i += 1; + /// class_bound.borrow_mut().i += 1; /// /// // Alternatively you can get a `PyRefMut` directly /// let class_ref: PyRefMut<'_, Class> = class.extract()?; @@ -1112,8 +1112,8 @@ pub trait PyAnyMethods<'py> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let a: &PyInt = 0_u8.into_py(py).into_ref(py).downcast()?; - /// let b: &PyInt = 42_u8.into_py(py).into_ref(py).downcast()?; + /// let a: Bound<'_, PyInt> = 0_u8.into_py(py).into_bound(py).downcast_into()?; + /// let b: Bound<'_, PyInt> = 42_u8.into_py(py).into_bound(py).downcast_into()?; /// assert!(a.rich_compare(b, CompareOp::Le)?.is_truthy()?); /// Ok(()) /// })?; @@ -1543,11 +1543,11 @@ pub trait PyAnyMethods<'py> { /// } /// /// Python::with_gil(|py| { - /// let class: &PyAny = Py::new(py, Class { i: 0 }).unwrap().into_ref(py); + /// let class = Py::new(py, Class { i: 0 }).unwrap().into_bound(py).into_any(); /// - /// let class_cell: &PyCell = class.downcast()?; + /// let class_bound: &Bound<'_, Class> = class.downcast()?; /// - /// class_cell.borrow_mut().i += 1; + /// class_bound.borrow_mut().i += 1; /// /// // Alternatively you can get a `PyRefMut` directly /// let class_ref: PyRefMut<'_, Class> = class.extract()?; diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 33a1ec7c749..24b935e24c2 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -173,7 +173,7 @@ def f(): let f = locals.get_item("f").unwrap().unwrap(); let err = f.call0().unwrap_err(); let traceback = err.traceback_bound(py).unwrap(); - let err_object = err.clone_ref(py).into_py(py).into_ref(py); + let err_object = err.clone_ref(py).into_py(py).into_bound(py); assert!(err_object.getattr("__traceback__").unwrap().is(&traceback)); }) diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index 15c8a0e019c..c57ca6a3e0c 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -64,8 +64,8 @@ impl ExampleClass { } } -fn make_example(py: Python<'_>) -> &PyCell { - Py::new( +fn make_example(py: Python<'_>) -> Bound<'_, ExampleClass> { + Bound::new( py, ExampleClass { value: 5, @@ -73,7 +73,6 @@ fn make_example(py: Python<'_>) -> &PyCell { }, ) .unwrap() - .into_ref(py) } #[test] @@ -82,6 +81,7 @@ fn test_getattr() { let example_py = make_example(py); assert_eq!( example_py + .as_any() .getattr("value") .unwrap() .extract::() @@ -90,6 +90,7 @@ fn test_getattr() { ); assert_eq!( example_py + .as_any() .getattr("special_custom_attr") .unwrap() .extract::() @@ -97,6 +98,7 @@ fn test_getattr() { 20, ); assert!(example_py + .as_any() .getattr("other_attr") .unwrap_err() .is_instance_of::(py)); @@ -107,9 +109,13 @@ fn test_getattr() { fn test_setattr() { Python::with_gil(|py| { let example_py = make_example(py); - example_py.setattr("special_custom_attr", 15).unwrap(); + example_py + .as_any() + .setattr("special_custom_attr", 15) + .unwrap(); assert_eq!( example_py + .as_any() .getattr("special_custom_attr") .unwrap() .extract::() @@ -123,8 +129,12 @@ fn test_setattr() { fn test_delattr() { Python::with_gil(|py| { let example_py = make_example(py); - example_py.delattr("special_custom_attr").unwrap(); - assert!(example_py.getattr("special_custom_attr").unwrap().is_none()); + example_py.as_any().delattr("special_custom_attr").unwrap(); + assert!(example_py + .as_any() + .getattr("special_custom_attr") + .unwrap() + .is_none()); }) } @@ -132,7 +142,7 @@ fn test_delattr() { fn test_str() { Python::with_gil(|py| { let example_py = make_example(py); - assert_eq!(example_py.str().unwrap().to_str().unwrap(), "5"); + assert_eq!(example_py.as_any().str().unwrap().to_cow().unwrap(), "5"); }) } @@ -141,7 +151,7 @@ fn test_repr() { Python::with_gil(|py| { let example_py = make_example(py); assert_eq!( - example_py.repr().unwrap().to_str().unwrap(), + example_py.as_any().repr().unwrap().to_cow().unwrap(), "ExampleClass(value=5)" ); }) @@ -151,7 +161,7 @@ fn test_repr() { fn test_hash() { Python::with_gil(|py| { let example_py = make_example(py); - assert_eq!(example_py.hash().unwrap(), 5); + assert_eq!(example_py.as_any().hash().unwrap(), 5); }) } @@ -159,9 +169,9 @@ fn test_hash() { fn test_bool() { Python::with_gil(|py| { let example_py = make_example(py); - assert!(example_py.is_truthy().unwrap()); + assert!(example_py.as_any().is_truthy().unwrap()); example_py.borrow_mut().value = 0; - assert!(!example_py.is_truthy().unwrap()); + assert!(!example_py.as_any().is_truthy().unwrap()); }) } From 61bc02d9276928708e6eacc14ed2f512fc7541f5 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 20 Feb 2024 08:45:47 +0100 Subject: [PATCH 157/349] deprecate `PyCell::new` in favor of `Py::new` or `Bound::new` (#3872) * deprecate `PyCell::new` in favor of `Py::new` or `Bound::new` * update deprecation warning Co-authored-by: David Hewitt --------- Co-authored-by: David Hewitt --- guide/src/class.md | 5 +++-- guide/src/class/object.md | 4 ++-- guide/src/class/protocols.md | 2 +- guide/src/migration.md | 4 ++-- guide/src/python_from_rust.md | 2 +- guide/src/types.md | 2 ++ src/macros.rs | 2 +- src/pycell.rs | 30 ++++++++++++++++++++++++++---- tests/test_arithmetics.rs | 22 +++++++++++----------- tests/test_class_basics.rs | 14 +++++++------- tests/test_class_conversion.rs | 18 ++++++++++-------- tests/test_class_new.rs | 12 +++--------- tests/test_gc.rs | 8 ++++---- tests/test_inheritance.rs | 8 ++++---- tests/test_methods.rs | 10 +++++----- tests/test_proto_methods.rs | 12 ++++++------ tests/test_pyself.rs | 2 +- tests/test_sequence.rs | 18 ++++++++++-------- tests/test_various.rs | 10 +++++----- 19 files changed, 104 insertions(+), 81 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index 9a2d91c1cb1..9ebba38a820 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -216,6 +216,7 @@ struct MyClass { num: i32, } Python::with_gil(|py| { +# #[allow(deprecated)] let obj = PyCell::new(py, MyClass { num: 3 }).unwrap(); { let obj_ref = obj.borrow(); // Get PyRef @@ -397,7 +398,7 @@ impl SubSubClass { } } # Python::with_gil(|py| { -# let subsub = pyo3::PyCell::new(py, SubSubClass::new()).unwrap(); +# let subsub = pyo3::Py::new(py, SubSubClass::new()).unwrap(); # pyo3::py_run!(py, subsub, "assert subsub.method3() == 3000"); # let subsub = SubSubClass::factory_method(py, 2).unwrap(); # let subsubsub = SubSubClass::factory_method(py, 3).unwrap(); @@ -441,7 +442,7 @@ impl DictWithCounter { } } # Python::with_gil(|py| { -# let cnt = pyo3::PyCell::new(py, DictWithCounter::new()).unwrap(); +# let cnt = pyo3::Py::new(py, DictWithCounter::new()).unwrap(); # pyo3::py_run!(py, cnt, "cnt.set('abc', 10); assert cnt['abc'] == 10") # }); # } diff --git a/guide/src/class/object.md b/guide/src/class/object.md index 5d4f53a17d4..471889391e2 100644 --- a/guide/src/class/object.md +++ b/guide/src/class/object.md @@ -217,8 +217,8 @@ impl Number { # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let x = PyCell::new(py, Number(4))?; -# let y = PyCell::new(py, Number(4))?; +# let x = &Bound::new(py, Number(4))?.into_any(); +# let y = &Bound::new(py, Number(4))?.into_any(); # assert!(x.eq(y)?); # assert!(!x.ne(y)?); # Ok(()) diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 411978f0567..516c051664b 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -207,7 +207,7 @@ impl Container { # Python::with_gil(|py| { # let container = Container { iter: vec![1, 2, 3, 4] }; -# let inst = pyo3::PyCell::new(py, container).unwrap(); +# let inst = pyo3::Py::new(py, container).unwrap(); # pyo3::py_run!(py, inst, "assert list(inst) == [1, 2, 3, 4]"); # pyo3::py_run!(py, inst, "assert list(iter(iter(inst))) == [1, 2, 3, 4]"); # }); diff --git a/guide/src/migration.md b/guide/src/migration.md index 33b465de8dc..9a958f35458 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -1317,7 +1317,7 @@ impl Names { } } # Python::with_gil(|py| { -# let names = PyCell::new(py, Names::new()).unwrap(); +# let names = Py::new(py, Names::new()).unwrap(); # pyo3::py_run!(py, names, r" # try: # names.merge(names) @@ -1352,7 +1352,7 @@ let obj_ref = PyRef::new(py, MyClass {}).unwrap(); ``` After: -```rust +```rust,ignore # use pyo3::prelude::*; # #[pyclass] # struct MyClass {} diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 4306795e5b3..4b81e2f62fd 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -206,7 +206,7 @@ Python::with_gil(|py| { id: 34, name: "Yu".to_string(), }; - let userdata = PyCell::new(py, userdata).unwrap(); + let userdata = Py::new(py, userdata).unwrap(); let userdata_as_tuple = (34, "Yu"); py_run!(py, userdata userdata_as_tuple, r#" assert repr(userdata) == "User Yu(id: 34)" diff --git a/guide/src/types.md b/guide/src/types.md index 1a926f43dab..f768a31f019 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -245,6 +245,7 @@ so it also exposes all of the methods on `PyAny`. # use pyo3::prelude::*; # #[pyclass] struct MyClass { } # Python::with_gil(|py| -> PyResult<()> { +# #[allow(deprecated)] let cell: &PyCell = PyCell::new(py, MyClass {})?; // To PyRef with .borrow() or .try_borrow() @@ -265,6 +266,7 @@ let _: &mut MyClass = &mut *py_ref_mut; # use pyo3::prelude::*; # #[pyclass] struct MyClass { } # Python::with_gil(|py| -> PyResult<()> { +# #[allow(deprecated)] let cell: &PyCell = PyCell::new(py, MyClass {})?; // Use methods from PyAny on PyCell with Deref implementation diff --git a/src/macros.rs b/src/macros.rs index 0779fb98a4b..29c2033dd4b 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -45,7 +45,7 @@ /// } /// /// Python::with_gil(|py| { -/// let time = PyCell::new(py, Time {hour: 8, minute: 43, second: 16}).unwrap(); +/// let time = Py::new(py, Time {hour: 8, minute: 43, second: 16}).unwrap(); /// let time_as_tuple = (8, 43, 16); /// py_run!(py, time time_as_tuple, r#" /// assert time.hour == 8 diff --git a/src/pycell.rs b/src/pycell.rs index 8409c5cc679..2a3f73dde78 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -249,6 +249,7 @@ unsafe impl PyLayout for PyCellBase where U: PySizedLayout {} /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { +/// # #[allow(deprecated)] /// let n = PyCell::new(py, Number { inner: 0 })?; /// /// let n_mutable: &mut Number = &mut n.borrow_mut(); @@ -284,6 +285,13 @@ impl PyCell { /// /// In cases where the value in the cell does not need to be accessed immediately after /// creation, consider [`Py::new`](crate::Py::new) as a more efficient alternative. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `Bound::new(py, value)` or `Py::new(py, value)` instead of `PyCell::new(py, value)`" + ) + )] pub fn new(py: Python<'_>, value: impl Into>) -> PyResult<&Self> { unsafe { let initializer = value.into(); @@ -332,6 +340,7 @@ impl PyCell { /// struct Class {} /// /// Python::with_gil(|py| { + /// # #[allow(deprecated)] /// let c = PyCell::new(py, Class {}).unwrap(); /// { /// let m = c.borrow_mut(); @@ -371,6 +380,7 @@ impl PyCell { /// #[pyclass] /// struct Class {} /// Python::with_gil(|py| { + /// # #[allow(deprecated)] /// let c = PyCell::new(py, Class {}).unwrap(); /// { /// let m = c.borrow(); @@ -406,6 +416,7 @@ impl PyCell { /// #[pyclass] /// struct Class {} /// Python::with_gil(|py| { + /// # #[allow(deprecated)] /// let c = PyCell::new(py, Class {}).unwrap(); /// /// { @@ -448,6 +459,7 @@ impl PyCell { /// Python::with_gil(|py| { /// let counter = FrozenCounter { value: AtomicUsize::new(0) }; /// + /// # #[allow(deprecated)] /// let cell = PyCell::new(py, counter).unwrap(); /// /// cell.get().value.fetch_add(1, Ordering::Relaxed); @@ -640,7 +652,7 @@ impl fmt::Debug for PyCell { /// } /// } /// # Python::with_gil(|py| { -/// # let sub = PyCell::new(py, Child::new()).unwrap(); +/// # let sub = Py::new(py, Child::new()).unwrap(); /// # pyo3::py_run!(py, sub, "assert sub.format() == 'Caterpillar(base: Butterfly, cnt: 3)'"); /// # }); /// ``` @@ -739,7 +751,7 @@ where /// } /// } /// # Python::with_gil(|py| { - /// # let sub = PyCell::new(py, Sub::new()).unwrap(); + /// # let sub = Py::new(py, Sub::new()).unwrap(); /// # pyo3::py_run!(py, sub, "assert sub.name() == 'base1 base2 sub'") /// # }); /// ``` @@ -1069,6 +1081,7 @@ mod tests { #[test] fn pycell_replace() { Python::with_gil(|py| { + #[allow(deprecated)] let cell = PyCell::new(py, SomeClass(0)).unwrap(); assert_eq!(*cell.borrow(), SomeClass(0)); @@ -1082,6 +1095,7 @@ mod tests { #[should_panic(expected = "Already borrowed: PyBorrowMutError")] fn pycell_replace_panic() { Python::with_gil(|py| { + #[allow(deprecated)] let cell = PyCell::new(py, SomeClass(0)).unwrap(); let _guard = cell.borrow(); @@ -1092,6 +1106,7 @@ mod tests { #[test] fn pycell_replace_with() { Python::with_gil(|py| { + #[allow(deprecated)] let cell = PyCell::new(py, SomeClass(0)).unwrap(); assert_eq!(*cell.borrow(), SomeClass(0)); @@ -1108,6 +1123,7 @@ mod tests { #[should_panic(expected = "Already borrowed: PyBorrowMutError")] fn pycell_replace_with_panic() { Python::with_gil(|py| { + #[allow(deprecated)] let cell = PyCell::new(py, SomeClass(0)).unwrap(); let _guard = cell.borrow(); @@ -1118,7 +1134,9 @@ mod tests { #[test] fn pycell_swap() { Python::with_gil(|py| { + #[allow(deprecated)] let cell = PyCell::new(py, SomeClass(0)).unwrap(); + #[allow(deprecated)] let cell2 = PyCell::new(py, SomeClass(123)).unwrap(); assert_eq!(*cell.borrow(), SomeClass(0)); assert_eq!(*cell2.borrow(), SomeClass(123)); @@ -1133,7 +1151,9 @@ mod tests { #[should_panic(expected = "Already borrowed: PyBorrowMutError")] fn pycell_swap_panic() { Python::with_gil(|py| { + #[allow(deprecated)] let cell = PyCell::new(py, SomeClass(0)).unwrap(); + #[allow(deprecated)] let cell2 = PyCell::new(py, SomeClass(123)).unwrap(); let _guard = cell.borrow(); @@ -1145,7 +1165,9 @@ mod tests { #[should_panic(expected = "Already borrowed: PyBorrowMutError")] fn pycell_swap_panic_other_borrowed() { Python::with_gil(|py| { + #[allow(deprecated)] let cell = PyCell::new(py, SomeClass(0)).unwrap(); + #[allow(deprecated)] let cell2 = PyCell::new(py, SomeClass(123)).unwrap(); let _guard = cell2.borrow(); @@ -1156,7 +1178,7 @@ mod tests { #[test] fn test_as_ptr() { Python::with_gil(|py| { - let cell = PyCell::new(py, SomeClass(0)).unwrap(); + let cell = Bound::new(py, SomeClass(0)).unwrap(); let ptr = cell.as_ptr(); assert_eq!(cell.borrow().as_ptr(), ptr); @@ -1167,7 +1189,7 @@ mod tests { #[test] fn test_into_ptr() { Python::with_gil(|py| { - let cell = PyCell::new(py, SomeClass(0)).unwrap(); + let cell = Bound::new(py, SomeClass(0)).unwrap(); let ptr = cell.as_ptr(); assert_eq!(cell.borrow().into_ptr(), ptr); diff --git a/tests/test_arithmetics.rs b/tests/test_arithmetics.rs index f336140e432..88f9ca44c28 100644 --- a/tests/test_arithmetics.rs +++ b/tests/test_arithmetics.rs @@ -43,7 +43,7 @@ impl UnaryArithmetic { #[test] fn unary_arithmetic() { Python::with_gil(|py| { - let c = PyCell::new(py, UnaryArithmetic::new(2.7)).unwrap(); + let c = Py::new(py, UnaryArithmetic::new(2.7)).unwrap(); py_run!(py, c, "assert repr(-c) == 'UA(-2.7)'"); py_run!(py, c, "assert repr(+c) == 'UA(2.7)'"); py_run!(py, c, "assert repr(abs(c)) == 'UA(2.7)'"); @@ -77,7 +77,7 @@ impl Indexable { #[test] fn indexable() { Python::with_gil(|py| { - let i = PyCell::new(py, Indexable(5)).unwrap(); + let i = Py::new(py, Indexable(5)).unwrap(); py_run!(py, i, "assert int(i) == 5"); py_run!(py, i, "assert [0, 1, 2, 3, 4, 5][i] == 5"); py_run!(py, i, "assert float(i) == 5.0"); @@ -137,7 +137,7 @@ impl InPlaceOperations { fn inplace_operations() { Python::with_gil(|py| { let init = |value, code| { - let c = PyCell::new(py, InPlaceOperations { value }).unwrap(); + let c = Py::new(py, InPlaceOperations { value }).unwrap(); py_run!(py, c, code); }; @@ -210,7 +210,7 @@ impl BinaryArithmetic { #[test] fn binary_arithmetic() { Python::with_gil(|py| { - let c = PyCell::new(py, BinaryArithmetic {}).unwrap(); + let c = Py::new(py, BinaryArithmetic {}).unwrap(); py_run!(py, c, "assert c + c == 'BA + BA'"); py_run!(py, c, "assert c.__add__(c) == 'BA + BA'"); py_run!(py, c, "assert c + 1 == 'BA + 1'"); @@ -238,7 +238,7 @@ fn binary_arithmetic() { py_run!(py, c, "assert pow(c, 1, 100) == 'BA ** 1 (mod: Some(100))'"); - let c: Bound<'_, PyAny> = c.extract().unwrap(); + let c: Bound<'_, PyAny> = c.extract(py).unwrap(); assert_py_eq!(c.add(&c).unwrap(), "BA + BA"); assert_py_eq!(c.sub(&c).unwrap(), "BA - BA"); assert_py_eq!(c.mul(&c).unwrap(), "BA * BA"); @@ -297,7 +297,7 @@ impl RhsArithmetic { #[test] fn rhs_arithmetic() { Python::with_gil(|py| { - let c = PyCell::new(py, RhsArithmetic {}).unwrap(); + let c = Py::new(py, RhsArithmetic {}).unwrap(); py_run!(py, c, "assert c.__radd__(1) == '1 + RA'"); py_run!(py, c, "assert 1 + c == '1 + RA'"); py_run!(py, c, "assert c.__rsub__(1) == '1 - RA'"); @@ -426,7 +426,7 @@ impl LhsAndRhs { #[test] fn lhs_fellback_to_rhs() { Python::with_gil(|py| { - let c = PyCell::new(py, LhsAndRhs {}).unwrap(); + let c = Py::new(py, LhsAndRhs {}).unwrap(); // If the light hand value is `LhsAndRhs`, LHS is used. py_run!(py, c, "assert c + 1 == 'LR + 1'"); py_run!(py, c, "assert c - 1 == 'LR - 1'"); @@ -494,7 +494,7 @@ impl RichComparisons2 { #[test] fn rich_comparisons() { Python::with_gil(|py| { - let c = PyCell::new(py, RichComparisons {}).unwrap(); + let c = Py::new(py, RichComparisons {}).unwrap(); py_run!(py, c, "assert (c < c) == 'RC < RC'"); py_run!(py, c, "assert (c < 1) == 'RC < 1'"); py_run!(py, c, "assert (1 < c) == 'RC > 1'"); @@ -519,7 +519,7 @@ fn rich_comparisons() { #[test] fn rich_comparisons_python_3_type_error() { Python::with_gil(|py| { - let c2 = PyCell::new(py, RichComparisons2 {}).unwrap(); + let c2 = Py::new(py, RichComparisons2 {}).unwrap(); py_expect_exception!(py, c2, "c2 < c2", PyTypeError); py_expect_exception!(py, c2, "c2 < 1", PyTypeError); py_expect_exception!(py, c2, "1 < c2", PyTypeError); @@ -620,7 +620,7 @@ mod return_not_implemented { fn _test_binary_dunder(dunder: &str) { Python::with_gil(|py| { - let c2 = PyCell::new(py, RichComparisonToSelf {}).unwrap(); + let c2 = Py::new(py, RichComparisonToSelf {}).unwrap(); py_run!( py, c2, @@ -636,7 +636,7 @@ mod return_not_implemented { _test_binary_dunder(dunder); Python::with_gil(|py| { - let c2 = PyCell::new(py, RichComparisonToSelf {}).unwrap(); + let c2 = Py::new(py, RichComparisonToSelf {}).unwrap(); py_expect_exception!( py, c2, diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 6d282765c31..fe80625e86b 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -376,7 +376,7 @@ struct DunderDictSupport { #[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)] fn dunder_dict_support() { Python::with_gil(|py| { - let inst = PyCell::new( + let inst = Py::new( py, DunderDictSupport { _pad: *b"DEADBEEFDEADBEEFDEADBEEFDEADBEEF", @@ -399,7 +399,7 @@ fn dunder_dict_support() { #[cfg_attr(all(Py_LIMITED_API, not(Py_3_10)), ignore)] fn access_dunder_dict() { Python::with_gil(|py| { - let inst = PyCell::new( + let inst = Py::new( py, DunderDictSupport { _pad: *b"DEADBEEFDEADBEEFDEADBEEFDEADBEEF", @@ -427,7 +427,7 @@ struct InheritDict { #[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)] fn inherited_dict() { Python::with_gil(|py| { - let inst = PyCell::new( + let inst = Py::new( py, ( InheritDict { _value: 0 }, @@ -458,7 +458,7 @@ struct WeakRefDunderDictSupport { #[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)] fn weakref_dunder_dict_support() { Python::with_gil(|py| { - let inst = PyCell::new( + let inst = Py::new( py, WeakRefDunderDictSupport { _pad: *b"DEADBEEFDEADBEEFDEADBEEFDEADBEEF", @@ -482,7 +482,7 @@ struct WeakRefSupport { #[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)] fn weakref_support() { Python::with_gil(|py| { - let inst = PyCell::new( + let inst = Py::new( py, WeakRefSupport { _pad: *b"DEADBEEFDEADBEEFDEADBEEFDEADBEEF", @@ -507,7 +507,7 @@ struct InheritWeakRef { #[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)] fn inherited_weakref() { Python::with_gil(|py| { - let inst = PyCell::new( + let inst = Py::new( py, ( InheritWeakRef { _value: 0 }, @@ -539,7 +539,7 @@ fn access_frozen_class_without_gil() { value: AtomicUsize::new(0), }; - let cell = PyCell::new(py, counter).unwrap(); + let cell = Bound::new(py, counter).unwrap(); cell.get().value.fetch_add(1, Ordering::Relaxed); diff --git a/tests/test_class_conversion.rs b/tests/test_class_conversion.rs index 29723b4ad97..ccb974097e1 100644 --- a/tests/test_class_conversion.rs +++ b/tests/test_class_conversion.rs @@ -63,7 +63,7 @@ struct PolymorphicContainer { #[test] fn test_polymorphic_container_stores_base_class() { Python::with_gil(|py| { - let p = PyCell::new( + let p = Py::new( py, PolymorphicContainer { inner: Py::new(py, BaseClass::default()).unwrap(), @@ -79,7 +79,7 @@ fn test_polymorphic_container_stores_base_class() { #[test] fn test_polymorphic_container_stores_sub_class() { Python::with_gil(|py| { - let p = PyCell::new( + let p = Py::new( py, PolymorphicContainer { inner: Py::new(py, BaseClass::default()).unwrap(), @@ -91,7 +91,7 @@ fn test_polymorphic_container_stores_sub_class() { p.bind(py) .setattr( "inner", - PyCell::new( + Py::new( py, PyClassInitializer::from(BaseClass::default()).add_subclass(SubClass {}), ) @@ -106,7 +106,7 @@ fn test_polymorphic_container_stores_sub_class() { #[test] fn test_polymorphic_container_does_not_accept_other_types() { Python::with_gil(|py| { - let p = PyCell::new( + let p = Py::new( py, PolymorphicContainer { inner: Py::new(py, BaseClass::default()).unwrap(), @@ -126,7 +126,7 @@ fn test_polymorphic_container_does_not_accept_other_types() { #[test] fn test_pyref_as_base() { Python::with_gil(|py| { - let cell = PyCell::new(py, (SubClass {}, BaseClass { value: 120 })).unwrap(); + let cell = Bound::new(py, (SubClass {}, BaseClass { value: 120 })).unwrap(); // First try PyRefMut let sub: PyRefMut<'_, SubClass> = cell.borrow_mut(); @@ -146,12 +146,14 @@ fn test_pyref_as_base() { #[test] fn test_pycell_deref() { Python::with_gil(|py| { - let cell = PyCell::new(py, (SubClass {}, BaseClass { value: 120 })).unwrap(); + let cell = Bound::new(py, (SubClass {}, BaseClass { value: 120 })).unwrap(); // Should be able to deref as PyAny + // FIXME: This deref does _not_ work assert_eq!( - cell.call_method0("foo") - .and_then(PyAny::extract::<&str>) + cell.as_any() + .call_method0("foo") + .and_then(|e| e.extract::<&str>()) .unwrap(), "SubClass" ); diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index b1a0e594799..9037590f678 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -222,12 +222,8 @@ impl NewExisting { static PRE_BUILT: GILOnceCell<[pyo3::Py; 2]> = GILOnceCell::new(); let existing = PRE_BUILT.get_or_init(py, || { [ - pyo3::PyCell::new(py, NewExisting { num: 0 }) - .unwrap() - .into(), - pyo3::PyCell::new(py, NewExisting { num: 1 }) - .unwrap() - .into(), + pyo3::Py::new(py, NewExisting { num: 0 }).unwrap(), + pyo3::Py::new(py, NewExisting { num: 1 }).unwrap(), ] }); @@ -235,9 +231,7 @@ impl NewExisting { return existing[val].clone_ref(py); } - pyo3::PyCell::new(py, NewExisting { num: val }) - .unwrap() - .into() + pyo3::Py::new(py, NewExisting { num: val }).unwrap() } } diff --git a/tests/test_gc.rs b/tests/test_gc.rs index 637596fd0c8..7a8bc9ded2d 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -99,7 +99,7 @@ fn gc_integration() { let drop_called = Arc::new(AtomicBool::new(false)); Python::with_gil(|py| { - let inst = PyCell::new( + let inst = Bound::new( py, GcIntegration { self_ref: py.None(), @@ -260,7 +260,7 @@ fn gc_during_borrow() { // create an object and check that traversing it works normally // when it's not borrowed - let cell = PyCell::new(py, TraversableClass::new()).unwrap(); + let cell = Bound::new(py, TraversableClass::new()).unwrap(); let obj = cell.to_object(py); assert!(!cell.borrow().traversed.load(Ordering::Relaxed)); traverse(obj.as_ptr(), novisit, std::ptr::null_mut()); @@ -268,7 +268,7 @@ fn gc_during_borrow() { // create an object and check that it is not traversed if the GC // is invoked while it is already borrowed mutably - let cell2 = PyCell::new(py, TraversableClass::new()).unwrap(); + let cell2 = Bound::new(py, TraversableClass::new()).unwrap(); let obj2 = cell2.to_object(py); let guard = cell2.borrow_mut(); assert!(!guard.traversed.load(Ordering::Relaxed)); @@ -417,7 +417,7 @@ fn traverse_cannot_be_hijacked() { let ty = py.get_type_bound::(); let traverse = get_type_traverse(ty.as_type_ptr()).unwrap(); - let cell = PyCell::new(py, HijackedTraverse::new()).unwrap(); + let cell = Bound::new(py, HijackedTraverse::new()).unwrap(); let obj = cell.to_object(py); assert_eq!(cell.borrow().traversed_and_hijacked(), (false, false)); traverse(obj.as_ptr(), novisit, std::ptr::null_mut()); diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 64c4eeffe89..9b29fefdbda 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -81,7 +81,7 @@ fn inheritance_with_new_methods() { #[test] fn call_base_and_sub_methods() { Python::with_gil(|py| { - let obj = PyCell::new(py, SubClass::new()).unwrap(); + let obj = Py::new(py, SubClass::new()).unwrap(); py_run!( py, obj, @@ -96,7 +96,7 @@ fn call_base_and_sub_methods() { #[test] fn mutation_fails() { Python::with_gil(|py| { - let obj = PyCell::new(py, SubClass::new()).unwrap(); + let obj = Py::new(py, SubClass::new()).unwrap(); let global = [("obj", obj)].into_py_dict_bound(py); let e = py .run_bound( @@ -202,7 +202,7 @@ mod inheriting_native_type { } Python::with_gil(|py| { - let set_sub = pyo3::PyCell::new(py, SetWithName::new()).unwrap(); + let set_sub = pyo3::Py::new(py, SetWithName::new()).unwrap(); py_run!( py, set_sub, @@ -229,7 +229,7 @@ mod inheriting_native_type { #[test] fn inherit_dict() { Python::with_gil(|py| { - let dict_sub = pyo3::PyCell::new(py, DictWithName::new()).unwrap(); + let dict_sub = pyo3::Py::new(py, DictWithName::new()).unwrap(); py_run!( py, dict_sub, diff --git a/tests/test_methods.rs b/tests/test_methods.rs index 7ca75016aec..68a004bd4bc 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -28,7 +28,7 @@ impl InstanceMethod { #[test] fn instance_method() { Python::with_gil(|py| { - let obj = PyCell::new(py, InstanceMethod { member: 42 }).unwrap(); + let obj = Bound::new(py, InstanceMethod { member: 42 }).unwrap(); let obj_ref = obj.borrow(); assert_eq!(obj_ref.method(), 42); py_assert!(py, obj, "obj.method() == 42"); @@ -52,7 +52,7 @@ impl InstanceMethodWithArgs { #[test] fn instance_method_with_args() { Python::with_gil(|py| { - let obj = PyCell::new(py, InstanceMethodWithArgs { member: 7 }).unwrap(); + let obj = Bound::new(py, InstanceMethodWithArgs { member: 7 }).unwrap(); let obj_ref = obj.borrow(); assert_eq!(obj_ref.method(6), 42); py_assert!(py, obj, "obj.method(3) == 21"); @@ -710,7 +710,7 @@ impl MethodWithLifeTime { #[test] fn method_with_lifetime() { Python::with_gil(|py| { - let obj = PyCell::new(py, MethodWithLifeTime {}).unwrap(); + let obj = Py::new(py, MethodWithLifeTime {}).unwrap(); py_run!( py, obj, @@ -758,8 +758,8 @@ impl MethodWithPyClassArg { #[test] fn method_with_pyclassarg() { Python::with_gil(|py| { - let obj1 = PyCell::new(py, MethodWithPyClassArg { value: 10 }).unwrap(); - let obj2 = PyCell::new(py, MethodWithPyClassArg { value: 10 }).unwrap(); + let obj1 = Py::new(py, MethodWithPyClassArg { value: 10 }).unwrap(); + let obj2 = Py::new(py, MethodWithPyClassArg { value: 10 }).unwrap(); let d = [("obj1", obj1), ("obj2", obj2)].into_py_dict_bound(py); py_run!(py, *d, "obj = obj1.add(obj2); assert obj.value == 20"); py_run!(py, *d, "obj = obj1.add_pyref(obj2); assert obj.value == 20"); diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index c57ca6a3e0c..fb813b6e52d 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -447,7 +447,7 @@ impl SetItem { #[test] fn setitem() { Python::with_gil(|py| { - let c = PyCell::new(py, SetItem { key: 0, val: 0 }).unwrap(); + let c = Bound::new(py, SetItem { key: 0, val: 0 }).unwrap(); py_run!(py, c, "c[1] = 2"); { let c = c.borrow(); @@ -473,7 +473,7 @@ impl DelItem { #[test] fn delitem() { Python::with_gil(|py| { - let c = PyCell::new(py, DelItem { key: 0 }).unwrap(); + let c = Bound::new(py, DelItem { key: 0 }).unwrap(); py_run!(py, c, "del c[1]"); { let c = c.borrow(); @@ -502,7 +502,7 @@ impl SetDelItem { #[test] fn setdelitem() { Python::with_gil(|py| { - let c = PyCell::new(py, SetDelItem { val: None }).unwrap(); + let c = Bound::new(py, SetDelItem { val: None }).unwrap(); py_run!(py, c, "c[1] = 2"); { let c = c.borrow(); @@ -580,7 +580,7 @@ impl ClassWithGetAttr { #[test] fn getattr_doesnt_override_member() { Python::with_gil(|py| { - let inst = PyCell::new(py, ClassWithGetAttr { data: 4 }).unwrap(); + let inst = Py::new(py, ClassWithGetAttr { data: 4 }).unwrap(); py_assert!(py, inst, "inst.data == 4"); py_assert!(py, inst, "inst.a == 8"); }); @@ -602,7 +602,7 @@ impl ClassWithGetAttribute { #[test] fn getattribute_overrides_member() { Python::with_gil(|py| { - let inst = PyCell::new(py, ClassWithGetAttribute { data: 4 }).unwrap(); + let inst = Py::new(py, ClassWithGetAttribute { data: 4 }).unwrap(); py_assert!(py, inst, "inst.data == 8"); py_assert!(py, inst, "inst.y == 8"); }); @@ -635,7 +635,7 @@ impl ClassWithGetAttrAndGetAttribute { #[test] fn getattr_and_getattribute() { Python::with_gil(|py| { - let inst = PyCell::new(py, ClassWithGetAttrAndGetAttribute).unwrap(); + let inst = Py::new(py, ClassWithGetAttrAndGetAttribute).unwrap(); py_assert!(py, inst, "inst.exists == 42"); py_assert!(py, inst, "inst.lucky == 57"); py_expect_exception!(py, inst, "inst.error", PyValueError); diff --git a/tests/test_pyself.rs b/tests/test_pyself.rs index 5019360236e..fa736f68455 100644 --- a/tests/test_pyself.rs +++ b/tests/test_pyself.rs @@ -111,7 +111,7 @@ fn test_clone_ref() { #[test] fn test_nested_iter_reset() { Python::with_gil(|py| { - let reader = PyCell::new(py, reader()).unwrap(); + let reader = Bound::new(py, reader()).unwrap(); py_assert!( py, reader, diff --git a/tests/test_sequence.rs b/tests/test_sequence.rs index d73d2d110b4..363c27f5eb5 100644 --- a/tests/test_sequence.rs +++ b/tests/test_sequence.rs @@ -268,7 +268,7 @@ fn test_generic_list_get() { #[test] fn test_generic_list_set() { Python::with_gil(|py| { - let list = PyCell::new(py, GenericList { items: vec![] }).unwrap(); + let list = Bound::new(py, GenericList { items: vec![] }).unwrap(); py_run!(py, list, "list.items = [1, 2, 3]"); assert!(list @@ -304,7 +304,7 @@ impl OptionList { fn test_option_list_get() { // Regression test for #798 Python::with_gil(|py| { - let list = PyCell::new( + let list = Py::new( py, OptionList { items: vec![Some(1), None], @@ -321,31 +321,33 @@ fn test_option_list_get() { #[test] fn sequence_is_not_mapping() { Python::with_gil(|py| { - let list = PyCell::new( + let list = Bound::new( py, OptionList { items: vec![Some(1), None], }, ) - .unwrap(); + .unwrap() + .into_any(); PySequence::register::(py).unwrap(); - assert!(list.as_ref().downcast::().is_err()); - assert!(list.as_ref().downcast::().is_ok()); + assert!(list.downcast::().is_err()); + assert!(list.downcast::().is_ok()); }) } #[test] fn sequence_length() { Python::with_gil(|py| { - let list = PyCell::new( + let list = Bound::new( py, OptionList { items: vec![Some(1), None], }, ) - .unwrap(); + .unwrap() + .into_any(); assert_eq!(list.len().unwrap(), 2); assert_eq!(unsafe { ffi::PySequence_Length(list.as_ptr()) }, 2); diff --git a/tests/test_various.rs b/tests/test_various.rs index a1c53fd28db..9ed1e1cf546 100644 --- a/tests/test_various.rs +++ b/tests/test_various.rs @@ -79,8 +79,8 @@ struct SimplePyClass {} fn intopytuple_pyclass() { Python::with_gil(|py| { let tup = ( - PyCell::new(py, SimplePyClass {}).unwrap(), - PyCell::new(py, SimplePyClass {}).unwrap(), + Py::new(py, SimplePyClass {}).unwrap(), + Py::new(py, SimplePyClass {}).unwrap(), ); py_assert!(py, tup, "type(tup[0]).__name__ == 'SimplePyClass'"); py_assert!(py, tup, "type(tup[0]).__name__ == type(tup[1]).__name__"); @@ -102,8 +102,8 @@ fn pytuple_pyclass_iter() { let tup = PyTuple::new_bound( py, [ - PyCell::new(py, SimplePyClass {}).unwrap(), - PyCell::new(py, SimplePyClass {}).unwrap(), + Py::new(py, SimplePyClass {}).unwrap(), + Py::new(py, SimplePyClass {}).unwrap(), ] .iter(), ); @@ -150,7 +150,7 @@ fn test_pickle() { let module = PyModule::new(py, "test_module").unwrap(); module.add_class::().unwrap(); add_module(py, module).unwrap(); - let inst = PyCell::new(py, PickleSupport {}).unwrap(); + let inst = Py::new(py, PickleSupport {}).unwrap(); py_run!( py, inst, From 885883bf68aacd6d5703283a12e899636939a06a Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Thu, 22 Feb 2024 00:56:03 +0200 Subject: [PATCH 158/349] Add `Py::drop_ref` method (#3871) * add Py::drop_ref method * add changelog entry * fix ffi import * integrate review feedback * Add a test * Fix some build errors * Fix some more build errors --- newsfragments/3871.added.md | 1 + src/instance.rs | 50 +++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 newsfragments/3871.added.md diff --git a/newsfragments/3871.added.md b/newsfragments/3871.added.md new file mode 100644 index 00000000000..f90e92fdfff --- /dev/null +++ b/newsfragments/3871.added.md @@ -0,0 +1 @@ +Add `Py::drop_ref` to explicitly drop a `Py`` and immediately decrease the Python reference count if the GIL is already held. diff --git a/src/instance.rs b/src/instance.rs index 3f19637e3b3..8d57bb9ea81 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -822,6 +822,10 @@ impl IntoPy for Borrowed<'_, '_, T> { /// Otherwise, the reference count will be decreased the next time the GIL is /// reacquired. /// +/// If you happen to be already holding the GIL, [`Py::drop_ref`] will decrease +/// the Python reference count immediately and will execute slightly faster than +/// relying on implicit [`Drop`]s. +/// /// # A note on `Send` and `Sync` /// /// Accessing this object is threadsafe, since any access to its API requires a [`Python<'py>`](crate::Python) token. @@ -1228,6 +1232,35 @@ impl Py { unsafe { Py::from_borrowed_ptr(py, self.0.as_ptr()) } } + /// Drops `self` and immediately decreases its reference count. + /// + /// This method is a micro-optimisation over [`Drop`] if you happen to be holding the GIL + /// already. + /// + /// Note that if you are using [`Bound`], you do not need to use [`Self::drop_ref`] since + /// [`Bound`] guarantees that the GIL is held. + /// + /// # Examples + /// + /// ```rust + /// use pyo3::prelude::*; + /// use pyo3::types::PyDict; + /// + /// # fn main() { + /// Python::with_gil(|py| { + /// let object: Py = PyDict::new_bound(py).unbind(); + /// + /// // some usage of object + /// + /// object.drop_ref(py); + /// }); + /// # } + /// ``` + #[inline] + pub fn drop_ref(self, py: Python<'_>) { + let _ = self.into_bound(py); + } + /// Returns whether the object is considered to be None. /// /// This is equivalent to the Python expression `self is None`. @@ -2142,6 +2175,23 @@ a = A() }) } + #[test] + fn explicit_drop_ref() { + Python::with_gil(|py| { + let object: Py = PyDict::new_bound(py).unbind(); + let object2 = object.clone_ref(py); + + assert_eq!(object.as_ptr(), object2.as_ptr()); + assert_eq!(object.get_refcnt(py), 2); + + object.drop_ref(py); + + assert_eq!(object2.get_refcnt(py), 1); + + object2.drop_ref(py); + }); + } + #[cfg(feature = "macros")] mod using_macros { use crate::PyCell; From 5ddcd46980834339e8809869496ba0a97278f0a3 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Thu, 22 Feb 2024 00:05:08 +0000 Subject: [PATCH 159/349] Deprecate `py.from_owned_ptr` methods (#3875) * Deprecate `py.from_owned_ptr` methods * Refactor PyString.to_str Co-authored-by: David Hewitt --------- Co-authored-by: David Hewitt --- src/conversion.rs | 31 +++++++++++++++++++++++++++++++ src/exceptions.rs | 1 + src/ffi/tests.rs | 10 +++++----- src/instance.rs | 10 ++++++++-- src/marker.rs | 24 ++++++++++++++++++++++++ src/pycell.rs | 1 + src/types/bytearray.rs | 5 +---- src/types/complex.rs | 6 +----- src/types/memoryview.rs | 5 +---- src/types/module.rs | 11 +++++++++-- src/types/slice.rs | 5 +---- src/types/string.rs | 5 +---- 12 files changed, 84 insertions(+), 30 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index d9f79ffa104..00b6ceae389 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -434,13 +434,28 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// Implementations must ensure the object does not get freed during `'p` /// and ensure that `ptr` is of the correct type. /// Note that it must be safe to decrement the reference count of `ptr`. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" + ) + )] unsafe fn from_owned_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self>; /// Convert from an arbitrary `PyObject` or panic. /// /// # Safety /// /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" + ) + )] unsafe fn from_owned_ptr_or_panic(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { + #[allow(deprecated)] Self::from_owned_ptr_or_opt(py, ptr).unwrap_or_else(|| err::panic_after_error(py)) } /// Convert from an arbitrary `PyObject` or panic. @@ -448,7 +463,15 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" + ) + )] unsafe fn from_owned_ptr(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { + #[allow(deprecated)] Self::from_owned_ptr_or_panic(py, ptr) } /// Convert from an arbitrary `PyObject`. @@ -456,7 +479,15 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" + ) + )] unsafe fn from_owned_ptr_or_err(py: Python<'p>, ptr: *mut ffi::PyObject) -> PyResult<&'p Self> { + #[allow(deprecated)] Self::from_owned_ptr_or_opt(py, ptr).ok_or_else(|| err::PyErr::fetch(py)) } /// Convert from an arbitrary borrowed `PyObject`. diff --git a/src/exceptions.rs b/src/exceptions.rs index 70809448bff..da48475e0d6 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -43,6 +43,7 @@ macro_rules! impl_exception_boilerplate { impl ::std::error::Error for $name { fn source(&self) -> ::std::option::Option<&(dyn ::std::error::Error + 'static)> { unsafe { + #[allow(deprecated)] let cause: &$crate::exceptions::PyBaseException = self .py() .from_owned_ptr_or_opt($crate::ffi::PyException_GetCause(self.as_ptr()))?; diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index 610edb1c92c..d67bd8485d8 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -5,7 +5,7 @@ use crate::Python; #[cfg(not(Py_LIMITED_API))] use crate::{ types::{PyDict, PyString}, - IntoPy, Py, PyAny, + Bound, IntoPy, Py, PyAny, }; #[cfg(not(any(Py_3_12, Py_LIMITED_API)))] use libc::wchar_t; @@ -16,9 +16,9 @@ use libc::wchar_t; fn test_datetime_fromtimestamp() { Python::with_gil(|py| { let args: Py = (100,).into_py(py); - let dt: &PyAny = unsafe { + let dt = unsafe { PyDateTime_IMPORT(); - py.from_owned_ptr(PyDateTime_FromTimestamp(args.as_ptr())) + Bound::from_owned_ptr(py, PyDateTime_FromTimestamp(args.as_ptr())) }; let locals = PyDict::new_bound(py); locals.set_item("dt", dt).unwrap(); @@ -37,9 +37,9 @@ fn test_datetime_fromtimestamp() { fn test_date_fromtimestamp() { Python::with_gil(|py| { let args: Py = (100,).into_py(py); - let dt: &PyAny = unsafe { + let dt = unsafe { PyDateTime_IMPORT(); - py.from_owned_ptr(PyDate_FromTimestamp(args.as_ptr())) + Bound::from_owned_ptr(py, PyDate_FromTimestamp(args.as_ptr())) }; let locals = PyDict::new_bound(py); locals.set_item("dt", dt).unwrap(); diff --git a/src/instance.rs b/src/instance.rs index 8d57bb9ea81..84b4cb62d9d 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -488,7 +488,10 @@ impl<'py, T> Bound<'py, T> { where T: HasPyGilRef, { - unsafe { self.py().from_owned_ptr(self.into_ptr()) } + #[allow(deprecated)] + unsafe { + self.py().from_owned_ptr(self.into_ptr()) + } } } @@ -974,7 +977,10 @@ where ) )] pub fn into_ref(self, py: Python<'_>) -> &T::AsRefTarget { - unsafe { py.from_owned_ptr(self.into_ptr()) } + #[allow(deprecated)] + unsafe { + py.from_owned_ptr(self.into_ptr()) + } } } diff --git a/src/marker.rs b/src/marker.rs index 29583e8e080..059a5662b5d 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -885,10 +885,18 @@ impl<'py> Python<'py> { /// /// Callers must ensure that ensure that the cast is valid. #[allow(clippy::wrong_self_convention)] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" + ) + )] pub unsafe fn from_owned_ptr(self, ptr: *mut ffi::PyObject) -> &'py T where T: FromPyPointer<'py>, { + #[allow(deprecated)] FromPyPointer::from_owned_ptr(self, ptr) } @@ -901,10 +909,18 @@ impl<'py> Python<'py> { /// /// Callers must ensure that ensure that the cast is valid. #[allow(clippy::wrong_self_convention)] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" + ) + )] pub unsafe fn from_owned_ptr_or_err(self, ptr: *mut ffi::PyObject) -> PyResult<&'py T> where T: FromPyPointer<'py>, { + #[allow(deprecated)] FromPyPointer::from_owned_ptr_or_err(self, ptr) } @@ -917,10 +933,18 @@ impl<'py> Python<'py> { /// /// Callers must ensure that ensure that the cast is valid. #[allow(clippy::wrong_self_convention)] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" + ) + )] pub unsafe fn from_owned_ptr_or_opt(self, ptr: *mut ffi::PyObject) -> Option<&'py T> where T: FromPyPointer<'py>, { + #[allow(deprecated)] FromPyPointer::from_owned_ptr_or_opt(self, ptr) } diff --git a/src/pycell.rs b/src/pycell.rs index 2a3f73dde78..5a42dfa514a 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -296,6 +296,7 @@ impl PyCell { unsafe { let initializer = value.into(); let self_ = initializer.create_cell(py)?; + #[allow(deprecated)] FromPyPointer::from_owned_ptr_or_err(py, self_ as _) } } diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 7f0fdf9ebbe..2a21509d87f 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -112,10 +112,7 @@ impl PyByteArray { ) )] pub fn from(src: &PyAny) -> PyResult<&PyByteArray> { - unsafe { - src.py() - .from_owned_ptr_or_err(ffi::PyByteArray_FromObject(src.as_ptr())) - } + PyByteArray::from_bound(&src.as_borrowed()).map(Bound::into_gil_ref) } /// Creates a new Python `bytearray` object from another Python object that diff --git a/src/types/complex.rs b/src/types/complex.rs index dafda97dbd5..6d4bc7a2244 100644 --- a/src/types/complex.rs +++ b/src/types/complex.rs @@ -141,11 +141,7 @@ mod not_limited_impls { impl<'py> Neg for &'py PyComplex { type Output = &'py PyComplex; fn neg(self) -> &'py PyComplex { - unsafe { - let val = (*self.as_ptr().cast::()).cval; - self.py() - .from_owned_ptr(ffi::PyComplex_FromCComplex(ffi::_Py_c_neg(val))) - } + (-self.as_borrowed()).into_gil_ref() } } diff --git a/src/types/memoryview.rs b/src/types/memoryview.rs index 414bfc69cfa..c04a98e7886 100644 --- a/src/types/memoryview.rs +++ b/src/types/memoryview.rs @@ -19,10 +19,7 @@ impl PyMemoryView { ) )] pub fn from(src: &PyAny) -> PyResult<&PyMemoryView> { - unsafe { - src.py() - .from_owned_ptr_or_err(ffi::PyMemoryView_FromObject(src.as_ptr())) - } + PyMemoryView::from_bound(&src.as_borrowed()).map(Bound::into_gil_ref) } /// Creates a new Python `memoryview` object from another Python object that diff --git a/src/types/module.rs b/src/types/module.rs index 8824dfcf030..137b1b37e3a 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -41,7 +41,10 @@ impl PyModule { pub fn new<'p>(py: Python<'p>, name: &str) -> PyResult<&'p PyModule> { // Could use PyModule_NewObject, but it doesn't exist on PyPy. let name = CString::new(name)?; - unsafe { py.from_owned_ptr_or_err(ffi::PyModule_New(name.as_ptr())) } + #[allow(deprecated)] + unsafe { + py.from_owned_ptr_or_err(ffi::PyModule_New(name.as_ptr())) + } } /// Imports the Python module with the specified name. @@ -67,7 +70,10 @@ impl PyModule { N: IntoPy>, { let name: Py = name.into_py(py); - unsafe { py.from_owned_ptr_or_err(ffi::PyImport_Import(name.as_ptr())) } + #[allow(deprecated)] + unsafe { + py.from_owned_ptr_or_err(ffi::PyImport_Import(name.as_ptr())) + } } /// Creates and loads a module named `module_name`, @@ -146,6 +152,7 @@ impl PyModule { return Err(PyErr::fetch(py)); } + #[allow(deprecated)] <&PyModule as FromPyObject>::extract(py.from_owned_ptr_or_err(mptr)?) } } diff --git a/src/types/slice.rs b/src/types/slice.rs index 8e86ff7ceee..b4b6731d695 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -78,10 +78,7 @@ impl PySlice { ) )] pub fn full(py: Python<'_>) -> &PySlice { - unsafe { - let ptr = ffi::PySlice_New(ffi::Py_None(), ffi::Py_None(), ffi::Py_None()); - py.from_owned_ptr(ptr) - } + PySlice::full_bound(py).into_gil_ref() } /// Constructs a new full slice that is equivalent to `::`. diff --git a/src/types/string.rs b/src/types/string.rs index bc4419944bc..0a7847d2959 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -241,10 +241,7 @@ impl PyString { #[cfg(not(any(Py_3_10, not(Py_LIMITED_API))))] { - let bytes = unsafe { - self.py() - .from_owned_ptr_or_err::(ffi::PyUnicode_AsUTF8String(self.as_ptr())) - }?; + let bytes = self.as_borrowed().encode_utf8()?.into_gil_ref(); Ok(unsafe { std::str::from_utf8_unchecked(bytes.as_bytes()) }) } } From c4f66657c58401023b54a858ebf04400827baa7c Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 22 Feb 2024 08:05:37 +0000 Subject: [PATCH 160/349] fix `either` feature conditional compilation, again (#3834) * fix `either` feature conditional compilation, again * test feature powerset in CI * install `rust-src` for feature powerset tests * review: adamreichold feedback * Fix one more case of redundant imports. * just check feature powerset for now --------- Co-authored-by: Adam Reichold --- .github/workflows/ci.yml | 17 ++++++++ Cargo.toml | 17 ++++---- newsfragments/3834.fixed.md | 1 + noxfile.py | 82 +++++++++++++++++++++++++++++++++++-- pytests/src/comparisons.rs | 1 - src/conversions/either.rs | 10 ++++- 6 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 newsfragments/3834.fixed.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22281185caa..6331a79c645 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -480,6 +480,22 @@ jobs: - run: python3 -m pip install --upgrade pip && pip install nox - run: python3 -m nox -s test-version-limits + check-feature-powerset: + needs: [fmt] + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: Swatinem/rust-cache@v2 + continue-on-error: true + - uses: dtolnay/rust-toolchain@stable + with: + components: rust-src + - uses: taiki-e/install-action@cargo-hack + - run: python3 -m pip install --upgrade pip && pip install nox + - run: python3 -m nox -s check-feature-powerset + conclusion: needs: - fmt @@ -494,6 +510,7 @@ jobs: - emscripten - test-debug - test-version-limits + - check-feature-powerset if: always() runs-on: ubuntu-latest steps: diff --git a/Cargo.toml b/Cargo.toml index 5386b76f573..4d1899cbdb9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,20 +107,21 @@ nightly = [] # This is mostly intended for testing purposes - activating *all* of these isn't particularly useful. full = [ "macros", + "gil-refs", # "multiple-pymethods", # TODO re-add this when MSRV is greater than 1.62 + "anyhow", "chrono", "chrono-tz", - "num-bigint", - "num-complex", - "hashbrown", - "smallvec", - "serde", - "indexmap", "either", - "eyre", - "anyhow", "experimental-inspect", + "eyre", + "hashbrown", + "indexmap", + "num-bigint", + "num-complex", "rust_decimal", + "serde", + "smallvec", ] [workspace] diff --git a/newsfragments/3834.fixed.md b/newsfragments/3834.fixed.md new file mode 100644 index 00000000000..fa77e84f36e --- /dev/null +++ b/newsfragments/3834.fixed.md @@ -0,0 +1 @@ +Fix compilation failure with `either` feature enabled without `experimental-inspect` enabled. diff --git a/noxfile.py b/noxfile.py index 3981e62e100..00288273aec 100644 --- a/noxfile.py +++ b/noxfile.py @@ -13,6 +13,14 @@ import nox import nox.command +try: + import tomllib as toml +except ImportError: + try: + import toml + except ImportError: + toml = None + nox.options.sessions = ["test", "clippy", "rustfmt", "ruff", "docs"] @@ -479,10 +487,8 @@ def check_changelog(session: nox.Session): def set_minimal_package_versions(session: nox.Session): from collections import defaultdict - try: - import tomllib as toml - except ImportError: - import toml + if toml is None: + session.error("requires Python 3.11 or `toml` to be installed") projects = ( None, @@ -593,6 +599,74 @@ def test_version_limits(session: nox.Session): _run_cargo(session, "check", env=env, expect_error=True) +@nox.session(name="check-feature-powerset", venv_backend="none") +def check_feature_powerset(session: nox.Session): + if toml is None: + session.error("requires Python 3.11 or `toml` to be installed") + + with (PYO3_DIR / "Cargo.toml").open("rb") as cargo_toml_file: + cargo_toml = toml.load(cargo_toml_file) + + EXCLUDED_FROM_FULL = { + "nightly", + "extension-module", + "full", + "default", + "auto-initialize", + "generate-import-lib", + "multiple-pymethods", # TODO add this after MSRV 1.62 + } + + features = cargo_toml["features"] + + full_feature = set(features["full"]) + abi3_features = {feature for feature in features if feature.startswith("abi3")} + abi3_version_features = abi3_features - {"abi3"} + + expected_full_feature = features.keys() - EXCLUDED_FROM_FULL - abi3_features + + uncovered_features = expected_full_feature - full_feature + if uncovered_features: + session.error( + f"some features missing from `full` meta feature: {uncovered_features}" + ) + + experimental_features = { + feature for feature in features if feature.startswith("experimental-") + } + full_without_experimental = full_feature - experimental_features + + if len(experimental_features) >= 2: + # justification: we always assume that feature within these groups are + # mutually exclusive to simplify CI + features_to_group = [ + full_without_experimental, + experimental_features, + ] + elif len(experimental_features) == 1: + # no need to make an experimental features group + features_to_group = [full_without_experimental] + else: + session.error("no experimental features exist; please simplify the noxfile") + + features_to_skip = [ + *EXCLUDED_FROM_FULL, + *abi3_version_features, + ] + + comma_join = ",".join + _run_cargo( + session, + "hack", + "--feature-powerset", + '--optional-deps=""', + f'--skip="{comma_join(features_to_skip)}"', + *(f"--group-features={comma_join(group)}" for group in features_to_group), + "check", + "--all-targets", + ) + + def _build_docs_for_ffi_check(session: nox.Session) -> None: # pyo3-ffi-check needs to scrape docs of pyo3-ffi _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") diff --git a/pytests/src/comparisons.rs b/pytests/src/comparisons.rs index d8c2f5a6a52..b3ba293186a 100644 --- a/pytests/src/comparisons.rs +++ b/pytests/src/comparisons.rs @@ -1,5 +1,4 @@ use pyo3::prelude::*; -use pyo3::{types::PyModule, Python}; #[pyclass] struct Eq(i64); diff --git a/src/conversions/either.rs b/src/conversions/either.rs index c763cdf95e5..16d5ad491eb 100644 --- a/src/conversions/either.rs +++ b/src/conversions/either.rs @@ -94,7 +94,13 @@ where } else if let Ok(r) = obj.extract::() { Ok(Either::Right(r)) } else { - let err_msg = format!("failed to convert the value to '{}'", Self::type_input()); + // TODO: it might be nice to use the `type_input()` name here once `type_input` + // is not experimental, rather than the Rust type names. + let err_msg = format!( + "failed to convert the value to 'Union[{}, {}]'", + std::any::type_name::(), + std::any::type_name::() + ); Err(PyTypeError::new_err(err_msg)) } } @@ -134,7 +140,7 @@ mod tests { assert!(err.is_instance_of::(py)); assert_eq!( err.to_string(), - "TypeError: failed to convert the value to 'Union[int, float]'" + "TypeError: failed to convert the value to 'Union[i32, f32]'" ); let obj_i = 42.to_object(py); From 9e74c858c272b6fab4193ae1fd7b7dc1ee6b8a2a Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 22 Feb 2024 09:35:47 +0000 Subject: [PATCH 161/349] add `PyModule::new_bound` and `PyModule::import_bound` (#3775) * add `PyModule::new` and `PyModule::import_bound` * review: Icxolu feedback --- examples/maturin-starter/src/lib.rs | 4 +- examples/plugin/src/main.rs | 2 +- examples/setuptools-rust-starter/src/lib.rs | 4 +- guide/src/class.md | 8 +- guide/src/class/numeric.md | 2 +- guide/src/conversions/traits.md | 8 +- guide/src/function/signature.md | 8 +- guide/src/module.md | 6 +- guide/src/python_from_rust.md | 26 ++-- pyo3-benches/benches/bench_call.rs | 13 +- pyo3-benches/benches/bench_intern.rs | 10 +- pytests/src/lib.rs | 4 +- src/conversions/anyhow.rs | 2 +- src/conversions/chrono.rs | 2 +- src/conversions/chrono_tz.rs | 2 + src/conversions/eyre.rs | 2 +- src/conversions/num_bigint.rs | 9 +- src/conversions/num_complex.rs | 17 ++- src/instance.rs | 17 +-- src/marker.rs | 8 +- src/pyclass_init.rs | 2 +- src/types/any.rs | 67 +++++----- src/types/capsule.rs | 6 +- src/types/module.rs | 131 ++++++++++++++------ src/types/traceback.rs | 5 +- tests/test_class_basics.rs | 2 +- tests/test_class_new.rs | 5 +- tests/test_coroutine.rs | 2 +- tests/test_inheritance.rs | 2 +- tests/test_module.rs | 20 +-- tests/test_proto_methods.rs | 15 +-- tests/test_various.rs | 8 +- 32 files changed, 229 insertions(+), 190 deletions(-) diff --git a/examples/maturin-starter/src/lib.rs b/examples/maturin-starter/src/lib.rs index 96ace0f97d5..8b0748f8311 100644 --- a/examples/maturin-starter/src/lib.rs +++ b/examples/maturin-starter/src/lib.rs @@ -27,8 +27,8 @@ fn maturin_starter(py: Python<'_>, m: &PyModule) -> PyResult<()> { // Inserting to sys.modules allows importing submodules nicely from Python // e.g. from maturin_starter.submodule import SubmoduleClass - let sys = PyModule::import(py, "sys")?; - let sys_modules: &PyDict = sys.getattr("modules")?.downcast()?; + let sys = PyModule::import_bound(py, "sys")?; + let sys_modules: Bound<'_, PyDict> = sys.getattr("modules")?.downcast_into()?; sys_modules.set_item("maturin_starter.submodule", m.getattr("submodule")?)?; Ok(()) diff --git a/examples/plugin/src/main.rs b/examples/plugin/src/main.rs index b50b54548e5..5a54a1837cb 100644 --- a/examples/plugin/src/main.rs +++ b/examples/plugin/src/main.rs @@ -19,7 +19,7 @@ fn main() -> Result<(), Box> { // Now we can load our python_plugin/gadget_init_plugin.py file. // It can in turn import other stuff as it deems appropriate - let plugin = PyModule::import(py, "gadget_init_plugin")?; + let plugin = PyModule::import_bound(py, "gadget_init_plugin")?; // and call start function there, which will return a python reference to Gadget. // Gadget here is a "pyclass" object reference let gadget = plugin.getattr("start")?.call0()?; diff --git a/examples/setuptools-rust-starter/src/lib.rs b/examples/setuptools-rust-starter/src/lib.rs index fbfeccc1555..2bcd411c238 100644 --- a/examples/setuptools-rust-starter/src/lib.rs +++ b/examples/setuptools-rust-starter/src/lib.rs @@ -27,8 +27,8 @@ fn _setuptools_rust_starter(py: Python<'_>, m: &PyModule) -> PyResult<()> { // Inserting to sys.modules allows importing submodules nicely from Python // e.g. from setuptools_rust_starter.submodule import SubmoduleClass - let sys = PyModule::import(py, "sys")?; - let sys_modules: &PyDict = sys.getattr("modules")?.downcast()?; + let sys = PyModule::import_bound(py, "sys")?; + let sys_modules: Bound<'_, PyDict> = sys.getattr("modules")?.downcast_into()?; sys_modules.set_item("setuptools_rust_starter.submodule", m.getattr("submodule")?)?; Ok(()) diff --git a/guide/src/class.md b/guide/src/class.md index 9ebba38a820..9662e8f6e09 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -941,8 +941,8 @@ impl MyClass { # # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; -# let module = PyModule::new(py, "my_module")?; +# let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; +# let module = PyModule::new_bound(py, "my_module")?; # module.add_class::()?; # let class = module.getattr("MyClass")?; # @@ -951,7 +951,7 @@ impl MyClass { # assert_eq!(doc, ""); # # let sig: String = inspect -# .call1((class,))? +# .call1((&class,))? # .call_method0("__str__")? # .extract()?; # assert_eq!(sig, "(c, d)"); @@ -959,7 +959,7 @@ impl MyClass { # let doc: String = class.getattr("__doc__")?.extract()?; # assert_eq!(doc, ""); # -# inspect.call1((class,)).expect_err("`text_signature` on classes is not compatible with compilation in `abi3` mode until Python 3.10 or greater"); +# inspect.call1((&class,)).expect_err("`text_signature` on classes is not compatible with compilation in `abi3` mode until Python 3.10 or greater"); # } # # { diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index cbd9db824a0..2ffb0a543ef 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -386,7 +386,7 @@ fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { # # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let globals = PyModule::import(py, "__main__")?.dict().as_borrowed(); +# let globals = PyModule::import_bound(py, "__main__")?.dict(); # globals.set_item("Number", Number::type_object_bound(py))?; # # py.run_bound(SCRIPT, Some(&globals), None)?; diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index b46e5c02f4c..5cdf2c590b9 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -54,7 +54,7 @@ struct RustyStruct { # # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let module = PyModule::from_code( +# let module = PyModule::from_code_bound( # py, # "class Foo: # def __init__(self): @@ -111,7 +111,7 @@ struct RustyStruct { # # fn main() -> PyResult<()> { # Python::with_gil(|py| -> PyResult<()> { -# let module = PyModule::from_code( +# let module = PyModule::from_code_bound( # py, # "class Foo(dict): # def __init__(self): @@ -339,7 +339,7 @@ enum RustyEnum<'a> { # ); # } # { -# let module = PyModule::from_code( +# let module = PyModule::from_code_bound( # py, # "class Foo(dict): # def __init__(self): @@ -364,7 +364,7 @@ enum RustyEnum<'a> { # } # # { -# let module = PyModule::from_code( +# let module = PyModule::from_code_bound( # py, # "class Foo(dict): # def __init__(self): diff --git a/guide/src/function/signature.md b/guide/src/function/signature.md index 8341f51c38c..d92767e7bde 100644 --- a/guide/src/function/signature.md +++ b/guide/src/function/signature.md @@ -138,7 +138,7 @@ fn increment(x: u64, amount: Option) -> u64 { # Python::with_gil(|py| { # let fun = pyo3::wrap_pyfunction!(increment, py)?; # -# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; +# let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; # let sig: String = inspect # .call1((fun,))? # .call_method0("__str__")? @@ -166,7 +166,7 @@ fn increment(x: u64, amount: Option) -> u64 { # Python::with_gil(|py| { # let fun = pyo3::wrap_pyfunction!(increment, py)?; # -# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; +# let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; # let sig: String = inspect # .call1((fun,))? # .call_method0("__str__")? @@ -209,7 +209,7 @@ fn add(a: u64, b: u64) -> u64 { # let doc: String = fun.getattr("__doc__")?.extract()?; # assert_eq!(doc, "This function adds two unsigned 64-bit integers."); # -# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; +# let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; # let sig: String = inspect # .call1((fun,))? # .call_method0("__str__")? @@ -257,7 +257,7 @@ fn add(a: u64, b: u64) -> u64 { # let doc: String = fun.getattr("__doc__")?.extract()?; # assert_eq!(doc, "This function adds two unsigned 64-bit integers."); # -# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?; +# let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; # let sig: String = inspect # .call1((fun,))? # .call_method0("__str__")? diff --git a/guide/src/module.md b/guide/src/module.md index 3d984f60d39..789c3d91ccb 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -78,9 +78,9 @@ fn parent_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { } fn register_child_module(py: Python<'_>, parent_module: &PyModule) -> PyResult<()> { - let child_module = PyModule::new(py, "child_module")?; - child_module.add_function(wrap_pyfunction!(func, child_module)?)?; - parent_module.add_submodule(child_module)?; + let child_module = PyModule::new_bound(py, "child_module")?; + child_module.add_function(&wrap_pyfunction!(func, child_module.as_gil_ref())?.as_borrowed())?; + parent_module.add_submodule(child_module.as_gil_ref())?; Ok(()) } diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 4b81e2f62fd..973f997cbf8 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -32,7 +32,7 @@ fn main() -> PyResult<()> { let arg3 = "arg3"; Python::with_gil(|py| { - let fun: Py = PyModule::from_code( + let fun: Py = PyModule::from_code_bound( py, "def example(*args, **kwargs): if args != (): @@ -78,7 +78,7 @@ fn main() -> PyResult<()> { let val2 = 2; Python::with_gil(|py| { - let fun: Py = PyModule::from_code( + let fun: Py = PyModule::from_code_bound( py, "def example(*args, **kwargs): if args != (): @@ -134,7 +134,7 @@ use pyo3::prelude::*; fn main() -> PyResult<()> { Python::with_gil(|py| { - let builtins = PyModule::import(py, "builtins")?; + let builtins = PyModule::import_bound(py, "builtins")?; let total: i32 = builtins .getattr("sum")? .call1((vec![1, 2, 3],))? @@ -233,7 +233,7 @@ use pyo3::{ # fn main() -> PyResult<()> { Python::with_gil(|py| { - let activators = PyModule::from_code( + let activators = PyModule::from_code_bound( py, r#" def relu(x): @@ -253,7 +253,7 @@ def leaky_relu(x, slope=0.01): let kwargs = [("slope", 0.2)].into_py_dict_bound(py); let lrelu_result: f64 = activators .getattr("leaky_relu")? - .call((-1.0,), Some(kwargs.as_gil_ref()))? + .call((-1.0,), Some(&kwargs))? .extract()?; assert_eq!(lrelu_result, -0.2); # Ok(()) @@ -310,12 +310,12 @@ pub fn add_one(x: i64) -> i64 { fn main() -> PyResult<()> { Python::with_gil(|py| { // Create new module - let foo_module = PyModule::new(py, "foo")?; - foo_module.add_function(wrap_pyfunction!(add_one, foo_module)?)?; + let foo_module = PyModule::new_bound(py, "foo")?; + foo_module.add_function(&wrap_pyfunction!(add_one, foo_module.as_gil_ref())?.as_borrowed())?; // Import and get sys.modules - let sys = PyModule::import(py, "sys")?; - let py_modules: &PyDict = sys.getattr("modules")?.downcast()?; + let sys = PyModule::import_bound(py, "sys")?; + let py_modules: Bound<'_, PyDict> = sys.getattr("modules")?.downcast_into()?; // Insert foo into sys.modules py_modules.set_item("foo", foo_module)?; @@ -381,8 +381,8 @@ fn main() -> PyResult<()> { )); let py_app = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/python_app/app.py")); let from_python = Python::with_gil(|py| -> PyResult> { - PyModule::from_code(py, py_foo, "utils.foo", "utils.foo")?; - let app: Py = PyModule::from_code(py, py_app, "", "")? + PyModule::from_code_bound(py, py_foo, "utils.foo", "utils.foo")?; + let app: Py = PyModule::from_code_bound(py, py_app, "", "")? .getattr("run")? .into(); app.call0(py) @@ -416,7 +416,7 @@ fn main() -> PyResult<()> { let from_python = Python::with_gil(|py| -> PyResult> { let syspath = py.import_bound("sys")?.getattr("path")?.downcast_into::()?; syspath.insert(0, &path)?; - let app: Py = PyModule::from_code(py, &py_app, "", "")? + let app: Py = PyModule::from_code_bound(py, &py_app, "", "")? .getattr("run")? .into(); app.call0(py) @@ -440,7 +440,7 @@ use pyo3::prelude::*; fn main() { Python::with_gil(|py| { - let custom_manager = PyModule::from_code( + let custom_manager = PyModule::from_code_bound( py, r#" class House(object): diff --git a/pyo3-benches/benches/bench_call.rs b/pyo3-benches/benches/bench_call.rs index 50772097961..8470c8768d3 100644 --- a/pyo3-benches/benches/bench_call.rs +++ b/pyo3-benches/benches/bench_call.rs @@ -1,10 +1,13 @@ +use std::hint::black_box; + use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; macro_rules! test_module { ($py:ident, $code:literal) => { - PyModule::from_code($py, $code, file!(), "test_module").expect("module creation failed") + PyModule::from_code_bound($py, $code, file!(), "test_module") + .expect("module creation failed") }; } @@ -12,11 +15,11 @@ fn bench_call_0(b: &mut Bencher<'_>) { Python::with_gil(|py| { let module = test_module!(py, "def foo(): pass"); - let foo_module = module.getattr("foo").unwrap(); + let foo_module = &module.getattr("foo").unwrap(); b.iter(|| { for _ in 0..1000 { - foo_module.call0().unwrap(); + black_box(foo_module).call0().unwrap(); } }); }) @@ -33,11 +36,11 @@ class Foo: " ); - let foo_module = module.getattr("Foo").unwrap().call0().unwrap(); + let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); b.iter(|| { for _ in 0..1000 { - foo_module.call_method0("foo").unwrap(); + black_box(foo_module).call_method0("foo").unwrap(); } }); }) diff --git a/pyo3-benches/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs index 806c4e95a9d..f9f9162a5ee 100644 --- a/pyo3-benches/benches/bench_intern.rs +++ b/pyo3-benches/benches/bench_intern.rs @@ -1,3 +1,5 @@ +use std::hint::black_box; + use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; @@ -6,17 +8,17 @@ use pyo3::intern; fn getattr_direct(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let sys = py.import_bound("sys").unwrap(); + let sys = &py.import_bound("sys").unwrap(); - b.iter(|| sys.getattr("version").unwrap()); + b.iter(|| black_box(sys).getattr("version").unwrap()); }); } fn getattr_intern(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let sys = py.import_bound("sys").unwrap(); + let sys = &py.import_bound("sys").unwrap(); - b.iter(|| sys.getattr(intern!(py, "version")).unwrap()); + b.iter(|| black_box(sys).getattr(intern!(py, "version")).unwrap()); }); } diff --git a/pytests/src/lib.rs b/pytests/src/lib.rs index e65385bf679..bfd80edb719 100644 --- a/pytests/src/lib.rs +++ b/pytests/src/lib.rs @@ -39,8 +39,8 @@ fn pyo3_pytests(py: Python<'_>, m: &PyModule) -> PyResult<()> { // Inserting to sys.modules allows importing submodules nicely from Python // e.g. import pyo3_pytests.buf_and_str as bas - let sys = PyModule::import(py, "sys")?; - let sys_modules: &PyDict = sys.getattr("modules")?.downcast()?; + let sys = PyModule::import_bound(py, "sys")?; + let sys_modules = sys.getattr("modules")?.downcast_into::()?; sys_modules.set_item("pyo3_pytests.awaitable", m.getattr("awaitable")?)?; sys_modules.set_item("pyo3_pytests.buf_and_str", m.getattr("buf_and_str")?)?; sys_modules.set_item("pyo3_pytests.comparisons", m.getattr("comparisons")?)?; diff --git a/src/conversions/anyhow.rs b/src/conversions/anyhow.rs index a490bd4ce31..fba8816d23a 100644 --- a/src/conversions/anyhow.rs +++ b/src/conversions/anyhow.rs @@ -75,7 +75,7 @@ //! // could call inside an application... //! // This might return a `PyErr`. //! let res = Python::with_gil(|py| { -//! let zlib = PyModule::import(py, "zlib")?; +//! let zlib = PyModule::import_bound(py, "zlib")?; //! let decompress = zlib.getattr("decompress")?; //! let bytes = PyBytes::new_bound(py, bytes); //! let value = decompress.call1((bytes,))?; diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 3aa6dbeb4ca..0c96ef9c705 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -564,7 +564,7 @@ fn timezone_utc_bound(py: Python<'_>) -> Bound<'_, PyAny> { #[cfg(test)] mod tests { use super::*; - use crate::{types::PyTuple, Py}; + use crate::{types::PyTuple, Bound, Py}; use std::{cmp::Ordering, panic}; #[test] diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index b4d0e7edf8c..986bb8313d2 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -69,6 +69,8 @@ impl FromPyObject<'_> for Tz { #[cfg(all(test, not(windows)))] // Troubles loading timezones on Windows mod tests { + use crate::{types::any::PyAnyMethods, Bound}; + use super::*; #[test] diff --git a/src/conversions/eyre.rs b/src/conversions/eyre.rs index 64ed4a0320f..236e2c8bc92 100644 --- a/src/conversions/eyre.rs +++ b/src/conversions/eyre.rs @@ -74,7 +74,7 @@ //! // could call inside an application... //! // This might return a `PyErr`. //! let res = Python::with_gil(|py| { -//! let zlib = PyModule::import(py, "zlib")?; +//! let zlib = PyModule::import_bound(py, "zlib")?; //! let decompress = zlib.getattr("decompress")?; //! let bytes = PyBytes::new_bound(py, bytes); //! let value = decompress.call1((bytes,))?; diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 652df154258..48a4ee60d73 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -262,7 +262,10 @@ fn int_n_bits(long: &Bound<'_, PyLong>) -> PyResult { #[cfg(test)] mod tests { use super::*; - use crate::types::{PyDict, PyModule}; + use crate::{ + types::{PyDict, PyModule}, + Bound, + }; use indoc::indoc; fn rust_fib() -> impl Iterator @@ -323,7 +326,7 @@ mod tests { }); } - fn python_index_class(py: Python<'_>) -> &PyModule { + fn python_index_class(py: Python<'_>) -> Bound<'_, PyModule> { let index_code = indoc!( r#" class C: @@ -333,7 +336,7 @@ mod tests { return self.x "# ); - PyModule::from_code(py, index_code, "index.py", "index").unwrap() + PyModule::from_code_bound(py, index_code, "index.py", "index").unwrap() } #[test] diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index 3b4fe0fc217..c736a7baa77 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -27,7 +27,7 @@ //! ```ignore //! # // not tested because nalgebra isn't supported on msrv //! # // please file an issue if it breaks! -//! use nalgebra::base::{dimension::Const, storage::Storage, Matrix}; +//! use nalgebra::base::{dimension::Const, Matrix}; //! use num_complex::Complex; //! use pyo3::prelude::*; //! @@ -55,9 +55,9 @@ //! # //! # fn main() -> PyResult<()> { //! # Python::with_gil(|py| -> PyResult<()> { -//! # let module = PyModule::new(py, "my_module")?; +//! # let module = PyModule::new_bound(py, "my_module")?; //! # -//! # module.add_function(wrap_pyfunction!(get_eigenvalues, module)?)?; +//! # module.add_function(&wrap_pyfunction!(get_eigenvalues, module.as_gil_ref())?.as_borrowed())?; //! # //! # let m11 = PyComplex::from_doubles_bound(py, 0_f64, -1_f64); //! # let m12 = PyComplex::from_doubles_bound(py, 1_f64, 0_f64); @@ -199,8 +199,7 @@ complex_conversion!(f64); #[cfg(test)] mod tests { use super::*; - use crate::types::complex::PyComplexMethods; - use crate::types::PyModule; + use crate::types::{any::PyAnyMethods, complex::PyComplexMethods, PyModule}; #[test] fn from_complex() { @@ -229,7 +228,7 @@ mod tests { #[test] fn from_python_magic() { Python::with_gil(|py| { - let module = PyModule::from_code( + let module = PyModule::from_code_bound( py, r#" class A: @@ -267,7 +266,7 @@ class C: #[test] fn from_python_inherited_magic() { Python::with_gil(|py| { - let module = PyModule::from_code( + let module = PyModule::from_code_bound( py, r#" class First: pass @@ -311,7 +310,7 @@ class C(First, IndexMixin): pass // `type(inst).attr(inst)` equivalent to `inst.attr()` for methods, but this isn't the only // way the descriptor protocol might be implemented. Python::with_gil(|py| { - let module = PyModule::from_code( + let module = PyModule::from_code_bound( py, r#" class A: @@ -334,7 +333,7 @@ class A: fn from_python_nondescriptor_magic() { // Magic methods don't need to implement the descriptor protocol, if they're callable. Python::with_gil(|py| { - let module = PyModule::from_code( + let module = PyModule::from_code_bound( py, r#" class MyComplex: diff --git a/src/instance.rs b/src/instance.rs index 84b4cb62d9d..f35ab4bbb3a 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -722,12 +722,12 @@ impl IntoPy for Borrowed<'_, '_, T> { /// # /// # fn main() -> PyResult<()> { /// # Python::with_gil(|py| { -/// # let m = pyo3::types::PyModule::new(py, "test")?; +/// # let m = pyo3::types::PyModule::new_bound(py, "test")?; /// # m.add_class::()?; /// # -/// # let foo: &PyCell = m.getattr("Foo")?.call0()?.downcast()?; +/// # let foo: Bound<'_, Foo> = m.getattr("Foo")?.call0()?.downcast_into()?; /// # let dict = &foo.borrow().inner; -/// # let dict: &PyDict = dict.as_ref(py); +/// # let dict: &Bound<'_, PyDict> = dict.bind(py); /// # /// # Ok(()) /// # }) @@ -759,10 +759,10 @@ impl IntoPy for Borrowed<'_, '_, T> { /// # /// # fn main() -> PyResult<()> { /// # Python::with_gil(|py| { -/// # let m = pyo3::types::PyModule::new(py, "test")?; +/// # let m = pyo3::types::PyModule::new_bound(py, "test")?; /// # m.add_class::()?; /// # -/// # let foo: &PyCell = m.getattr("Foo")?.call0()?.downcast()?; +/// # let foo: Bound<'_, Foo> = m.getattr("Foo")?.call0()?.downcast_into()?; /// # let bar = &foo.borrow().inner; /// # let bar: &Bar = &*bar.borrow(py); /// # @@ -1357,7 +1357,7 @@ impl Py { /// } /// # /// # Python::with_gil(|py| { - /// # let ob = PyModule::new(py, "empty").unwrap().into_py(py); + /// # let ob = PyModule::new_bound(py, "empty").unwrap().into_py(py); /// # set_answer(ob, py).unwrap(); /// # }); /// ``` @@ -1905,6 +1905,7 @@ impl PyObject { #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::{Bound, Py, PyObject}; + use crate::types::any::PyAnyMethods; use crate::types::PyCapsule; use crate::types::{dict::IntoPyDict, PyDict, PyString}; use crate::{ffi, Borrowed, PyAny, PyNativeType, PyResult, Python, ToPyObject}; @@ -1978,7 +1979,7 @@ class A: pass a = A() "#; - let module = PyModule::from_code(py, CODE, "", "")?; + let module = PyModule::from_code_bound(py, CODE, "", "")?; let instance: Py = module.getattr("a")?.into(); instance.getattr(py, "foo").unwrap_err(); @@ -2005,7 +2006,7 @@ class A: pass a = A() "#; - let module = PyModule::from_code(py, CODE, "", "")?; + let module = PyModule::from_code_bound(py, CODE, "", "")?; let instance: Py = module.getattr("a")?.into(); let foo = crate::intern!(py, "foo"); diff --git a/src/marker.rs b/src/marker.rs index 059a5662b5d..449ea2bde6b 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -764,7 +764,7 @@ impl<'py> Python<'py> { where N: IntoPy>, { - PyModule::import(self, name) + Self::import_bound(self, name).map(Bound::into_gil_ref) } /// Imports the Python module with the specified name. @@ -772,11 +772,7 @@ impl<'py> Python<'py> { where N: IntoPy>, { - // FIXME: This should be replaced by `PyModule::import_bound` once thats - // implemented. - PyModule::import(self, name) - .map(PyNativeType::as_borrowed) - .map(crate::Borrowed::to_owned) + PyModule::import_bound(self, name) } /// Gets the Python builtin value `None`. diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index ac3aa3ef852..94d377a732c 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -184,7 +184,7 @@ impl PyClassInitializer { /// /// fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let m = PyModule::new(py, "example")?; + /// let m = PyModule::new_bound(py, "example")?; /// m.add_class::()?; /// m.add_class::()?; /// diff --git a/src/types/any.rs b/src/types/any.rs index 4d371764d29..15f2fd3135c 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -146,16 +146,16 @@ impl PyAny { /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{prelude::*, intern}; /// # /// #[pyfunction] - /// fn set_answer(ob: &PyAny) -> PyResult<()> { + /// fn set_answer(ob: &Bound<'_, PyAny>) -> PyResult<()> { /// ob.setattr(intern!(ob.py(), "answer"), 42) /// } /// # /// # Python::with_gil(|py| { - /// # let ob = PyModule::new(py, "empty").unwrap(); - /// # set_answer(ob).unwrap(); + /// # let ob = PyModule::new_bound(py, "empty").unwrap(); + /// # set_answer(&ob).unwrap(); /// # }); /// ``` pub fn setattr(&self, attr_name: N, value: V) -> PyResult<()> @@ -346,7 +346,7 @@ impl PyAny { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let builtins = PyModule::import(py, "builtins")?; + /// let builtins = PyModule::import_bound(py, "builtins")?; /// let print = builtins.getattr("print")?; /// assert!(print.is_callable()); /// Ok(()) @@ -385,12 +385,12 @@ impl PyAny { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let module = PyModule::from_code(py, CODE, "", "")?; + /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let fun = module.getattr("function")?; /// let args = ("hello",); /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; - /// let result = fun.call(args, Some(kwargs.as_gil_ref()))?; + /// let result = fun.call(args, Some(&kwargs))?; /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); /// Ok(()) /// }) @@ -417,7 +417,7 @@ impl PyAny { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let module = PyModule::import(py, "builtins")?; + /// let module = PyModule::import_bound(py, "builtins")?; /// let help = module.getattr("help")?; /// help.call0()?; /// Ok(()) @@ -448,7 +448,7 @@ impl PyAny { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let module = PyModule::from_code(py, CODE, "", "")?; + /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let fun = module.getattr("function")?; /// let args = ("hello",); /// let result = fun.call1(args)?; @@ -485,12 +485,12 @@ impl PyAny { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let module = PyModule::from_code(py, CODE, "", "")?; + /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let instance = module.getattr("a")?; /// let args = ("hello",); /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; - /// let result = instance.call_method("method", args, Some(kwargs.as_gil_ref()))?; + /// let result = instance.call_method("method", args, Some(&kwargs))?; /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); /// Ok(()) /// }) @@ -529,7 +529,7 @@ impl PyAny { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let module = PyModule::from_code(py, CODE, "", "")?; + /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let instance = module.getattr("a")?; /// let result = instance.call_method0("method")?; /// assert_eq!(result.extract::<&str>()?, "called with no arguments"); @@ -569,7 +569,7 @@ impl PyAny { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let module = PyModule::from_code(py, CODE, "", "")?; + /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let instance = module.getattr("a")?; /// let args = ("hello",); /// let result = instance.call_method1("method", args)?; @@ -1008,16 +1008,16 @@ pub trait PyAnyMethods<'py> { /// # Example: `intern!`ing the attribute name /// /// ``` - /// # use pyo3::{intern, pyfunction, types::PyModule, PyAny, Python, PyResult}; + /// # use pyo3::{prelude::*, intern}; /// # /// #[pyfunction] - /// fn set_answer(ob: &PyAny) -> PyResult<()> { + /// fn set_answer(ob: &Bound<'_, PyAny>) -> PyResult<()> { /// ob.setattr(intern!(ob.py(), "answer"), 42) /// } /// # /// # Python::with_gil(|py| { - /// # let ob = PyModule::new(py, "empty").unwrap(); - /// # set_answer(ob).unwrap(); + /// # let ob = PyModule::new_bound(py, "empty").unwrap(); + /// # set_answer(&ob).unwrap(); /// # }); /// ``` fn setattr(&self, attr_name: N, value: V) -> PyResult<()> @@ -1228,7 +1228,7 @@ pub trait PyAnyMethods<'py> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let builtins = PyModule::import(py, "builtins")?; + /// let builtins = PyModule::import_bound(py, "builtins")?; /// let print = builtins.getattr("print")?; /// assert!(print.is_callable()); /// Ok(()) @@ -1265,12 +1265,12 @@ pub trait PyAnyMethods<'py> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let module = PyModule::from_code(py, CODE, "", "")?; + /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let fun = module.getattr("function")?; /// let args = ("hello",); /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; - /// let result = fun.call(args, Some(kwargs.as_gil_ref()))?; + /// let result = fun.call(args, Some(&kwargs))?; /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); /// Ok(()) /// }) @@ -1293,7 +1293,7 @@ pub trait PyAnyMethods<'py> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let module = PyModule::import(py, "builtins")?; + /// let module = PyModule::import_bound(py, "builtins")?; /// let help = module.getattr("help")?; /// help.call0()?; /// Ok(()) @@ -1322,7 +1322,7 @@ pub trait PyAnyMethods<'py> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let module = PyModule::from_code(py, CODE, "", "")?; + /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let fun = module.getattr("function")?; /// let args = ("hello",); /// let result = fun.call1(args)?; @@ -1357,12 +1357,12 @@ pub trait PyAnyMethods<'py> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let module = PyModule::from_code(py, CODE, "", "")?; + /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let instance = module.getattr("a")?; /// let args = ("hello",); /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; - /// let result = instance.call_method("method", args, Some(kwargs.as_gil_ref()))?; + /// let result = instance.call_method("method", args, Some(&kwargs))?; /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); /// Ok(()) /// }) @@ -1401,7 +1401,7 @@ pub trait PyAnyMethods<'py> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let module = PyModule::from_code(py, CODE, "", "")?; + /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let instance = module.getattr("a")?; /// let result = instance.call_method0("method")?; /// assert_eq!(result.extract::<&str>()?, "called with no arguments"); @@ -1436,7 +1436,7 @@ pub trait PyAnyMethods<'py> { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| { - /// let module = PyModule::from_code(py, CODE, "", "")?; + /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let instance = module.getattr("a")?; /// let args = ("hello",); /// let result = instance.call_method1("method", args)?; @@ -2326,13 +2326,13 @@ mod tests { use crate::{ basic::CompareOp, types::{any::PyAnyMethods, IntoPyDict, PyAny, PyBool, PyList, PyLong, PyModule}, - PyNativeType, PyTypeInfo, Python, ToPyObject, + Bound, PyNativeType, PyTypeInfo, Python, ToPyObject, }; #[test] fn test_lookup_special() { Python::with_gil(|py| { - let module = PyModule::from_code( + let module = PyModule::from_code_bound( py, r#" class CustomCallable: @@ -2371,13 +2371,8 @@ class NonHeapNonDescriptorInt: .unwrap(); let int = crate::intern!(py, "__int__"); - let eval_int = |obj: &PyAny| { - obj.as_borrowed() - .lookup_special(int)? - .unwrap() - .call0()? - .extract::() - }; + let eval_int = + |obj: Bound<'_, PyAny>| obj.lookup_special(int)?.unwrap().call0()?.extract::(); let simple = module.getattr("SimpleInt").unwrap().call0().unwrap(); assert_eq!(eval_int(simple).unwrap(), 1); @@ -2430,7 +2425,7 @@ class NonHeapNonDescriptorInt: #[test] fn test_call_method0() { Python::with_gil(|py| { - let module = PyModule::from_code( + let module = PyModule::from_code_bound( py, r#" class SimpleClass: diff --git a/src/types/capsule.rs b/src/types/capsule.rs index 342ee9b44bc..aa1a910d21b 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -31,7 +31,7 @@ use std::os::raw::{c_char, c_int, c_void}; /// /// let capsule = PyCapsule::new_bound(py, foo, Some(name.clone()))?; /// -/// let module = PyModule::import(py, "builtins")?; +/// let module = PyModule::import_bound(py, "builtins")?; /// module.add("capsule", capsule)?; /// /// let cap: &Foo = unsafe { PyCapsule::import(py, name.as_ref())? }; @@ -441,11 +441,13 @@ fn name_ptr_ignore_error(slf: &Bound<'_, PyCapsule>) -> *const c_char { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use libc::c_void; use crate::prelude::PyModule; use crate::types::capsule::PyCapsuleMethods; + use crate::types::module::PyModuleMethods; use crate::{types::PyCapsule, Py, PyResult, Python}; use std::ffi::CString; use std::sync::mpsc::{channel, Sender}; @@ -528,7 +530,7 @@ mod tests { let capsule = PyCapsule::new_bound(py, foo, Some(name.clone()))?; - let module = PyModule::import(py, "builtins")?; + let module = PyModule::import_bound(py, "builtins")?; module.add("capsule", capsule)?; // check error when wrong named passed for capsule. diff --git a/src/types/module.rs b/src/types/module.rs index 137b1b37e3a..245d38d9e08 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -1,11 +1,12 @@ use crate::callback::IntoPyCallbackOutput; use crate::err::{PyErr, PyResult}; use crate::ffi_ptr_ext::FfiPtrExt; +use crate::py_result_ext::PyResultExt; use crate::pyclass::PyClass; use crate::types::{ any::PyAnyMethods, list::PyListMethods, PyAny, PyCFunction, PyDict, PyList, PyString, }; -use crate::{exceptions, ffi, Bound, FromPyObject, IntoPy, Py, PyNativeType, PyObject, Python}; +use crate::{exceptions, ffi, Bound, IntoPy, Py, PyNativeType, PyObject, Python}; use std::ffi::CString; use std::str; @@ -22,6 +23,19 @@ pub struct PyModule(PyAny); pyobject_native_type_core!(PyModule, pyobject_native_static_type_object!(ffi::PyModule_Type), #checkfunction=ffi::PyModule_Check); impl PyModule { + /// Deprecated form of [`PyModule::new_bound`]. + #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyModule::new` will be replaced by `PyModule::new_bound` in a future PyO3 version" + ) + )] + pub fn new<'py>(py: Python<'py>, name: &str) -> PyResult<&'py PyModule> { + Self::new_bound(py, name).map(Bound::into_gil_ref) + } + /// Creates a new module object with the `__name__` attribute set to `name`. /// /// # Examples @@ -31,22 +45,39 @@ impl PyModule { /// /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { - /// let module = PyModule::new(py, "my_module")?; + /// let module = PyModule::new_bound(py, "my_module")?; /// - /// assert_eq!(module.name()?, "my_module"); + /// assert_eq!(module.name()?.to_cow()?, "my_module"); /// Ok(()) /// })?; /// # Ok(())} /// ``` - pub fn new<'p>(py: Python<'p>, name: &str) -> PyResult<&'p PyModule> { + pub fn new_bound<'py>(py: Python<'py>, name: &str) -> PyResult> { // Could use PyModule_NewObject, but it doesn't exist on PyPy. let name = CString::new(name)?; - #[allow(deprecated)] unsafe { - py.from_owned_ptr_or_err(ffi::PyModule_New(name.as_ptr())) + ffi::PyModule_New(name.as_ptr()) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } + /// Deprecated form of [`PyModule::import_bound`]. + #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyModule::import` will be replaced by `PyModule::import_bound` in a future PyO3 version" + ) + )] + pub fn import(py: Python<'_>, name: N) -> PyResult<&PyModule> + where + N: IntoPy>, + { + Self::import_bound(py, name).map(Bound::into_gil_ref) + } + /// Imports the Python module with the specified name. /// /// # Examples @@ -56,7 +87,7 @@ impl PyModule { /// use pyo3::prelude::*; /// /// Python::with_gil(|py| { - /// let module = PyModule::import(py, "antigravity").expect("No flying for you."); + /// let module = PyModule::import_bound(py, "antigravity").expect("No flying for you."); /// }); /// # } /// ``` @@ -65,17 +96,36 @@ impl PyModule { /// ```python /// import antigravity /// ``` - pub fn import(py: Python<'_>, name: N) -> PyResult<&PyModule> + pub fn import_bound(py: Python<'_>, name: N) -> PyResult> where N: IntoPy>, { let name: Py = name.into_py(py); - #[allow(deprecated)] unsafe { - py.from_owned_ptr_or_err(ffi::PyImport_Import(name.as_ptr())) + ffi::PyImport_Import(name.as_ptr()) + .assume_owned_or_err(py) + .downcast_into_unchecked() } } + /// Deprecated form of [`PyModule::from_code_bound`]. + #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyModule::from_code` will be replaced by `PyModule::from_code_bound` in a future PyO3 version" + ) + )] + pub fn from_code<'py>( + py: Python<'py>, + code: &str, + file_name: &str, + module_name: &str, + ) -> PyResult<&'py PyModule> { + Self::from_code_bound(py, code, file_name, module_name).map(Bound::into_gil_ref) + } + /// Creates and loads a module named `module_name`, /// containing the Python code passed to `code` /// and pretending to live at `file_name`. @@ -105,7 +155,7 @@ impl PyModule { /// let code = include_str!("../../assets/script.py"); /// /// Python::with_gil(|py| -> PyResult<()> { - /// PyModule::from_code(py, code, "example.py", "example")?; + /// PyModule::from_code_bound(py, code, "example.py", "example")?; /// Ok(()) /// })?; /// # Ok(()) @@ -124,36 +174,29 @@ impl PyModule { /// let code = std::fs::read_to_string("assets/script.py")?; /// /// Python::with_gil(|py| -> PyResult<()> { - /// PyModule::from_code(py, &code, "example.py", "example")?; + /// PyModule::from_code_bound(py, &code, "example.py", "example")?; /// Ok(()) /// })?; /// Ok(()) /// # } /// ``` - pub fn from_code<'p>( - py: Python<'p>, + pub fn from_code_bound<'py>( + py: Python<'py>, code: &str, file_name: &str, module_name: &str, - ) -> PyResult<&'p PyModule> { + ) -> PyResult> { let data = CString::new(code)?; let filename = CString::new(file_name)?; let module = CString::new(module_name)?; unsafe { - let cptr = ffi::Py_CompileString(data.as_ptr(), filename.as_ptr(), ffi::Py_file_input); - if cptr.is_null() { - return Err(PyErr::fetch(py)); - } - - let mptr = ffi::PyImport_ExecCodeModuleEx(module.as_ptr(), cptr, filename.as_ptr()); - ffi::Py_DECREF(cptr); - if mptr.is_null() { - return Err(PyErr::fetch(py)); - } + let code = ffi::Py_CompileString(data.as_ptr(), filename.as_ptr(), ffi::Py_file_input) + .assume_owned_or_err(py)?; - #[allow(deprecated)] - <&PyModule as FromPyObject>::extract(py.from_owned_ptr_or_err(mptr)?) + ffi::PyImport_ExecCodeModuleEx(module.as_ptr(), code.as_ptr(), filename.as_ptr()) + .assume_owned_or_err(py) + .downcast_into() } } @@ -293,10 +336,10 @@ impl PyModule { /// /// #[pymodule] /// fn my_module(py: Python<'_>, module: &PyModule) -> PyResult<()> { - /// let submodule = PyModule::new(py, "submodule")?; + /// let submodule = PyModule::new_bound(py, "submodule")?; /// submodule.add("super_useful_constant", "important")?; /// - /// module.add_submodule(submodule)?; + /// module.add_submodule(submodule.as_gil_ref())?; /// Ok(()) /// } /// ``` @@ -487,10 +530,10 @@ pub trait PyModuleMethods<'py> { /// /// #[pymodule] /// fn my_module(py: Python<'_>, module: &PyModule) -> PyResult<()> { - /// let submodule = PyModule::new(py, "submodule")?; + /// let submodule = PyModule::new_bound(py, "submodule")?; /// submodule.add("super_useful_constant", "important")?; /// - /// module.add_submodule(submodule)?; + /// module.add_submodule(submodule.as_gil_ref())?; /// Ok(()) /// } /// ``` @@ -580,8 +623,6 @@ impl<'py> PyModuleMethods<'py> for Bound<'py, PyModule> { fn name(&self) -> PyResult> { #[cfg(not(PyPy))] { - use crate::py_result_ext::PyResultExt; - unsafe { ffi::PyModule_GetNameObject(self.as_ptr()) .assume_owned_or_err(self.py()) @@ -601,8 +642,6 @@ impl<'py> PyModuleMethods<'py> for Bound<'py, PyModule> { #[cfg(not(PyPy))] fn filename(&self) -> PyResult> { - use crate::py_result_ext::PyResultExt; - unsafe { ffi::PyModule_GetFilenameObject(self.as_ptr()) .assume_owned_or_err(self.py()) @@ -676,14 +715,21 @@ fn __name__(py: Python<'_>) -> &Bound<'_, PyString> { } #[cfg(test)] +#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { - use crate::{types::PyModule, Python}; + use crate::{ + types::{module::PyModuleMethods, string::PyStringMethods, PyModule}, + Python, + }; #[test] fn module_import_and_name() { Python::with_gil(|py| { - let builtins = PyModule::import(py, "builtins").unwrap(); - assert_eq!(builtins.name().unwrap(), "builtins"); + let builtins = PyModule::import_bound(py, "builtins").unwrap(); + assert_eq!( + builtins.name().unwrap().to_cow().unwrap().as_ref(), + "builtins" + ); }) } @@ -691,8 +737,13 @@ mod tests { #[cfg(not(PyPy))] fn module_filename() { Python::with_gil(|py| { - let site = PyModule::import(py, "site").unwrap(); - assert!(site.filename().unwrap().ends_with("site.py")); + let site = PyModule::import_bound(py, "site").unwrap(); + assert!(site + .filename() + .unwrap() + .to_cow() + .unwrap() + .ends_with("site.py")); }) } } diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 24b935e24c2..b8782463367 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -1,11 +1,8 @@ use crate::err::{error_on_minusone, PyResult}; -use crate::types::PyString; +use crate::types::{any::PyAnyMethods, string::PyStringMethods, PyString}; use crate::{ffi, Bound}; use crate::{PyAny, PyNativeType}; -use super::any::PyAnyMethods; -use super::string::PyStringMethods; - /// Represents a Python traceback. #[repr(transparent)] pub struct PyTraceback(PyAny); diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index fe80625e86b..21ccc4a6bb6 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -154,7 +154,7 @@ struct EmptyClassInModule {} #[ignore] fn empty_class_in_module() { Python::with_gil(|py| { - let module = PyModule::new(py, "test_module.nested").unwrap(); + let module = PyModule::new_bound(py, "test_module.nested").unwrap(); module.add_class::().unwrap(); let ty = module.getattr("EmptyClassInModule").unwrap(); diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index 9037590f678..9e16631bb83 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -169,10 +169,7 @@ c = Class() assert c.from_rust is False "# ); - let globals = PyModule::import(py, "__main__") - .unwrap() - .dict() - .as_borrowed(); + let globals = PyModule::import_bound(py, "__main__").unwrap().dict(); globals.set_item("SuperClass", super_cls).unwrap(); py.run_bound(source, Some(&globals), None) .map_err(|e| e.display(py)) diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index b3b8ba7be1d..e04aafda24d 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -262,7 +262,7 @@ fn test_async_method_receiver() { Python::with_gil(|gil| { let test = r#" import asyncio - + obj = Counter() coro1 = obj.get() coro2 = obj.get() diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 9b29fefdbda..7a465f1a995 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -354,7 +354,7 @@ fn module_add_class_inherit_bool_fails() { struct ExtendsBool; Python::with_gil(|py| { - let m = PyModule::new(py, "test_module").unwrap(); + let m = PyModule::new_bound(py, "test_module").unwrap(); let err = m.add_class::().unwrap_err(); assert_eq!( diff --git a/tests/test_module.rs b/tests/test_module.rs index 9a59770e047..9d14f243d50 100644 --- a/tests/test_module.rs +++ b/tests/test_module.rs @@ -135,9 +135,9 @@ fn test_module_renaming() { } #[test] -fn test_module_from_code() { +fn test_module_from_code_bound() { Python::with_gil(|py| { - let adder_mod = PyModule::from_code( + let adder_mod = PyModule::from_code_bound( py, "def add(a,b):\n\treturn a+b", "adder_mod.py", @@ -239,8 +239,8 @@ fn subfunction() -> String { "Subfunction".to_string() } -fn submodule(module: &PyModule) -> PyResult<()> { - module.add_function(wrap_pyfunction!(subfunction, module)?)?; +fn submodule(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(&wrap_pyfunction!(subfunction, module.as_gil_ref())?.as_borrowed())?; Ok(()) } @@ -258,12 +258,12 @@ fn superfunction() -> String { #[pymodule] fn supermodule(py: Python<'_>, module: &PyModule) -> PyResult<()> { module.add_function(wrap_pyfunction!(superfunction, module)?)?; - let module_to_add = PyModule::new(py, "submodule")?; - submodule(module_to_add)?; - module.add_submodule(module_to_add)?; - let module_to_add = PyModule::new(py, "submodule_with_init_fn")?; - submodule_with_init_fn(py, module_to_add)?; - module.add_submodule(module_to_add)?; + let module_to_add = PyModule::new_bound(py, "submodule")?; + submodule(&module_to_add)?; + module.add_submodule(module_to_add.as_gil_ref())?; + let module_to_add = PyModule::new_bound(py, "submodule_with_init_fn")?; + submodule_with_init_fn(py, module_to_add.as_gil_ref())?; + module.add_submodule(module_to_add.as_gil_ref())?; Ok(()) } diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index fb813b6e52d..61b912431e8 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -697,10 +697,7 @@ if sys.platform == "win32" and sys.version_info >= (3, 8, 0): asyncio.run(main()) "#; - let globals = PyModule::import(py, "__main__") - .unwrap() - .dict() - .as_borrowed(); + let globals = PyModule::import_bound(py, "__main__").unwrap().dict(); globals.set_item("Once", once).unwrap(); py.run_bound(source, Some(&globals), None) .map_err(|e| e.display(py)) @@ -754,10 +751,7 @@ if sys.platform == "win32" and sys.version_info >= (3, 8, 0): asyncio.run(main()) "#; - let globals = PyModule::import(py, "__main__") - .unwrap() - .dict() - .as_borrowed(); + let globals = PyModule::import_bound(py, "__main__").unwrap().dict(); globals.set_item("Once", once).unwrap(); globals .set_item("AsyncIterator", py.get_type_bound::()) @@ -829,10 +823,7 @@ del c.counter assert c.counter.count == 1 "# ); - let globals = PyModule::import(py, "__main__") - .unwrap() - .dict() - .as_borrowed(); + let globals = PyModule::import_bound(py, "__main__").unwrap().dict(); globals.set_item("Counter", counter).unwrap(); py.run_bound(source, Some(&globals), None) .map_err(|e| e.display(py)) diff --git a/tests/test_various.rs b/tests/test_various.rs index 9ed1e1cf546..6f1bfd908a7 100644 --- a/tests/test_various.rs +++ b/tests/test_various.rs @@ -133,8 +133,8 @@ impl PickleSupport { } } -fn add_module(py: Python<'_>, module: &PyModule) -> PyResult<()> { - py.import_bound("sys")? +fn add_module(module: Bound<'_, PyModule>) -> PyResult<()> { + PyModule::import_bound(module.py(), "sys")? .dict() .get_item("modules") .unwrap() @@ -147,9 +147,9 @@ fn add_module(py: Python<'_>, module: &PyModule) -> PyResult<()> { #[cfg_attr(all(Py_LIMITED_API, not(Py_3_10)), ignore)] fn test_pickle() { Python::with_gil(|py| { - let module = PyModule::new(py, "test_module").unwrap(); + let module = PyModule::new_bound(py, "test_module").unwrap(); module.add_class::().unwrap(); - add_module(py, module).unwrap(); + add_module(module).unwrap(); let inst = Py::new(py, PickleSupport {}).unwrap(); py_run!( py, From 4f8ee968812977c1ecc9909143ecea75b114870d Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Thu, 22 Feb 2024 23:38:42 +0100 Subject: [PATCH 162/349] fix `AsRef` and `Deref` impls on `Bound` (#3879) * fix `AsRef` and `Deref` of `Bound` to `Bound` * cleanup unnessesary `.as_any()` calls * remove trait bound on `AsRef` impl * add comment for `Deref` trait bound * rename marker trait --- guide/src/class/object.md | 4 ++-- pyo3-macros-backend/src/pyclass.rs | 2 ++ src/instance.rs | 12 +++++----- src/types/boolobject.rs | 4 ++-- src/types/mod.rs | 27 ++++++++++++++++++++++ tests/test_buffer.rs | 12 +++++----- tests/test_class_conversion.rs | 6 ++--- tests/test_field_cfg.rs | 10 ++------ tests/test_inheritance.rs | 2 +- tests/test_mapping.rs | 4 ++-- tests/test_proto_methods.rs | 37 +++++++++++------------------- 11 files changed, 65 insertions(+), 55 deletions(-) diff --git a/guide/src/class/object.md b/guide/src/class/object.md index 471889391e2..10867976df0 100644 --- a/guide/src/class/object.md +++ b/guide/src/class/object.md @@ -217,8 +217,8 @@ impl Number { # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let x = &Bound::new(py, Number(4))?.into_any(); -# let y = &Bound::new(py, Number(4))?.into_any(); +# let x = &Bound::new(py, Number(4))?; +# let y = &Bound::new(py, Number(4))?; # assert!(x.eq(y)?); # assert!(!x.ne(y)?); # Ok(()) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 806df88eee5..ce39cb01196 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -365,6 +365,8 @@ fn impl_class( const _: () = { use #krate as _pyo3; + impl _pyo3::types::DerefToPyAny for #cls {} + #pytypeinfo_impl #py_class_impl diff --git a/src/instance.rs b/src/instance.rs index f35ab4bbb3a..ecf128112a2 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -4,7 +4,7 @@ use crate::pycell::{PyBorrowError, PyBorrowMutError, PyCell}; use crate::pyclass::boolean_struct::{False, True}; use crate::type_object::HasPyGilRef; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; -use crate::types::{PyDict, PyString, PyTuple}; +use crate::types::{DerefToPyAny, PyDict, PyString, PyTuple}; use crate::{ ffi, AsPyPointer, DowncastError, FromPyObject, IntoPy, PyAny, PyClass, PyClassInitializer, PyRef, PyRefMut, PyTypeInfo, Python, ToPyObject, @@ -366,9 +366,12 @@ fn python_format( } } +// The trait bound is needed to avoid running into the auto-deref recursion +// limit (error[E0055]), because `Bound` would deref into itself. See: +// https://github.com/rust-lang/rust/issues/19509 impl<'py, T> Deref for Bound<'py, T> where - T: AsRef, + T: DerefToPyAny, { type Target = Bound<'py, PyAny>; @@ -378,10 +381,7 @@ where } } -impl<'py, T> AsRef> for Bound<'py, T> -where - T: AsRef, -{ +impl<'py, T> AsRef> for Bound<'py, T> { #[inline] fn as_ref(&self) -> &Bound<'py, PyAny> { self.as_any() diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 01bc14b4431..906a967069a 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -168,7 +168,7 @@ mod tests { Python::with_gil(|py| { assert!(PyBool::new_bound(py, true).is_true()); let t = PyBool::new_bound(py, true); - assert!(t.as_any().extract::().unwrap()); + assert!(t.extract::().unwrap()); assert!(true.to_object(py).is(&*PyBool::new_bound(py, true))); }); } @@ -178,7 +178,7 @@ mod tests { Python::with_gil(|py| { assert!(!PyBool::new_bound(py, false).is_true()); let t = PyBool::new_bound(py, false); - assert!(!t.as_any().extract::().unwrap()); + assert!(!t.extract::().unwrap()); assert!(false.to_object(py).is(&*PyBool::new_bound(py, false))); }); } diff --git a/src/types/mod.rs b/src/types/mod.rs index 4ebf050b4b8..217e240129f 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -86,6 +86,31 @@ pub mod iter { pub use super::tuple::{BorrowedTupleIterator, BoundTupleIterator, PyTupleIterator}; } +/// Python objects that have a base type. +/// +/// This marks types that can be upcast into a [`PyAny`] and used in its place. +/// This essentially includes every Python object except [`PyAny`] itself. +/// +/// This is used to provide the [`Deref>`](std::ops::Deref) +/// implementations for [`Bound<'_, T>`](crate::Bound). +/// +/// Users should not need to implement this trait directly. It's implementation +/// is provided by the [`#[pyclass]`](macro@crate::pyclass) attribute. +/// +/// ## Note +/// This is needed because the compiler currently tries to figure out all the +/// types in a deref-chain before starting to look for applicable method calls. +/// So we need to prevent [`Bound<'_, PyAny`](crate::Bound) dereferencing to +/// itself in order to avoid running into the recursion limit. This trait is +/// used to exclude this from our blanket implementation. See [this Rust +/// issue][1] for more details. If the compiler limitation gets resolved, this +/// trait will be removed. +/// +/// [1]: https://github.com/rust-lang/rust/issues/19509 +pub trait DerefToPyAny { + // Empty. +} + // Implementations core to all native types #[doc(hidden)] #[macro_export] @@ -183,6 +208,8 @@ macro_rules! pyobject_native_type_named ( unsafe{&*(ob as *const $name as *const $crate::PyAny)} } } + + impl $crate::types::DerefToPyAny for $name {} }; ); diff --git a/tests/test_buffer.rs b/tests/test_buffer.rs index 72f94b3bcc8..0b3da881884 100644 --- a/tests/test_buffer.rs +++ b/tests/test_buffer.rs @@ -96,11 +96,11 @@ fn test_get_buffer_errors() { ) .unwrap(); - assert!(PyBuffer::::get_bound(instance.bind(py).as_any()).is_ok()); + assert!(PyBuffer::::get_bound(instance.bind(py)).is_ok()); instance.borrow_mut(py).error = Some(TestGetBufferError::NullShape); assert_eq!( - PyBuffer::::get_bound(instance.bind(py).as_any()) + PyBuffer::::get_bound(instance.bind(py)) .unwrap_err() .to_string(), "BufferError: shape is null" @@ -108,7 +108,7 @@ fn test_get_buffer_errors() { instance.borrow_mut(py).error = Some(TestGetBufferError::NullStrides); assert_eq!( - PyBuffer::::get_bound(instance.bind(py).as_any()) + PyBuffer::::get_bound(instance.bind(py)) .unwrap_err() .to_string(), "BufferError: strides is null" @@ -116,7 +116,7 @@ fn test_get_buffer_errors() { instance.borrow_mut(py).error = Some(TestGetBufferError::IncorrectItemSize); assert_eq!( - PyBuffer::::get_bound(instance.bind(py).as_any()) + PyBuffer::::get_bound(instance.bind(py)) .unwrap_err() .to_string(), "BufferError: buffer contents are not compatible with u32" @@ -124,7 +124,7 @@ fn test_get_buffer_errors() { instance.borrow_mut(py).error = Some(TestGetBufferError::IncorrectFormat); assert_eq!( - PyBuffer::::get_bound(instance.bind(py).as_any()) + PyBuffer::::get_bound(instance.bind(py)) .unwrap_err() .to_string(), "BufferError: buffer contents are not compatible with u32" @@ -132,7 +132,7 @@ fn test_get_buffer_errors() { instance.borrow_mut(py).error = Some(TestGetBufferError::IncorrectAlignment); assert_eq!( - PyBuffer::::get_bound(instance.bind(py).as_any()) + PyBuffer::::get_bound(instance.bind(py)) .unwrap_err() .to_string(), "BufferError: buffer contents are insufficiently aligned for u32" diff --git a/tests/test_class_conversion.rs b/tests/test_class_conversion.rs index ccb974097e1..a09195278b0 100644 --- a/tests/test_class_conversion.rs +++ b/tests/test_class_conversion.rs @@ -146,13 +146,11 @@ fn test_pyref_as_base() { #[test] fn test_pycell_deref() { Python::with_gil(|py| { - let cell = Bound::new(py, (SubClass {}, BaseClass { value: 120 })).unwrap(); + let obj = Bound::new(py, (SubClass {}, BaseClass { value: 120 })).unwrap(); // Should be able to deref as PyAny - // FIXME: This deref does _not_ work assert_eq!( - cell.as_any() - .call_method0("foo") + obj.call_method0("foo") .and_then(|e| e.extract::<&str>()) .unwrap(), "SubClass" diff --git a/tests/test_field_cfg.rs b/tests/test_field_cfg.rs index be400f2d749..c5fc4958dbf 100644 --- a/tests/test_field_cfg.rs +++ b/tests/test_field_cfg.rs @@ -22,14 +22,8 @@ fn test_cfg() { Python::with_gil(|py| { let cfg = CfgClass { b: 3 }; let py_cfg = Py::new(py, cfg).unwrap(); - assert!(py_cfg.bind(py).as_any().getattr("a").is_err()); - let b: u32 = py_cfg - .bind(py) - .as_any() - .getattr("b") - .unwrap() - .extract() - .unwrap(); + assert!(py_cfg.bind(py).getattr("a").is_err()); + let b: u32 = py_cfg.bind(py).getattr("b").unwrap().extract().unwrap(); assert_eq!(b, 3); }); } diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 7a465f1a995..1209713e487 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -247,7 +247,7 @@ mod inheriting_native_type { let item = &py.eval_bound("object()", None, None).unwrap(); assert_eq!(item.get_refcnt(), 1); - dict_sub.bind(py).as_any().set_item("foo", item).unwrap(); + dict_sub.bind(py).set_item("foo", item).unwrap(); assert_eq!(item.get_refcnt(), 2); drop(dict_sub); diff --git a/tests/test_mapping.rs b/tests/test_mapping.rs index 06928e74183..4e24db97793 100644 --- a/tests/test_mapping.rs +++ b/tests/test_mapping.rs @@ -123,7 +123,7 @@ fn mapping_is_not_sequence() { PyMapping::register::(py).unwrap(); - assert!(m.bind(py).as_any().downcast::().is_ok()); - assert!(m.bind(py).as_any().downcast::().is_err()); + assert!(m.bind(py).downcast::().is_ok()); + assert!(m.bind(py).downcast::().is_err()); }); } diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index 61b912431e8..a1cfde2badf 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -81,7 +81,6 @@ fn test_getattr() { let example_py = make_example(py); assert_eq!( example_py - .as_any() .getattr("value") .unwrap() .extract::() @@ -90,7 +89,6 @@ fn test_getattr() { ); assert_eq!( example_py - .as_any() .getattr("special_custom_attr") .unwrap() .extract::() @@ -98,7 +96,6 @@ fn test_getattr() { 20, ); assert!(example_py - .as_any() .getattr("other_attr") .unwrap_err() .is_instance_of::(py)); @@ -109,13 +106,9 @@ fn test_getattr() { fn test_setattr() { Python::with_gil(|py| { let example_py = make_example(py); - example_py - .as_any() - .setattr("special_custom_attr", 15) - .unwrap(); + example_py.setattr("special_custom_attr", 15).unwrap(); assert_eq!( example_py - .as_any() .getattr("special_custom_attr") .unwrap() .extract::() @@ -129,12 +122,8 @@ fn test_setattr() { fn test_delattr() { Python::with_gil(|py| { let example_py = make_example(py); - example_py.as_any().delattr("special_custom_attr").unwrap(); - assert!(example_py - .as_any() - .getattr("special_custom_attr") - .unwrap() - .is_none()); + example_py.delattr("special_custom_attr").unwrap(); + assert!(example_py.getattr("special_custom_attr").unwrap().is_none()); }) } @@ -142,7 +131,7 @@ fn test_delattr() { fn test_str() { Python::with_gil(|py| { let example_py = make_example(py); - assert_eq!(example_py.as_any().str().unwrap().to_cow().unwrap(), "5"); + assert_eq!(example_py.str().unwrap().to_cow().unwrap(), "5"); }) } @@ -151,7 +140,7 @@ fn test_repr() { Python::with_gil(|py| { let example_py = make_example(py); assert_eq!( - example_py.as_any().repr().unwrap().to_cow().unwrap(), + example_py.repr().unwrap().to_cow().unwrap(), "ExampleClass(value=5)" ); }) @@ -161,7 +150,7 @@ fn test_repr() { fn test_hash() { Python::with_gil(|py| { let example_py = make_example(py); - assert_eq!(example_py.as_any().hash().unwrap(), 5); + assert_eq!(example_py.hash().unwrap(), 5); }) } @@ -169,9 +158,9 @@ fn test_hash() { fn test_bool() { Python::with_gil(|py| { let example_py = make_example(py); - assert!(example_py.as_any().is_truthy().unwrap()); + assert!(example_py.is_truthy().unwrap()); example_py.borrow_mut().value = 0; - assert!(!example_py.as_any().is_truthy().unwrap()); + assert!(!example_py.is_truthy().unwrap()); }) } @@ -231,7 +220,7 @@ fn mapping() { ) .unwrap(); - let mapping: &Bound<'_, PyMapping> = inst.bind(py).as_any().downcast().unwrap(); + let mapping: &Bound<'_, PyMapping> = inst.bind(py).downcast().unwrap(); py_assert!(py, inst, "len(inst) == 0"); @@ -333,7 +322,7 @@ fn sequence() { let inst = Py::new(py, Sequence { values: vec![] }).unwrap(); - let sequence: &Bound<'_, PySequence> = inst.bind(py).as_any().downcast().unwrap(); + let sequence: &Bound<'_, PySequence> = inst.bind(py).downcast().unwrap(); py_assert!(py, inst, "len(inst) == 0"); @@ -360,16 +349,16 @@ fn sequence() { // indices. assert!(sequence.len().is_err()); // however regular python len() works thanks to mp_len slot - assert_eq!(inst.bind(py).as_any().len().unwrap(), 0); + assert_eq!(inst.bind(py).len().unwrap(), 0); py_run!(py, inst, "inst.append(0)"); sequence.set_item(0, 5).unwrap(); - assert_eq!(inst.bind(py).as_any().len().unwrap(), 1); + assert_eq!(inst.bind(py).len().unwrap(), 1); assert_eq!(sequence.get_item(0).unwrap().extract::().unwrap(), 5); sequence.del_item(0).unwrap(); - assert_eq!(inst.bind(py).as_any().len().unwrap(), 0); + assert_eq!(inst.bind(py).len().unwrap(), 0); }); } From 8bd82da93923c42acac860bc182398c8ad9c617a Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Thu, 22 Feb 2024 23:52:25 +0100 Subject: [PATCH 163/349] add missing deprecation for `PyDict::from_sequence` (#3884) --- src/types/dict.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/types/dict.rs b/src/types/dict.rs index 39a0c45e35e..2d215c1f8b9 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -75,6 +75,13 @@ impl PyDict { } /// Deprecated form of [`from_sequence_bound`][PyDict::from_sequence_bound]. + #[cfg_attr( + all(not(PyPy), not(feature = "gil-refs")), + deprecated( + since = "0.21.0", + note = "`PyDict::from_sequence` will be replaced by `PyDict::from_sequence_bound` in a future PyO3 version" + ) + )] #[inline] #[cfg(not(PyPy))] pub fn from_sequence(seq: &PyAny) -> PyResult<&PyDict> { From 22a23ffb3150037fc62a5b74e3a0a8cec9cbdf6f Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Thu, 22 Feb 2024 23:06:55 +0000 Subject: [PATCH 164/349] Tidy some usage of `py.from_borrowed_ptr` and `py.from_borrowed_ptr_or_opt` (#3877) * Tidy some usage of py.from_borrowed_ptr * Add BoundRef::ref_from_ptr_or_opt --- pyo3-macros-backend/src/pymethod.rs | 6 +++--- src/ffi/tests.rs | 14 ++++++-------- src/impl_/pyclass.rs | 8 +++++--- src/impl_/pymethods.rs | 7 +++++++ src/instance.rs | 12 ++++++++++++ 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index d45d2e12f26..74072f2a745 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -572,12 +572,12 @@ pub fn impl_py_setter_def( _slf: *mut _pyo3::ffi::PyObject, _value: *mut _pyo3::ffi::PyObject, ) -> _pyo3::PyResult<::std::os::raw::c_int> { - let _value = py - .from_borrowed_ptr_or_opt(_value) + use ::std::convert::Into; + let _value = _pyo3::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_value) .ok_or_else(|| { _pyo3::exceptions::PyAttributeError::new_err("can't delete attribute") })?; - let _val = _pyo3::FromPyObject::extract(_value)?; + let _val = _pyo3::FromPyObject::extract_bound(_value.into())?; #( #holders )* _pyo3::callback::convert(py, #setter_impl) } diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index d67bd8485d8..3532172c933 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -57,9 +57,9 @@ fn test_date_fromtimestamp() { #[test] fn test_utc_timezone() { Python::with_gil(|py| { - let utc_timezone: &PyAny = unsafe { + let utc_timezone: Bound<'_, PyAny> = unsafe { PyDateTime_IMPORT(); - py.from_borrowed_ptr(PyDateTime_TimeZone_UTC()) + Bound::from_borrowed_ptr(py, PyDateTime_TimeZone_UTC()) }; let locals = PyDict::new_bound(py); locals.set_item("utc_timezone", utc_timezone).unwrap(); @@ -254,35 +254,33 @@ fn test_get_tzinfo() { crate::Python::with_gil(|py| { use crate::types::{PyDateTime, PyTime}; - use crate::PyAny; let utc = &timezone_utc_bound(py); let dt = PyDateTime::new_bound(py, 2018, 1, 1, 0, 0, 0, 0, Some(utc)).unwrap(); assert!( - unsafe { py.from_borrowed_ptr::(PyDateTime_DATE_GET_TZINFO(dt.as_ptr())) } + unsafe { Bound::from_borrowed_ptr(py, PyDateTime_DATE_GET_TZINFO(dt.as_ptr())) } .is(utc) ); let dt = PyDateTime::new_bound(py, 2018, 1, 1, 0, 0, 0, 0, None).unwrap(); assert!( - unsafe { py.from_borrowed_ptr::(PyDateTime_DATE_GET_TZINFO(dt.as_ptr())) } + unsafe { Bound::from_borrowed_ptr(py, PyDateTime_DATE_GET_TZINFO(dt.as_ptr())) } .is_none() ); let t = PyTime::new_bound(py, 0, 0, 0, 0, Some(utc)).unwrap(); assert!( - unsafe { py.from_borrowed_ptr::(PyDateTime_TIME_GET_TZINFO(t.as_ptr())) } - .is(utc) + unsafe { Bound::from_borrowed_ptr(py, PyDateTime_TIME_GET_TZINFO(t.as_ptr())) }.is(utc) ); let t = PyTime::new_bound(py, 0, 0, 0, 0, None).unwrap(); assert!( - unsafe { py.from_borrowed_ptr::(PyDateTime_TIME_GET_TZINFO(t.as_ptr())) } + unsafe { Bound::from_borrowed_ptr(py, PyDateTime_TIME_GET_TZINFO(t.as_ptr())) } .is_none() ); }) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 2bd39dda1a0..e2204fabb02 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -6,8 +6,10 @@ use crate::{ internal_tricks::extract_c_string, pycell::PyCellLayout, pyclass_init::PyObjectInit, + types::any::PyAnyMethods, types::PyBool, - Py, PyAny, PyCell, PyClass, PyErr, PyMethodDefType, PyNativeType, PyResult, PyTypeInfo, Python, + Borrowed, Py, PyAny, PyCell, PyClass, PyErr, PyMethodDefType, PyNativeType, PyResult, + PyTypeInfo, Python, }; use std::{ borrow::Cow, @@ -811,8 +813,8 @@ slot_fragment_trait! { other: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { // By default `__ne__` will try `__eq__` and invert the result - let slf: &PyAny = py.from_borrowed_ptr(slf); - let other: &PyAny = py.from_borrowed_ptr(other); + let slf = Borrowed::from_ptr(py, slf); + let other = Borrowed::from_ptr(py, other); slf.eq(other).map(|is_eq| PyBool::new_bound(py, !is_eq).to_owned().into_ptr()) } } diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index eef3b569d08..196766d0034 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -483,6 +483,13 @@ impl<'a, 'py> BoundRef<'a, 'py, PyAny> { BoundRef(Bound::ref_from_ptr(py, ptr)) } + pub unsafe fn ref_from_ptr_or_opt( + py: Python<'py>, + ptr: &'a *mut ffi::PyObject, + ) -> Option { + Bound::ref_from_ptr_or_opt(py, ptr).as_ref().map(BoundRef) + } + pub unsafe fn downcast_unchecked(self) -> BoundRef<'a, 'py, T> { BoundRef(self.0.downcast_unchecked::()) } diff --git a/src/instance.rs b/src/instance.rs index ecf128112a2..307b341fac7 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -191,6 +191,18 @@ impl<'py> Bound<'py, PyAny> { ) -> &'a Self { &*(ptr as *const *mut ffi::PyObject).cast::>() } + + /// Variant of the above which returns `None` for null pointers. + /// + /// # Safety + /// - `ptr` must be a valid pointer to a Python object for the lifetime `'a, or null. + #[inline] + pub(crate) unsafe fn ref_from_ptr_or_opt<'a>( + _py: Python<'py>, + ptr: &'a *mut ffi::PyObject, + ) -> &'a Option { + &*(ptr as *const *mut ffi::PyObject).cast::>>() + } } impl<'py, T> Bound<'py, T> From 5ca810236dcd3dac183d60b33a7115b29226bf65 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 23 Feb 2024 00:51:50 +0000 Subject: [PATCH 165/349] ci: rework GitHub caching strategy (#3886) * ci: rework GitHub caching strategy * clean up some more redundant imports flagged by nightly --- .github/workflows/benches.yml | 11 +++---- .github/workflows/build.yml | 3 +- .github/workflows/cache-cleanup.yml | 29 ++++++++++++++++++ .github/workflows/ci.yml | 46 ++++++++++++----------------- src/conversions/chrono.rs | 2 +- src/conversions/chrono_tz.rs | 2 -- src/conversions/num_bigint.rs | 5 +--- src/conversions/num_complex.rs | 2 +- 8 files changed, 56 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/cache-cleanup.yml diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 6c4d31e074a..572f77efd76 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -23,14 +23,11 @@ jobs: with: components: rust-src - - uses: actions/cache@v4 + - uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - pyo3-benches/target - target - key: cargo-${{ runner.os }}-bench-${{ hashFiles('**/Cargo.toml') }} + workspaces: | + . + pyo3-benches continue-on-error: true - name: Install cargo-codspeed diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af05eb20376..0f3e707bfba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,8 +47,7 @@ jobs: - uses: Swatinem/rust-cache@v2 with: - key: cargo-${{ inputs.python-architecture }} - continue-on-error: true + save-if: ${{ github.event_name != 'merge_group' }} - if: inputs.os == 'ubuntu-latest' name: Prepare LD_LIBRARY_PATH (Ubuntu only) diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml new file mode 100644 index 00000000000..2833ed093ee --- /dev/null +++ b/.github/workflows/cache-cleanup.yml @@ -0,0 +1,29 @@ +name: CI Cache Cleanup +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Cleanup + run: | + gh extension install actions/gh-actions-cache + + echo "Fetching list of cache key" + cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + BRANCH: refs/pull/${{ github.event.pull_request.number }}/merge diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6331a79c645..a61da751394 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,6 @@ jobs: check-msrv: needs: [fmt] runs-on: ubuntu-latest - if: github.ref != 'refs/heads/main' steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master @@ -57,8 +56,7 @@ jobs: architecture: "x64" - uses: Swatinem/rust-cache@v2 with: - key: check-msrv-1.56.0 - continue-on-error: true + save-if: ${{ github.event_name != 'merge_group' }} - run: python -m pip install --upgrade pip && pip install nox - name: Prepare minimal package versions run: nox -s set-minimal-package-versions @@ -70,7 +68,6 @@ jobs: clippy: needs: [fmt] runs-on: ${{ matrix.platform.os }} - if: github.ref != 'refs/heads/main' strategy: # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} @@ -136,8 +133,7 @@ jobs: architecture: ${{ matrix.platform.python-architecture }} - uses: Swatinem/rust-cache@v2 with: - key: clippy-${{ matrix.platform.rust-target }}-${{ matrix.platform.os }}-${{ matrix.rust }} - continue-on-error: true + save-if: ${{ github.event_name != 'merge_group' }} - run: python -m pip install --upgrade pip && pip install nox - run: nox -s clippy-all env: @@ -199,7 +195,7 @@ jobs: } extra-features: "nightly multiple-pymethods" build-full: - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} needs: [fmt] uses: ./.github/workflows/build.yml @@ -293,7 +289,7 @@ jobs: extra-features: "multiple-pymethods" valgrind: - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] runs-on: ubuntu-latest steps: @@ -301,8 +297,7 @@ jobs: - uses: actions/setup-python@v5 - uses: Swatinem/rust-cache@v2 with: - key: cargo-valgrind - continue-on-error: true + save-if: ${{ github.event_name != 'merge_group' }} - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@valgrind - run: python -m pip install --upgrade pip && pip install nox @@ -313,7 +308,7 @@ jobs: TRYBUILD: overwrite careful: - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] runs-on: ubuntu-latest steps: @@ -321,8 +316,7 @@ jobs: - uses: actions/setup-python@v5 - uses: Swatinem/rust-cache@v2 with: - key: cargo-careful - continue-on-error: true + save-if: ${{ github.event_name != 'merge_group' }} - uses: dtolnay/rust-toolchain@nightly with: components: rust-src @@ -334,7 +328,7 @@ jobs: TRYBUILD: overwrite docsrs: - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] runs-on: ubuntu-latest steps: @@ -342,8 +336,7 @@ jobs: - uses: actions/setup-python@v5 - uses: Swatinem/rust-cache@v2 with: - key: cargo-careful - continue-on-error: true + save-if: ${{ github.event_name != 'merge_group' }} - uses: dtolnay/rust-toolchain@nightly with: components: rust-src @@ -368,8 +361,7 @@ jobs: - uses: Swatinem/rust-cache@v2 if: steps.should-skip.outputs.skip != 'true' with: - key: coverage-cargo-${{ matrix.os }} - continue-on-error: true + save-if: ${{ github.event_name != 'merge_group' }} - uses: dtolnay/rust-toolchain@stable if: steps.should-skip.outputs.skip != 'true' with: @@ -390,7 +382,7 @@ jobs: emscripten: name: emscripten - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -416,7 +408,7 @@ jobs: key: ${{ hashFiles('emscripten/*') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }} - uses: Swatinem/rust-cache@v2 with: - key: cargo-emscripten-wasm32 + save-if: ${{ github.event_name != 'merge_group' }} - name: Build if: steps.cache.outputs.cache-hit != 'true' run: nox -s build-emscripten @@ -425,14 +417,12 @@ jobs: test-debug: needs: [fmt] - if: github.ref != 'refs/heads/main' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 with: - key: cargo-test-debug - continue-on-error: true + save-if: ${{ github.event_name != 'merge_group' }} - uses: dtolnay/rust-toolchain@stable with: components: rust-src @@ -470,25 +460,27 @@ jobs: test-version-limits: needs: [fmt] - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 - continue-on-error: true + with: + save-if: ${{ github.event_name != 'merge_group' }} - uses: dtolnay/rust-toolchain@stable - run: python3 -m pip install --upgrade pip && pip install nox - run: python3 -m nox -s test-version-limits check-feature-powerset: needs: [fmt] - if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 - uses: Swatinem/rust-cache@v2 - continue-on-error: true + with: + save-if: ${{ github.event_name != 'merge_group' }} - uses: dtolnay/rust-toolchain@stable with: components: rust-src diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 0c96ef9c705..3aa6dbeb4ca 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -564,7 +564,7 @@ fn timezone_utc_bound(py: Python<'_>) -> Bound<'_, PyAny> { #[cfg(test)] mod tests { use super::*; - use crate::{types::PyTuple, Bound, Py}; + use crate::{types::PyTuple, Py}; use std::{cmp::Ordering, panic}; #[test] diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index 986bb8313d2..b4d0e7edf8c 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -69,8 +69,6 @@ impl FromPyObject<'_> for Tz { #[cfg(all(test, not(windows)))] // Troubles loading timezones on Windows mod tests { - use crate::{types::any::PyAnyMethods, Bound}; - use super::*; #[test] diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 48a4ee60d73..e6f345fd200 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -262,10 +262,7 @@ fn int_n_bits(long: &Bound<'_, PyLong>) -> PyResult { #[cfg(test)] mod tests { use super::*; - use crate::{ - types::{PyDict, PyModule}, - Bound, - }; + use crate::types::{PyDict, PyModule}; use indoc::indoc; fn rust_fib() -> impl Iterator diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index c736a7baa77..369888af85f 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -199,7 +199,7 @@ complex_conversion!(f64); #[cfg(test)] mod tests { use super::*; - use crate::types::{any::PyAnyMethods, complex::PyComplexMethods, PyModule}; + use crate::types::{complex::PyComplexMethods, PyModule}; #[test] fn from_complex() { From 6a815875a0d010aea40b65d7348764998e33ec23 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 23 Feb 2024 07:31:51 +0100 Subject: [PATCH 166/349] port `PyErr::from_type` to `Bound` API (#3885) --- src/err/mod.rs | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/err/mod.rs b/src/err/mod.rs index 46f482a6879..ab39e8cd46f 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -182,6 +182,21 @@ impl PyErr { }))) } + /// Deprecated form of [`PyErr::from_type_bound`] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyErr::from_type` will be replaced by `PyErr::from_type_bound` in a future PyO3 version" + ) + )] + pub fn from_type(ty: &PyType, args: A) -> PyErr + where + A: PyErrArguments + Send + Sync + 'static, + { + PyErr::from_state(PyErrState::lazy(ty.into(), args)) + } + /// Constructs a new PyErr from the given Python type and arguments. /// /// `ty` is the exception type; usually one of the standard exceptions @@ -192,11 +207,11 @@ impl PyErr { /// If `ty` does not inherit from `BaseException`, then a `TypeError` will be returned. /// /// If calling `ty` with `args` raises an exception, that exception will be returned. - pub fn from_type(ty: &PyType, args: A) -> PyErr + pub fn from_type_bound(ty: Bound<'_, PyType>, args: A) -> PyErr where A: PyErrArguments + Send + Sync + 'static, { - PyErr::from_state(PyErrState::lazy(ty.into(), args)) + PyErr::from_state(PyErrState::lazy(ty.unbind().into_any(), args)) } /// Deprecated form of [`PyErr::from_value_bound`]. @@ -1231,10 +1246,8 @@ mod tests { assert!(!err.matches(py, PyTypeError::type_object_bound(py))); // String is not a valid exception class, so we should get a TypeError - let err: PyErr = PyErr::from_type( - crate::types::PyString::type_object_bound(py).as_gil_ref(), - "foo", - ); + let err: PyErr = + PyErr::from_type_bound(crate::types::PyString::type_object_bound(py), "foo"); assert!(err.matches(py, PyTypeError::type_object_bound(py))); }) } From 0f92b670b2a474acc2a8faf44b1f1926ddfb741a Mon Sep 17 00:00:00 2001 From: Alexander Hill Date: Fri, 23 Feb 2024 01:49:44 -0500 Subject: [PATCH 167/349] docs: Add lcov / codecov options for nox coverage and update docs (#3825) * docs: Clarify use of llmv-cov plugin * Mention first-run * Update Contributing.md * Add options * fix --- .gitignore | 1 + Contributing.md | 9 +++++++-- noxfile.py | 12 ++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 4240d326f71..d27dfa6f9a5 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,6 @@ pip-wheel-metadata valgrind-python.supp *.pyd lcov.info +coverage.json netlify_build/ .nox/ diff --git a/Contributing.md b/Contributing.md index dad543856cc..f87fffc0906 100644 --- a/Contributing.md +++ b/Contributing.md @@ -177,9 +177,14 @@ Second, there is a Python-based benchmark contained in the `pytests` subdirector You can view what code is and isn't covered by PyO3's tests. We aim to have 100% coverage - please check coverage and add tests if you notice a lack of coverage! -- First, generate a `lcov.info` file with +- First, ensure the llmv-cov cargo plugin is installed. You may need to run the plugin through cargo once before using it with `nox`. ```shell -nox -s coverage +cargo install cargo-llvm-cov +cargo llvm-cov +``` +- Then, generate an `lcov.info` file with +```shell +nox -s coverage -- lcov ``` You can install an IDE plugin to view the coverage. For example, if you use VSCode: - Add the [coverage-gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) plugin. diff --git a/noxfile.py b/noxfile.py index 00288273aec..add737e5f15 100644 --- a/noxfile.py +++ b/noxfile.py @@ -61,6 +61,14 @@ def coverage(session: nox.Session) -> None: session.env.update(_get_coverage_env()) _run_cargo(session, "llvm-cov", "clean", "--workspace") test(session) + + cov_format = "codecov" + output_file = "coverage.json" + + if "lcov" in session.posargs: + cov_format = "lcov" + output_file = "lcov.info" + _run_cargo( session, "llvm-cov", @@ -70,9 +78,9 @@ def coverage(session: nox.Session) -> None: "--package=pyo3-macros", "--package=pyo3-ffi", "report", - "--codecov", + f"--{cov_format}", "--output-path", - "coverage.json", + output_file, ) From 9186da38924cd9dbbf8b98282e16e4382beccc7f Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 23 Feb 2024 08:16:37 +0000 Subject: [PATCH 168/349] ci: run fmt unconditionally (#3888) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a61da751394..d22dc2236b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,6 @@ env: jobs: fmt: - if: github.ref != 'refs/heads/main' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -383,6 +382,7 @@ jobs: emscripten: name: emscripten if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} + needs: [fmt] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 11d143d0c9cb33cc67e4933f54d0a0d8680899cb Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 23 Feb 2024 12:30:38 +0000 Subject: [PATCH 169/349] release: 0.20.3 (#3890) --- CHANGELOG.md | 15 ++++++++++++++- README.md | 4 ++-- examples/decorator/.template/pre-script.rhai | 2 +- .../maturin-starter/.template/pre-script.rhai | 2 +- examples/plugin/.template/pre-script.rhai | 2 +- .../.template/pre-script.rhai | 2 +- examples/word-count/.template/pre-script.rhai | 2 +- newsfragments/3619.fixed.md | 1 - newsfragments/3821.packaging.md | 1 - newsfragments/3834.fixed.md | 1 - 10 files changed, 21 insertions(+), 11 deletions(-) delete mode 100644 newsfragments/3619.fixed.md delete mode 100644 newsfragments/3821.packaging.md delete mode 100644 newsfragments/3834.fixed.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9104db2921f..295a035370a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,18 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.20.3] - 2024-02-23 + +### Packaging + +- Add `portable-atomic` dependency. [#3619](https://github.com/PyO3/pyo3/pull/3619) +- Check maximum version of Python at build time and for versions not yet supported require opt-in to the `abi3` stable ABI by the environment variable `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1`. [#3821](https://github.com/PyO3/pyo3/pull/3821) + +### Fixed + +- Use `portable-atomic` to support platforms without 64-bit atomics. [#3619](https://github.com/PyO3/pyo3/pull/3619) +- Fix compilation failure with `either` feature enabled without `experimental-inspect` enabled. [#3834](https://github.com/PyO3/pyo3/pull/3834) + ## [0.20.2] - 2024-01-04 ### Packaging @@ -1628,7 +1640,8 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.20.2...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.20.3...HEAD +[0.20.3]: https://github.com/pyo3/pyo3/compare/v0.20.2...v0.20.3 [0.20.2]: https://github.com/pyo3/pyo3/compare/v0.20.1...v0.20.2 [0.20.1]: https://github.com/pyo3/pyo3/compare/v0.20.0...v0.20.1 [0.20.0]: https://github.com/pyo3/pyo3/compare/v0.19.2...v0.20.0 diff --git a/README.md b/README.md index 21e09157d8d..6769025e9fd 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.20.2", features = ["extension-module"] } +pyo3 = { version = "0.20.3", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.20.2" +version = "0.20.3" features = ["auto-initialize"] ``` diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 12b203c3bb4..f059903cc18 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.2"); +variable::set("PYO3_VERSION", "0.20.3"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 12b203c3bb4..f059903cc18 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.2"); +variable::set("PYO3_VERSION", "0.20.3"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 72cfe2be91d..90989a891ef 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.2"); +variable::set("PYO3_VERSION", "0.20.3"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 78a656558f8..d5520ae1d28 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.2"); +variable::set("PYO3_VERSION", "0.20.3"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 12b203c3bb4..f059903cc18 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.2"); +variable::set("PYO3_VERSION", "0.20.3"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/newsfragments/3619.fixed.md b/newsfragments/3619.fixed.md deleted file mode 100644 index 690542409f4..00000000000 --- a/newsfragments/3619.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Use portable-atomic to support platforms without 64-bit atomics diff --git a/newsfragments/3821.packaging.md b/newsfragments/3821.packaging.md deleted file mode 100644 index 4bd89355086..00000000000 --- a/newsfragments/3821.packaging.md +++ /dev/null @@ -1 +0,0 @@ -Check maximum version of Python at build time and for versions not yet supported require opt-in to the `abi3` stable ABI by the environment variable `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1`. diff --git a/newsfragments/3834.fixed.md b/newsfragments/3834.fixed.md deleted file mode 100644 index fa77e84f36e..00000000000 --- a/newsfragments/3834.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix compilation failure with `either` feature enabled without `experimental-inspect` enabled. From fbf2e919147009f46efee3ceb1b4feea764349f5 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 23 Feb 2024 12:30:46 +0000 Subject: [PATCH 170/349] macros: exact dependency on `pyo3-build-config` (#3889) --- pyo3-macros-backend/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 458b280f881..0e83cc29fd4 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -16,7 +16,7 @@ edition = "2021" [dependencies] heck = "0.4" proc-macro2 = { version = "1", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "0.21.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0-dev", features = ["resolve-config"] } quote = { version = "1", default-features = false } [dependencies.syn] From e145ae851adca553f62b0f9a0466605cd55fe34a Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Fri, 23 Feb 2024 14:07:54 +0000 Subject: [PATCH 171/349] Update new_closure_bound closure signature (#3883) * Update new_closure_bound closure signature Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> * Use anonymous lifetimes in closure bounds Co-authored-by: David Hewitt * Take &Bound in PyCFunction closures --------- Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> Co-authored-by: David Hewitt --- src/types/function.rs | 28 ++++++++++++++++++---------- tests/test_pyfunction.rs | 17 ++++++++++------- tests/ui/invalid_closure.rs | 9 +++++---- tests/ui/invalid_closure.stderr | 4 ++-- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/types/function.rs b/src/types/function.rs index 6c7db09aed3..7cbb05e2a48 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -101,7 +101,10 @@ impl PyCFunction { F: Fn(&PyTuple, Option<&PyDict>) -> R + Send + 'static, R: crate::callback::IntoPyCallbackOutput<*mut ffi::PyObject>, { - Self::new_closure_bound(py, name, doc, closure).map(Bound::into_gil_ref) + Self::new_closure_bound(py, name, doc, move |args, kwargs| { + closure(args.as_gil_ref(), kwargs.map(Bound::as_gil_ref)) + }) + .map(Bound::into_gil_ref) } /// Create a new function from a closure. @@ -113,7 +116,7 @@ impl PyCFunction { /// # use pyo3::{py_run, types::{PyCFunction, PyDict, PyTuple}}; /// /// Python::with_gil(|py| { - /// let add_one = |args: &PyTuple, _kwargs: Option<&PyDict>| -> PyResult<_> { + /// let add_one = |args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>| -> PyResult<_> { /// let i = args.extract::<(i64,)>()?.0; /// Ok(i+1) /// }; @@ -121,14 +124,14 @@ impl PyCFunction { /// py_run!(py, add_one, "assert add_one(42) == 43"); /// }); /// ``` - pub fn new_closure_bound<'a, F, R>( - py: Python<'a>, + pub fn new_closure_bound<'py, F, R>( + py: Python<'py>, name: Option<&'static str>, doc: Option<&'static str>, closure: F, - ) -> PyResult> + ) -> PyResult> where - F: Fn(&PyTuple, Option<&PyDict>) -> R + Send + 'static, + F: Fn(&Bound<'_, PyTuple>, Option<&Bound<'_, PyDict>>) -> R + Send + 'static, R: crate::callback::IntoPyCallbackOutput<*mut ffi::PyObject>, { let method_def = pymethods::PyMethodDef::cfunction_with_keywords( @@ -199,9 +202,11 @@ unsafe extern "C" fn run_closure( kwargs: *mut ffi::PyObject, ) -> *mut ffi::PyObject where - F: Fn(&PyTuple, Option<&PyDict>) -> R + Send + 'static, + F: Fn(&Bound<'_, PyTuple>, Option<&Bound<'_, PyDict>>) -> R + Send + 'static, R: crate::callback::IntoPyCallbackOutput<*mut ffi::PyObject>, { + use crate::types::any::PyAnyMethods; + crate::impl_::trampoline::cfunction_with_keywords( capsule_ptr, args, @@ -210,9 +215,12 @@ where let boxed_fn: &ClosureDestructor = &*(ffi::PyCapsule_GetPointer(capsule_ptr, closure_capsule_name().as_ptr()) as *mut ClosureDestructor); - let args = py.from_borrowed_ptr::(args); - let kwargs = py.from_borrowed_ptr_or_opt::(kwargs); - crate::callback::convert(py, (boxed_fn.closure)(args, kwargs)) + let args = Bound::ref_from_ptr(py, &args).downcast_unchecked::(); + let kwargs = Bound::ref_from_ptr_or_opt(py, &kwargs) + .as_ref() + .map(|b| b.downcast_unchecked::()); + let result = (boxed_fn.closure)(args, kwargs); + crate::callback::convert(py, result) }, ) } diff --git a/tests/test_pyfunction.rs b/tests/test_pyfunction.rs index e06f7c2fb9c..14748418687 100644 --- a/tests/test_pyfunction.rs +++ b/tests/test_pyfunction.rs @@ -401,7 +401,9 @@ fn test_pycfunction_new_with_keywords() { #[test] fn test_closure() { Python::with_gil(|py| { - let f = |args: &types::PyTuple, _kwargs: Option<&types::PyDict>| -> PyResult<_> { + let f = |args: &Bound<'_, types::PyTuple>, + _kwargs: Option<&Bound<'_, types::PyDict>>| + -> PyResult<_> { Python::with_gil(|py| { let res: Vec<_> = args .iter() @@ -439,12 +441,13 @@ fn test_closure() { fn test_closure_counter() { Python::with_gil(|py| { let counter = std::cell::RefCell::new(0); - let counter_fn = - move |_args: &types::PyTuple, _kwargs: Option<&types::PyDict>| -> PyResult { - let mut counter = counter.borrow_mut(); - *counter += 1; - Ok(*counter) - }; + let counter_fn = move |_args: &Bound<'_, types::PyTuple>, + _kwargs: Option<&Bound<'_, types::PyDict>>| + -> PyResult { + let mut counter = counter.borrow_mut(); + *counter += 1; + Ok(*counter) + }; let counter_py = PyCFunction::new_closure_bound(py, None, None, counter_fn).unwrap(); py_assert!(py, counter_py, "counter_py() == 1"); diff --git a/tests/ui/invalid_closure.rs b/tests/ui/invalid_closure.rs index 0613b027665..eca988f1e57 100644 --- a/tests/ui/invalid_closure.rs +++ b/tests/ui/invalid_closure.rs @@ -6,10 +6,11 @@ fn main() { let local_data = vec![0, 1, 2, 3, 4]; let ref_: &[u8] = &local_data; - let closure_fn = |_args: &PyTuple, _kwargs: Option<&PyDict>| -> PyResult<()> { - println!("This is five: {:?}", ref_.len()); - Ok(()) - }; + let closure_fn = + |_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>| -> PyResult<()> { + println!("This is five: {:?}", ref_.len()); + Ok(()) + }; PyCFunction::new_closure_bound(py, None, None, closure_fn) .unwrap() .into() diff --git a/tests/ui/invalid_closure.stderr b/tests/ui/invalid_closure.stderr index 7fed8908583..890d7640502 100644 --- a/tests/ui/invalid_closure.stderr +++ b/tests/ui/invalid_closure.stderr @@ -6,8 +6,8 @@ error[E0597]: `local_data` does not live long enough 7 | let ref_: &[u8] = &local_data; | ^^^^^^^^^^^ borrowed value does not live long enough ... -13 | PyCFunction::new_closure_bound(py, None, None, closure_fn) +14 | PyCFunction::new_closure_bound(py, None, None, closure_fn) | ---------------------------------------------------------- argument requires that `local_data` is borrowed for `'static` ... -16 | }); +17 | }); | - `local_data` dropped here while still borrowed From 0f29feca8fe0a6fc969136f60064814611585a35 Mon Sep 17 00:00:00 2001 From: David Matos Date: Sat, 24 Feb 2024 14:25:06 +0100 Subject: [PATCH 172/349] Tidy up deprecation message on bound api (#3893) --- src/conversion.rs | 8 ++++---- src/instance.rs | 2 +- src/marker.rs | 10 +++++----- src/pycell.rs | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index 00b6ceae389..415f11089df 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -438,7 +438,7 @@ pub unsafe trait FromPyPointer<'p>: Sized { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" + note = "use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" ) )] unsafe fn from_owned_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self>; @@ -451,7 +451,7 @@ pub unsafe trait FromPyPointer<'p>: Sized { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" + note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" ) )] unsafe fn from_owned_ptr_or_panic(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { @@ -467,7 +467,7 @@ pub unsafe trait FromPyPointer<'p>: Sized { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" + note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" ) )] unsafe fn from_owned_ptr(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { @@ -483,7 +483,7 @@ pub unsafe trait FromPyPointer<'p>: Sized { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" + note = "use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" ) )] unsafe fn from_owned_ptr_or_err(py: Python<'p>, ptr: *mut ffi::PyObject) -> PyResult<&'p Self> { diff --git a/src/instance.rs b/src/instance.rs index 307b341fac7..3efdf58f597 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -985,7 +985,7 @@ where not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `obj.into_bound(py)` instead of `obj.into_ref(py)`" + note = "use `obj.into_bound(py)` instead of `obj.into_ref(py)`" ) )] pub fn into_ref(self, py: Python<'_>) -> &T::AsRefTarget { diff --git a/src/marker.rs b/src/marker.rs index 449ea2bde6b..5609601f440 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -842,7 +842,7 @@ impl<'py> Python<'py> { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `obj.downcast_bound::(py)` instead of `py.checked_cast_as::(obj)`" + note = "use `obj.downcast_bound::(py)` instead of `py.checked_cast_as::(obj)`" ) )] pub fn checked_cast_as(self, obj: PyObject) -> Result<&'py T, PyDowncastError<'py>> @@ -863,7 +863,7 @@ impl<'py> Python<'py> { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `obj.downcast_bound_unchecked::(py)` instead of `py.cast_as::(obj)`" + note = "use `obj.downcast_bound_unchecked::(py)` instead of `py.cast_as::(obj)`" ) )] pub unsafe fn cast_as(self, obj: PyObject) -> &'py T @@ -885,7 +885,7 @@ impl<'py> Python<'py> { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" + note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" ) )] pub unsafe fn from_owned_ptr(self, ptr: *mut ffi::PyObject) -> &'py T @@ -909,7 +909,7 @@ impl<'py> Python<'py> { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" + note = "use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" ) )] pub unsafe fn from_owned_ptr_or_err(self, ptr: *mut ffi::PyObject) -> PyResult<&'py T> @@ -933,7 +933,7 @@ impl<'py> Python<'py> { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" + note = "use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" ) )] pub unsafe fn from_owned_ptr_or_opt(self, ptr: *mut ffi::PyObject) -> Option<&'py T> diff --git a/src/pycell.rs b/src/pycell.rs index 5a42dfa514a..989d039f9d2 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -289,7 +289,7 @@ impl PyCell { not(feature = "gil-refs"), deprecated( since = "0.21.0", - note = "part of the deprecated GIL Ref API; to migrate use `Bound::new(py, value)` or `Py::new(py, value)` instead of `PyCell::new(py, value)`" + note = "use `Bound::new(py, value)` or `Py::new(py, value)` instead of `PyCell::new(py, value)`" ) )] pub fn new(py: Python<'_>, value: impl Into>) -> PyResult<&Self> { From c06bb8f1f1188dad78ec96baa3fa687731aba9b4 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 24 Feb 2024 14:46:59 +0100 Subject: [PATCH 173/349] reexport `PyAnyMethods` and friends from `pyo3::types` (#3895) * reexport `PyAnyMethods` and friends from `pyo3::types` * remove duplicated imports --- src/conversions/indexmap.rs | 4 ---- src/prelude.rs | 1 + src/types/mod.rs | 40 ++++++++++++++++++------------------- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index 5a8cd3d6951..e908cddb621 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -87,8 +87,6 @@ //! # if another hash table was used, the order could be random //! ``` -use crate::types::any::PyAnyMethods; -use crate::types::dict::PyDictMethods; use crate::types::*; use crate::{Bound, FromPyObject, IntoPy, PyErr, PyObject, Python, ToPyObject}; use std::{cmp, hash}; @@ -137,8 +135,6 @@ where #[cfg(test)] mod test_indexmap { - use crate::types::any::PyAnyMethods; - use crate::types::dict::PyDictMethods; use crate::types::*; use crate::{IntoPy, PyObject, Python, ToPyObject}; diff --git a/src/prelude.rs b/src/prelude.rs index 5a342281eeb..1de7c3acd2d 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -39,6 +39,7 @@ pub use crate::types::mapping::PyMappingMethods; pub use crate::types::module::PyModuleMethods; pub use crate::types::sequence::PySequenceMethods; pub use crate::types::set::PySetMethods; +pub use crate::types::slice::PySliceMethods; pub use crate::types::string::PyStringMethods; pub use crate::types::traceback::PyTracebackMethods; pub use crate::types::tuple::PyTupleMethods; diff --git a/src/types/mod.rs b/src/types/mod.rs index 217e240129f..4baa086ab79 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,13 +1,13 @@ //! Various types defined by the Python interpreter such as `int`, `str` and `tuple`. -pub use self::any::PyAny; -pub use self::boolobject::PyBool; -pub use self::bytearray::PyByteArray; -pub use self::bytes::PyBytes; -pub use self::capsule::PyCapsule; +pub use self::any::{PyAny, PyAnyMethods}; +pub use self::boolobject::{PyBool, PyBoolMethods}; +pub use self::bytearray::{PyByteArray, PyByteArrayMethods}; +pub use self::bytes::{PyBytes, PyBytesMethods}; +pub use self::capsule::{PyCapsule, PyCapsuleMethods}; #[cfg(not(Py_LIMITED_API))] pub use self::code::PyCode; -pub use self::complex::PyComplex; +pub use self::complex::{PyComplex, PyComplexMethods}; #[allow(deprecated)] #[cfg(not(Py_LIMITED_API))] pub use self::datetime::timezone_utc; @@ -16,37 +16,37 @@ pub use self::datetime::{ timezone_utc_bound, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, PyTimeAccess, PyTzInfo, PyTzInfoAccess, }; -pub use self::dict::{IntoPyDict, PyDict}; +pub use self::dict::{IntoPyDict, PyDict, PyDictMethods}; #[cfg(not(PyPy))] pub use self::dict::{PyDictItems, PyDictKeys, PyDictValues}; pub use self::ellipsis::PyEllipsis; -pub use self::float::PyFloat; +pub use self::float::{PyFloat, PyFloatMethods}; #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] pub use self::frame::PyFrame; -pub use self::frozenset::{PyFrozenSet, PyFrozenSetBuilder}; +pub use self::frozenset::{PyFrozenSet, PyFrozenSetBuilder, PyFrozenSetMethods}; pub use self::function::PyCFunction; #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] pub use self::function::PyFunction; pub use self::iterator::PyIterator; -pub use self::list::PyList; -pub use self::mapping::PyMapping; +pub use self::list::{PyList, PyListMethods}; +pub use self::mapping::{PyMapping, PyMappingMethods}; pub use self::memoryview::PyMemoryView; -pub use self::module::PyModule; +pub use self::module::{PyModule, PyModuleMethods}; pub use self::none::PyNone; pub use self::notimplemented::PyNotImplemented; pub use self::num::PyLong; pub use self::num::PyLong as PyInt; #[cfg(not(PyPy))] pub use self::pysuper::PySuper; -pub use self::sequence::PySequence; -pub use self::set::PySet; -pub use self::slice::{PySlice, PySliceIndices}; +pub use self::sequence::{PySequence, PySequenceMethods}; +pub use self::set::{PySet, PySetMethods}; +pub use self::slice::{PySlice, PySliceIndices, PySliceMethods}; #[cfg(not(Py_LIMITED_API))] pub use self::string::PyStringData; -pub use self::string::{PyString, PyString as PyUnicode}; -pub use self::traceback::PyTraceback; -pub use self::tuple::PyTuple; -pub use self::typeobject::PyType; +pub use self::string::{PyString, PyString as PyUnicode, PyStringMethods}; +pub use self::traceback::{PyTraceback, PyTracebackMethods}; +pub use self::tuple::{PyTuple, PyTupleMethods}; +pub use self::typeobject::{PyType, PyTypeMethods}; /// Iteration over Python collections. /// @@ -332,7 +332,7 @@ mod num; mod pysuper; pub(crate) mod sequence; pub(crate) mod set; -mod slice; +pub(crate) mod slice; pub(crate) mod string; pub(crate) mod traceback; pub(crate) mod tuple; From e0e3981e17c7b0f9b221ac1566e5a2eadf1a4f6b Mon Sep 17 00:00:00 2001 From: Thomas Tanon Date: Sat, 24 Feb 2024 14:50:18 +0100 Subject: [PATCH 174/349] #[pymodule] mod some_module { ... } v3 (#3815) * #[pymodule] mod some_module { ... } v3 Based on #2367 and #3294 Allows to export classes, native classes, functions and submodules and provide an init function See test/test_module.rs for an example Future work: - update examples, README and guide - investigate having #[pyclass] and #[pyfunction] directly in the #[pymodule] Co-authored-by: David Hewitt Co-authored-by: Georg Brandl * tests: group exported imports * Consolidate pymodule macro code to avoid duplicates * Makes pymodule_init take Bound<'_, PyModule> * Renames #[pyo3] to #[pymodule_export] * Gates #[pymodule] mod behind the experimental-declarative-modules feature * Properly fails on functions inside of declarative modules --------- Co-authored-by: David Hewitt Co-authored-by: Georg Brandl --- Cargo.toml | 4 + newsfragments/3815.added.md | 2 + pyo3-macros-backend/src/lib.rs | 2 +- pyo3-macros-backend/src/module.rs | 219 +++++++++++++++--- pyo3-macros-backend/src/pyfunction.rs | 6 + pyo3-macros/Cargo.toml | 1 + pyo3-macros/src/lib.rs | 40 ++-- src/impl_/pymodule.rs | 32 ++- src/macros.rs | 4 +- tests/test_append_to_inittab.rs | 37 ++- tests/test_compile_error.rs | 8 + tests/test_declarative_module.rs | 101 ++++++++ tests/ui/invalid_pymodule_glob.rs | 14 ++ tests/ui/invalid_pymodule_glob.stderr | 5 + tests/ui/invalid_pymodule_in_root.rs | 6 + tests/ui/invalid_pymodule_in_root.stderr | 13 ++ tests/ui/invalid_pymodule_trait.rs | 9 + tests/ui/invalid_pymodule_trait.stderr | 5 + .../ui/invalid_pymodule_two_pymodule_init.rs | 16 ++ .../invalid_pymodule_two_pymodule_init.stderr | 5 + 20 files changed, 458 insertions(+), 71 deletions(-) create mode 100644 newsfragments/3815.added.md create mode 100644 tests/test_declarative_module.rs create mode 100644 tests/ui/invalid_pymodule_glob.rs create mode 100644 tests/ui/invalid_pymodule_glob.stderr create mode 100644 tests/ui/invalid_pymodule_in_root.rs create mode 100644 tests/ui/invalid_pymodule_in_root.stderr create mode 100644 tests/ui/invalid_pymodule_trait.rs create mode 100644 tests/ui/invalid_pymodule_trait.stderr create mode 100644 tests/ui/invalid_pymodule_two_pymodule_init.rs create mode 100644 tests/ui/invalid_pymodule_two_pymodule_init.stderr diff --git a/Cargo.toml b/Cargo.toml index 4d1899cbdb9..e7364a7c9f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,9 @@ default = ["macros"] # and IntoPy traits experimental-inspect = [] +# Enables annotating Rust inline modules with #[pymodule] to build Python modules declaratively +experimental-declarative-modules = ["pyo3-macros/experimental-declarative-modules", "macros"] + # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc. macros = ["pyo3-macros", "indoc", "unindent"] @@ -114,6 +117,7 @@ full = [ "chrono-tz", "either", "experimental-inspect", + "experimental-declarative-modules", "eyre", "hashbrown", "indexmap", diff --git a/newsfragments/3815.added.md b/newsfragments/3815.added.md new file mode 100644 index 00000000000..e4fd3e9315a --- /dev/null +++ b/newsfragments/3815.added.md @@ -0,0 +1,2 @@ +The ability to create Python modules with a Rust `mod` block +behind the `experimental-declarative-modules` feature. \ No newline at end of file diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index 745a8471c2b..a9d75a2a6fe 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -22,7 +22,7 @@ mod pymethod; mod quotes; pub use frompyobject::build_derive_from_pyobject; -pub use module::{process_functions_in_module, pymodule_impl, PyModuleOptions}; +pub use module::{pymodule_function_impl, pymodule_module_impl, PyModuleOptions}; pub use pyclass::{build_py_class, build_py_enum, PyClassArgs}; pub use pyfunction::{build_py_function, PyFunctionOptions}; pub use pyimpl::{build_py_methods, PyClassMethodsType}; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index ccd84bb363a..6907e484f71 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -2,8 +2,9 @@ use crate::{ attributes::{self, take_attributes, take_pyo3_options, CrateAttribute, NameAttribute}, + get_doc, pyfunction::{impl_wrap_pyfunction, PyFunctionOptions}, - utils::{get_pyo3_crate, PythonDoc}, + utils::get_pyo3_crate, }; use proc_macro2::TokenStream; use quote::quote; @@ -12,7 +13,7 @@ use syn::{ parse::{Parse, ParseStream}, spanned::Spanned, token::Comma, - Ident, Path, Result, Visibility, + Item, Path, Result, }; #[derive(Default)] @@ -56,33 +57,154 @@ impl PyModuleOptions { } } +pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { + let syn::ItemMod { + attrs, + vis, + unsafety: _, + ident, + mod_token: _, + content, + semi: _, + } = &mut module; + let items = if let Some((_, items)) = content { + items + } else { + bail_spanned!(module.span() => "`#[pymodule]` can only be used on inline modules") + }; + let options = PyModuleOptions::from_attrs(attrs)?; + let krate = get_pyo3_crate(&options.krate); + let doc = get_doc(attrs, None); + + let mut module_items = Vec::new(); + let mut module_items_cfg_attrs = Vec::new(); + + fn extract_use_items( + source: &syn::UseTree, + cfg_attrs: &[syn::Attribute], + target_items: &mut Vec, + target_cfg_attrs: &mut Vec>, + ) -> Result<()> { + match source { + syn::UseTree::Name(name) => { + target_items.push(name.ident.clone()); + target_cfg_attrs.push(cfg_attrs.to_vec()); + } + syn::UseTree::Path(path) => { + extract_use_items(&path.tree, cfg_attrs, target_items, target_cfg_attrs)? + } + syn::UseTree::Group(group) => { + for tree in &group.items { + extract_use_items(tree, cfg_attrs, target_items, target_cfg_attrs)? + } + } + syn::UseTree::Glob(glob) => { + bail_spanned!(glob.span() => "#[pymodule] cannot import glob statements") + } + syn::UseTree::Rename(rename) => { + target_items.push(rename.rename.clone()); + target_cfg_attrs.push(cfg_attrs.to_vec()); + } + } + Ok(()) + } + + let mut pymodule_init = None; + + for item in &mut *items { + match item { + Item::Use(item_use) => { + let mut is_pyo3 = false; + item_use.attrs.retain(|attr| { + let found = attr.path().is_ident("pymodule_export"); + is_pyo3 |= found; + !found + }); + if is_pyo3 { + let cfg_attrs = item_use + .attrs + .iter() + .filter(|attr| attr.path().is_ident("cfg")) + .cloned() + .collect::>(); + extract_use_items( + &item_use.tree, + &cfg_attrs, + &mut module_items, + &mut module_items_cfg_attrs, + )?; + } + } + Item::Fn(item_fn) => { + let mut is_module_init = false; + item_fn.attrs.retain(|attr| { + let found = attr.path().is_ident("pymodule_init"); + is_module_init |= found; + !found + }); + if is_module_init { + ensure_spanned!(pymodule_init.is_none(), item_fn.span() => "only one pymodule_init may be specified"); + let ident = &item_fn.sig.ident; + pymodule_init = Some(quote! { #ident(module)?; }); + } else { + bail_spanned!(item.span() => "only 'use' statements and and pymodule_init functions are allowed in #[pymodule]") + } + } + item => { + bail_spanned!(item.span() => "only 'use' statements and and pymodule_init functions are allowed in #[pymodule]") + } + } + } + + let initialization = module_initialization(options, ident); + Ok(quote!( + #vis mod #ident { + #(#items)* + + #initialization + + impl MakeDef { + const fn make_def() -> #krate::impl_::pymodule::ModuleDef { + use #krate::impl_::pymodule as impl_; + const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule); + unsafe { + impl_::ModuleDef::new( + __PYO3_NAME, + #doc, + INITIALIZER + ) + } + } + } + + fn __pyo3_pymodule(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { + use #krate::impl_::pymodule::PyAddToModule; + #( + #(#module_items_cfg_attrs)* + #module_items::add_to_module(module)?; + )* + #pymodule_init + Ok(()) + } + } + )) +} + /// Generates the function that is called by the python interpreter to initialize the native /// module -pub fn pymodule_impl( - fnname: &Ident, - options: PyModuleOptions, - doc: PythonDoc, - visibility: &Visibility, -) -> TokenStream { - let name = options.name.unwrap_or_else(|| fnname.unraw()); +pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result { + let options = PyModuleOptions::from_attrs(&mut function.attrs)?; + process_functions_in_module(&options, &mut function)?; let krate = get_pyo3_crate(&options.krate); - let pyinit_symbol = format!("PyInit_{}", name); + let ident = &function.sig.ident; + let vis = &function.vis; + let doc = get_doc(&function.attrs, None); - quote! { - // Create a module with the same name as the `#[pymodule]` - this way `use ` - // will actually bring both the module and the function into scope. - #[doc(hidden)] - #visibility mod #fnname { - pub(crate) struct MakeDef; - pub static DEF: #krate::impl_::pymodule::ModuleDef = MakeDef::make_def(); - pub const NAME: &'static str = concat!(stringify!(#name), "\0"); - - /// This autogenerated function is called by the python interpreter when importing - /// the module. - #[export_name = #pyinit_symbol] - pub unsafe extern "C" fn init() -> *mut #krate::ffi::PyObject { - #krate::impl_::trampoline::module_init(|py| DEF.make_module(py)) - } + let initialization = module_initialization(options, ident); + Ok(quote! { + #function + #vis mod #ident { + #initialization } // Generate the definition inside an anonymous function in the same scope as the original function - @@ -91,28 +213,59 @@ pub fn pymodule_impl( // inside a function body) const _: () = { use #krate::impl_::pymodule as impl_; - impl #fnname::MakeDef { + + fn __pyo3_pymodule(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { + #ident(module.py(), module.as_gil_ref()) + } + + impl #ident::MakeDef { const fn make_def() -> impl_::ModuleDef { - const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(#fnname); unsafe { - impl_::ModuleDef::new(#fnname::NAME, #doc, INITIALIZER) + const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule); + impl_::ModuleDef::new( + #ident::__PYO3_NAME, + #doc, + INITIALIZER + ) } } } }; + }) +} + +fn module_initialization(options: PyModuleOptions, ident: &syn::Ident) -> TokenStream { + let name = options.name.unwrap_or_else(|| ident.unraw()); + let krate = get_pyo3_crate(&options.krate); + let pyinit_symbol = format!("PyInit_{}", name); + + quote! { + pub const __PYO3_NAME: &'static str = concat!(stringify!(#name), "\0"); + + pub(super) struct MakeDef; + pub static DEF: #krate::impl_::pymodule::ModuleDef = MakeDef::make_def(); + + pub fn add_to_module(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { + use #krate::prelude::PyModuleMethods; + module.add_submodule(DEF.make_module(module.py())?.bind(module.py())) + } + + /// This autogenerated function is called by the python interpreter when importing + /// the module. + #[export_name = #pyinit_symbol] + pub unsafe extern "C" fn __pyo3_init() -> *mut #krate::ffi::PyObject { + #krate::impl_::trampoline::module_init(|py| DEF.make_module(py)) + } } } /// Finds and takes care of the #[pyfn(...)] in `#[pymodule]` -pub fn process_functions_in_module( - options: &PyModuleOptions, - func: &mut syn::ItemFn, -) -> syn::Result<()> { +fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn) -> Result<()> { let mut stmts: Vec = Vec::new(); let krate = get_pyo3_crate(&options.krate); for mut stmt in func.block.stmts.drain(..) { - if let syn::Stmt::Item(syn::Item::Fn(func)) = &mut stmt { + if let syn::Stmt::Item(Item::Fn(func)) = &mut stmt { if let Some(pyfn_args) = get_pyfn_attr(&mut func.attrs)? { let module_name = pyfn_args.modname; let wrapped_function = impl_wrap_pyfunction(func, pyfn_args.options)?; diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index b265a34d39f..7b48585cddc 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -269,6 +269,12 @@ pub fn impl_wrap_pyfunction( #vis mod #name { pub(crate) struct MakeDef; pub const DEF: #krate::impl_::pyfunction::PyMethodDef = MakeDef::DEF; + + pub fn add_to_module(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { + use #krate::prelude::PyModuleMethods; + use ::std::convert::Into; + module.add_function(&#krate::types::PyCFunction::internal_new(&DEF, module.as_gil_ref().into())?) + } } // Generate the definition inside an anonymous function in the same scope as the original function - diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 576c94a2bc1..a0368a5f364 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -15,6 +15,7 @@ proc-macro = true [features] multiple-pymethods = [] +experimental-declarative-modules = [] [dependencies] proc-macro2 = { version = "1", default-features = false } diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index d00ede89143..64756a1c73b 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -6,11 +6,11 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use pyo3_macros_backend::{ build_derive_from_pyobject, build_py_class, build_py_enum, build_py_function, build_py_methods, - get_doc, process_functions_in_module, pymodule_impl, PyClassArgs, PyClassMethodsType, - PyFunctionOptions, PyModuleOptions, + pymodule_function_impl, pymodule_module_impl, PyClassArgs, PyClassMethodsType, + PyFunctionOptions, }; use quote::quote; -use syn::{parse::Nothing, parse_macro_input}; +use syn::{parse::Nothing, parse_macro_input, Item}; /// A proc macro used to implement Python modules. /// @@ -36,31 +36,27 @@ use syn::{parse::Nothing, parse_macro_input}; #[proc_macro_attribute] pub fn pymodule(args: TokenStream, input: TokenStream) -> TokenStream { parse_macro_input!(args as Nothing); - - let mut ast = parse_macro_input!(input as syn::ItemFn); - let options = match PyModuleOptions::from_attrs(&mut ast.attrs) { - Ok(options) => options, - Err(e) => return e.into_compile_error().into(), - }; - - if let Err(err) = process_functions_in_module(&options, &mut ast) { - return err.into_compile_error().into(); + match parse_macro_input!(input as Item) { + Item::Mod(module) => if cfg!(feature = "experimental-declarative-modules") { + pymodule_module_impl(module) + } else { + Err(syn::Error::new_spanned( + module, + "#[pymodule] requires the 'experimental-declarative-modules' feature to be used on Rust modules.", + )) + }, + Item::Fn(function) => pymodule_function_impl(function), + unsupported => Err(syn::Error::new_spanned( + unsupported, + "#[pymodule] only supports modules and functions.", + )), } - - let doc = get_doc(&ast.attrs, None); - - let expanded = pymodule_impl(&ast.sig.ident, options, doc, &ast.vis); - - quote!( - #ast - #expanded - ) + .unwrap_or_compile_error() .into() } #[proc_macro_attribute] pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream { - use syn::Item; let item = parse_macro_input!(input as Item); match item { Item::Struct(struct_) => pyclass_impl(attr, struct_, methods_type()), diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index 7103f8b3938..9fff799c37b 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -7,7 +7,8 @@ use portable_atomic::{AtomicI64, Ordering}; #[cfg(not(PyPy))] use crate::exceptions::PyImportError; -use crate::{ffi, sync::GILOnceCell, types::PyModule, Py, PyResult, Python}; +use crate::types::module::PyModuleMethods; +use crate::{ffi, sync::GILOnceCell, types::PyModule, Bound, Py, PyResult, PyTypeInfo, Python}; /// `Sync` wrapper of `ffi::PyModuleDef`. pub struct ModuleDef { @@ -22,7 +23,7 @@ pub struct ModuleDef { } /// Wrapper to enable initializer to be used in const fns. -pub struct ModuleInitializer(pub for<'py> fn(Python<'py>, &PyModule) -> PyResult<()>); +pub struct ModuleInitializer(pub for<'py> fn(&Bound<'py, PyModule>) -> PyResult<()>); unsafe impl Sync for ModuleDef {} @@ -126,18 +127,34 @@ impl ModuleDef { ffi::PyModule_Create(self.ffi_def.get()), )? }; - (self.initializer.0)(py, module.as_ref(py))?; + self.initializer.0(module.bind(py))?; Ok(module) }) .map(|py_module| py_module.clone_ref(py)) } } +/// Trait to add an element (class, function...) to a module. +/// +/// Currently only implemented for classes. +pub trait PyAddToModule { + fn add_to_module(module: &Bound<'_, PyModule>) -> PyResult<()>; +} + +impl PyAddToModule for T { + fn add_to_module(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add(Self::NAME, Self::type_object_bound(module.py())) + } +} + #[cfg(test)] mod tests { use std::sync::atomic::{AtomicBool, Ordering}; - use crate::{types::any::PyAnyMethods, types::PyModule, PyResult, Python}; + use crate::{ + types::{any::PyAnyMethods, module::PyModuleMethods, PyModule}, + Bound, PyResult, Python, + }; use super::{ModuleDef, ModuleInitializer}; @@ -147,7 +164,7 @@ mod tests { ModuleDef::new( "test_module\0", "some doc\0", - ModuleInitializer(|_, m| { + ModuleInitializer(|m| { m.add("SOME_CONSTANT", 42)?; Ok(()) }), @@ -192,7 +209,7 @@ mod tests { static INIT_CALLED: AtomicBool = AtomicBool::new(false); #[allow(clippy::unnecessary_wraps)] - fn init(_: Python<'_>, _: &PyModule) -> PyResult<()> { + fn init(_: &Bound<'_, PyModule>) -> PyResult<()> { INIT_CALLED.store(true, Ordering::SeqCst); Ok(()) } @@ -203,8 +220,7 @@ mod tests { assert_eq!((*module_def.ffi_def.get()).m_doc, DOC.as_ptr() as _); Python::with_gil(|py| { - module_def.initializer.0(py, py.import_bound("builtins").unwrap().into_gil_ref()) - .unwrap(); + module_def.initializer.0(&py.import_bound("builtins").unwrap()).unwrap(); assert!(INIT_CALLED.load(Ordering::SeqCst)); }) } diff --git a/src/macros.rs b/src/macros.rs index 29c2033dd4b..9b0d2816882 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -169,8 +169,8 @@ macro_rules! append_to_inittab { ); } $crate::ffi::PyImport_AppendInittab( - $module::NAME.as_ptr() as *const ::std::os::raw::c_char, - ::std::option::Option::Some($module::init), + $module::__PYO3_NAME.as_ptr() as *const ::std::os::raw::c_char, + ::std::option::Option::Some($module::__pyo3_init), ); } }; diff --git a/tests/test_append_to_inittab.rs b/tests/test_append_to_inittab.rs index 00cccdbb49e..59ecaf42909 100644 --- a/tests/test_append_to_inittab.rs +++ b/tests/test_append_to_inittab.rs @@ -1,4 +1,5 @@ #![cfg(all(feature = "macros", not(PyPy)))] + use pyo3::prelude::*; #[pyfunction] @@ -7,26 +8,52 @@ fn foo() -> usize { } #[pymodule] -fn module_with_functions(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn module_fn_with_functions(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(foo, m)?).unwrap(); Ok(()) } +#[cfg(feature = "experimental-declarative-modules")] +#[pymodule] +mod module_mod_with_functions { + #[pymodule_export] + use super::foo; +} + #[cfg(not(PyPy))] #[test] fn test_module_append_to_inittab() { use pyo3::append_to_inittab; - append_to_inittab!(module_with_functions); + + append_to_inittab!(module_fn_with_functions); + + #[cfg(feature = "experimental-declarative-modules")] + append_to_inittab!(module_mod_with_functions); + + Python::with_gil(|py| { + py.run_bound( + r#" +import module_fn_with_functions +assert module_fn_with_functions.foo() == 123 +"#, + None, + None, + ) + .map_err(|e| e.display(py)) + .unwrap(); + }); + + #[cfg(feature = "experimental-declarative-modules")] Python::with_gil(|py| { py.run_bound( r#" -import module_with_functions -assert module_with_functions.foo() == 123 +import module_mod_with_functions +assert module_mod_with_functions.foo() == 123 "#, None, None, ) .map_err(|e| e.display(py)) .unwrap(); - }) + }); } diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index adcef887f5c..5f2d25db92f 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -40,4 +40,12 @@ fn test_compile_errors() { t.compile_fail("tests/ui/not_send2.rs"); t.compile_fail("tests/ui/get_set_all.rs"); t.compile_fail("tests/ui/traverse.rs"); + #[cfg(feature = "experimental-declarative-modules")] + t.compile_fail("tests/ui/invalid_pymodule_in_root.rs"); + #[cfg(feature = "experimental-declarative-modules")] + t.compile_fail("tests/ui/invalid_pymodule_glob.rs"); + #[cfg(feature = "experimental-declarative-modules")] + t.compile_fail("tests/ui/invalid_pymodule_trait.rs"); + #[cfg(feature = "experimental-declarative-modules")] + t.compile_fail("tests/ui/invalid_pymodule_two_pymodule_init.rs"); } diff --git a/tests/test_declarative_module.rs b/tests/test_declarative_module.rs new file mode 100644 index 00000000000..86913d9b800 --- /dev/null +++ b/tests/test_declarative_module.rs @@ -0,0 +1,101 @@ +#![cfg(feature = "experimental-declarative-modules")] + +use pyo3::create_exception; +use pyo3::exceptions::PyException; +use pyo3::prelude::*; + +#[path = "../src/tests/common.rs"] +mod common; + +#[pyclass] +struct ValueClass { + value: usize, +} + +#[pymethods] +impl ValueClass { + #[new] + fn new(value: usize) -> ValueClass { + ValueClass { value } + } +} + +#[pyclass(module = "module")] +struct LocatedClass {} + +#[pyfunction] +fn double(x: usize) -> usize { + x * 2 +} + +create_exception!( + declarative_module, + MyError, + PyException, + "Some description." +); + +/// A module written using declarative syntax. +#[pymodule] +mod declarative_module { + #[pymodule_export] + use super::declarative_submodule; + #[pymodule_export] + // This is not a real constraint but to test cfg attribute support + #[cfg(not(Py_LIMITED_API))] + use super::LocatedClass; + use super::*; + #[pymodule_export] + use super::{declarative_module2, double, MyError, ValueClass as Value}; + + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add("double2", m.getattr("double")?) + } +} + +#[pyfunction] +fn double_value(v: &ValueClass) -> usize { + v.value * 2 +} + +#[pymodule] +mod declarative_submodule { + #[pymodule_export] + use super::{double, double_value}; +} + +/// A module written using declarative syntax. +#[pymodule] +#[pyo3(name = "declarative_module_renamed")] +mod declarative_module2 { + #[pymodule_export] + use super::double; +} + +#[test] +fn test_declarative_module() { + Python::with_gil(|py| { + let m = pyo3::wrap_pymodule!(declarative_module)(py).into_bound(py); + py_assert!( + py, + m, + "m.__doc__ == 'A module written using declarative syntax.'" + ); + + py_assert!(py, m, "m.double(2) == 4"); + py_assert!(py, m, "m.double2(3) == 6"); + py_assert!(py, m, "m.declarative_submodule.double(4) == 8"); + py_assert!( + py, + m, + "m.declarative_submodule.double_value(m.ValueClass(1)) == 2" + ); + py_assert!(py, m, "str(m.MyError('foo')) == 'foo'"); + py_assert!(py, m, "m.declarative_module_renamed.double(2) == 4"); + #[cfg(Py_LIMITED_API)] + py_assert!(py, m, "not hasattr(m, 'LocatedClass')"); + #[cfg(not(Py_LIMITED_API))] + py_assert!(py, m, "hasattr(m, 'LocatedClass')"); + }) +} diff --git a/tests/ui/invalid_pymodule_glob.rs b/tests/ui/invalid_pymodule_glob.rs new file mode 100644 index 00000000000..107cdf9382a --- /dev/null +++ b/tests/ui/invalid_pymodule_glob.rs @@ -0,0 +1,14 @@ +use pyo3::prelude::*; + +#[pyfunction] +fn foo() -> usize { + 0 +} + +#[pymodule] +mod module { + #[pymodule_export] + use super::*; +} + +fn main() {} diff --git a/tests/ui/invalid_pymodule_glob.stderr b/tests/ui/invalid_pymodule_glob.stderr new file mode 100644 index 00000000000..237e02037aa --- /dev/null +++ b/tests/ui/invalid_pymodule_glob.stderr @@ -0,0 +1,5 @@ +error: #[pymodule] cannot import glob statements + --> tests/ui/invalid_pymodule_glob.rs:11:16 + | +11 | use super::*; + | ^ diff --git a/tests/ui/invalid_pymodule_in_root.rs b/tests/ui/invalid_pymodule_in_root.rs new file mode 100644 index 00000000000..47af4205f71 --- /dev/null +++ b/tests/ui/invalid_pymodule_in_root.rs @@ -0,0 +1,6 @@ +use pyo3::prelude::*; + +#[pymodule] +mod invalid_pymodule_in_root_module; + +fn main() {} diff --git a/tests/ui/invalid_pymodule_in_root.stderr b/tests/ui/invalid_pymodule_in_root.stderr new file mode 100644 index 00000000000..91783be0e97 --- /dev/null +++ b/tests/ui/invalid_pymodule_in_root.stderr @@ -0,0 +1,13 @@ +error[E0658]: non-inline modules in proc macro input are unstable + --> tests/ui/invalid_pymodule_in_root.rs:4:1 + | +4 | mod invalid_pymodule_in_root_module; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: see issue #54727 for more information + +error: `#[pymodule]` can only be used on inline modules + --> tests/ui/invalid_pymodule_in_root.rs:4:1 + | +4 | mod invalid_pymodule_in_root_module; + | ^^^ diff --git a/tests/ui/invalid_pymodule_trait.rs b/tests/ui/invalid_pymodule_trait.rs new file mode 100644 index 00000000000..6649a3547a0 --- /dev/null +++ b/tests/ui/invalid_pymodule_trait.rs @@ -0,0 +1,9 @@ +use pyo3::prelude::*; + +#[pymodule] +mod module { + #[pymodule_export] + trait Foo {} +} + +fn main() {} diff --git a/tests/ui/invalid_pymodule_trait.stderr b/tests/ui/invalid_pymodule_trait.stderr new file mode 100644 index 00000000000..3ed128617f5 --- /dev/null +++ b/tests/ui/invalid_pymodule_trait.stderr @@ -0,0 +1,5 @@ +error: only 'use' statements and and pymodule_init functions are allowed in #[pymodule] + --> tests/ui/invalid_pymodule_trait.rs:5:5 + | +5 | #[pymodule_export] + | ^ diff --git a/tests/ui/invalid_pymodule_two_pymodule_init.rs b/tests/ui/invalid_pymodule_two_pymodule_init.rs new file mode 100644 index 00000000000..d676b0fa277 --- /dev/null +++ b/tests/ui/invalid_pymodule_two_pymodule_init.rs @@ -0,0 +1,16 @@ +use pyo3::prelude::*; + +#[pymodule] +mod module { + #[pymodule_init] + fn init(m: &PyModule) -> PyResult<()> { + Ok(()) + } + + #[pymodule_init] + fn init2(m: &PyModule) -> PyResult<()> { + Ok(()) + } +} + +fn main() {} diff --git a/tests/ui/invalid_pymodule_two_pymodule_init.stderr b/tests/ui/invalid_pymodule_two_pymodule_init.stderr new file mode 100644 index 00000000000..9f0900f9348 --- /dev/null +++ b/tests/ui/invalid_pymodule_two_pymodule_init.stderr @@ -0,0 +1,5 @@ +error: only one pymodule_init may be specified + --> tests/ui/invalid_pymodule_two_pymodule_init.rs:11:5 + | +11 | fn init2(m: &PyModule) -> PyResult<()> { + | ^^ From 8f1b99e1e955a333ba9382d3c46d74bd82b1eab9 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 24 Feb 2024 22:35:01 +0000 Subject: [PATCH 175/349] move chat discussions to Discord (#3892) * move chat discussions to Discord * guide: add some more signposting to the PyO3 Discord --- .github/ISSUE_TEMPLATE/config.yml | 6 +++--- Contributing.md | 4 ++-- README.md | 4 ++-- guide/src/faq.md | 4 +++- guide/src/getting_started.md | 2 ++ 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4d31afe411d..ca4239b6fa7 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,7 +5,7 @@ contact_links: about: Ask and answer questions about PyO3 on Discussions - name: 🔧 Troubleshooting url: https://github.com/PyO3/pyo3/discussions - about: For troubleshooting help, see the Discussions + about: For troubleshooting help, see the Discussions - name: 👋 Chat - url: https://gitter.im/PyO3/Lobby - about: Engage with PyO3's users and developers on Gitter \ No newline at end of file + url: https://discord.gg/c4BwayXQ + about: Engage with PyO3's users and developers on Discord diff --git a/Contributing.md b/Contributing.md index f87fffc0906..9392c0fc2de 100644 --- a/Contributing.md +++ b/Contributing.md @@ -29,7 +29,7 @@ To work and develop PyO3, you need Python & Rust installed on your system. ### Help users identify bugs -The [PyO3 Gitter channel](https://gitter.im/PyO3/Lobby) is very active with users who are new to PyO3, and often completely new to Rust. Helping them debug is a great way to get experience with the PyO3 codebase. +The [PyO3 Discord server](https://discord.gg/c4BwayXQ) is very active with users who are new to PyO3, and often completely new to Rust. Helping them debug is a great way to get experience with the PyO3 codebase. Helping others often reveals bugs, documentation weaknesses, and missing APIs. It's a good idea to open GitHub issues for these immediately so the resolution can be designed and implemented! @@ -203,7 +203,7 @@ You can install an IDE plugin to view the coverage. For example, if you use VSCo ## Sponsor this project -At the moment there is no official organisation that accepts sponsorship on PyO3's behalf. If you're seeking to provide significant funding to the PyO3 ecosystem, please reach out to us on [GitHub](https://github.com/PyO3/pyo3/issues/new) or [Gitter](https://gitter.im/PyO3/Lobby) and we can discuss. +At the moment there is no official organisation that accepts sponsorship on PyO3's behalf. If you're seeking to provide significant funding to the PyO3 ecosystem, please reach out to us on [GitHub](https://github.com/PyO3/pyo3/issues/new) or [Discord](https://discord.gg/c4BwayXQ) and we can discuss. In the meanwhile, some of our maintainers have personal GitHub sponsorship pages and would be grateful for your support: diff --git a/README.md b/README.md index 6769025e9fd..c139f13693e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![codecov](https://img.shields.io/codecov/c/gh/PyO3/pyo3?logo=codecov)](https://codecov.io/gh/PyO3/pyo3) [![crates.io](https://img.shields.io/crates/v/pyo3?logo=rust)](https://crates.io/crates/pyo3) [![minimum rustc 1.56](https://img.shields.io/badge/rustc-1.56+-blue?logo=rust)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) -[![dev chat](https://img.shields.io/gitter/room/PyO3/Lobby?logo=gitter)](https://gitter.im/PyO3/Lobby) +[![discord server](https://img.shields.io/discord/1209263839632424990?logo=discord)](https://discord.gg/c4BwayXQ) [![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green?logo=github)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) [Rust](https://www.rust-lang.org/) bindings for [Python](https://www.python.org/), including tools for creating native Python extension modules. Running and interacting with Python code from a Rust binary is also supported. @@ -237,7 +237,7 @@ about this topic. Everyone is welcomed to contribute to PyO3! There are many ways to support the project, such as: -- help PyO3 users with issues on GitHub and Gitter +- help PyO3 users with issues on GitHub and [Discord](https://discord.gg/c4BwayXQ) - improve documentation - write features and bugfixes - publish blogs and examples of how to use PyO3 diff --git a/guide/src/faq.md b/guide/src/faq.md index 8308acc64de..6f5188c73be 100644 --- a/guide/src/faq.md +++ b/guide/src/faq.md @@ -1,5 +1,7 @@ # Frequently Asked Questions and troubleshooting +Sorry that you're having trouble using PyO3. If you can't find the answer to your problem in the list below, you can also reach out for help on [GitHub Discussions](https://github.com/PyO3/pyo3/discussions) and on [Discord](https://discord.gg/c4BwayXQ). + ## I'm experiencing deadlocks using PyO3 with lazy_static or once_cell! `lazy_static` and `once_cell::sync` both use locks to ensure that initialization is performed only by a single thread. Because the Python GIL is an additional lock this can lead to deadlocks in the following way: @@ -183,7 +185,7 @@ struct MyClass; ## I'm trying to call Python from Rust but I get `STATUS_DLL_NOT_FOUND` or `STATUS_ENTRYPOINT_NOT_FOUND`! -This happens on Windows when linking to the python DLL fails or the wrong one is linked. The Python DLL on Windows will usually be called something like: +This happens on Windows when linking to the python DLL fails or the wrong one is linked. The Python DLL on Windows will usually be called something like: - `python3X.dll` for Python 3.X, e.g. `python310.dll` for Python 3.10 - `python3.dll` when using PyO3's `abi3` feature diff --git a/guide/src/getting_started.md b/guide/src/getting_started.md index 22bce336d80..235272dab20 100644 --- a/guide/src/getting_started.md +++ b/guide/src/getting_started.md @@ -2,6 +2,8 @@ To get started using PyO3 you will need three things: a Rust toolchain, a Python environment, and a way to build. We'll cover each of these below. +> If you'd like to chat to the PyO3 maintainers and other PyO3 users, consider joining the [PyO3 Discord server](https://discord.gg/c4BwayXQ). We're keen to hear about your experience getting started, so we can make PyO3 as accessible as possible for everyone! + ## Rust First, make sure you have Rust installed on your system. If you haven't already done so, try following the instructions [here](https://www.rust-lang.org/tools/install). PyO3 runs on both the `stable` and `nightly` versions so you can choose whichever one fits you best. The minimum required Rust version is 1.56. From 7c10ff4327b613b6239b0c270e2aaf490b0d6b45 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sun, 25 Feb 2024 08:13:36 +0100 Subject: [PATCH 176/349] allow `Bound<'_, T>` in #[pymethods] `self` position (#3896) * allow `Bound<'_, T>` in #[pymethods] `self` position * rename `TryFromPyCell` -> `TryFromBoundRef` * remove unneccessary unsafe --- pyo3-macros-backend/src/method.rs | 11 +++---- pyo3-macros-backend/src/pyclass.rs | 2 +- src/impl_/pymethods.rs | 40 +++++++++++++++++++++-- src/pycell.rs | 2 +- tests/ui/invalid_pymethod_receiver.stderr | 8 ++--- 5 files changed, 49 insertions(+), 14 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index f492a330c92..2d21c265683 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -151,7 +151,7 @@ impl FnType { #[derive(Clone, Debug)] pub enum SelfType { Receiver { mutable: bool, span: Span }, - TryFromPyCell(Span), + TryFromBoundRef(Span), } #[derive(Clone, Copy)] @@ -204,15 +204,14 @@ impl SelfType { ) }) } - SelfType::TryFromPyCell(span) => { + SelfType::TryFromBoundRef(span) => { error_mode.handle_error( quote_spanned! { *span => - #py.from_borrowed_ptr::<_pyo3::PyAny>(#slf).downcast::<_pyo3::PyCell<#cls>>() + _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf).downcast::<#cls>() .map_err(::std::convert::Into::<_pyo3::PyErr>::into) .and_then( - #[allow(clippy::useless_conversion)] // In case slf is PyCell #[allow(unknown_lints, clippy::unnecessary_fallible_conversions)] // In case slf is Py (unknown_lints can be removed when MSRV is 1.75+) - |cell| ::std::convert::TryFrom::try_from(cell).map_err(::std::convert::Into::into) + |bound| ::std::convert::TryFrom::try_from(bound).map_err(::std::convert::Into::into) ) } @@ -291,7 +290,7 @@ pub fn parse_method_receiver(arg: &syn::FnArg) -> Result { if let syn::Type::ImplTrait(_) = &**ty { bail_spanned!(ty.span() => IMPL_TRAIT_ERR); } - Ok(SelfType::TryFromPyCell(ty.span())) + Ok(SelfType::TryFromBoundRef(ty.span())) } } } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index ce39cb01196..784c39f71aa 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1188,7 +1188,7 @@ fn complex_enum_variant_field_getter<'a>( ) -> Result { let signature = crate::pyfunction::FunctionSignature::from_arguments(vec![])?; - let self_type = crate::method::SelfType::TryFromPyCell(field_span); + let self_type = crate::method::SelfType::TryFromBoundRef(field_span); let spec = FnSpec { tp: crate::method::FnType::Getter(self_type.clone()), diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 196766d0034..9afb6699269 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -3,10 +3,12 @@ use crate::exceptions::PyStopAsyncIteration; use crate::gil::LockGIL; use crate::impl_::panic::PanicTrap; use crate::internal_tricks::extract_c_string; +use crate::pycell::{PyBorrowError, PyBorrowMutError}; +use crate::pyclass::boolean_struct::False; use crate::types::{any::PyAnyMethods, PyModule, PyType}; use crate::{ - ffi, Bound, Py, PyAny, PyCell, PyClass, PyErr, PyObject, PyResult, PyTraverseError, PyVisit, - Python, + ffi, Bound, DowncastError, Py, PyAny, PyCell, PyClass, PyErr, PyObject, PyRef, PyRefMut, + PyResult, PyTraverseError, PyTypeCheck, PyVisit, Python, }; use std::borrow::Cow; use std::ffi::CStr; @@ -490,6 +492,10 @@ impl<'a, 'py> BoundRef<'a, 'py, PyAny> { Bound::ref_from_ptr_or_opt(py, ptr).as_ref().map(BoundRef) } + pub fn downcast(self) -> Result, DowncastError<'a, 'py>> { + self.0.downcast::().map(BoundRef) + } + pub unsafe fn downcast_unchecked(self) -> BoundRef<'a, 'py, T> { BoundRef(self.0.downcast_unchecked::()) } @@ -511,6 +517,36 @@ impl<'a> From> for &'a PyModule { } } +impl<'a, 'py, T: PyClass> From> for &'a PyCell { + #[inline] + fn from(bound: BoundRef<'a, 'py, T>) -> Self { + bound.0.as_gil_ref() + } +} + +impl<'a, 'py, T: PyClass> TryFrom> for PyRef<'py, T> { + type Error = PyBorrowError; + #[inline] + fn try_from(value: BoundRef<'a, 'py, T>) -> Result { + value.0.clone().into_gil_ref().try_into() + } +} + +impl<'a, 'py, T: PyClass> TryFrom> for PyRefMut<'py, T> { + type Error = PyBorrowMutError; + #[inline] + fn try_from(value: BoundRef<'a, 'py, T>) -> Result { + value.0.clone().into_gil_ref().try_into() + } +} + +impl<'a, 'py, T> From> for Bound<'py, T> { + #[inline] + fn from(bound: BoundRef<'a, 'py, T>) -> Self { + bound.0.clone() + } +} + impl<'a, 'py, T> From> for &'a Bound<'py, T> { #[inline] fn from(bound: BoundRef<'a, 'py, T>) -> Self { diff --git a/src/pycell.rs b/src/pycell.rs index 989d039f9d2..71f2f7cc655 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -654,7 +654,7 @@ impl fmt::Debug for PyCell { /// } /// # Python::with_gil(|py| { /// # let sub = Py::new(py, Child::new()).unwrap(); -/// # pyo3::py_run!(py, sub, "assert sub.format() == 'Caterpillar(base: Butterfly, cnt: 3)'"); +/// # pyo3::py_run!(py, sub, "assert sub.format() == 'Caterpillar(base: Butterfly, cnt: 4)'"); /// # }); /// ``` /// diff --git a/tests/ui/invalid_pymethod_receiver.stderr b/tests/ui/invalid_pymethod_receiver.stderr index b7a7880dbde..b6dd44bd9bf 100644 --- a/tests/ui/invalid_pymethod_receiver.stderr +++ b/tests/ui/invalid_pymethod_receiver.stderr @@ -1,8 +1,8 @@ -error[E0277]: the trait bound `i32: From<&PyCell>` is not satisfied +error[E0277]: the trait bound `i32: From>` is not satisfied --> tests/ui/invalid_pymethod_receiver.rs:8:43 | 8 | fn method_with_invalid_self_type(slf: i32, py: Python<'_>, index: u32) {} - | ^^^ the trait `From<&PyCell>` is not implemented for `i32` + | ^^^ the trait `From>` is not implemented for `i32` | = help: the following other types implement trait `From`: > @@ -11,5 +11,5 @@ error[E0277]: the trait bound `i32: From<&PyCell>` is not satisfied > > > - = note: required for `&PyCell` to implement `Into` - = note: required for `i32` to implement `TryFrom<&PyCell>` + = note: required for `BoundRef<'_, '_, MyClass>` to implement `Into` + = note: required for `i32` to implement `TryFrom>` From 404161c96972c6d6aa5a2b08096472a92b9600cb Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 25 Feb 2024 09:49:22 +0000 Subject: [PATCH 177/349] ci: apply correct permissions for cache cleanup job (#3898) --- .github/workflows/cache-cleanup.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml index 2833ed093ee..2b81a69ce88 100644 --- a/.github/workflows/cache-cleanup.yml +++ b/.github/workflows/cache-cleanup.yml @@ -7,6 +7,8 @@ on: jobs: cleanup: runs-on: ubuntu-latest + permissions: + actions: write steps: - name: Cleanup run: | From 8e2219b0d9685244b615bfaa1acab8d56498a866 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 26 Feb 2024 20:28:04 +0000 Subject: [PATCH 178/349] silence non-local-definitions nightly lint (#3901) * silence non-local-definitions nightly lint * add newsfragment * add FIXMEs for `non_local_definitions` * also allow `non_local_definitions` in doctests --- newsfragments/3901.fixed.md | 1 + pyo3-macros-backend/src/frompyobject.rs | 2 ++ pyo3-macros-backend/src/module.rs | 2 ++ pyo3-macros-backend/src/pyclass.rs | 6 ++++++ pyo3-macros-backend/src/pyfunction.rs | 3 +++ pyo3-macros-backend/src/pyimpl.rs | 2 ++ pyo3-macros-backend/src/pymethod.rs | 23 ++++++++++++----------- src/exceptions.rs | 2 ++ src/lib.rs | 9 ++++++++- src/types/mod.rs | 8 ++++++++ 10 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3901.fixed.md diff --git a/newsfragments/3901.fixed.md b/newsfragments/3901.fixed.md new file mode 100644 index 00000000000..0845c2bbcf5 --- /dev/null +++ b/newsfragments/3901.fixed.md @@ -0,0 +1 @@ +Fix `non_local_definitions` lint warning triggered by many PyO3 macros. diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 5e193bf4a24..c1410180d05 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -604,6 +604,8 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { let ident = &tokens.ident; Ok(quote!( + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] const _: () = { use #krate as _pyo3; use _pyo3::prelude::PyAnyMethods; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 6907e484f71..148cea6f8dd 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -211,6 +211,8 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result // this avoids complications around the fact that the generated module has a different scope // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pymodule] is // inside a function body) + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] const _: () = { use #krate::impl_::pymodule as impl_; diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 784c39f71aa..1547e78b4c2 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -362,6 +362,8 @@ fn impl_class( .impl_all()?; Ok(quote! { + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] const _: () = { use #krate as _pyo3; @@ -783,6 +785,8 @@ fn impl_simple_enum( .impl_all()?; Ok(quote! { + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] const _: () = { use #krate as _pyo3; @@ -917,6 +921,8 @@ fn impl_complex_enum( } Ok(quote! { + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] const _: () = { use #krate as _pyo3; diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 7b48585cddc..38362d08a00 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -281,8 +281,11 @@ pub fn impl_wrap_pyfunction( // this avoids complications around the fact that the generated module has a different scope // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pyfunction] is // inside a function body) + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] const _: () = { use #krate as _pyo3; + impl #name::MakeDef { const DEF: #krate::impl_::pyfunction::PyMethodDef = #methoddef; } diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index f5ae111bf48..5802638e340 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -168,6 +168,8 @@ pub fn impl_methods( }; Ok(quote! { + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] const _: () = { use #krate as _pyo3; diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 74072f2a745..358d327ed0b 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -1241,6 +1241,18 @@ impl SlotFragmentDef { )?; let ret_ty = ret_ty.ffi_type(); Ok(quote! { + impl #cls { + unsafe fn #wrapper_ident( + py: _pyo3::Python, + _raw_slf: *mut _pyo3::ffi::PyObject, + #(#arg_idents: #arg_types),* + ) -> _pyo3::PyResult<#ret_ty> { + let _slf = _raw_slf; + #( #holders )* + #body + } + } + impl _pyo3::impl_::pyclass::#fragment_trait<#cls> for _pyo3::impl_::pyclass::PyClassImplCollector<#cls> { #[inline] @@ -1250,17 +1262,6 @@ impl SlotFragmentDef { _raw_slf: *mut _pyo3::ffi::PyObject, #(#arg_idents: #arg_types),* ) -> _pyo3::PyResult<#ret_ty> { - impl #cls { - unsafe fn #wrapper_ident( - py: _pyo3::Python, - _raw_slf: *mut _pyo3::ffi::PyObject, - #(#arg_idents: #arg_types),* - ) -> _pyo3::PyResult<#ret_ty> { - let _slf = _raw_slf; - #( #holders )* - #body - } - } #cls::#wrapper_ident(py, _raw_slf, #(#arg_idents),*) } } diff --git a/src/exceptions.rs b/src/exceptions.rs index da48475e0d6..071470160ae 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -19,6 +19,8 @@ use std::os::raw::c_char; #[macro_export] macro_rules! impl_exception_boilerplate { ($name: ident) => { + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] impl ::std::convert::From<&$name> for $crate::PyErr { #[inline] fn from(err: &$name) -> $crate::PyErr { diff --git a/src/lib.rs b/src/lib.rs index 0a0b4eae684..6b39b534794 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,7 +10,14 @@ rust_2021_prelude_collisions, warnings ), - allow(unused_variables, unused_assignments, unused_extern_crates) + allow( + unused_variables, + unused_assignments, + unused_extern_crates, + // FIXME https://github.com/rust-lang/rust/issues/121621#issuecomment-1965156376 + unknown_lints, + non_local_definitions, + ) )))] //! Rust bindings to the Python interpreter. diff --git a/src/types/mod.rs b/src/types/mod.rs index 4baa086ab79..938716f78f4 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -188,6 +188,8 @@ macro_rules! pyobject_native_type_named ( } } + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] impl<$($generics,)*> $crate::IntoPy<$crate::Py<$name>> for &'_ $name { #[inline] fn into_py(self, py: $crate::Python<'_>) -> $crate::Py<$name> { @@ -195,6 +197,8 @@ macro_rules! pyobject_native_type_named ( } } + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] impl<$($generics,)*> ::std::convert::From<&'_ $name> for $crate::Py<$name> { #[inline] fn from(other: &$name) -> Self { @@ -203,6 +207,8 @@ macro_rules! pyobject_native_type_named ( } } + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] impl<'a, $($generics,)*> ::std::convert::From<&'a $name> for &'a $crate::PyAny { fn from(ob: &'a $name) -> Self { unsafe{&*(ob as *const $name as *const $crate::PyAny)} @@ -252,6 +258,8 @@ macro_rules! pyobject_native_type_info( #[macro_export] macro_rules! pyobject_native_type_extract { ($name:ty $(;$generics:ident)*) => { + // FIXME https://github.com/PyO3/pyo3/issues/3903 + #[allow(unknown_lints, non_local_definitions)] impl<'py, $($generics,)*> $crate::FromPyObject<'py> for &'py $name { #[inline] fn extract_bound(obj: &$crate::Bound<'py, $crate::PyAny>) -> $crate::PyResult { From cd1c0dbf39c636321e6679d66b82294dca36969b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 26 Feb 2024 21:47:35 +0000 Subject: [PATCH 179/349] ci: move cross compile tests to their own jobs (#3887) * ci: move cross compile tests to their own jobs * don't run cross-compile jobs on regular PRs --- .github/workflows/build.yml | 54 ----------------------- .github/workflows/ci.yml | 86 +++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 54 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f3e707bfba..25c8014f762 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -138,60 +138,6 @@ jobs: if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }} run: nox -s ffi-check - - - name: Test cross compilation - if: ${{ inputs.os == 'ubuntu-latest' && inputs.python-version == '3.9' }} - uses: PyO3/maturin-action@v1 - env: - PYO3_CROSS_LIB_DIR: /opt/python/cp39-cp39/lib - with: - target: aarch64-unknown-linux-gnu - manylinux: auto - args: --release -i python3.9 -m examples/maturin-starter/Cargo.toml - - - run: sudo rm -rf examples/maturin-starter/target - if: ${{ inputs.os == 'ubuntu-latest' && inputs.python-version == '3.9' }} - - name: Test cross compile to same architecture - if: ${{ inputs.os == 'ubuntu-latest' && inputs.python-version == '3.9' }} - uses: PyO3/maturin-action@v1 - env: - PYO3_CROSS_LIB_DIR: /opt/python/cp39-cp39/lib - with: - target: x86_64-unknown-linux-gnu - manylinux: auto - args: --release -i python3.9 -m examples/maturin-starter/Cargo.toml - - - name: Test cross compilation - if: ${{ inputs.os == 'macos-latest' && inputs.python-version == '3.9' }} - uses: PyO3/maturin-action@v1 - with: - target: aarch64-apple-darwin - args: --release -i python3.9 -m examples/maturin-starter/Cargo.toml - - - name: Test cross compile to Windows - if: ${{ inputs.os == 'ubuntu-latest' && inputs.python-version == '3.8' }} - env: - XWIN_ARCH: x86_64 - run: | - set -ex - sudo apt-get install -y mingw-w64 llvm - rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc - pip install cargo-xwin - # abi3 - cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu - cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc - # non-abi3 - export PYO3_CROSS_PYTHON_VERSION=3.9 - cargo build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-gnu - cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc - - - name: Test cross compile to Windows with maturin - if: ${{ inputs.os == 'ubuntu-latest' && inputs.python-version == '3.8' }} - uses: PyO3/maturin-action@v1 - with: - target: x86_64-pc-windows-gnu - args: -i python3.8 -m examples/maturin-starter/Cargo.toml --features abi3 - env: CARGO_TERM_VERBOSE: true CARGO_BUILD_TARGET: ${{ inputs.rust-target }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d22dc2236b1..5f4347aa6f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -488,6 +488,90 @@ jobs: - run: python3 -m pip install --upgrade pip && pip install nox - run: python3 -m nox -s check-feature-powerset + test-cross-compilation: + needs: [fmt] + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} + runs-on: ${{ matrix.os }} + name: test-cross-compilation ${{ matrix.os }} -> ${{ matrix.target }} + strategy: + # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present + fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} + matrix: + include: + # ubuntu "cross compile" to itself + - os: "ubuntu-latest" + target: "x86_64-unknown-linux-gnu" + flags: "-i python3.12" + manylinux: auto + # ubuntu x86_64 -> aarch64 + - os: "ubuntu-latest" + target: "aarch64-unknown-linux-gnu" + flags: "-i python3.12" + manylinux: auto + # ubuntu x86_64 -> windows x86_64 + - os: "ubuntu-latest" + target: "x86_64-pc-windows-gnu" + flags: "-i python3.12 --features abi3 --features generate-import-lib" + manylinux: off + # macos x86_64 -> aarch64 + - os: "macos-13" # last x86_64 macos runners + target: "aarch64-apple-darwin" + # macos aarch64 -> x86_64 + - os: "macos-14" # aarch64 macos runners + target: "x86_64-apple-darwin" + steps: + - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: + examples/maturin-starter + save-if: ${{ github.event_name != 'merge_group' }} + - name: Setup cross-compiler + if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }} + run: sudo apt-get install -y mingw-w64 llvm + - uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux }} + args: --release -m examples/maturin-starter/Cargo.toml ${{ matrix.flags }} + + test-cross-compilation-windows: + needs: [fmt] + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: + examples/maturin-starter + save-if: ${{ github.event_name != 'merge_group' }} + - uses: actions/cache/restore@v4 + with: + # https://github.com/PyO3/maturin/discussions/1953 + path: ~/.cache/cargo-xwin + key: cargo-xwin-cache + - name: Test cross compile to Windows + env: + XWIN_ARCH: x86_64 + run: | + set -ex + sudo apt-get install -y mingw-w64 llvm + rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc + pip install cargo-xwin + # abi3 + cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu + cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc + # non-abi3 + export PYO3_CROSS_PYTHON_VERSION=3.12 + cargo build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-gnu + cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc + - if: ${{ github.ref == 'refs/heads/main' }} + uses: actions/cache/save@v4 + with: + path: ~/.cache/cargo-xwin + key: cargo-xwin-cache conclusion: needs: - fmt @@ -503,6 +587,8 @@ jobs: - test-debug - test-version-limits - check-feature-powerset + - test-cross-compilation + - test-cross-compilation-windows if: always() runs-on: ubuntu-latest steps: From 5c41ea0aded350d7c70c15cfacc3862bc5dce8b2 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Mon, 26 Feb 2024 23:14:41 +0000 Subject: [PATCH 180/349] Implement `From>` for PyErr (#3881) * Implement `From>` for PyErr * Replace PyErr::from_value_bound calls with .into * Fix From expected error message * Add a trait bound to From> for PyErr --- src/err/mod.rs | 18 ++++++++++ src/exceptions.rs | 4 ++- src/lib.rs | 2 +- src/types/string.rs | 40 ++++++++++------------- tests/ui/invalid_result_conversion.stderr | 2 +- 5 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/err/mod.rs b/src/err/mod.rs index ab39e8cd46f..5e054449bc9 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -982,6 +982,24 @@ impl PyErrArguments for PyDowncastErrorArguments { } } +/// Python exceptions that can be converted to [`PyErr`]. +/// +/// This is used to implement [`From> for PyErr`]. +/// +/// Users should not need to implement this trait directly. It is implemented automatically in the +/// [`crate::import_exception!`] and [`crate::create_exception!`] macros. +pub trait ToPyErr {} + +impl<'py, T> std::convert::From> for PyErr +where + T: ToPyErr, +{ + #[inline] + fn from(err: Bound<'py, T>) -> PyErr { + PyErr::from_value_bound(err.into_any()) + } +} + /// Convert `PyDowncastError` to Python `TypeError`. impl<'a> std::convert::From> for PyErr { fn from(err: PyDowncastError<'_>) -> PyErr { diff --git a/src/exceptions.rs b/src/exceptions.rs index 071470160ae..1c952393241 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -54,6 +54,8 @@ macro_rules! impl_exception_boilerplate { } } } + + impl $crate::ToPyErr for $name {} }; } @@ -1074,7 +1076,7 @@ mod tests { ); // Restoring should preserve the same error - let e = PyErr::from_value_bound(decode_err.into_any()); + let e: PyErr = decode_err.into(); e.restore(py); assert_eq!( diff --git a/src/lib.rs b/src/lib.rs index 6b39b534794..d0c53c4a0c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -308,7 +308,7 @@ pub use crate::conversion::{AsPyPointer, FromPyObject, FromPyPointer, IntoPy, To #[allow(deprecated)] pub use crate::conversion::{PyTryFrom, PyTryInto}; pub use crate::err::{ - DowncastError, DowncastIntoError, PyDowncastError, PyErr, PyErrArguments, PyResult, + DowncastError, DowncastIntoError, PyDowncastError, PyErr, PyErrArguments, PyResult, ToPyErr, }; pub use crate::gil::GILPool; #[cfg(not(PyPy))] diff --git a/src/types/string.rs b/src/types/string.rs index 0a7847d2959..4a33236b47b 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -73,9 +73,7 @@ impl<'a> PyStringData<'a> { match self { Self::Ucs1(data) => match str::from_utf8(data) { Ok(s) => Ok(Cow::Borrowed(s)), - Err(e) => Err(crate::PyErr::from_value_bound( - PyUnicodeDecodeError::new_utf8_bound(py, data, e)?.into_any(), - )), + Err(e) => Err(PyUnicodeDecodeError::new_utf8_bound(py, data, e)?.into()), }, Self::Ucs2(data) => match String::from_utf16(data) { Ok(s) => Ok(Cow::Owned(s)), @@ -83,30 +81,26 @@ impl<'a> PyStringData<'a> { let mut message = e.to_string().as_bytes().to_vec(); message.push(0); - Err(crate::PyErr::from_value_bound( - PyUnicodeDecodeError::new_bound( - py, - CStr::from_bytes_with_nul(b"utf-16\0").unwrap(), - self.as_bytes(), - 0..self.as_bytes().len(), - CStr::from_bytes_with_nul(&message).unwrap(), - )? - .into_any(), - )) - } - }, - Self::Ucs4(data) => match data.iter().map(|&c| std::char::from_u32(c)).collect() { - Some(s) => Ok(Cow::Owned(s)), - None => Err(crate::PyErr::from_value_bound( - PyUnicodeDecodeError::new_bound( + Err(PyUnicodeDecodeError::new_bound( py, - CStr::from_bytes_with_nul(b"utf-32\0").unwrap(), + CStr::from_bytes_with_nul(b"utf-16\0").unwrap(), self.as_bytes(), 0..self.as_bytes().len(), - CStr::from_bytes_with_nul(b"error converting utf-32\0").unwrap(), + CStr::from_bytes_with_nul(&message).unwrap(), )? - .into_any(), - )), + .into()) + } + }, + Self::Ucs4(data) => match data.iter().map(|&c| std::char::from_u32(c)).collect() { + Some(s) => Ok(Cow::Owned(s)), + None => Err(PyUnicodeDecodeError::new_bound( + py, + CStr::from_bytes_with_nul(b"utf-32\0").unwrap(), + self.as_bytes(), + 0..self.as_bytes().len(), + CStr::from_bytes_with_nul(b"error converting utf-32\0").unwrap(), + )? + .into()), }, } } diff --git a/tests/ui/invalid_result_conversion.stderr b/tests/ui/invalid_result_conversion.stderr index b3e65517e36..73ed2cba1aa 100644 --- a/tests/ui/invalid_result_conversion.stderr +++ b/tests/ui/invalid_result_conversion.stderr @@ -5,6 +5,7 @@ error[E0277]: the trait bound `PyErr: From` is not satisfied | ^^^^^^^^^^^^^ the trait `From` is not implemented for `PyErr` | = help: the following other types implement trait `From`: + >> > > > @@ -12,7 +13,6 @@ error[E0277]: the trait bound `PyErr: From` is not satisfied >> >> > - > and $N others = note: required for `MyError` to implement `Into` = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From 93704047a58335cc9de584325826aeac7abf84f7 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 27 Feb 2024 18:56:22 +0000 Subject: [PATCH 181/349] store `Bound` inside `PyRef` and `PyRefMut` (#3860) * store `Bound` inside `PyRef` and `PyRefMut` * update `FromPyObject` for `PyRef` to use `extract_bound` * review: Icxolu feedback --- src/conversion.rs | 11 ++-- src/impl_/pymethods.rs | 8 +-- src/instance.rs | 18 +++---- src/pycell.rs | 114 ++++++++++++++++++++++++++--------------- 4 files changed, 91 insertions(+), 60 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index 415f11089df..fc407fa5bc8 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -4,6 +4,7 @@ use crate::err::{self, PyDowncastError, PyResult}; use crate::inspect::types::TypeInfo; use crate::pyclass::boolean_struct::False; use crate::type_object::PyTypeInfo; +use crate::types::any::PyAnyMethods; use crate::types::PyTuple; use crate::{ ffi, gil, Bound, Py, PyAny, PyCell, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, @@ -287,9 +288,8 @@ impl<'py, T> FromPyObject<'py> for PyRef<'py, T> where T: PyClass, { - fn extract(obj: &'py PyAny) -> PyResult { - let cell: &PyCell = obj.downcast()?; - cell.try_borrow().map_err(Into::into) + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { + obj.downcast::()?.try_borrow().map_err(Into::into) } } @@ -297,9 +297,8 @@ impl<'py, T> FromPyObject<'py> for PyRefMut<'py, T> where T: PyClass, { - fn extract(obj: &'py PyAny) -> PyResult { - let cell: &PyCell = obj.downcast()?; - cell.try_borrow_mut().map_err(Into::into) + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { + obj.downcast::()?.try_borrow_mut().map_err(Into::into) } } diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 9afb6699269..0b73ce9bc73 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -7,8 +7,8 @@ use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; use crate::types::{any::PyAnyMethods, PyModule, PyType}; use crate::{ - ffi, Bound, DowncastError, Py, PyAny, PyCell, PyClass, PyErr, PyObject, PyRef, PyRefMut, - PyResult, PyTraverseError, PyTypeCheck, PyVisit, Python, + ffi, Borrowed, Bound, DowncastError, Py, PyAny, PyCell, PyClass, PyErr, PyObject, PyRef, + PyRefMut, PyResult, PyTraverseError, PyTypeCheck, PyVisit, Python, }; use std::borrow::Cow; use std::ffi::CStr; @@ -272,8 +272,8 @@ where let trap = PanicTrap::new("uncaught panic inside __traverse__ handler"); let py = Python::assume_gil_acquired(); - let slf = py.from_borrowed_ptr::>(slf); - let borrow = slf.try_borrow_threadsafe(); + let slf = Borrowed::from_ptr_unchecked(py, slf).downcast_unchecked::(); + let borrow = PyRef::try_borrow_threadsafe(&slf); let visit = PyVisit::from_raw(visit, arg, py); let retval = if let Ok(borrow) = borrow { diff --git a/src/instance.rs b/src/instance.rs index 3efdf58f597..cb33ed32c75 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -242,8 +242,8 @@ where /// /// Panics if the value is currently mutably borrowed. For a non-panicking variant, use /// [`try_borrow`](#method.try_borrow). - pub fn borrow(&'py self) -> PyRef<'py, T> { - self.get_cell().borrow() + pub fn borrow(&self) -> PyRef<'py, T> { + PyRef::borrow(self) } /// Mutably borrows the value `T`. @@ -275,11 +275,11 @@ where /// # Panics /// Panics if the value is currently borrowed. For a non-panicking variant, use /// [`try_borrow_mut`](#method.try_borrow_mut). - pub fn borrow_mut(&'py self) -> PyRefMut<'py, T> + pub fn borrow_mut(&self) -> PyRefMut<'py, T> where T: PyClass, { - self.get_cell().borrow_mut() + PyRefMut::borrow(self) } /// Attempts to immutably borrow the value `T`, returning an error if the value is currently mutably borrowed. @@ -289,8 +289,8 @@ where /// This is the non-panicking variant of [`borrow`](#method.borrow). /// /// For frozen classes, the simpler [`get`][Self::get] is available. - pub fn try_borrow(&'py self) -> Result, PyBorrowError> { - self.get_cell().try_borrow() + pub fn try_borrow(&self) -> Result, PyBorrowError> { + PyRef::try_borrow(self) } /// Attempts to mutably borrow the value `T`, returning an error if the value is currently borrowed. @@ -298,11 +298,11 @@ where /// The borrow lasts while the returned [`PyRefMut`] exists. /// /// This is the non-panicking variant of [`borrow_mut`](#method.borrow_mut). - pub fn try_borrow_mut(&'py self) -> Result, PyBorrowMutError> + pub fn try_borrow_mut(&self) -> Result, PyBorrowMutError> where T: PyClass, { - self.get_cell().try_borrow_mut() + PyRefMut::try_borrow(self) } /// Provide an immutable borrow of the value `T` without acquiring the GIL. @@ -337,7 +337,7 @@ where unsafe { &*cell.get_ptr() } } - fn get_cell(&'py self) -> &'py PyCell { + pub(crate) fn get_cell(&'py self) -> &'py PyCell { let cell = self.as_ptr().cast::>(); // SAFETY: Bound is known to contain an object which is laid out in memory as a // PyCell. diff --git a/src/pycell.rs b/src/pycell.rs index 71f2f7cc655..29fd1b37886 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -192,6 +192,7 @@ //! [Interior Mutability]: https://doc.rust-lang.org/book/ch15-05-interior-mutability.html "RefCell and the Interior Mutability Pattern - The Rust Programming Language" use crate::exceptions::PyRuntimeError; +use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::pyclass::{ PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, }; @@ -201,6 +202,7 @@ use crate::pyclass::{ }; use crate::pyclass_init::PyClassInitializer; use crate::type_object::{PyLayout, PySizedLayout}; +use crate::types::any::PyAnyMethods; use crate::types::PyAny; use crate::{ conversion::{AsPyPointer, FromPyPointer, ToPyObject}, @@ -310,7 +312,7 @@ impl PyCell { /// Panics if the value is currently mutably borrowed. For a non-panicking variant, use /// [`try_borrow`](#method.try_borrow). pub fn borrow(&self) -> PyRef<'_, T> { - self.try_borrow().expect("Already mutably borrowed") + PyRef::borrow(&self.as_borrowed()) } /// Mutably borrows the value `T`. This borrow lasts as long as the returned `PyRefMut` exists. @@ -323,7 +325,7 @@ impl PyCell { where T: PyClass, { - self.try_borrow_mut().expect("Already borrowed") + PyRefMut::borrow(&self.as_borrowed()) } /// Immutably borrows the value `T`, returning an error if the value is currently @@ -355,18 +357,7 @@ impl PyCell { /// }); /// ``` pub fn try_borrow(&self) -> Result, PyBorrowError> { - self.ensure_threadsafe(); - self.borrow_checker() - .try_borrow() - .map(|_| PyRef { inner: self }) - } - - /// Variant of [`try_borrow`][Self::try_borrow] which fails instead of panicking if called from the wrong thread - pub(crate) fn try_borrow_threadsafe(&self) -> Result, PyBorrowError> { - self.check_threadsafe()?; - self.borrow_checker() - .try_borrow() - .map(|_| PyRef { inner: self }) + PyRef::try_borrow(&self.as_borrowed()) } /// Mutably borrows the value `T`, returning an error if the value is currently borrowed. @@ -395,10 +386,7 @@ impl PyCell { where T: PyClass, { - self.ensure_threadsafe(); - self.borrow_checker() - .try_borrow_mut() - .map(|_| PyRefMut { inner: self }) + PyRefMut::try_borrow(&self.as_borrowed()) } /// Immutably borrows the value `T`, returning an error if the value is @@ -654,13 +642,15 @@ impl fmt::Debug for PyCell { /// } /// # Python::with_gil(|py| { /// # let sub = Py::new(py, Child::new()).unwrap(); -/// # pyo3::py_run!(py, sub, "assert sub.format() == 'Caterpillar(base: Butterfly, cnt: 4)'"); +/// # pyo3::py_run!(py, sub, "assert sub.format() == 'Caterpillar(base: Butterfly, cnt: 5)', sub.format()"); /// # }); /// ``` /// /// See the [module-level documentation](self) for more information. pub struct PyRef<'p, T: PyClass> { - inner: &'p PyCell, + // TODO: once the GIL Ref API is removed, consider adding a lifetime parameter to `PyRef` to + // store `Borrowed` here instead, avoiding reference counting overhead. + inner: Bound<'p, T>, } impl<'p, T: PyClass> PyRef<'p, T> { @@ -676,11 +666,11 @@ where U: PyClass, { fn as_ref(&self) -> &T::BaseType { - unsafe { &*self.inner.ob_base.get_ptr() } + unsafe { &*self.inner.get_cell().ob_base.get_ptr() } } } -impl<'p, T: PyClass> PyRef<'p, T> { +impl<'py, T: PyClass> PyRef<'py, T> { /// Returns the raw FFI pointer represented by self. /// /// # Safety @@ -702,7 +692,27 @@ impl<'p, T: PyClass> PyRef<'p, T> { /// of the pointer or decrease the reference count (e.g. with [`pyo3::ffi::Py_DecRef`](crate::ffi::Py_DecRef)). #[inline] pub fn into_ptr(self) -> *mut ffi::PyObject { - self.inner.into_ptr() + self.inner.clone().into_ptr() + } + + pub(crate) fn borrow(obj: &Bound<'py, T>) -> Self { + Self::try_borrow(obj).expect("Already mutably borrowed") + } + + pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { + let cell = obj.get_cell(); + cell.ensure_threadsafe(); + cell.borrow_checker() + .try_borrow() + .map(|_| Self { inner: obj.clone() }) + } + + pub(crate) fn try_borrow_threadsafe(obj: &Bound<'py, T>) -> Result { + let cell = obj.get_cell(); + cell.check_threadsafe()?; + cell.borrow_checker() + .try_borrow() + .map(|_| Self { inner: obj.clone() }) } } @@ -757,10 +767,14 @@ where /// # }); /// ``` pub fn into_super(self) -> PyRef<'p, U> { - let PyRef { inner } = self; - std::mem::forget(self); + let py = self.py(); PyRef { - inner: &inner.ob_base, + inner: unsafe { + ManuallyDrop::new(self) + .as_ptr() + .assume_owned(py) + .downcast_into_unchecked() + }, } } } @@ -770,13 +784,13 @@ impl<'p, T: PyClass> Deref for PyRef<'p, T> { #[inline] fn deref(&self) -> &T { - unsafe { &*self.inner.get_ptr() } + unsafe { &*self.inner.get_cell().get_ptr() } } } impl<'p, T: PyClass> Drop for PyRef<'p, T> { fn drop(&mut self) { - self.inner.borrow_checker().release_borrow() + self.inner.get_cell().borrow_checker().release_borrow() } } @@ -788,7 +802,7 @@ impl IntoPy for PyRef<'_, T> { impl IntoPy for &'_ PyRef<'_, T> { fn into_py(self, py: Python<'_>) -> PyObject { - self.inner.into_py(py) + unsafe { PyObject::from_borrowed_ptr(py, self.inner.as_ptr()) } } } @@ -815,7 +829,9 @@ impl fmt::Debug for PyRef<'_, T> { /// /// See the [module-level documentation](self) for more information. pub struct PyRefMut<'p, T: PyClass> { - inner: &'p PyCell, + // TODO: once the GIL Ref API is removed, consider adding a lifetime parameter to `PyRef` to + // store `Borrowed` here instead, avoiding reference counting overhead. + inner: Bound<'p, T>, } impl<'p, T: PyClass> PyRefMut<'p, T> { @@ -831,7 +847,7 @@ where U: PyClass, { fn as_ref(&self) -> &T::BaseType { - unsafe { &*self.inner.ob_base.get_ptr() } + unsafe { &*self.inner.get_cell().ob_base.get_ptr() } } } @@ -841,11 +857,11 @@ where U: PyClass, { fn as_mut(&mut self) -> &mut T::BaseType { - unsafe { &mut *self.inner.ob_base.get_ptr() } + unsafe { &mut *self.inner.get_cell().ob_base.get_ptr() } } } -impl<'p, T: PyClass> PyRefMut<'p, T> { +impl<'py, T: PyClass> PyRefMut<'py, T> { /// Returns the raw FFI pointer represented by self. /// /// # Safety @@ -867,7 +883,19 @@ impl<'p, T: PyClass> PyRefMut<'p, T> { /// of the pointer or decrease the reference count (e.g. with [`pyo3::ffi::Py_DecRef`](crate::ffi::Py_DecRef)). #[inline] pub fn into_ptr(self) -> *mut ffi::PyObject { - self.inner.into_ptr() + self.inner.clone().into_ptr() + } + + pub(crate) fn borrow(obj: &Bound<'py, T>) -> Self { + Self::try_borrow(obj).expect("Already borrowed") + } + + pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { + let cell = obj.get_cell(); + cell.ensure_threadsafe(); + cell.borrow_checker() + .try_borrow_mut() + .map(|_| Self { inner: obj.clone() }) } } @@ -880,10 +908,14 @@ where /// /// See [`PyRef::into_super`] for more. pub fn into_super(self) -> PyRefMut<'p, U> { - let PyRefMut { inner } = self; - std::mem::forget(self); + let py = self.py(); PyRefMut { - inner: &inner.ob_base, + inner: unsafe { + ManuallyDrop::new(self) + .as_ptr() + .assume_owned(py) + .downcast_into_unchecked() + }, } } } @@ -893,20 +925,20 @@ impl<'p, T: PyClass> Deref for PyRefMut<'p, T> { #[inline] fn deref(&self) -> &T { - unsafe { &*self.inner.get_ptr() } + unsafe { &*self.inner.get_cell().get_ptr() } } } impl<'p, T: PyClass> DerefMut for PyRefMut<'p, T> { #[inline] fn deref_mut(&mut self) -> &mut T { - unsafe { &mut *self.inner.get_ptr() } + unsafe { &mut *self.inner.get_cell().get_ptr() } } } impl<'p, T: PyClass> Drop for PyRefMut<'p, T> { fn drop(&mut self) { - self.inner.borrow_checker().release_borrow_mut() + self.inner.get_cell().borrow_checker().release_borrow_mut() } } @@ -918,7 +950,7 @@ impl> IntoPy for PyRefMut<'_, T> { impl> IntoPy for &'_ PyRefMut<'_, T> { fn into_py(self, py: Python<'_>) -> PyObject { - self.inner.into_py(py) + self.inner.clone().into_py(py) } } From a3ad28b70ca847459b3f5d8b8644beb826268cc2 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Tue, 27 Feb 2024 19:19:52 +0000 Subject: [PATCH 182/349] Pymodule bound (#3897) * Support Bound in pymodule and pyfunction macros Co-authored-by: David Hewitt * Remove spurious $ character Co-authored-by: Matthew Neeley * Rework PyCFunction::new_bound signatures This allows us to remove the awkward `PyFunctionArgumentsBound` enum. * Use BoundRef instead of BoundModule * support argument deduction for `wrap_pyfunction_bound!` * support `wrap_pyfunction!` with `Bound` input/output * Fix docs link to `wrap_pyfunction_bound!` * Revert back to wrap_pyfunction! --------- Co-authored-by: David Hewitt Co-authored-by: Matthew Neeley --- guide/src/module.md | 2 +- guide/src/python_from_rust.md | 2 +- pyo3-macros-backend/src/module.rs | 13 +++-- pyo3-macros-backend/src/pyfunction.rs | 6 +-- src/impl_/pyfunction.rs | 70 +++++++++++++++++++++--- src/macros.rs | 45 ++++++++++++++-- src/prelude.rs | 2 +- src/types/function.rs | 72 ++++++++++++++++++++----- src/types/module.rs | 7 +-- tests/test_module.rs | 12 ++--- tests/test_pyfunction.rs | 6 ++- tests/test_wrap_pyfunction_deduction.rs | 9 ++++ 12 files changed, 203 insertions(+), 43 deletions(-) diff --git a/guide/src/module.md b/guide/src/module.md index 789c3d91ccb..53c390bec06 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -79,7 +79,7 @@ fn parent_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { fn register_child_module(py: Python<'_>, parent_module: &PyModule) -> PyResult<()> { let child_module = PyModule::new_bound(py, "child_module")?; - child_module.add_function(&wrap_pyfunction!(func, child_module.as_gil_ref())?.as_borrowed())?; + child_module.add_function(wrap_pyfunction!(func, &child_module)?)?; parent_module.add_submodule(child_module.as_gil_ref())?; Ok(()) } diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index 973f997cbf8..b51bbe592eb 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -311,7 +311,7 @@ fn main() -> PyResult<()> { Python::with_gil(|py| { // Create new module let foo_module = PyModule::new_bound(py, "foo")?; - foo_module.add_function(&wrap_pyfunction!(add_one, foo_module.as_gil_ref())?.as_borrowed())?; + foo_module.add_function(wrap_pyfunction!(add_one, &foo_module)?)?; // Import and get sys.modules let sys = PyModule::import_bound(py, "sys")?; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 148cea6f8dd..40cf34f22fd 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -215,15 +215,16 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result #[allow(unknown_lints, non_local_definitions)] const _: () = { use #krate::impl_::pymodule as impl_; + use #krate::impl_::pymethods::BoundRef; fn __pyo3_pymodule(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { - #ident(module.py(), module.as_gil_ref()) + #ident(module.py(), ::std::convert::Into::into(BoundRef(module))) } impl #ident::MakeDef { const fn make_def() -> impl_::ModuleDef { + const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule); unsafe { - const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule); impl_::ModuleDef::new( #ident::__PYO3_NAME, #doc, @@ -263,9 +264,13 @@ fn module_initialization(options: PyModuleOptions, ident: &syn::Ident) -> TokenS /// Finds and takes care of the #[pyfn(...)] in `#[pymodule]` fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn) -> Result<()> { - let mut stmts: Vec = Vec::new(); let krate = get_pyo3_crate(&options.krate); + let mut stmts: Vec = vec![syn::parse_quote!( + #[allow(unknown_lints, unused_imports, redundant_imports)] + use #krate::{PyNativeType, types::PyModuleMethods}; + )]; + for mut stmt in func.block.stmts.drain(..) { if let syn::Stmt::Item(Item::Fn(func)) = &mut stmt { if let Some(pyfn_args) = get_pyfn_attr(&mut func.attrs)? { @@ -274,7 +279,7 @@ fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn let name = &func.sig.ident; let statements: Vec = syn::parse_quote! { #wrapped_function - #module_name.add_function(#krate::impl_::pyfunction::_wrap_pyfunction(&#name::DEF, #module_name)?)?; + #module_name.as_borrowed().add_function(#krate::wrap_pyfunction!(#name, #module_name.as_borrowed())?)?; }; stmts.extend(statements); } diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 38362d08a00..265a4824109 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -268,12 +268,12 @@ pub fn impl_wrap_pyfunction( #[doc(hidden)] #vis mod #name { pub(crate) struct MakeDef; - pub const DEF: #krate::impl_::pyfunction::PyMethodDef = MakeDef::DEF; + pub const DEF: #krate::impl_::pymethods::PyMethodDef = MakeDef::DEF; pub fn add_to_module(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { use #krate::prelude::PyModuleMethods; use ::std::convert::Into; - module.add_function(&#krate::types::PyCFunction::internal_new(&DEF, module.as_gil_ref().into())?) + module.add_function(#krate::types::PyCFunction::internal_new(&DEF, module.as_gil_ref().into())?) } } @@ -287,7 +287,7 @@ pub fn impl_wrap_pyfunction( use #krate as _pyo3; impl #name::MakeDef { - const DEF: #krate::impl_::pyfunction::PyMethodDef = #methoddef; + const DEF: #krate::impl_::pymethods::PyMethodDef = #methoddef; } #[allow(non_snake_case)] diff --git a/src/impl_/pyfunction.rs b/src/impl_/pyfunction.rs index 464beeb8ce5..531e0c93192 100644 --- a/src/impl_/pyfunction.rs +++ b/src/impl_/pyfunction.rs @@ -1,10 +1,68 @@ -use crate::{derive_utils::PyFunctionArguments, types::PyCFunction, PyResult}; +use crate::{ + types::{PyCFunction, PyModule}, + Borrowed, Bound, PyResult, Python, +}; pub use crate::impl_::pymethods::PyMethodDef; -pub fn _wrap_pyfunction<'a>( - method_def: &PyMethodDef, - py_or_module: impl Into>, -) -> PyResult<&'a PyCFunction> { - PyCFunction::internal_new(method_def, py_or_module.into()).map(|x| x.into_gil_ref()) +/// Trait to enable the use of `wrap_pyfunction` with both `Python` and `PyModule`, +/// and also to infer the return type of either `&'py PyCFunction` or `Bound<'py, PyCFunction>`. +pub trait WrapPyFunctionArg<'py, T> { + fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult; +} + +impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for Bound<'py, PyModule> { + fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { + PyCFunction::internal_new_bound(self.py(), method_def, Some(&self)) + } +} + +impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for &'_ Bound<'py, PyModule> { + fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { + PyCFunction::internal_new_bound(self.py(), method_def, Some(self)) + } +} + +impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for Borrowed<'_, 'py, PyModule> { + fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { + PyCFunction::internal_new_bound(self.py(), method_def, Some(&self)) + } +} + +impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for &'_ Borrowed<'_, 'py, PyModule> { + fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { + PyCFunction::internal_new_bound(self.py(), method_def, Some(self)) + } +} + +// For Python<'py>, only the GIL Ref form exists to avoid causing type inference to kick in. +// The `wrap_pyfunction_bound!` macro is needed for the Bound form. +impl<'py> WrapPyFunctionArg<'py, &'py PyCFunction> for Python<'py> { + fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult<&'py PyCFunction> { + PyCFunction::internal_new(method_def, self.into()).map(Bound::into_gil_ref) + } +} + +impl<'py> WrapPyFunctionArg<'py, &'py PyCFunction> for &'py PyModule { + fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult<&'py PyCFunction> { + PyCFunction::internal_new(method_def, self.into()).map(Bound::into_gil_ref) + } +} + +/// Helper for `wrap_pyfunction_bound!` to guarantee return type of `Bound<'py, PyCFunction>`. +pub struct OnlyBound(pub T); + +impl<'py, T> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for OnlyBound +where + T: WrapPyFunctionArg<'py, Bound<'py, PyCFunction>>, +{ + fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { + WrapPyFunctionArg::wrap_pyfunction(self.0, method_def) + } +} + +impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for OnlyBound> { + fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { + PyCFunction::internal_new_bound(self.0, method_def, None) + } } diff --git a/src/macros.rs b/src/macros.rs index 9b0d2816882..33f378e7b83 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -121,18 +121,57 @@ macro_rules! py_run_impl { /// Wraps a Rust function annotated with [`#[pyfunction]`](macro@crate::pyfunction). /// /// This can be used with [`PyModule::add_function`](crate::types::PyModule::add_function) to add free -/// functions to a [`PyModule`](crate::types::PyModule) - see its documentation for more information. +/// functions to a [`PyModule`](crate::types::PyModule) - see its documentation for more +/// information. +/// +/// During the migration from the GIL Ref API to the Bound API, the return type of this macro will +/// be either the `&'py PyModule` GIL Ref or `Bound<'py, PyModule>` according to the second +/// argument. +/// +/// For backwards compatibility, if the second argument is `Python<'py>` then the return type will +/// be `&'py PyModule` GIL Ref. To get `Bound<'py, PyModule>`, use the [`crate::wrap_pyfunction_bound!`] +/// macro instead. #[macro_export] macro_rules! wrap_pyfunction { ($function:path) => { &|py_or_module| { use $function as wrapped_pyfunction; - $crate::impl_::pyfunction::_wrap_pyfunction(&wrapped_pyfunction::DEF, py_or_module) + $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( + py_or_module, + &wrapped_pyfunction::DEF, + ) + } + }; + ($function:path, $py_or_module:expr) => {{ + use $function as wrapped_pyfunction; + $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( + $py_or_module, + &wrapped_pyfunction::DEF, + ) + }}; +} + +/// Wraps a Rust function annotated with [`#[pyfunction]`](macro@crate::pyfunction). +/// +/// This can be used with [`PyModule::add_function`](crate::types::PyModule::add_function) to add free +/// functions to a [`PyModule`](crate::types::PyModule) - see its documentation for more information. +#[macro_export] +macro_rules! wrap_pyfunction_bound { + ($function:path) => { + &|py_or_module| { + use $function as wrapped_pyfunction; + $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( + $crate::impl_::pyfunction::OnlyBound(py_or_module), + &wrapped_pyfunction::DEF, + ) } }; ($function:path, $py_or_module:expr) => {{ use $function as wrapped_pyfunction; - $crate::impl_::pyfunction::_wrap_pyfunction(&wrapped_pyfunction::DEF, $py_or_module) + $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( + $crate::impl_::pyfunction::OnlyBound($py_or_module), + &wrapped_pyfunction::DEF, + ) }}; } diff --git a/src/prelude.rs b/src/prelude.rs index 1de7c3acd2d..dcf4fe71cdd 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -23,7 +23,7 @@ pub use crate::PyNativeType; pub use pyo3_macros::{pyclass, pyfunction, pymethods, pymodule, FromPyObject}; #[cfg(feature = "macros")] -pub use crate::wrap_pyfunction; +pub use crate::{wrap_pyfunction, wrap_pyfunction_bound}; pub use crate::types::any::PyAnyMethods; pub use crate::types::boolobject::PyBoolMethods; diff --git a/src/types/function.rs b/src/types/function.rs index 7cbb05e2a48..75173d2aadb 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -3,10 +3,11 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::methods::PyMethodDefDestructor; use crate::py_result_ext::PyResultExt; use crate::types::capsule::PyCapsuleMethods; +use crate::types::module::PyModuleMethods; use crate::{ ffi, impl_::pymethods::{self, PyMethodDef}, - types::{PyCapsule, PyDict, PyString, PyTuple}, + types::{PyCapsule, PyDict, PyModule, PyString, PyTuple}, }; use crate::{Bound, IntoPy, Py, PyAny, PyResult, Python}; use std::cell::UnsafeCell; @@ -33,23 +34,33 @@ impl PyCFunction { doc: &'static str, py_or_module: PyFunctionArguments<'a>, ) -> PyResult<&'a Self> { - Self::new_with_keywords_bound(fun, name, doc, py_or_module).map(Bound::into_gil_ref) + Self::internal_new( + &PyMethodDef::cfunction_with_keywords( + name, + pymethods::PyCFunctionWithKeywords(fun), + doc, + ), + py_or_module, + ) + .map(Bound::into_gil_ref) } /// Create a new built-in function with keywords (*args and/or **kwargs). - pub fn new_with_keywords_bound<'a>( + pub fn new_with_keywords_bound<'py>( + py: Python<'py>, fun: ffi::PyCFunctionWithKeywords, name: &'static str, doc: &'static str, - py_or_module: PyFunctionArguments<'a>, - ) -> PyResult> { - Self::internal_new( + module: Option<&Bound<'py, PyModule>>, + ) -> PyResult> { + Self::internal_new_bound( + py, &PyMethodDef::cfunction_with_keywords( name, pymethods::PyCFunctionWithKeywords(fun), doc, ), - py_or_module, + module, ) } @@ -67,19 +78,25 @@ impl PyCFunction { doc: &'static str, py_or_module: PyFunctionArguments<'a>, ) -> PyResult<&'a Self> { - Self::new_bound(fun, name, doc, py_or_module).map(Bound::into_gil_ref) + Self::internal_new( + &PyMethodDef::noargs(name, pymethods::PyCFunction(fun), doc), + py_or_module, + ) + .map(Bound::into_gil_ref) } /// Create a new built-in function which takes no arguments. - pub fn new_bound<'a>( + pub fn new_bound<'py>( + py: Python<'py>, fun: ffi::PyCFunction, name: &'static str, doc: &'static str, - py_or_module: PyFunctionArguments<'a>, - ) -> PyResult> { - Self::internal_new( + module: Option<&Bound<'py, PyModule>>, + ) -> PyResult> { + Self::internal_new_bound( + py, &PyMethodDef::noargs(name, pymethods::PyCFunction(fun), doc), - py_or_module, + module, ) } @@ -189,6 +206,35 @@ impl PyCFunction { .downcast_into_unchecked() } } + + #[doc(hidden)] + pub(crate) fn internal_new_bound<'py>( + py: Python<'py>, + method_def: &PyMethodDef, + module: Option<&Bound<'py, PyModule>>, + ) -> PyResult> { + let (mod_ptr, module_name): (_, Option>) = if let Some(m) = module { + let mod_ptr = m.as_ptr(); + (mod_ptr, Some(m.name()?.into_py(py))) + } else { + (std::ptr::null_mut(), None) + }; + let (def, destructor) = method_def.as_method_def()?; + + // FIXME: stop leaking the def and destructor + let def = Box::into_raw(Box::new(def)); + std::mem::forget(destructor); + + let module_name_ptr = module_name + .as_ref() + .map_or(std::ptr::null_mut(), Py::as_ptr); + + unsafe { + ffi::PyCFunction_NewEx(def, mod_ptr, module_name_ptr) + .assume_owned_or_err(py) + .downcast_into_unchecked() + } + } } fn closure_capsule_name() -> &'static CStr { diff --git a/src/types/module.rs b/src/types/module.rs index 245d38d9e08..75fc6625846 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -399,7 +399,8 @@ impl PyModule { /// [1]: crate::prelude::pyfunction /// [2]: crate::wrap_pyfunction pub fn add_function<'a>(&'a self, fun: &'a PyCFunction) -> PyResult<()> { - self.as_borrowed().add_function(&fun.as_borrowed()) + let name = fun.getattr(__name__(self.py()))?.extract()?; + self.add(name, fun) } } @@ -590,7 +591,7 @@ pub trait PyModuleMethods<'py> { /// /// [1]: crate::prelude::pyfunction /// [2]: crate::wrap_pyfunction - fn add_function(&self, fun: &Bound<'_, PyCFunction>) -> PyResult<()>; + fn add_function(&self, fun: Bound<'_, PyCFunction>) -> PyResult<()>; } impl<'py> PyModuleMethods<'py> for Bound<'py, PyModule> { @@ -700,7 +701,7 @@ impl<'py> PyModuleMethods<'py> for Bound<'py, PyModule> { self.add(name, module) } - fn add_function(&self, fun: &Bound<'_, PyCFunction>) -> PyResult<()> { + fn add_function(&self, fun: Bound<'_, PyCFunction>) -> PyResult<()> { let name = fun.getattr(__name__(self.py()))?; self.add(name.downcast_into::()?, fun) } diff --git a/tests/test_module.rs b/tests/test_module.rs index 9d14f243d50..5763043e57d 100644 --- a/tests/test_module.rs +++ b/tests/test_module.rs @@ -240,7 +240,7 @@ fn subfunction() -> String { } fn submodule(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(&wrap_pyfunction!(subfunction, module.as_gil_ref())?.as_borrowed())?; + module.add_function(wrap_pyfunction!(subfunction, module)?)?; Ok(()) } @@ -295,14 +295,14 @@ fn test_module_nesting() { // Test that argument parsing specification works for pyfunctions #[pyfunction(signature = (a=5, *args))] -fn ext_vararg_fn(py: Python<'_>, a: i32, args: &PyTuple) -> PyObject { - [a.to_object(py), args.into()].to_object(py) +fn ext_vararg_fn(py: Python<'_>, a: i32, args: &Bound<'_, PyTuple>) -> PyObject { + [a.to_object(py), args.into_py(py)].to_object(py) } #[pymodule] -fn vararg_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn vararg_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { #[pyfn(m, signature = (a=5, *args))] - fn int_vararg_fn(py: Python<'_>, a: i32, args: &PyTuple) -> PyObject { + fn int_vararg_fn(py: Python<'_>, a: i32, args: &Bound<'_, PyTuple>) -> PyObject { ext_vararg_fn(py, a, args) } @@ -410,7 +410,7 @@ fn pyfunction_with_pass_module_in_attribute(module: &PyModule) -> PyResult<&str> } #[pymodule] -fn module_with_functions_with_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn module_with_functions_with_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(pyfunction_with_module, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_gil_ref, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_owned, m)?)?; diff --git a/tests/test_pyfunction.rs b/tests/test_pyfunction.rs index 14748418687..838f7353737 100644 --- a/tests/test_pyfunction.rs +++ b/tests/test_pyfunction.rs @@ -324,10 +324,11 @@ fn test_pycfunction_new() { } let py_fn = PyCFunction::new_bound( + py, c_fn, "py_fn", "py_fn for test (this is the docstring)", - py.into(), + None, ) .unwrap(); @@ -381,10 +382,11 @@ fn test_pycfunction_new_with_keywords() { } let py_fn = PyCFunction::new_with_keywords_bound( + py, c_fn, "py_fn", "py_fn for test (this is the docstring)", - py.into(), + None, ) .unwrap(); diff --git a/tests/test_wrap_pyfunction_deduction.rs b/tests/test_wrap_pyfunction_deduction.rs index 8723ad43fbd..52c9adcb6d7 100644 --- a/tests/test_wrap_pyfunction_deduction.rs +++ b/tests/test_wrap_pyfunction_deduction.rs @@ -13,3 +13,12 @@ pub fn add_wrapped(wrapper: &impl Fn(Python<'_>) -> PyResult<&PyCFunction>) { fn wrap_pyfunction_deduction() { add_wrapped(wrap_pyfunction!(f)); } + +pub fn add_wrapped_bound(wrapper: &impl Fn(Python<'_>) -> PyResult>) { + let _ = wrapper; +} + +#[test] +fn wrap_pyfunction_deduction_bound() { + add_wrapped_bound(wrap_pyfunction_bound!(f)); +} From 6f03a5464f53a7f76e406eb7d09610324cd8667c Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 27 Feb 2024 23:15:35 +0100 Subject: [PATCH 183/349] cleans up `PyCFunction::internal_new` (#3910) This deduplicates some code around `PyCFunction::internal_new` --- pyo3-macros-backend/src/pyfunction.rs | 2 +- src/impl_/pyfunction.rs | 17 ++++++----- src/types/function.rs | 43 ++++++--------------------- 3 files changed, 19 insertions(+), 43 deletions(-) diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 265a4824109..45de94e4a10 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -273,7 +273,7 @@ pub fn impl_wrap_pyfunction( pub fn add_to_module(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { use #krate::prelude::PyModuleMethods; use ::std::convert::Into; - module.add_function(#krate::types::PyCFunction::internal_new(&DEF, module.as_gil_ref().into())?) + module.add_function(#krate::types::PyCFunction::internal_new(module.py(), &DEF, module.into())?) } } diff --git a/src/impl_/pyfunction.rs b/src/impl_/pyfunction.rs index 531e0c93192..cb838fea6c2 100644 --- a/src/impl_/pyfunction.rs +++ b/src/impl_/pyfunction.rs @@ -1,6 +1,6 @@ use crate::{ types::{PyCFunction, PyModule}, - Borrowed, Bound, PyResult, Python, + Borrowed, Bound, PyNativeType, PyResult, Python, }; pub use crate::impl_::pymethods::PyMethodDef; @@ -13,25 +13,25 @@ pub trait WrapPyFunctionArg<'py, T> { impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for Bound<'py, PyModule> { fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { - PyCFunction::internal_new_bound(self.py(), method_def, Some(&self)) + PyCFunction::internal_new(self.py(), method_def, Some(&self)) } } impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for &'_ Bound<'py, PyModule> { fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { - PyCFunction::internal_new_bound(self.py(), method_def, Some(self)) + PyCFunction::internal_new(self.py(), method_def, Some(self)) } } impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for Borrowed<'_, 'py, PyModule> { fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { - PyCFunction::internal_new_bound(self.py(), method_def, Some(&self)) + PyCFunction::internal_new(self.py(), method_def, Some(&self)) } } impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for &'_ Borrowed<'_, 'py, PyModule> { fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { - PyCFunction::internal_new_bound(self.py(), method_def, Some(self)) + PyCFunction::internal_new(self.py(), method_def, Some(self)) } } @@ -39,13 +39,14 @@ impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for &'_ Borrowed<'_, ' // The `wrap_pyfunction_bound!` macro is needed for the Bound form. impl<'py> WrapPyFunctionArg<'py, &'py PyCFunction> for Python<'py> { fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult<&'py PyCFunction> { - PyCFunction::internal_new(method_def, self.into()).map(Bound::into_gil_ref) + PyCFunction::internal_new(self, method_def, None).map(Bound::into_gil_ref) } } impl<'py> WrapPyFunctionArg<'py, &'py PyCFunction> for &'py PyModule { fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult<&'py PyCFunction> { - PyCFunction::internal_new(method_def, self.into()).map(Bound::into_gil_ref) + PyCFunction::internal_new(self.py(), method_def, Some(&self.as_borrowed())) + .map(Bound::into_gil_ref) } } @@ -63,6 +64,6 @@ where impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for OnlyBound> { fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { - PyCFunction::internal_new_bound(self.0, method_def, None) + PyCFunction::internal_new(self.0, method_def, None) } } diff --git a/src/types/function.rs b/src/types/function.rs index 75173d2aadb..ea8201fb131 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -9,7 +9,7 @@ use crate::{ impl_::pymethods::{self, PyMethodDef}, types::{PyCapsule, PyDict, PyModule, PyString, PyTuple}, }; -use crate::{Bound, IntoPy, Py, PyAny, PyResult, Python}; +use crate::{Bound, IntoPy, Py, PyAny, PyNativeType, PyResult, Python}; use std::cell::UnsafeCell; use std::ffi::CStr; @@ -34,13 +34,15 @@ impl PyCFunction { doc: &'static str, py_or_module: PyFunctionArguments<'a>, ) -> PyResult<&'a Self> { + let (py, module) = py_or_module.into_py_and_maybe_module(); Self::internal_new( + py, &PyMethodDef::cfunction_with_keywords( name, pymethods::PyCFunctionWithKeywords(fun), doc, ), - py_or_module, + module.map(PyNativeType::as_borrowed).as_deref(), ) .map(Bound::into_gil_ref) } @@ -53,7 +55,7 @@ impl PyCFunction { doc: &'static str, module: Option<&Bound<'py, PyModule>>, ) -> PyResult> { - Self::internal_new_bound( + Self::internal_new( py, &PyMethodDef::cfunction_with_keywords( name, @@ -78,9 +80,11 @@ impl PyCFunction { doc: &'static str, py_or_module: PyFunctionArguments<'a>, ) -> PyResult<&'a Self> { + let (py, module) = py_or_module.into_py_and_maybe_module(); Self::internal_new( + py, &PyMethodDef::noargs(name, pymethods::PyCFunction(fun), doc), - py_or_module, + module.map(PyNativeType::as_borrowed).as_deref(), ) .map(Bound::into_gil_ref) } @@ -93,7 +97,7 @@ impl PyCFunction { doc: &'static str, module: Option<&Bound<'py, PyModule>>, ) -> PyResult> { - Self::internal_new_bound( + Self::internal_new( py, &PyMethodDef::noargs(name, pymethods::PyCFunction(fun), doc), module, @@ -180,35 +184,6 @@ impl PyCFunction { #[doc(hidden)] pub fn internal_new<'py>( - method_def: &PyMethodDef, - py_or_module: PyFunctionArguments<'py>, - ) -> PyResult> { - let (py, module) = py_or_module.into_py_and_maybe_module(); - let (mod_ptr, module_name): (_, Option>) = if let Some(m) = module { - let mod_ptr = m.as_ptr(); - (mod_ptr, Some(m.name()?.into_py(py))) - } else { - (std::ptr::null_mut(), None) - }; - let (def, destructor) = method_def.as_method_def()?; - - // FIXME: stop leaking the def and destructor - let def = Box::into_raw(Box::new(def)); - std::mem::forget(destructor); - - let module_name_ptr = module_name - .as_ref() - .map_or(std::ptr::null_mut(), Py::as_ptr); - - unsafe { - ffi::PyCFunction_NewEx(def, mod_ptr, module_name_ptr) - .assume_owned_or_err(py) - .downcast_into_unchecked() - } - } - - #[doc(hidden)] - pub(crate) fn internal_new_bound<'py>( py: Python<'py>, method_def: &PyMethodDef, module: Option<&Bound<'py, PyModule>>, From a15e4b1a116cb57edaf2f18c3a6bbd75dfb2c003 Mon Sep 17 00:00:00 2001 From: Matthew Neeley Date: Tue, 27 Feb 2024 14:24:14 -0800 Subject: [PATCH 184/349] Allow pymodule functions to take a single Bound<'_, PyModule> arg (#3905) --- newsfragments/3905.changed.md | 1 + pyo3-macros-backend/src/module.rs | 10 +++++++++- src/tests/hygiene/pyfunction.rs | 8 ++++++++ src/tests/hygiene/pymodule.rs | 15 +++++++++++++++ tests/test_no_imports.rs | 12 ++++++++++++ 5 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3905.changed.md diff --git a/newsfragments/3905.changed.md b/newsfragments/3905.changed.md new file mode 100644 index 00000000000..917584eb72a --- /dev/null +++ b/newsfragments/3905.changed.md @@ -0,0 +1 @@ +The `#[pymodule]` macro now supports module functions that take a single argument as a `&Bound<'_, PyModule>`. diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 40cf34f22fd..0bdb1cbc50c 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -201,6 +201,14 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result let doc = get_doc(&function.attrs, None); let initialization = module_initialization(options, ident); + + // Module function called with optional Python<'_> marker as first arg, followed by the module. + let mut module_args = Vec::new(); + if function.sig.inputs.len() == 2 { + module_args.push(quote!(module.py())); + } + module_args.push(quote!(::std::convert::Into::into(BoundRef(module)))); + Ok(quote! { #function #vis mod #ident { @@ -218,7 +226,7 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result use #krate::impl_::pymethods::BoundRef; fn __pyo3_pymodule(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { - #ident(module.py(), ::std::convert::Into::into(BoundRef(module))) + #ident(#(#module_args),*) } impl #ident::MakeDef { diff --git a/src/tests/hygiene/pyfunction.rs b/src/tests/hygiene/pyfunction.rs index 19fe2739407..9cfad0db6c6 100644 --- a/src/tests/hygiene/pyfunction.rs +++ b/src/tests/hygiene/pyfunction.rs @@ -14,3 +14,11 @@ fn invoke_wrap_pyfunction() { crate::py_run!(py, func, r#"func(5)"#); }); } + +#[test] +fn invoke_wrap_pyfunction_bound() { + crate::Python::with_gil(|py| { + let func = crate::wrap_pyfunction_bound!(do_something, py).unwrap(); + crate::py_run!(py, func, r#"func(5)"#); + }); +} diff --git a/src/tests/hygiene/pymodule.rs b/src/tests/hygiene/pymodule.rs index 0b37c440325..bb49d3823c1 100644 --- a/src/tests/hygiene/pymodule.rs +++ b/src/tests/hygiene/pymodule.rs @@ -21,3 +21,18 @@ fn my_module(_py: crate::Python<'_>, m: &crate::types::PyModule) -> crate::PyRes ::std::result::Result::Ok(()) } + +#[crate::pymodule] +#[pyo3(crate = "crate")] +fn my_module_bound(m: &crate::Bound<'_, crate::types::PyModule>) -> crate::PyResult<()> { + as crate::types::PyModuleMethods>::add_function( + m, + crate::wrap_pyfunction_bound!(do_something, m)?, + )?; + as crate::types::PyModuleMethods>::add_wrapped( + m, + crate::wrap_pymodule!(foo), + )?; + + ::std::result::Result::Ok(()) +} diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index 88932ed282a..69f4b6e4651 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -22,6 +22,18 @@ fn basic_module(_py: pyo3::Python<'_>, m: &pyo3::types::PyModule) -> pyo3::PyRes Ok(()) } +#[pyo3::pymodule] +fn basic_module_bound(m: &pyo3::Bound<'_, pyo3::types::PyModule>) -> pyo3::PyResult<()> { + #[pyfn(m)] + fn answer() -> usize { + 42 + } + + m.add_function(pyo3::wrap_pyfunction_bound!(basic_function, m)?)?; + + Ok(()) +} + #[pyo3::pyclass] struct BasicClass { #[pyo3(get)] From 8a12970c9694f8340318235173e53b80ee9d4ff7 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 28 Feb 2024 19:36:20 +0000 Subject: [PATCH 185/349] update `extract_argument` to use Bound APIs (#3708) * update `extract_argument` to use `Bound` APIs * tidy up borrow in macros expression * update `trybuild` output * more concise form for `DowncastError::new` Co-authored-by: Lily Foote * use `Borrowed` instead of newtype * use `Borrowed::from_ptr` methods in extract_argument * update UI tests * avoid double-negative `#[cfg]` clauses Co-authored-by: Lily Foote * review: LilyFoote, Icxolu feedback --------- Co-authored-by: Lily Foote --- guide/src/class.md | 4 +- pyo3-benches/benches/bench_comparisons.rs | 8 +- pyo3-macros-backend/src/method.rs | 12 +- pyo3-macros-backend/src/params.rs | 18 +-- pyo3-macros-backend/src/pyclass.rs | 6 +- pyo3-macros-backend/src/pymethod.rs | 31 ++-- pytests/src/pyfunctions.rs | 74 +++++----- src/impl_/extract_argument.rs | 172 ++++++++++++++-------- src/impl_/pymethods.rs | 17 ++- src/types/dict.rs | 68 +++++++++ src/types/string.rs | 2 +- src/types/tuple.rs | 6 +- 12 files changed, 274 insertions(+), 144 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index 9662e8f6e09..8ce896f5a61 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -1254,7 +1254,7 @@ impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a type Holder = ::std::option::Option>; #[inline] - fn extract(obj: &'py pyo3::PyAny, holder: &'a mut Self::Holder) -> pyo3::PyResult { + fn extract(obj: &'a pyo3::Bound<'py, PyAny>, holder: &'a mut Self::Holder) -> pyo3::PyResult { pyo3::impl_::extract_argument::extract_pyclass_ref(obj, holder) } } @@ -1264,7 +1264,7 @@ impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a type Holder = ::std::option::Option>; #[inline] - fn extract(obj: &'py pyo3::PyAny, holder: &'a mut Self::Holder) -> pyo3::PyResult { + fn extract(obj: &'a pyo3::Bound<'py, PyAny>, holder: &'a mut Self::Holder) -> pyo3::PyResult { pyo3::impl_::extract_argument::extract_pyclass_ref_mut(obj, holder) } } diff --git a/pyo3-benches/benches/bench_comparisons.rs b/pyo3-benches/benches/bench_comparisons.rs index ffd4c1a452f..fbd473f06cf 100644 --- a/pyo3-benches/benches/bench_comparisons.rs +++ b/pyo3-benches/benches/bench_comparisons.rs @@ -45,8 +45,8 @@ impl OrderedRichcmp { fn bench_ordered_dunder_methods(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let obj1 = Py::new(py, OrderedDunderMethods(0)).unwrap().into_ref(py); - let obj2 = Py::new(py, OrderedDunderMethods(1)).unwrap().into_ref(py); + let obj1 = &Bound::new(py, OrderedDunderMethods(0)).unwrap().into_any(); + let obj2 = &Bound::new(py, OrderedDunderMethods(1)).unwrap().into_any(); b.iter(|| obj2.gt(obj1).unwrap()); }); @@ -54,8 +54,8 @@ fn bench_ordered_dunder_methods(b: &mut Bencher<'_>) { fn bench_ordered_richcmp(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let obj1 = Py::new(py, OrderedRichcmp(0)).unwrap().into_ref(py); - let obj2 = Py::new(py, OrderedRichcmp(1)).unwrap().into_ref(py); + let obj1 = &Bound::new(py, OrderedRichcmp(0)).unwrap().into_any(); + let obj2 = &Bound::new(py, OrderedRichcmp(1)).unwrap().into_any(); b.iter(|| obj2.gt(obj1).unwrap()); }); diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 2d21c265683..6ee87fb99a6 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -196,10 +196,11 @@ impl SelfType { holders.push(quote_spanned! { *span => #[allow(clippy::let_unit_value)] let mut #holder = _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT; + let mut #slf = _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf); }); error_mode.handle_error(quote_spanned! { *span => _pyo3::impl_::extract_argument::#method::<#cls>( - #py.from_borrowed_ptr::<_pyo3::PyAny>(#slf), + &#slf, &mut #holder, ) }) @@ -582,7 +583,8 @@ impl<'a> FnSpec<'a> { ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #( #holders )* - #call + let result = #call; + result } } } @@ -601,7 +603,8 @@ impl<'a> FnSpec<'a> { let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert #( #holders )* - #call + let result = #call; + result } } } @@ -619,7 +622,8 @@ impl<'a> FnSpec<'a> { let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert #( #holders )* - #call + let result = #call; + result } } } diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 3781b41b765..679d3e4260a 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -44,8 +44,8 @@ pub fn impl_arg_params( .collect::>()?; return Ok(( quote! { - let _args = py.from_borrowed_ptr::<_pyo3::types::PyTuple>(_args); - let _kwargs: ::std::option::Option<&_pyo3::types::PyDict> = py.from_borrowed_ptr_or_opt(_kwargs); + let _args = _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(py, &_args); + let _kwargs = _pyo3::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_kwargs); }, arg_convert, )); @@ -180,7 +180,7 @@ fn impl_arg_param( let holder = push_holder(); return Ok(quote_arg_span! { _pyo3::impl_::extract_argument::extract_argument( - _args, + &_args, &mut #holder, #name_str )? @@ -193,7 +193,7 @@ fn impl_arg_param( let holder = push_holder(); return Ok(quote_arg_span! { _pyo3::impl_::extract_argument::extract_optional_argument( - _kwargs.map(::std::convert::AsRef::as_ref), + _kwargs.as_deref(), &mut #holder, #name_str, || ::std::option::Option::None @@ -217,7 +217,7 @@ fn impl_arg_param( quote_arg_span! { #[allow(clippy::redundant_closure)] _pyo3::impl_::extract_argument::from_py_with_with_default( - #arg_value.map(_pyo3::PyNativeType::as_borrowed).as_deref(), + #arg_value.as_deref(), #name_str, #expr_path as fn(_) -> _, || #default @@ -226,7 +226,7 @@ fn impl_arg_param( } else { quote_arg_span! { _pyo3::impl_::extract_argument::from_py_with( - &_pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value).as_borrowed(), + &_pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value), #name_str, #expr_path as fn(_) -> _, )? @@ -237,7 +237,7 @@ fn impl_arg_param( quote_arg_span! { #[allow(clippy::redundant_closure)] _pyo3::impl_::extract_argument::extract_optional_argument( - #arg_value, + #arg_value.as_deref(), &mut #holder, #name_str, || #default @@ -248,7 +248,7 @@ fn impl_arg_param( quote_arg_span! { #[allow(clippy::redundant_closure)] _pyo3::impl_::extract_argument::extract_argument_with_default( - #arg_value, + #arg_value.as_deref(), &mut #holder, #name_str, || #default @@ -258,7 +258,7 @@ fn impl_arg_param( let holder = push_holder(); quote_arg_span! { _pyo3::impl_::extract_argument::extract_argument( - _pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value), + &_pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value), &mut #holder, #name_str )? diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 1547e78b4c2..5ec6ac172e6 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1378,7 +1378,7 @@ impl<'a> PyClassImplsBuilder<'a> { type Holder = ::std::option::Option<_pyo3::PyRef<'py, #cls>>; #[inline] - fn extract(obj: &'py _pyo3::PyAny, holder: &'a mut Self::Holder) -> _pyo3::PyResult { + fn extract(obj: &'a _pyo3::Bound<'py, _pyo3::PyAny>, holder: &'a mut Self::Holder) -> _pyo3::PyResult { _pyo3::impl_::extract_argument::extract_pyclass_ref(obj, holder) } } @@ -1390,7 +1390,7 @@ impl<'a> PyClassImplsBuilder<'a> { type Holder = ::std::option::Option<_pyo3::PyRef<'py, #cls>>; #[inline] - fn extract(obj: &'py _pyo3::PyAny, holder: &'a mut Self::Holder) -> _pyo3::PyResult { + fn extract(obj: &'a _pyo3::Bound<'py, _pyo3::PyAny>, holder: &'a mut Self::Holder) -> _pyo3::PyResult { _pyo3::impl_::extract_argument::extract_pyclass_ref(obj, holder) } } @@ -1400,7 +1400,7 @@ impl<'a> PyClassImplsBuilder<'a> { type Holder = ::std::option::Option<_pyo3::PyRefMut<'py, #cls>>; #[inline] - fn extract(obj: &'py _pyo3::PyAny, holder: &'a mut Self::Holder) -> _pyo3::PyResult { + fn extract(obj: &'a _pyo3::Bound<'py, _pyo3::PyAny>, holder: &'a mut Self::Holder) -> _pyo3::PyResult { _pyo3::impl_::extract_argument::extract_pyclass_ref_mut(obj, holder) } } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 358d327ed0b..4cb07a9ad2a 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -698,7 +698,8 @@ pub fn impl_py_getter_def( _slf: *mut _pyo3::ffi::PyObject ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { #( #holders )* - #body + let result = #body; + result } }; @@ -930,39 +931,31 @@ impl Ty { extract_error_mode, holders, &name_str, - quote! { - py.from_borrowed_ptr::<_pyo3::PyAny>(#ident) - }, + quote! { #ident }, ), Ty::MaybeNullObject => extract_object( extract_error_mode, holders, &name_str, quote! { - py.from_borrowed_ptr::<_pyo3::PyAny>( - if #ident.is_null() { - _pyo3::ffi::Py_None() - } else { - #ident - } - ) + if #ident.is_null() { + _pyo3::ffi::Py_None() + } else { + #ident + } }, ), Ty::NonNullObject => extract_object( extract_error_mode, holders, &name_str, - quote! { - py.from_borrowed_ptr::<_pyo3::PyAny>(#ident.as_ptr()) - }, + quote! { #ident.as_ptr() }, ), Ty::IPowModulo => extract_object( extract_error_mode, holders, &name_str, - quote! { - #ident.to_borrowed_any(py) - }, + quote! { #ident.as_ptr() }, ), Ty::CompareOp => extract_error_mode.handle_error( quote! { @@ -988,7 +981,7 @@ fn extract_object( extract_error_mode: ExtractErrorMode, holders: &mut Vec, name: &str, - source: TokenStream, + source_ptr: TokenStream, ) -> TokenStream { let holder = syn::Ident::new(&format!("holder_{}", holders.len()), Span::call_site()); holders.push(quote! { @@ -997,7 +990,7 @@ fn extract_object( }); extract_error_mode.handle_error(quote! { _pyo3::impl_::extract_argument::extract_argument( - #source, + &_pyo3::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr), &mut #holder, #name ) diff --git a/pytests/src/pyfunctions.rs b/pytests/src/pyfunctions.rs index 1eef970430e..a92733c2898 100644 --- a/pytests/src/pyfunctions.rs +++ b/pytests/src/pyfunctions.rs @@ -4,62 +4,66 @@ use pyo3::types::{PyDict, PyTuple}; #[pyfunction(signature = ())] fn none() {} +type Any<'py> = Bound<'py, PyAny>; +type Dict<'py> = Bound<'py, PyDict>; +type Tuple<'py> = Bound<'py, PyTuple>; + #[pyfunction(signature = (a, b = None, *, c = None))] -fn simple<'a>( - a: &'a PyAny, - b: Option<&'a PyAny>, - c: Option<&'a PyAny>, -) -> (&'a PyAny, Option<&'a PyAny>, Option<&'a PyAny>) { +fn simple<'py>( + a: Any<'py>, + b: Option>, + c: Option>, +) -> (Any<'py>, Option>, Option>) { (a, b, c) } #[pyfunction(signature = (a, b = None, *args, c = None))] -fn simple_args<'a>( - a: &'a PyAny, - b: Option<&'a PyAny>, - args: &'a PyTuple, - c: Option<&'a PyAny>, -) -> (&'a PyAny, Option<&'a PyAny>, &'a PyTuple, Option<&'a PyAny>) { +fn simple_args<'py>( + a: Any<'py>, + b: Option>, + args: Tuple<'py>, + c: Option>, +) -> (Any<'py>, Option>, Tuple<'py>, Option>) { (a, b, args, c) } #[pyfunction(signature = (a, b = None, c = None, **kwargs))] -fn simple_kwargs<'a>( - a: &'a PyAny, - b: Option<&'a PyAny>, - c: Option<&'a PyAny>, - kwargs: Option<&'a PyDict>, +fn simple_kwargs<'py>( + a: Any<'py>, + b: Option>, + c: Option>, + kwargs: Option>, ) -> ( - &'a PyAny, - Option<&'a PyAny>, - Option<&'a PyAny>, - Option<&'a PyDict>, + Any<'py>, + Option>, + Option>, + Option>, ) { (a, b, c, kwargs) } #[pyfunction(signature = (a, b = None, *args, c = None, **kwargs))] -fn simple_args_kwargs<'a>( - a: &'a PyAny, - b: Option<&'a PyAny>, - args: &'a PyTuple, - c: Option<&'a PyAny>, - kwargs: Option<&'a PyDict>, +fn simple_args_kwargs<'py>( + a: Any<'py>, + b: Option>, + args: Tuple<'py>, + c: Option>, + kwargs: Option>, ) -> ( - &'a PyAny, - Option<&'a PyAny>, - &'a PyTuple, - Option<&'a PyAny>, - Option<&'a PyDict>, + Any<'py>, + Option>, + Tuple<'py>, + Option>, + Option>, ) { (a, b, args, c, kwargs) } #[pyfunction(signature = (*args, **kwargs))] -fn args_kwargs<'a>( - args: &'a PyTuple, - kwargs: Option<&'a PyDict>, -) -> (&'a PyTuple, Option<&'a PyDict>) { +fn args_kwargs<'py>( + args: Tuple<'py>, + kwargs: Option>, +) -> (Tuple<'py>, Option>) { (args, kwargs) } diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index 27728c6d46f..d2e2b18cf05 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -2,10 +2,16 @@ use crate::{ exceptions::PyTypeError, ffi, pyclass::boolean_struct::False, - types::{PyDict, PyString, PyTuple}, - Bound, FromPyObject, PyAny, PyClass, PyErr, PyRef, PyRefMut, PyResult, PyTypeCheck, Python, + types::{any::PyAnyMethods, dict::PyDictMethods, tuple::PyTupleMethods, PyDict, PyTuple}, + Borrowed, Bound, FromPyObject, PyAny, PyClass, PyErr, PyRef, PyRefMut, PyResult, PyTypeCheck, + Python, }; +/// Helper type used to keep implementation more concise. +/// +/// (Function argument extraction borrows input arguments.) +type PyArg<'py> = Borrowed<'py, 'py, PyAny>; + /// A trait which is used to help PyO3 macros extract function arguments. /// /// `#[pyclass]` structs need to extract as `PyRef` and `PyRefMut` @@ -16,7 +22,7 @@ use crate::{ /// There exists a trivial blanket implementation for `T: FromPyObject` with `Holder = ()`. pub trait PyFunctionArgument<'a, 'py>: Sized + 'a { type Holder: FunctionArgumentHolder; - fn extract(obj: &'py PyAny, holder: &'a mut Self::Holder) -> PyResult; + fn extract(obj: &'a Bound<'py, PyAny>, holder: &'a mut Self::Holder) -> PyResult; } impl<'a, 'py, T> PyFunctionArgument<'a, 'py> for T @@ -26,20 +32,23 @@ where type Holder = (); #[inline] - fn extract(obj: &'py PyAny, _: &'a mut ()) -> PyResult { + fn extract(obj: &'a Bound<'py, PyAny>, _: &'a mut ()) -> PyResult { obj.extract() } } -impl<'a, 'py, T> PyFunctionArgument<'a, 'py> for &'a Bound<'py, T> +impl<'a, 'py, T: 'py> PyFunctionArgument<'a, 'py> for &'a Bound<'py, T> where T: PyTypeCheck, { - type Holder = Option>; + type Holder = Option<&'a Bound<'py, T>>; #[inline] - fn extract(obj: &'py PyAny, holder: &'a mut Option>) -> PyResult { - Ok(&*holder.insert(obj.extract()?)) + fn extract( + obj: &'a Bound<'py, PyAny>, + holder: &'a mut Option<&'a Bound<'py, T>>, + ) -> PyResult { + Ok(holder.insert(obj.downcast()?)) } } @@ -59,7 +68,7 @@ impl FunctionArgumentHolder for Option { #[inline] pub fn extract_pyclass_ref<'a, 'py: 'a, T: PyClass>( - obj: &'py PyAny, + obj: &'a Bound<'py, PyAny>, holder: &'a mut Option>, ) -> PyResult<&'a T> { Ok(&*holder.insert(obj.extract()?)) @@ -67,7 +76,7 @@ pub fn extract_pyclass_ref<'a, 'py: 'a, T: PyClass>( #[inline] pub fn extract_pyclass_ref_mut<'a, 'py: 'a, T: PyClass>( - obj: &'py PyAny, + obj: &'a Bound<'py, PyAny>, holder: &'a mut Option>, ) -> PyResult<&'a mut T> { Ok(&mut *holder.insert(obj.extract()?)) @@ -76,7 +85,7 @@ pub fn extract_pyclass_ref_mut<'a, 'py: 'a, T: PyClass>( /// The standard implementation of how PyO3 extracts a `#[pyfunction]` or `#[pymethod]` function argument. #[doc(hidden)] pub fn extract_argument<'a, 'py, T>( - obj: &'py PyAny, + obj: &'a Bound<'py, PyAny>, holder: &'a mut T::Holder, arg_name: &str, ) -> PyResult @@ -93,7 +102,7 @@ where /// does not implement `PyFunctionArgument` for `T: PyClass`. #[doc(hidden)] pub fn extract_optional_argument<'a, 'py, T>( - obj: Option<&'py PyAny>, + obj: Option<&'a Bound<'py, PyAny>>, holder: &'a mut T::Holder, arg_name: &str, default: fn() -> Option, @@ -117,7 +126,7 @@ where /// Alternative to [`extract_argument`] used when the argument has a default value provided by an annotation. #[doc(hidden)] pub fn extract_argument_with_default<'a, 'py, T>( - obj: Option<&'py PyAny>, + obj: Option<&'a Bound<'py, PyAny>>, holder: &'a mut T::Holder, arg_name: &str, default: fn() -> T, @@ -165,7 +174,6 @@ pub fn from_py_with_with_default<'a, 'py, T>( #[doc(hidden)] #[cold] pub fn argument_extraction_error(py: Python<'_>, arg_name: &str, error: PyErr) -> PyErr { - use crate::types::any::PyAnyMethods; if error .get_type_bound(py) .is(&py.get_type_bound::()) @@ -189,7 +197,7 @@ pub fn argument_extraction_error(py: Python<'_>, arg_name: &str, error: PyErr) - /// `argument` must not be `None` #[doc(hidden)] #[inline] -pub unsafe fn unwrap_required_argument(argument: Option<&PyAny>) -> &PyAny { +pub unsafe fn unwrap_required_argument(argument: Option>) -> PyArg<'_> { match argument { Some(value) => value, #[cfg(debug_assertions)] @@ -236,7 +244,7 @@ impl FunctionDescription { args: *const *mut ffi::PyObject, nargs: ffi::Py_ssize_t, kwnames: *mut ffi::PyObject, - output: &mut [Option<&'py PyAny>], + output: &mut [Option>], ) -> PyResult<(V::Varargs, K::Varkeywords)> where V: VarargsHandler<'py>, @@ -253,8 +261,10 @@ impl FunctionDescription { ); // Handle positional arguments - // Safety: Option<&PyAny> has the same memory layout as `*mut ffi::PyObject` - let args: *const Option<&PyAny> = args.cast(); + // Safety: + // - Option has the same memory layout as `*mut ffi::PyObject` + // - we both have the GIL and can borrow these input references for the `'py` lifetime. + let args: *const Option> = args.cast(); let positional_args_provided = nargs as usize; let remaining_positional_args = if args.is_null() { debug_assert_eq!(positional_args_provided, 0); @@ -274,13 +284,20 @@ impl FunctionDescription { // Handle keyword arguments let mut varkeywords = K::Varkeywords::default(); - if let Some(kwnames) = py.from_borrowed_ptr_or_opt::(kwnames) { - // Safety: &PyAny has the same memory layout as `*mut ffi::PyObject` - let kwargs = - ::std::slice::from_raw_parts((args as *const &PyAny).offset(nargs), kwnames.len()); + + // Safety: kwnames is known to be a pointer to a tuple, or null + // - we both have the GIL and can borrow this input reference for the `'py` lifetime. + let kwnames: Option> = + Borrowed::from_ptr_or_opt(py, kwnames).map(|kwnames| kwnames.downcast_unchecked()); + if let Some(kwnames) = kwnames { + // Safety: PyArg has the same memory layout as `*mut ffi::PyObject` + let kwargs = ::std::slice::from_raw_parts( + (args as *const PyArg<'py>).offset(nargs), + kwnames.len(), + ); self.handle_kwargs::( - kwnames.iter().zip(kwargs.iter().copied()), + kwnames.iter_borrowed().zip(kwargs.iter().copied()), &mut varkeywords, num_positional_parameters, output, @@ -312,14 +329,20 @@ impl FunctionDescription { py: Python<'py>, args: *mut ffi::PyObject, kwargs: *mut ffi::PyObject, - output: &mut [Option<&'py PyAny>], + output: &mut [Option>], ) -> PyResult<(V::Varargs, K::Varkeywords)> where V: VarargsHandler<'py>, K: VarkeywordsHandler<'py>, { - let args = py.from_borrowed_ptr::(args); - let kwargs: ::std::option::Option<&PyDict> = py.from_borrowed_ptr_or_opt(kwargs); + // Safety: + // - `args` is known to be a tuple + // - `kwargs` is known to be a dict or null + // - we both have the GIL and can borrow these input references for the `'py` lifetime. + let args: Borrowed<'py, 'py, PyTuple> = + Borrowed::from_ptr(py, args).downcast_unchecked::(); + let kwargs: Option> = + Borrowed::from_ptr_or_opt(py, kwargs).map(|kwargs| kwargs.downcast_unchecked()); let num_positional_parameters = self.positional_parameter_names.len(); @@ -331,17 +354,26 @@ impl FunctionDescription { ); // Copy positional arguments into output - for (i, arg) in args.iter().take(num_positional_parameters).enumerate() { + for (i, arg) in args + .iter_borrowed() + .take(num_positional_parameters) + .enumerate() + { output[i] = Some(arg); } // If any arguments remain, push them to varargs (if possible) or error - let varargs = V::handle_varargs_tuple(args, self)?; + let varargs = V::handle_varargs_tuple(&args, self)?; // Handle keyword arguments let mut varkeywords = K::Varkeywords::default(); if let Some(kwargs) = kwargs { - self.handle_kwargs::(kwargs, &mut varkeywords, num_positional_parameters, output)? + self.handle_kwargs::( + kwargs.iter_borrowed(), + &mut varkeywords, + num_positional_parameters, + output, + )? } // Once all inputs have been processed, check that all required arguments have been provided. @@ -358,11 +390,11 @@ impl FunctionDescription { kwargs: I, varkeywords: &mut K::Varkeywords, num_positional_parameters: usize, - output: &mut [Option<&'py PyAny>], + output: &mut [Option>], ) -> PyResult<()> where K: VarkeywordsHandler<'py>, - I: IntoIterator, + I: IntoIterator, PyArg<'py>)>, { debug_assert_eq!( num_positional_parameters, @@ -374,11 +406,21 @@ impl FunctionDescription { ); let mut positional_only_keyword_arguments = Vec::new(); for (kwarg_name_py, value) in kwargs { - // All keyword arguments should be UTF-8 strings, but we'll check, just in case. - // If it isn't, then it will be handled below as a varkeyword (which may raise an - // error if this function doesn't accept **kwargs). Rust source is always UTF-8 - // and so all argument names in `#[pyfunction]` signature must be UTF-8. - if let Ok(kwarg_name) = kwarg_name_py.downcast::()?.to_str() { + // Safety: All keyword arguments should be UTF-8 strings, but if it's not, `.to_str()` + // will return an error anyway. + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + let kwarg_name = + unsafe { kwarg_name_py.downcast_unchecked::() }.to_str(); + + #[cfg(all(not(Py_3_10), Py_LIMITED_API))] + let kwarg_name = kwarg_name_py.extract::(); + + if let Ok(kwarg_name_owned) = kwarg_name { + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + let kwarg_name = kwarg_name_owned; + #[cfg(all(not(Py_3_10), Py_LIMITED_API))] + let kwarg_name: &str = &kwarg_name_owned; + // Try to place parameter in keyword only parameters if let Some(i) = self.find_keyword_parameter_in_keyword_only(kwarg_name) { if output[i + num_positional_parameters] @@ -397,7 +439,7 @@ impl FunctionDescription { // kwarg to conflict with a postional-only argument - the value // will go into **kwargs anyway. if K::handle_varkeyword(varkeywords, kwarg_name_py, value, self).is_err() { - positional_only_keyword_arguments.push(kwarg_name); + positional_only_keyword_arguments.push(kwarg_name_owned); } } else if output[i].replace(value).is_some() { return Err(self.multiple_values_for_argument(kwarg_name)); @@ -410,6 +452,11 @@ impl FunctionDescription { } if !positional_only_keyword_arguments.is_empty() { + #[cfg(all(not(Py_3_10), Py_LIMITED_API))] + let positional_only_keyword_arguments: Vec<_> = positional_only_keyword_arguments + .iter() + .map(std::ops::Deref::deref) + .collect(); return Err(self.positional_only_keyword_arguments(&positional_only_keyword_arguments)); } @@ -436,7 +483,7 @@ impl FunctionDescription { #[inline] fn ensure_no_missing_required_positional_arguments( &self, - output: &[Option<&PyAny>], + output: &[Option>], positional_args_provided: usize, ) -> PyResult<()> { if positional_args_provided < self.required_positional_parameters { @@ -452,7 +499,7 @@ impl FunctionDescription { #[inline] fn ensure_no_missing_required_keyword_arguments( &self, - output: &[Option<&PyAny>], + output: &[Option>], ) -> PyResult<()> { let keyword_output = &output[self.positional_parameter_names.len()..]; for (param, out) in self.keyword_only_parameters.iter().zip(keyword_output) { @@ -497,11 +544,11 @@ impl FunctionDescription { } #[cold] - fn unexpected_keyword_argument(&self, argument: &PyAny) -> PyErr { + fn unexpected_keyword_argument(&self, argument: PyArg<'_>) -> PyErr { PyTypeError::new_err(format!( "{} got an unexpected keyword argument '{}'", self.full_name(), - argument + argument.as_any() )) } @@ -534,7 +581,7 @@ impl FunctionDescription { } #[cold] - fn missing_required_keyword_arguments(&self, keyword_outputs: &[Option<&PyAny>]) -> PyErr { + fn missing_required_keyword_arguments(&self, keyword_outputs: &[Option>]) -> PyErr { debug_assert_eq!(self.keyword_only_parameters.len(), keyword_outputs.len()); let missing_keyword_only_arguments: Vec<_> = self @@ -555,7 +602,7 @@ impl FunctionDescription { } #[cold] - fn missing_required_positional_arguments(&self, output: &[Option<&PyAny>]) -> PyErr { + fn missing_required_positional_arguments(&self, output: &[Option>]) -> PyErr { let missing_positional_arguments: Vec<_> = self .positional_parameter_names .iter() @@ -575,14 +622,14 @@ pub trait VarargsHandler<'py> { /// Called by `FunctionDescription::extract_arguments_fastcall` with any additional arguments. fn handle_varargs_fastcall( py: Python<'py>, - varargs: &[Option<&PyAny>], + varargs: &[Option>], function_description: &FunctionDescription, ) -> PyResult; /// Called by `FunctionDescription::extract_arguments_tuple_dict` with the original tuple. /// /// Additional arguments are those in the tuple slice starting from `function_description.positional_parameter_names.len()`. fn handle_varargs_tuple( - args: &'py PyTuple, + args: &Bound<'py, PyTuple>, function_description: &FunctionDescription, ) -> PyResult; } @@ -596,7 +643,7 @@ impl<'py> VarargsHandler<'py> for NoVarargs { #[inline] fn handle_varargs_fastcall( _py: Python<'py>, - varargs: &[Option<&PyAny>], + varargs: &[Option>], function_description: &FunctionDescription, ) -> PyResult { let extra_arguments = varargs.len(); @@ -610,7 +657,7 @@ impl<'py> VarargsHandler<'py> for NoVarargs { #[inline] fn handle_varargs_tuple( - args: &'py PyTuple, + args: &Bound<'py, PyTuple>, function_description: &FunctionDescription, ) -> PyResult { let positional_parameter_count = function_description.positional_parameter_names.len(); @@ -627,19 +674,19 @@ impl<'py> VarargsHandler<'py> for NoVarargs { pub struct TupleVarargs; impl<'py> VarargsHandler<'py> for TupleVarargs { - type Varargs = &'py PyTuple; + type Varargs = Bound<'py, PyTuple>; #[inline] fn handle_varargs_fastcall( py: Python<'py>, - varargs: &[Option<&PyAny>], + varargs: &[Option>], _function_description: &FunctionDescription, ) -> PyResult { - Ok(PyTuple::new_bound(py, varargs).into_gil_ref()) + Ok(PyTuple::new_bound(py, varargs)) } #[inline] fn handle_varargs_tuple( - args: &'py PyTuple, + args: &Bound<'py, PyTuple>, function_description: &FunctionDescription, ) -> PyResult { let positional_parameters = function_description.positional_parameter_names.len(); @@ -652,8 +699,8 @@ pub trait VarkeywordsHandler<'py> { type Varkeywords: Default; fn handle_varkeyword( varkeywords: &mut Self::Varkeywords, - name: &'py PyAny, - value: &'py PyAny, + name: PyArg<'py>, + value: PyArg<'py>, function_description: &FunctionDescription, ) -> PyResult<()>; } @@ -666,8 +713,8 @@ impl<'py> VarkeywordsHandler<'py> for NoVarkeywords { #[inline] fn handle_varkeyword( _varkeywords: &mut Self::Varkeywords, - name: &'py PyAny, - _value: &'py PyAny, + name: PyArg<'py>, + _value: PyArg<'py>, function_description: &FunctionDescription, ) -> PyResult<()> { Err(function_description.unexpected_keyword_argument(name)) @@ -678,28 +725,29 @@ impl<'py> VarkeywordsHandler<'py> for NoVarkeywords { pub struct DictVarkeywords; impl<'py> VarkeywordsHandler<'py> for DictVarkeywords { - type Varkeywords = Option<&'py PyDict>; + type Varkeywords = Option>; #[inline] fn handle_varkeyword( varkeywords: &mut Self::Varkeywords, - name: &'py PyAny, - value: &'py PyAny, + name: PyArg<'py>, + value: PyArg<'py>, _function_description: &FunctionDescription, ) -> PyResult<()> { varkeywords - .get_or_insert_with(|| PyDict::new_bound(name.py()).into_gil_ref()) + .get_or_insert_with(|| PyDict::new_bound(name.py())) .set_item(name, value) } } fn push_parameter_list(msg: &mut String, parameter_names: &[&str]) { + let len = parameter_names.len(); for (i, parameter) in parameter_names.iter().enumerate() { if i != 0 { - if parameter_names.len() > 2 { + if len > 2 { msg.push(','); } - if i == parameter_names.len() - 1 { + if i == len - 1 { msg.push_str(" and ") } else { msg.push(' ') @@ -778,7 +826,7 @@ mod tests { }; assert_eq!( err.to_string(), - "TypeError: 'int' object cannot be converted to 'PyString'" + "TypeError: example() got an unexpected keyword argument '1'" ); }) } diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 0b73ce9bc73..70c95ca0883 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -38,14 +38,15 @@ pub type ipowfunc = unsafe extern "C" fn( impl IPowModulo { #[cfg(Py_3_8)] #[inline] - pub fn to_borrowed_any(self, py: Python<'_>) -> &PyAny { - unsafe { py.from_borrowed_ptr::(self.0) } + pub fn as_ptr(self) -> *mut ffi::PyObject { + self.0 } #[cfg(not(Py_3_8))] #[inline] - pub fn to_borrowed_any(self, py: Python<'_>) -> &PyAny { - unsafe { py.from_borrowed_ptr::(ffi::Py_None()) } + pub fn as_ptr(self) -> *mut ffi::PyObject { + // Safety: returning a borrowed pointer to Python `None` singleton + unsafe { ffi::Py_None() } } } @@ -560,3 +561,11 @@ impl From> for Py { bound.0.clone().unbind() } } + +impl<'py, T> std::ops::Deref for BoundRef<'_, 'py, T> { + type Target = Bound<'py, T>; + #[inline] + fn deref(&self) -> &Self::Target { + self.0 + } +} diff --git a/src/types/dict.rs b/src/types/dict.rs index 2d215c1f8b9..daae753c6cc 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -524,6 +524,16 @@ impl<'py> PyDictMethods<'py> for Bound<'py, PyDict> { } } +impl<'a, 'py> Borrowed<'a, 'py, PyDict> { + /// Iterates over the contents of this dictionary without incrementing reference counts. + /// + /// # Safety + /// It must be known that this dictionary will not be modified during iteration. + pub(crate) unsafe fn iter_borrowed(self) -> BorrowedDictIter<'a, 'py> { + BorrowedDictIter::new(self) + } +} + fn dict_len(dict: &Bound<'_, PyDict>) -> Py_ssize_t { #[cfg(any(not(Py_3_8), PyPy, Py_LIMITED_API))] unsafe { @@ -662,6 +672,64 @@ impl<'py> IntoIterator for Bound<'py, PyDict> { } } +mod borrowed_iter { + use super::*; + + /// Variant of the above which is used to iterate the items of the dictionary + /// without incrementing reference counts. This is only safe if it's known + /// that the dictionary will not be modified during iteration. + pub struct BorrowedDictIter<'a, 'py> { + dict: Borrowed<'a, 'py, PyDict>, + ppos: ffi::Py_ssize_t, + len: ffi::Py_ssize_t, + } + + impl<'a, 'py> Iterator for BorrowedDictIter<'a, 'py> { + type Item = (Borrowed<'a, 'py, PyAny>, Borrowed<'a, 'py, PyAny>); + + #[inline] + fn next(&mut self) -> Option { + let mut key: *mut ffi::PyObject = std::ptr::null_mut(); + let mut value: *mut ffi::PyObject = std::ptr::null_mut(); + + // Safety: self.dict lives sufficiently long that the pointer is not dangling + if unsafe { ffi::PyDict_Next(self.dict.as_ptr(), &mut self.ppos, &mut key, &mut value) } + != 0 + { + let py = self.dict.py(); + self.len -= 1; + // Safety: + // - PyDict_Next returns borrowed values + // - we have already checked that `PyDict_Next` succeeded, so we can assume these to be non-null + Some(unsafe { (key.assume_borrowed(py), value.assume_borrowed(py)) }) + } else { + None + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } + } + + impl ExactSizeIterator for BorrowedDictIter<'_, '_> { + fn len(&self) -> usize { + self.len as usize + } + } + + impl<'a, 'py> BorrowedDictIter<'a, 'py> { + pub(super) fn new(dict: Borrowed<'a, 'py, PyDict>) -> Self { + let len = dict_len(&dict); + BorrowedDictIter { dict, ppos: 0, len } + } + } +} + +pub(crate) use borrowed_iter::BorrowedDictIter; + /// Conversion trait that allows a sequence of tuples to be converted into `PyDict` /// Primary use case for this trait is `call` and `call_method` methods as keywords argument. pub trait IntoPyDict: Sized { diff --git a/src/types/string.rs b/src/types/string.rs index 4a33236b47b..88ebb30b2d1 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -354,7 +354,7 @@ impl<'py> PyStringMethods<'py> for Bound<'py, PyString> { impl<'a> Borrowed<'a, '_, PyString> { #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] #[allow(clippy::wrong_self_convention)] - fn to_str(self) -> PyResult<&'a str> { + pub(crate) fn to_str(self) -> PyResult<&'a str> { // PyUnicode_AsUTF8AndSize only available on limited API starting with 3.10. let mut size: ffi::Py_ssize_t = 0; let data: *const u8 = diff --git a/src/types/tuple.rs b/src/types/tuple.rs index caad7d9c0e9..51d33f93460 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -411,7 +411,7 @@ impl<'py> PyTupleMethods<'py> for Bound<'py, PyTuple> { } fn iter_borrowed<'a>(&'a self) -> BorrowedTupleIterator<'a, 'py> { - BorrowedTupleIterator::new(self.as_borrowed()) + self.as_borrowed().iter_borrowed() } fn to_list(&self) -> Bound<'py, PyList> { @@ -433,6 +433,10 @@ impl<'a, 'py> Borrowed<'a, 'py, PyTuple> { unsafe fn get_borrowed_item_unchecked(self, index: usize) -> Borrowed<'a, 'py, PyAny> { ffi::PyTuple_GET_ITEM(self.as_ptr(), index as Py_ssize_t).assume_borrowed(self.py()) } + + pub(crate) fn iter_borrowed(self) -> BorrowedTupleIterator<'a, 'py> { + BorrowedTupleIterator::new(self) + } } /// Used by `PyTuple::iter()`. From 55833365b54dec47c2a4cd4ccbda218be1141d16 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 28 Feb 2024 20:36:55 +0100 Subject: [PATCH 186/349] seals `PyAnyMethods` and friends (#3909) * seals `PyAnyMethods` and friends This seals these new traits, preventing downstream crates from implementing them on their types. These traits are mainly a workaround for arbitrary self receiver types, so this gives us more flexibility if these need to be changed in the future. * move `PyResultExt` seal --- src/ffi_ptr_ext.rs | 11 +---------- src/lib.rs | 1 + src/py_result_ext.rs | 12 +----------- src/sealed.rs | 34 ++++++++++++++++++++++++++++++++++ src/types/any.rs | 2 +- src/types/boolobject.rs | 2 +- src/types/bytearray.rs | 2 +- src/types/bytes.rs | 2 +- src/types/capsule.rs | 2 +- src/types/complex.rs | 2 +- src/types/dict.rs | 2 +- src/types/float.rs | 2 +- src/types/frozenset.rs | 2 +- src/types/list.rs | 2 +- src/types/mapping.rs | 2 +- src/types/module.rs | 2 +- src/types/sequence.rs | 2 +- src/types/set.rs | 2 +- src/types/slice.rs | 2 +- src/types/string.rs | 2 +- src/types/traceback.rs | 2 +- src/types/tuple.rs | 2 +- src/types/typeobject.rs | 2 +- 23 files changed, 56 insertions(+), 40 deletions(-) create mode 100644 src/sealed.rs diff --git a/src/ffi_ptr_ext.rs b/src/ffi_ptr_ext.rs index 8a45f5d70c0..3ca8671f1f6 100644 --- a/src/ffi_ptr_ext.rs +++ b/src/ffi_ptr_ext.rs @@ -1,19 +1,10 @@ +use crate::sealed::Sealed; use crate::{ ffi, instance::{Borrowed, Bound}, PyAny, PyResult, Python, }; -mod sealed { - use super::*; - - pub trait Sealed {} - - impl Sealed for *mut ffi::PyObject {} -} - -use sealed::Sealed; - pub(crate) trait FfiPtrExt: Sealed { unsafe fn assume_owned_or_err(self, py: Python<'_>) -> PyResult>; unsafe fn assume_owned_or_opt(self, py: Python<'_>) -> Option>; diff --git a/src/lib.rs b/src/lib.rs index d0c53c4a0c9..17dbd972d85 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -324,6 +324,7 @@ pub use crate::version::PythonVersionInfo; pub(crate) mod ffi_ptr_ext; pub(crate) mod py_result_ext; +pub(crate) mod sealed; /// Old module which contained some implementation details of the `#[pyproto]` module. /// diff --git a/src/py_result_ext.rs b/src/py_result_ext.rs index 66309988dc4..2ad079ed7ac 100644 --- a/src/py_result_ext.rs +++ b/src/py_result_ext.rs @@ -1,16 +1,6 @@ use crate::{types::any::PyAnyMethods, Bound, PyAny, PyResult, PyTypeCheck}; -mod sealed { - use super::*; - - pub trait Sealed {} - - impl Sealed for PyResult> {} -} - -use sealed::Sealed; - -pub(crate) trait PyResultExt<'py>: Sealed { +pub(crate) trait PyResultExt<'py>: crate::sealed::Sealed { fn downcast_into(self) -> PyResult>; unsafe fn downcast_into_unchecked(self) -> PyResult>; } diff --git a/src/sealed.rs b/src/sealed.rs new file mode 100644 index 00000000000..e2d5c5ccfed --- /dev/null +++ b/src/sealed.rs @@ -0,0 +1,34 @@ +use crate::types::{ + PyBool, PyByteArray, PyBytes, PyCapsule, PyComplex, PyDict, PyFloat, PyFrozenSet, PyList, + PyMapping, PyModule, PySequence, PySet, PySlice, PyString, PyTraceback, PyTuple, PyType, +}; +use crate::{ffi, Bound, PyAny, PyResult}; + +pub trait Sealed {} + +// for FfiPtrExt +impl Sealed for *mut ffi::PyObject {} + +// for PyResultExt +impl Sealed for PyResult> {} + +// for Py(...)Methods +impl Sealed for Bound<'_, PyAny> {} +impl Sealed for Bound<'_, PyBool> {} +impl Sealed for Bound<'_, PyByteArray> {} +impl Sealed for Bound<'_, PyBytes> {} +impl Sealed for Bound<'_, PyCapsule> {} +impl Sealed for Bound<'_, PyComplex> {} +impl Sealed for Bound<'_, PyDict> {} +impl Sealed for Bound<'_, PyFloat> {} +impl Sealed for Bound<'_, PyFrozenSet> {} +impl Sealed for Bound<'_, PyList> {} +impl Sealed for Bound<'_, PyMapping> {} +impl Sealed for Bound<'_, PyModule> {} +impl Sealed for Bound<'_, PySequence> {} +impl Sealed for Bound<'_, PySet> {} +impl Sealed for Bound<'_, PySlice> {} +impl Sealed for Bound<'_, PyString> {} +impl Sealed for Bound<'_, PyTraceback> {} +impl Sealed for Bound<'_, PyTuple> {} +impl Sealed for Bound<'_, PyType> {} diff --git a/src/types/any.rs b/src/types/any.rs index 15f2fd3135c..0207217ba96 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -939,7 +939,7 @@ impl PyAny { /// It is recommended you import this trait via `use pyo3::prelude::*` rather than /// by importing this trait directly. #[doc(alias = "PyAny")] -pub trait PyAnyMethods<'py> { +pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// Returns whether `self` and `other` point to the same object. To compare /// the equality of two objects (the `==` operator), use [`eq`](PyAny::eq). /// diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 906a967069a..3a2f60f6fa3 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -55,7 +55,7 @@ impl PyBool { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyBool")] -pub trait PyBoolMethods<'py> { +pub trait PyBoolMethods<'py>: crate::sealed::Sealed { /// Gets whether this boolean is `true`. fn is_true(&self) -> bool; } diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 2a21509d87f..a860b4d4cca 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -293,7 +293,7 @@ impl PyByteArray { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyByteArray")] -pub trait PyByteArrayMethods<'py> { +pub trait PyByteArrayMethods<'py>: crate::sealed::Sealed { /// Gets the length of the bytearray. fn len(&self) -> usize; diff --git a/src/types/bytes.rs b/src/types/bytes.rs index 0861af630a5..2a54c172d1f 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -143,7 +143,7 @@ impl PyBytes { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyBytes")] -pub trait PyBytesMethods<'py> { +pub trait PyBytesMethods<'py>: crate::sealed::Sealed { /// Gets the Python string as a byte slice. fn as_bytes(&self) -> &[u8]; } diff --git a/src/types/capsule.rs b/src/types/capsule.rs index aa1a910d21b..fe5a47e1796 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -263,7 +263,7 @@ impl PyCapsule { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyCapsule")] -pub trait PyCapsuleMethods<'py> { +pub trait PyCapsuleMethods<'py>: crate::sealed::Sealed { /// Sets the context pointer in the capsule. /// /// Returns an error if this capsule is not valid. diff --git a/src/types/complex.rs b/src/types/complex.rs index 6d4bc7a2244..80ecffdc5cf 100644 --- a/src/types/complex.rs +++ b/src/types/complex.rs @@ -258,7 +258,7 @@ mod not_limited_impls { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyComplex")] -pub trait PyComplexMethods<'py> { +pub trait PyComplexMethods<'py>: crate::sealed::Sealed { /// Returns the real part of the complex number. fn real(&self) -> c_double; /// Returns the imaginary part of the complex number. diff --git a/src/types/dict.rs b/src/types/dict.rs index daae753c6cc..5d1243463f2 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -290,7 +290,7 @@ impl PyDict { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyDict")] -pub trait PyDictMethods<'py> { +pub trait PyDictMethods<'py>: crate::sealed::Sealed { /// Returns a new dictionary that contains the same key-value pairs as self. /// /// This is equivalent to the Python expression `self.copy()`. diff --git a/src/types/float.rs b/src/types/float.rs index 766c597b2b4..6db1fdae038 100644 --- a/src/types/float.rs +++ b/src/types/float.rs @@ -58,7 +58,7 @@ impl PyFloat { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyFloat")] -pub trait PyFloatMethods<'py> { +pub trait PyFloatMethods<'py>: crate::sealed::Sealed { /// Gets the value of this float. fn value(&self) -> c_double; } diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 15f892588e5..cc16a9341c8 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -157,7 +157,7 @@ impl PyFrozenSet { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyFrozenSet")] -pub trait PyFrozenSetMethods<'py> { +pub trait PyFrozenSetMethods<'py>: crate::sealed::Sealed { /// Returns the number of items in the set. /// /// This is equivalent to the Python expression `len(self)`. diff --git a/src/types/list.rs b/src/types/list.rs index 3e3942bcc13..a8df26ddbc5 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -285,7 +285,7 @@ index_impls!(PyList, "list", PyList::len, PyList::get_slice); /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyList")] -pub trait PyListMethods<'py> { +pub trait PyListMethods<'py>: crate::sealed::Sealed { /// Returns the length of the list. fn len(&self) -> usize; diff --git a/src/types/mapping.rs b/src/types/mapping.rs index afbdae688b4..a5a93163bbd 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -109,7 +109,7 @@ impl PyMapping { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyMapping")] -pub trait PyMappingMethods<'py> { +pub trait PyMappingMethods<'py>: crate::sealed::Sealed { /// Returns the number of objects in the mapping. /// /// This is equivalent to the Python expression `len(self)`. diff --git a/src/types/module.rs b/src/types/module.rs index 75fc6625846..693df79cafa 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -410,7 +410,7 @@ impl PyModule { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyModule")] -pub trait PyModuleMethods<'py> { +pub trait PyModuleMethods<'py>: crate::sealed::Sealed { /// Returns the module's `__dict__` attribute, which contains the module's symbol table. fn dict(&self) -> Bound<'py, PyDict>; diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 80306cbf44b..62abb66fa6e 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -192,7 +192,7 @@ impl PySequence { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PySequence")] -pub trait PySequenceMethods<'py> { +pub trait PySequenceMethods<'py>: crate::sealed::Sealed { /// Returns the number of objects in sequence. /// /// This is equivalent to the Python expression `len(self)`. diff --git a/src/types/set.rs b/src/types/set.rs index 0c1733527cf..8e36ab81f8e 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -139,7 +139,7 @@ impl PySet { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PySet")] -pub trait PySetMethods<'py> { +pub trait PySetMethods<'py>: crate::sealed::Sealed { /// Removes all elements from the set. fn clear(&self); diff --git a/src/types/slice.rs b/src/types/slice.rs index b4b6731d695..8e7545208dc 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -105,7 +105,7 @@ impl PySlice { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PySlice")] -pub trait PySliceMethods<'py> { +pub trait PySliceMethods<'py>: crate::sealed::Sealed { /// Retrieves the start, stop, and step indices from the slice object, /// assuming a sequence of length `length`, and stores the length of the /// slice in its `slicelength` member. diff --git a/src/types/string.rs b/src/types/string.rs index 88ebb30b2d1..93f3682c3a4 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -282,7 +282,7 @@ impl PyString { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyString")] -pub trait PyStringMethods<'py> { +pub trait PyStringMethods<'py>: crate::sealed::Sealed { /// Gets the Python string as a Rust UTF-8 string slice. /// /// Returns a `UnicodeEncodeError` if the input is not valid unicode diff --git a/src/types/traceback.rs b/src/types/traceback.rs index b8782463367..c4cedd791f6 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -56,7 +56,7 @@ impl PyTraceback { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyTraceback")] -pub trait PyTracebackMethods<'py> { +pub trait PyTracebackMethods<'py>: crate::sealed::Sealed { /// Formats the traceback as a string. /// /// This does not include the exception type and value. The exception type and value can be diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 51d33f93460..3ffaf9c3224 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -248,7 +248,7 @@ index_impls!(PyTuple, "tuple", PyTuple::len, PyTuple::get_slice); /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyTuple")] -pub trait PyTupleMethods<'py> { +pub trait PyTupleMethods<'py>: crate::sealed::Sealed { /// Gets the length of the tuple. fn len(&self) -> usize; diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 4c1b17a2aa8..d75e39d022d 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -107,7 +107,7 @@ impl PyType { /// syntax these methods are separated into a trait, because stable Rust does not yet support /// `arbitrary_self_types`. #[doc(alias = "PyType")] -pub trait PyTypeMethods<'py> { +pub trait PyTypeMethods<'py>: crate::sealed::Sealed { /// Retrieves the underlying FFI pointer associated with this Python object. fn as_type_ptr(&self) -> *mut ffi::PyTypeObject; From a582fa01639df01d884d5ed8683b44a8d7bf0734 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 28 Feb 2024 20:51:40 +0000 Subject: [PATCH 187/349] docs: update discord invite to permanent one (#3913) --- .github/ISSUE_TEMPLATE/config.yml | 2 +- Contributing.md | 4 ++-- README.md | 4 ++-- guide/src/faq.md | 2 +- guide/src/getting_started.md | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index ca4239b6fa7..7919c91f0c6 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -7,5 +7,5 @@ contact_links: url: https://github.com/PyO3/pyo3/discussions about: For troubleshooting help, see the Discussions - name: 👋 Chat - url: https://discord.gg/c4BwayXQ + url: https://discord.gg/33kcChzH7f about: Engage with PyO3's users and developers on Discord diff --git a/Contributing.md b/Contributing.md index 9392c0fc2de..b34bf420072 100644 --- a/Contributing.md +++ b/Contributing.md @@ -29,7 +29,7 @@ To work and develop PyO3, you need Python & Rust installed on your system. ### Help users identify bugs -The [PyO3 Discord server](https://discord.gg/c4BwayXQ) is very active with users who are new to PyO3, and often completely new to Rust. Helping them debug is a great way to get experience with the PyO3 codebase. +The [PyO3 Discord server](https://discord.gg/33kcChzH7f) is very active with users who are new to PyO3, and often completely new to Rust. Helping them debug is a great way to get experience with the PyO3 codebase. Helping others often reveals bugs, documentation weaknesses, and missing APIs. It's a good idea to open GitHub issues for these immediately so the resolution can be designed and implemented! @@ -203,7 +203,7 @@ You can install an IDE plugin to view the coverage. For example, if you use VSCo ## Sponsor this project -At the moment there is no official organisation that accepts sponsorship on PyO3's behalf. If you're seeking to provide significant funding to the PyO3 ecosystem, please reach out to us on [GitHub](https://github.com/PyO3/pyo3/issues/new) or [Discord](https://discord.gg/c4BwayXQ) and we can discuss. +At the moment there is no official organisation that accepts sponsorship on PyO3's behalf. If you're seeking to provide significant funding to the PyO3 ecosystem, please reach out to us on [GitHub](https://github.com/PyO3/pyo3/issues/new) or [Discord](https://discord.gg/33kcChzH7f) and we can discuss. In the meanwhile, some of our maintainers have personal GitHub sponsorship pages and would be grateful for your support: diff --git a/README.md b/README.md index c139f13693e..eaf1ce8809b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![codecov](https://img.shields.io/codecov/c/gh/PyO3/pyo3?logo=codecov)](https://codecov.io/gh/PyO3/pyo3) [![crates.io](https://img.shields.io/crates/v/pyo3?logo=rust)](https://crates.io/crates/pyo3) [![minimum rustc 1.56](https://img.shields.io/badge/rustc-1.56+-blue?logo=rust)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) -[![discord server](https://img.shields.io/discord/1209263839632424990?logo=discord)](https://discord.gg/c4BwayXQ) +[![discord server](https://img.shields.io/discord/1209263839632424990?logo=discord)](https://discord.gg/33kcChzH7f) [![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green?logo=github)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) [Rust](https://www.rust-lang.org/) bindings for [Python](https://www.python.org/), including tools for creating native Python extension modules. Running and interacting with Python code from a Rust binary is also supported. @@ -237,7 +237,7 @@ about this topic. Everyone is welcomed to contribute to PyO3! There are many ways to support the project, such as: -- help PyO3 users with issues on GitHub and [Discord](https://discord.gg/c4BwayXQ) +- help PyO3 users with issues on GitHub and [Discord](https://discord.gg/33kcChzH7f) - improve documentation - write features and bugfixes - publish blogs and examples of how to use PyO3 diff --git a/guide/src/faq.md b/guide/src/faq.md index 6f5188c73be..1034e9ccc2a 100644 --- a/guide/src/faq.md +++ b/guide/src/faq.md @@ -1,6 +1,6 @@ # Frequently Asked Questions and troubleshooting -Sorry that you're having trouble using PyO3. If you can't find the answer to your problem in the list below, you can also reach out for help on [GitHub Discussions](https://github.com/PyO3/pyo3/discussions) and on [Discord](https://discord.gg/c4BwayXQ). +Sorry that you're having trouble using PyO3. If you can't find the answer to your problem in the list below, you can also reach out for help on [GitHub Discussions](https://github.com/PyO3/pyo3/discussions) and on [Discord](https://discord.gg/33kcChzH7f). ## I'm experiencing deadlocks using PyO3 with lazy_static or once_cell! diff --git a/guide/src/getting_started.md b/guide/src/getting_started.md index 235272dab20..7b76639c1c4 100644 --- a/guide/src/getting_started.md +++ b/guide/src/getting_started.md @@ -2,7 +2,7 @@ To get started using PyO3 you will need three things: a Rust toolchain, a Python environment, and a way to build. We'll cover each of these below. -> If you'd like to chat to the PyO3 maintainers and other PyO3 users, consider joining the [PyO3 Discord server](https://discord.gg/c4BwayXQ). We're keen to hear about your experience getting started, so we can make PyO3 as accessible as possible for everyone! +> If you'd like to chat to the PyO3 maintainers and other PyO3 users, consider joining the [PyO3 Discord server](https://discord.gg/33kcChzH7f). We're keen to hear about your experience getting started, so we can make PyO3 as accessible as possible for everyone! ## Rust From 68ec6de0c9196225fc5806825e04f8649341a27d Mon Sep 17 00:00:00 2001 From: Matthew Neeley Date: Wed, 28 Feb 2024 14:36:50 -0800 Subject: [PATCH 188/349] Use single-arg form of `#[pymodule]` function in docs and tests (#3899) * Use single-arg form for `#[pymodule]` functions in docs and tests * Update guide/src/function.md Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> * Add test of two-argument module function * Fix new test --------- Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> --- README.md | 2 +- examples/decorator/src/lib.rs | 2 +- examples/getitem/src/lib.rs | 2 +- examples/maturin-starter/src/submodule.rs | 2 +- examples/plugin/plugin_api/src/lib.rs | 2 +- .../setuptools-rust-starter/src/submodule.rs | 2 +- examples/word-count/src/lib.rs | 2 +- guide/src/class.md | 2 +- guide/src/class/numeric.md | 2 +- guide/src/class/object.md | 4 +- guide/src/ecosystem/async-await.md | 6 +- guide/src/ecosystem/logging.md | 4 +- guide/src/function.md | 12 ++-- guide/src/function/signature.md | 2 +- guide/src/getting_started.md | 2 +- guide/src/module.md | 4 +- guide/src/python_from_rust.md | 2 +- guide/src/trait_bounds.md | 4 +- pytests/src/awaitable.rs | 2 +- pytests/src/buf_and_str.rs | 2 +- pytests/src/comparisons.rs | 2 +- pytests/src/datetime.rs | 2 +- pytests/src/dict_iter.rs | 2 +- pytests/src/enums.rs | 4 +- pytests/src/misc.rs | 2 +- pytests/src/objstore.rs | 2 +- pytests/src/othermod.rs | 2 +- pytests/src/path.rs | 2 +- pytests/src/pyclasses.rs | 2 +- pytests/src/pyfunctions.rs | 2 +- pytests/src/sequence.rs | 2 +- pytests/src/subclassing.rs | 2 +- src/conversions/indexmap.rs | 2 +- src/conversions/num_bigint.rs | 2 +- src/conversions/num_complex.rs | 4 +- src/conversions/rust_decimal.rs | 2 +- src/exceptions.rs | 6 +- src/lib.rs | 2 +- src/types/module.rs | 12 ++-- tests/test_append_to_inittab.rs | 4 +- tests/test_module.rs | 68 ++++++++++++------- tests/test_text_signature.rs | 2 +- tests/ui/invalid_pymodule_args.rs | 2 +- 43 files changed, 108 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index eaf1ce8809b..21dc125d5ab 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ fn sum_as_string(a: usize, b: usize) -> PyResult { /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to /// import the module. #[pymodule] -fn string_sum(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn string_sum(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; Ok(()) } diff --git a/examples/decorator/src/lib.rs b/examples/decorator/src/lib.rs index fb2f2932dd2..9dccabc7341 100644 --- a/examples/decorator/src/lib.rs +++ b/examples/decorator/src/lib.rs @@ -60,7 +60,7 @@ impl PyCounter { } #[pymodule] -pub fn decorator(_py: Python<'_>, module: &PyModule) -> PyResult<()> { +pub fn decorator(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; Ok(()) } diff --git a/examples/getitem/src/lib.rs b/examples/getitem/src/lib.rs index 90a3e9fc52f..eed60076bb8 100644 --- a/examples/getitem/src/lib.rs +++ b/examples/getitem/src/lib.rs @@ -75,7 +75,7 @@ impl ExampleContainer { #[pymodule] #[pyo3(name = "getitem")] -fn example(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn example(m: &Bound<'_, PyModule>) -> PyResult<()> { // ? -https://github.com/PyO3/maturin/issues/475 m.add_class::()?; Ok(()) diff --git a/examples/maturin-starter/src/submodule.rs b/examples/maturin-starter/src/submodule.rs index 56540b2e469..f3eb174100b 100644 --- a/examples/maturin-starter/src/submodule.rs +++ b/examples/maturin-starter/src/submodule.rs @@ -16,7 +16,7 @@ impl SubmoduleClass { } #[pymodule] -pub fn submodule(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn submodule(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/examples/plugin/plugin_api/src/lib.rs b/examples/plugin/plugin_api/src/lib.rs index 59aae55699d..580c85a8c8e 100644 --- a/examples/plugin/plugin_api/src/lib.rs +++ b/examples/plugin/plugin_api/src/lib.rs @@ -26,7 +26,7 @@ impl Gadget { /// A Python module for plugin interface types #[pymodule] -pub fn plugin_api(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn plugin_api(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/examples/setuptools-rust-starter/src/submodule.rs b/examples/setuptools-rust-starter/src/submodule.rs index 56540b2e469..f3eb174100b 100644 --- a/examples/setuptools-rust-starter/src/submodule.rs +++ b/examples/setuptools-rust-starter/src/submodule.rs @@ -16,7 +16,7 @@ impl SubmoduleClass { } #[pymodule] -pub fn submodule(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn submodule(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/examples/word-count/src/lib.rs b/examples/word-count/src/lib.rs index b7d3a8033a6..5bc73df97a4 100644 --- a/examples/word-count/src/lib.rs +++ b/examples/word-count/src/lib.rs @@ -33,7 +33,7 @@ fn count_line(line: &str, needle: &str) -> usize { } #[pymodule] -fn word_count(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn word_count(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(search, m)?)?; m.add_function(wrap_pyfunction!(search_sequential, m)?)?; m.add_function(wrap_pyfunction!(search_sequential_allow_threads, m)?)?; diff --git a/guide/src/class.md b/guide/src/class.md index 8ce896f5a61..1e752700613 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -181,7 +181,7 @@ The next step is to create the module initializer and add our class to it: # struct Number(i32); # #[pymodule] -fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index 2ffb0a543ef..3e6a3cf47e9 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -326,7 +326,7 @@ impl Number { } #[pymodule] -fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/guide/src/class/object.md b/guide/src/class/object.md index 10867976df0..729815ade0b 100644 --- a/guide/src/class/object.md +++ b/guide/src/class/object.md @@ -18,7 +18,7 @@ impl Number { } #[pymodule] -fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } @@ -295,7 +295,7 @@ impl Number { } #[pymodule] -fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index f537ab90df1..ec46e872ba7 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -128,7 +128,7 @@ fn rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { } #[pymodule] -fn my_async_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_async_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; Ok(()) @@ -151,7 +151,7 @@ fn rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { } #[pymodule] -fn my_async_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_async_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; Ok(()) } @@ -475,7 +475,7 @@ fn rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { } #[pymodule] -fn my_async_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_async_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(rust_sleep, m)?)?; Ok(()) diff --git a/guide/src/ecosystem/logging.md b/guide/src/ecosystem/logging.md index 2e7d4a087c6..da95c4a7cd2 100644 --- a/guide/src/ecosystem/logging.md +++ b/guide/src/ecosystem/logging.md @@ -30,11 +30,11 @@ fn log_something() { } #[pymodule] -fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { // A good place to install the Rust -> Python logger. pyo3_log::init(); - m.add_function(wrap_pyfunction!(log_something))?; + m.add_function(wrap_pyfunction!(log_something, m)?)?; Ok(()) } ``` diff --git a/guide/src/function.md b/guide/src/function.md index f3955ba554b..2ed38f6256f 100644 --- a/guide/src/function.md +++ b/guide/src/function.md @@ -13,7 +13,7 @@ fn double(x: usize) -> usize { } #[pymodule] -fn my_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(double, m)?)?; Ok(()) } @@ -55,7 +55,7 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python } #[pymodule] - fn module_with_functions(py: Python<'_>, m: &PyModule) -> PyResult<()> { + fn module_with_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(no_args_py, m)?)?; Ok(()) } @@ -92,7 +92,7 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python } #[pymodule] - fn module_with_fn(py: Python<'_>, m: &PyModule) -> PyResult<()> { + fn module_with_fn(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(pyfunction_with_module, m)?) } ``` @@ -166,8 +166,8 @@ Python argument passing convention.) It then embeds the call to the Rust functio FFI-wrapper function. This wrapper handles extraction of the regular arguments and the keyword arguments from the input `PyObject`s. -The `wrap_pyfunction` macro can be used to directly get a `PyCFunction` given a -`#[pyfunction]` and a `PyModule`: `wrap_pyfunction!(rust_fun, module)`. +The `wrap_pyfunction` macro can be used to directly get a `Bound` given a +`#[pyfunction]` and a `Bound`: `wrap_pyfunction!(rust_fun, module)`. ## `#[pyfn]` shorthand @@ -197,7 +197,7 @@ documented in the rest of this chapter. The code above is expanded to the follow use pyo3::prelude::*; #[pymodule] -fn my_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { #[pyfunction] fn double(x: usize) -> usize { x * 2 diff --git a/guide/src/function/signature.md b/guide/src/function/signature.md index d92767e7bde..4fcfe958b19 100644 --- a/guide/src/function/signature.md +++ b/guide/src/function/signature.md @@ -21,7 +21,7 @@ fn num_kwds(kwds: Option<&PyDict>) -> usize { } #[pymodule] -fn module_with_functions(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn module_with_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(num_kwds, m)?).unwrap(); Ok(()) } diff --git a/guide/src/getting_started.md b/guide/src/getting_started.md index 7b76639c1c4..008da8109f6 100644 --- a/guide/src/getting_started.md +++ b/guide/src/getting_started.md @@ -163,7 +163,7 @@ fn sum_as_string(a: usize, b: usize) -> PyResult { /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to /// import the module. #[pymodule] -fn pyo3_example(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn pyo3_example(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; Ok(()) } diff --git a/guide/src/module.md b/guide/src/module.md index 53c390bec06..a403a3621e6 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -12,7 +12,7 @@ fn double(x: usize) -> usize { /// This module is implemented in Rust. #[pymodule] -fn my_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(double, m)?)?; Ok(()) } @@ -34,7 +34,7 @@ fn double(x: usize) -> usize { #[pymodule] #[pyo3(name = "custom_name")] -fn my_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(double, m)?)?; Ok(()) } diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index b51bbe592eb..d8f3214f58b 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -283,7 +283,7 @@ fn add_one(x: i64) -> i64 { } #[pymodule] -fn foo(_py: Python<'_>, foo_module: &PyModule) -> PyResult<()> { +fn foo(foo_module: &Bound<'_, PyModule>) -> PyResult<()> { foo_module.add_function(wrap_pyfunction!(add_one, foo_module)?)?; Ok(()) } diff --git a/guide/src/trait_bounds.md b/guide/src/trait_bounds.md index e0dd988412f..e05d44e982a 100644 --- a/guide/src/trait_bounds.md +++ b/guide/src/trait_bounds.md @@ -132,7 +132,7 @@ struct UserModel { } #[pymodule] -fn trait_exposure(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn trait_exposure(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } @@ -489,7 +489,7 @@ pub struct UserModel { } #[pymodule] -fn trait_exposure(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn trait_exposure(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_function(wrap_pyfunction!(solve_wrapper, m)?)?; Ok(()) diff --git a/pytests/src/awaitable.rs b/pytests/src/awaitable.rs index e1a70b42bb0..5e3b98e14ea 100644 --- a/pytests/src/awaitable.rs +++ b/pytests/src/awaitable.rs @@ -79,7 +79,7 @@ impl FutureAwaitable { } #[pymodule] -pub fn awaitable(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn awaitable(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; Ok(()) diff --git a/pytests/src/buf_and_str.rs b/pytests/src/buf_and_str.rs index 23db9f0625e..e9651e0cfd9 100644 --- a/pytests/src/buf_and_str.rs +++ b/pytests/src/buf_and_str.rs @@ -48,7 +48,7 @@ fn return_memoryview(py: Python<'_>) -> PyResult> { } #[pymodule] -pub fn buf_and_str(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn buf_and_str(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_function(wrap_pyfunction!(return_memoryview, m)?)?; Ok(()) diff --git a/pytests/src/comparisons.rs b/pytests/src/comparisons.rs index b3ba293186a..fa35acf8e1a 100644 --- a/pytests/src/comparisons.rs +++ b/pytests/src/comparisons.rs @@ -101,7 +101,7 @@ impl OrderedDefaultNe { } #[pymodule] -pub fn comparisons(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn comparisons(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/pytests/src/datetime.rs b/pytests/src/datetime.rs index 9d8f32a93c9..aeb57240c15 100644 --- a/pytests/src/datetime.rs +++ b/pytests/src/datetime.rs @@ -205,7 +205,7 @@ impl TzClass { } #[pymodule] -pub fn datetime(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn datetime(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(make_date, m)?)?; m.add_function(wrap_pyfunction!(get_date_tuple, m)?)?; m.add_function(wrap_pyfunction!(date_from_timestamp, m)?)?; diff --git a/pytests/src/dict_iter.rs b/pytests/src/dict_iter.rs index 5f5992b6efc..985c929792f 100644 --- a/pytests/src/dict_iter.rs +++ b/pytests/src/dict_iter.rs @@ -3,7 +3,7 @@ use pyo3::prelude::*; use pyo3::types::PyDict; #[pymodule] -pub fn dict_iter(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn dict_iter(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/pytests/src/enums.rs b/pytests/src/enums.rs index 11b592d3563..32478cbefc2 100644 --- a/pytests/src/enums.rs +++ b/pytests/src/enums.rs @@ -1,7 +1,7 @@ -use pyo3::{pyclass, pyfunction, pymodule, types::PyModule, wrap_pyfunction, PyResult, Python}; +use pyo3::{pyclass, pyfunction, pymodule, types::PyModule, wrap_pyfunction, Bound, PyResult}; #[pymodule] -pub fn enums(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn enums(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_wrapped(wrap_pyfunction!(do_simple_stuff))?; diff --git a/pytests/src/misc.rs b/pytests/src/misc.rs index bd941461e91..3b893ccd1f6 100644 --- a/pytests/src/misc.rs +++ b/pytests/src/misc.rs @@ -31,7 +31,7 @@ fn get_item_and_run_callback(dict: Bound<'_, PyDict>, callback: Bound<'_, PyAny> } #[pymodule] -pub fn misc(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn misc(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(issue_219, m)?)?; m.add_function(wrap_pyfunction!(get_type_full_name, m)?)?; m.add_function(wrap_pyfunction!(accepts_bool, m)?)?; diff --git a/pytests/src/objstore.rs b/pytests/src/objstore.rs index f7fc66edb84..440f29fad63 100644 --- a/pytests/src/objstore.rs +++ b/pytests/src/objstore.rs @@ -19,6 +19,6 @@ impl ObjStore { } #[pymodule] -pub fn objstore(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn objstore(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::() } diff --git a/pytests/src/othermod.rs b/pytests/src/othermod.rs index 763d38a878f..29ca8121890 100644 --- a/pytests/src/othermod.rs +++ b/pytests/src/othermod.rs @@ -29,7 +29,7 @@ fn double(x: i32) -> i32 { } #[pymodule] -pub fn othermod(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn othermod(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(double, m)?)?; m.add_class::()?; diff --git a/pytests/src/path.rs b/pytests/src/path.rs index b3e8f92bacf..0675e56d13a 100644 --- a/pytests/src/path.rs +++ b/pytests/src/path.rs @@ -12,7 +12,7 @@ fn take_pathbuf(path: PathBuf) -> PathBuf { } #[pymodule] -pub fn path(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn path(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(make_path, m)?)?; m.add_function(wrap_pyfunction!(take_pathbuf, m)?)?; diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index 9c7b2d250da..1f3baec2755 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -80,7 +80,7 @@ impl AssertingBaseClassGilRef { struct ClassWithoutConstructor; #[pymodule] -pub fn pyclasses(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn pyclasses(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/pytests/src/pyfunctions.rs b/pytests/src/pyfunctions.rs index a92733c2898..77496198bb9 100644 --- a/pytests/src/pyfunctions.rs +++ b/pytests/src/pyfunctions.rs @@ -68,7 +68,7 @@ fn args_kwargs<'py>( } #[pymodule] -pub fn pyfunctions(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn pyfunctions(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(none, m)?)?; m.add_function(wrap_pyfunction!(simple, m)?)?; m.add_function(wrap_pyfunction!(simple_args, m)?)?; diff --git a/pytests/src/sequence.rs b/pytests/src/sequence.rs index 5916414ee8f..0e48a161bd3 100644 --- a/pytests/src/sequence.rs +++ b/pytests/src/sequence.rs @@ -17,7 +17,7 @@ fn vec_to_vec_pystring(vec: Vec<&PyString>) -> Vec<&PyString> { } #[pymodule] -pub fn sequence(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn sequence(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(vec_to_vec_i32, m)?)?; m.add_function(wrap_pyfunction!(array_to_array_i32, m)?)?; m.add_function(wrap_pyfunction!(vec_to_vec_pystring, m)?)?; diff --git a/pytests/src/subclassing.rs b/pytests/src/subclassing.rs index 0033114ccea..8e451cd9183 100644 --- a/pytests/src/subclassing.rs +++ b/pytests/src/subclassing.rs @@ -18,7 +18,7 @@ impl Subclassable { } #[pymodule] -pub fn subclassing(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn subclassing(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index e908cddb621..bb9ae0126d0 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -71,7 +71,7 @@ //! } //! //! #[pymodule] -//! fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +//! fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { //! m.add_function(wrap_pyfunction!(calculate_statistics, m)?)?; //! Ok(()) //! } diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index e6f345fd200..69bc5493366 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -31,7 +31,7 @@ //! } //! //! #[pymodule] -//! fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +//! fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { //! m.add_function(wrap_pyfunction!(add_one, m)?)?; //! Ok(()) //! } diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index 369888af85f..a57b2745ec9 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -44,7 +44,7 @@ //! } //! //! #[pymodule] -//! fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +//! fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { //! m.add_function(wrap_pyfunction!(get_eigenvalues, m)?)?; //! Ok(()) //! } @@ -57,7 +57,7 @@ //! # Python::with_gil(|py| -> PyResult<()> { //! # let module = PyModule::new_bound(py, "my_module")?; //! # -//! # module.add_function(&wrap_pyfunction!(get_eigenvalues, module.as_gil_ref())?.as_borrowed())?; +//! # module.add_function(&wrap_pyfunction!(get_eigenvalues, module)?)?; //! # //! # let m11 = PyComplex::from_doubles_bound(py, 0_f64, -1_f64); //! # let m12 = PyComplex::from_doubles_bound(py, 1_f64, 0_f64); diff --git a/src/conversions/rust_decimal.rs b/src/conversions/rust_decimal.rs index 2e3151720e6..8bf2e33752d 100644 --- a/src/conversions/rust_decimal.rs +++ b/src/conversions/rust_decimal.rs @@ -30,7 +30,7 @@ //! } //! //! #[pymodule] -//! fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +//! fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { //! m.add_function(wrap_pyfunction!(add_one, m)?)?; //! Ok(()) //! } diff --git a/src/exceptions.rs b/src/exceptions.rs index 1c952393241..3401679b25e 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -164,9 +164,9 @@ macro_rules! import_exception { /// } /// /// #[pymodule] -/// fn my_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { -/// m.add("MyError", py.get_type_bound::())?; -/// m.add_function(wrap_pyfunction!(raise_myerror, py)?)?; +/// fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { +/// m.add("MyError", m.py().get_type_bound::())?; +/// m.add_function(wrap_pyfunction!(raise_myerror, m)?)?; /// Ok(()) /// } /// # fn main() -> PyResult<()> { diff --git a/src/lib.rs b/src/lib.rs index 17dbd972d85..c2591cec91e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -168,7 +168,7 @@ //! //! /// A Python module implemented in Rust. //! #[pymodule] -//! fn string_sum(py: Python<'_>, m: &PyModule) -> PyResult<()> { +//! fn string_sum(m: &Bound<'_, PyModule>) -> PyResult<()> { //! m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; //! //! Ok(()) diff --git a/src/types/module.rs b/src/types/module.rs index 693df79cafa..d6f3acb2567 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -239,7 +239,7 @@ impl PyModule { /// use pyo3::prelude::*; /// /// #[pymodule] - /// fn my_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// fn my_module(module: &Bound<'_, PyModule>) -> PyResult<()> { /// module.add("c", 299_792_458)?; /// Ok(()) /// } @@ -280,7 +280,7 @@ impl PyModule { /// struct Foo { /* fields omitted */ } /// /// #[pymodule] - /// fn my_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// fn my_module(module: &Bound<'_, PyModule>) -> PyResult<()> { /// module.add_class::()?; /// Ok(()) /// } @@ -377,7 +377,7 @@ impl PyModule { /// println!("Hello world!") /// } /// #[pymodule] - /// fn my_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// fn my_module(module: &Bound<'_, PyModule>) -> PyResult<()> { /// module.add_function(wrap_pyfunction!(say_hello, module)?) /// } /// ``` @@ -442,7 +442,7 @@ pub trait PyModuleMethods<'py>: crate::sealed::Sealed { /// use pyo3::prelude::*; /// /// #[pymodule] - /// fn my_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// fn my_module(module: &Bound<'_, PyModule>) -> PyResult<()> { /// module.add("c", 299_792_458)?; /// Ok(()) /// } @@ -481,7 +481,7 @@ pub trait PyModuleMethods<'py>: crate::sealed::Sealed { /// struct Foo { /* fields omitted */ } /// /// #[pymodule] - /// fn my_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// fn my_module(module: &Bound<'_, PyModule>) -> PyResult<()> { /// module.add_class::()?; /// Ok(()) /// } @@ -570,7 +570,7 @@ pub trait PyModuleMethods<'py>: crate::sealed::Sealed { /// println!("Hello world!") /// } /// #[pymodule] - /// fn my_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// fn my_module(module: &Bound<'_, PyModule>) -> PyResult<()> { /// module.add_function(wrap_pyfunction!(say_hello, module)?) /// } /// ``` diff --git a/tests/test_append_to_inittab.rs b/tests/test_append_to_inittab.rs index 59ecaf42909..da35298b4d9 100644 --- a/tests/test_append_to_inittab.rs +++ b/tests/test_append_to_inittab.rs @@ -8,8 +8,8 @@ fn foo() -> usize { } #[pymodule] -fn module_fn_with_functions(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(foo, m)?).unwrap(); +fn module_fn_with_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(foo, m)?)?; Ok(()) } diff --git a/tests/test_module.rs b/tests/test_module.rs index 5763043e57d..d4c4acca90f 100644 --- a/tests/test_module.rs +++ b/tests/test_module.rs @@ -36,7 +36,7 @@ fn double(x: usize) -> usize { /// This module is implemented in Rust. #[pymodule] -fn module_with_functions(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn module_with_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { #[pyfn(m)] #[pyo3(name = "no_parameters")] fn function_with_name() -> usize { @@ -54,14 +54,14 @@ fn module_with_functions(_py: Python<'_>, m: &PyModule) -> PyResult<()> { v.value * 2 } - m.add_class::().unwrap(); - m.add_class::().unwrap(); - m.add_class::().unwrap(); + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; - m.add("foo", "bar").unwrap(); + m.add("foo", "bar")?; - m.add_function(wrap_pyfunction!(double, m)?).unwrap(); - m.add("also_double", wrap_pyfunction!(double, m)?).unwrap(); + m.add_function(wrap_pyfunction!(double, m)?)?; + m.add("also_double", wrap_pyfunction!(double, m)?)?; Ok(()) } @@ -116,9 +116,31 @@ fn test_module_with_functions() { }); } +/// This module uses a legacy two-argument module function. +#[pymodule] +fn module_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(double, m)?)?; + Ok(()) +} + +#[test] +fn test_module_with_explicit_py_arg() { + use pyo3::wrap_pymodule; + + Python::with_gil(|py| { + let d = [( + "module_with_explicit_py_arg", + wrap_pymodule!(module_with_explicit_py_arg)(py), + )] + .into_py_dict_bound(py); + + py_assert!(py, *d, "module_with_explicit_py_arg.double(3) == 6"); + }); +} + #[pymodule] #[pyo3(name = "other_name")] -fn some_name(_: Python<'_>, m: &PyModule) -> PyResult<()> { +fn some_name(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("other_name", "other_name")?; Ok(()) } @@ -166,7 +188,7 @@ fn r#move() -> usize { } #[pymodule] -fn raw_ident_module(_py: Python<'_>, module: &PyModule) -> PyResult<()> { +fn raw_ident_module(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(r#move, module)?) } @@ -190,7 +212,7 @@ fn custom_named_fn() -> usize { #[test] fn test_custom_names() { #[pymodule] - fn custom_names(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + fn custom_names(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(custom_named_fn, m)?)?; Ok(()) } @@ -206,7 +228,7 @@ fn test_custom_names() { #[test] fn test_module_dict() { #[pymodule] - fn module_dict(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + fn module_dict(m: &Bound<'_, PyModule>) -> PyResult<()> { m.dict().set_item("yay", "me")?; Ok(()) } @@ -222,7 +244,7 @@ fn test_module_dict() { fn test_module_dunder_all() { Python::with_gil(|py| { #[pymodule] - fn dunder_all(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + fn dunder_all(m: &Bound<'_, PyModule>) -> PyResult<()> { m.dict().set_item("yay", "me")?; m.add_function(wrap_pyfunction!(custom_named_fn, m)?)?; Ok(()) @@ -245,7 +267,7 @@ fn submodule(module: &Bound<'_, PyModule>) -> PyResult<()> { } #[pymodule] -fn submodule_with_init_fn(_py: Python<'_>, module: &PyModule) -> PyResult<()> { +fn submodule_with_init_fn(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(subfunction, module)?)?; Ok(()) } @@ -256,14 +278,14 @@ fn superfunction() -> String { } #[pymodule] -fn supermodule(py: Python<'_>, module: &PyModule) -> PyResult<()> { +fn supermodule(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(superfunction, module)?)?; - let module_to_add = PyModule::new_bound(py, "submodule")?; + let module_to_add = PyModule::new_bound(module.py(), "submodule")?; submodule(&module_to_add)?; - module.add_submodule(module_to_add.as_gil_ref())?; - let module_to_add = PyModule::new_bound(py, "submodule_with_init_fn")?; - submodule_with_init_fn(py, module_to_add.as_gil_ref())?; - module.add_submodule(module_to_add.as_gil_ref())?; + module.add_submodule(&module_to_add)?; + let module_to_add = PyModule::new_bound(module.py(), "submodule_with_init_fn")?; + submodule_with_init_fn(&module_to_add)?; + module.add_submodule(&module_to_add)?; Ok(()) } @@ -300,7 +322,7 @@ fn ext_vararg_fn(py: Python<'_>, a: i32, args: &Bound<'_, PyTuple>) -> PyObject } #[pymodule] -fn vararg_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { +fn vararg_module(m: &Bound<'_, PyModule>) -> PyResult<()> { #[pyfn(m, signature = (a=5, *args))] fn int_vararg_fn(py: Python<'_>, a: i32, args: &Bound<'_, PyTuple>) -> PyObject { ext_vararg_fn(py, a, args) @@ -328,7 +350,7 @@ fn test_module_with_constant() { // Regression test for #1102 #[pymodule] - fn module_with_constant(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + fn module_with_constant(m: &Bound<'_, PyModule>) -> PyResult<()> { const ANON: AnonClass = AnonClass {}; m.add("ANON", ANON)?; @@ -410,7 +432,7 @@ fn pyfunction_with_pass_module_in_attribute(module: &PyModule) -> PyResult<&str> } #[pymodule] -fn module_with_functions_with_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { +fn module_with_functions_with_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(pyfunction_with_module, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_gil_ref, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_owned, m)?)?; @@ -475,7 +497,7 @@ fn test_module_doc_hidden() { #[doc(hidden)] #[allow(clippy::unnecessary_wraps)] #[pymodule] - fn my_module(_py: Python<'_>, _m: &PyModule) -> PyResult<()> { + fn my_module(_m: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } diff --git a/tests/test_text_signature.rs b/tests/test_text_signature.rs index cb2cd85e99b..6b56999c62f 100644 --- a/tests/test_text_signature.rs +++ b/tests/test_text_signature.rs @@ -335,7 +335,7 @@ fn test_auto_test_signature_opt_out() { #[test] fn test_pyfn() { #[pymodule] - fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { #[pyfn(m, signature = (a, b=None, *, c=42))] #[pyo3(text_signature = "(a, b=None, *, c=42)")] fn my_function(a: i32, b: Option, c: i32) { diff --git a/tests/ui/invalid_pymodule_args.rs b/tests/ui/invalid_pymodule_args.rs index ebd229eef2e..37e53960fd3 100644 --- a/tests/ui/invalid_pymodule_args.rs +++ b/tests/ui/invalid_pymodule_args.rs @@ -1,7 +1,7 @@ use pyo3::prelude::*; #[pymodule(some_arg)] -fn module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn module(m: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } From 56683ed553dfa413cc623425c655514b69c3992b Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Thu, 29 Feb 2024 07:15:34 +0000 Subject: [PATCH 189/349] deprecate Py::as_ref (#3864) * Deprecate Py::as_ref * Reword as_ref deprecation note Co-authored-by: David Hewitt * Tidy up remaining uses of Py::as_ref Co-authored-by: David Hewitt * Pass hello into println! explicitly --------- Co-authored-by: David Hewitt --- guide/src/class.md | 1 + guide/src/memory.md | 4 +- guide/src/performance.md | 10 +++- guide/src/trait_bounds.md | 15 ++++++ guide/src/types.md | 4 ++ src/impl_/coroutine.rs | 12 ++++- src/instance.rs | 9 ++++ src/pycell/impl_.rs | 50 +++++++++---------- src/pyclass.rs | 16 +++++- tests/ui/invalid_frozen_pyclass_borrow.rs | 4 +- tests/ui/invalid_frozen_pyclass_borrow.stderr | 28 +++++------ 11 files changed, 105 insertions(+), 48 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index 1e752700613..cc254973370 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -256,6 +256,7 @@ fn return_myclass() -> Py { let obj = return_myclass(); Python::with_gil(|py| { + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let cell = obj.as_ref(py); // Py::as_ref returns &PyCell let obj_ref = cell.borrow(); // Get PyRef assert_eq!(obj_ref.num, 1); diff --git a/guide/src/memory.md b/guide/src/memory.md index 2f5e5d9b0bd..46136e3f1a4 100644 --- a/guide/src/memory.md +++ b/guide/src/memory.md @@ -171,7 +171,9 @@ let hello: Py = Python::with_gil(|py| { // Do some stuff... // Now sometime later in the program we want to access `hello`. Python::with_gil(|py| { - println!("Python says: {}", hello.as_ref(py)); + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. + let hello = hello.as_ref(py); + println!("Python says: {}", hello); }); // Now we're done with `hello`. drop(hello); // Memory *not* released here. diff --git a/guide/src/performance.md b/guide/src/performance.md index 23fb59c4e90..fe362bed953 100644 --- a/guide/src/performance.md +++ b/guide/src/performance.md @@ -70,7 +70,11 @@ struct FooRef<'a>(&'a PyList); impl PartialEq for FooRef<'_> { fn eq(&self, other: &Foo) -> bool { - Python::with_gil(|py| self.0.len() == other.0.as_ref(py).len()) + Python::with_gil(|py| { + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. + let len = other.0.as_ref(py).len(); + self.0.len() == len + }) } } ``` @@ -88,7 +92,9 @@ impl PartialEq for FooRef<'_> { fn eq(&self, other: &Foo) -> bool { // Access to `&'a PyAny` implies access to `Python<'a>`. let py = self.0.py(); - self.0.len() == other.0.as_ref(py).len() + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. + let len = other.0.as_ref(py).len(); + self.0.len() == len } } ``` diff --git a/guide/src/trait_bounds.md b/guide/src/trait_bounds.md index e05d44e982a..6dfaa2e20aa 100644 --- a/guide/src/trait_bounds.md +++ b/guide/src/trait_bounds.md @@ -83,6 +83,7 @@ impl Model for UserModel { Python::with_gil(|py| { let values: Vec = var.clone(); let list: PyObject = values.into_py(py); + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let py_model = self.model.as_ref(py); py_model .call_method("set_variables", (list,), None) @@ -93,6 +94,7 @@ impl Model for UserModel { fn get_results(&self) -> Vec { println!("Rust calling Python to get the results"); Python::with_gil(|py| { + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. self.model .as_ref(py) .call_method("get_results", (), None) @@ -105,6 +107,7 @@ impl Model for UserModel { fn compute(&mut self) { println!("Rust calling Python to perform the computation"); Python::with_gil(|py| { + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. self.model .as_ref(py) .call_method("compute", (), None) @@ -183,6 +186,7 @@ This wrapper will also perform the type conversions between Python and Rust. # Python::with_gil(|py| { # let values: Vec = var.clone(); # let list: PyObject = values.into_py(py); +# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # let py_model = self.model.as_ref(py); # py_model # .call_method("set_variables", (list,), None) @@ -193,6 +197,7 @@ This wrapper will also perform the type conversions between Python and Rust. # fn get_results(&self) -> Vec { # println!("Rust calling Python to get the results"); # Python::with_gil(|py| { +# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # self.model # .as_ref(py) # .call_method("get_results", (), None) @@ -205,6 +210,7 @@ This wrapper will also perform the type conversions between Python and Rust. # fn compute(&mut self) { # println!("Rust calling Python to perform the computation"); # Python::with_gil(|py| { +# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # self.model # .as_ref(py) # .call_method("compute", (), None) @@ -349,6 +355,7 @@ We used in our `get_results` method the following call that performs the type co impl Model for UserModel { fn get_results(&self) -> Vec { println!("Rust calling Python to get the results"); + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. Python::with_gil(|py| { self.model .as_ref(py) @@ -363,6 +370,7 @@ impl Model for UserModel { # Python::with_gil(|py| { # let values: Vec = var.clone(); # let list: PyObject = values.into_py(py); +# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # let py_model = self.model.as_ref(py); # py_model # .call_method("set_variables", (list,), None) @@ -373,6 +381,7 @@ impl Model for UserModel { # fn compute(&mut self) { # println!("Rust calling Python to perform the computation"); # Python::with_gil(|py| { +# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # self.model # .as_ref(py) # .call_method("compute", (), None) @@ -403,6 +412,7 @@ impl Model for UserModel { fn get_results(&self) -> Vec { println!("Get results from Rust calling Python"); Python::with_gil(|py| { + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let py_result: &PyAny = self .model .as_ref(py) @@ -424,6 +434,7 @@ impl Model for UserModel { # Python::with_gil(|py| { # let values: Vec = var.clone(); # let list: PyObject = values.into_py(py); +# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # let py_model = self.model.as_ref(py); # py_model # .call_method("set_variables", (list,), None) @@ -434,6 +445,7 @@ impl Model for UserModel { # fn compute(&mut self) { # println!("Rust calling Python to perform the computation"); # Python::with_gil(|py| { +# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # self.model # .as_ref(py) # .call_method("compute", (), None) @@ -523,6 +535,7 @@ impl Model for UserModel { Python::with_gil(|py| { let values: Vec = var.clone(); let list: PyObject = values.into_py(py); + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let py_model = self.model.as_ref(py); py_model .call_method("set_variables", (list,), None) @@ -533,6 +546,7 @@ impl Model for UserModel { fn get_results(&self) -> Vec { println!("Get results from Rust calling Python"); Python::with_gil(|py| { + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let py_result: &PyAny = self .model .as_ref(py) @@ -553,6 +567,7 @@ impl Model for UserModel { fn compute(&mut self) { println!("Rust calling Python to perform the computation"); Python::with_gil(|py| { + #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. self.model .as_ref(py) .call_method("compute", (), None) diff --git a/guide/src/types.md b/guide/src/types.md index f768a31f019..51ea10f4545 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -145,6 +145,7 @@ let _ = list.repr()?; let _: &PyAny = list; // To &PyAny explicitly with .as_ref() +#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let _: &PyAny = list.as_ref(); // To Py with .into() or Py::from() @@ -179,6 +180,7 @@ For a `Py`, the conversions are as below: let list: Py = PyList::empty_bound(py).unbind(); // To &PyList with Py::as_ref() (borrows from the Py) +#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let _: &PyList = list.as_ref(py); # let list_clone = list.clone(); // Because `.into_ref()` will consume `list`. @@ -202,6 +204,7 @@ For a `#[pyclass] struct MyClass`, the conversions for `Py` are below: let my_class: Py = Py::new(py, MyClass { })?; // To &PyCell with Py::as_ref() (borrows from the Py) +#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let _: &PyCell = my_class.as_ref(py); # let my_class_clone = my_class.clone(); // Because `.into_ref()` will consume `my_class`. @@ -276,6 +279,7 @@ let _ = cell.repr()?; let _: &PyAny = cell; // To &PyAny explicitly with .as_ref() +#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let _: &PyAny = cell.as_ref(); # Ok(()) # }).unwrap(); diff --git a/src/impl_/coroutine.rs b/src/impl_/coroutine.rs index c9ca4873c6c..a59eddd0e8f 100644 --- a/src/impl_/coroutine.rs +++ b/src/impl_/coroutine.rs @@ -56,7 +56,11 @@ impl Deref for RefGuard { impl Drop for RefGuard { fn drop(&mut self) { - Python::with_gil(|gil| self.0.as_ref(gil).release_ref()) + Python::with_gil(|gil| { + #[allow(deprecated)] + let self_ref = self.0.bind(gil); + self_ref.release_ref() + }) } } @@ -87,6 +91,10 @@ impl> DerefMut for RefMutGuard { impl> Drop for RefMutGuard { fn drop(&mut self) { - Python::with_gil(|gil| self.0.as_ref(gil).release_mut()) + Python::with_gil(|gil| { + #[allow(deprecated)] + let self_ref = self.0.bind(gil); + self_ref.release_mut() + }) } } diff --git a/src/instance.rs b/src/instance.rs index cb33ed32c75..65c5ee3bc5f 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -914,6 +914,7 @@ where /// # /// Python::with_gil(|py| { /// let list: Py = PyList::empty_bound(py).into(); + /// # #[allow(deprecated)] /// let list: &PyList = list.as_ref(py); /// assert_eq!(list.len(), 0); /// }); @@ -929,10 +930,18 @@ where /// /// Python::with_gil(|py| { /// let my_class: Py = Py::new(py, MyClass {}).unwrap(); + /// # #[allow(deprecated)] /// let my_class_cell: &PyCell = my_class.as_ref(py); /// assert!(my_class_cell.try_borrow().is_ok()); /// }); /// ``` + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `obj.bind(py)` instead of `obj.as_ref(py)`" + ) + )] pub fn as_ref<'py>(&'py self, _py: Python<'py>) -> &'py T::AsRefTarget { let any = self.as_ptr() as *const PyAny; unsafe { PyNativeType::unchecked_downcast(&*any) } diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 62875e67ae4..29ba7eda7eb 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -284,43 +284,43 @@ mod tests { ) .unwrap(); - let mmm_cell: &PyCell = mmm.as_ref(py); + let mmm_bound: &Bound<'_, MutableChildOfMutableChildOfMutableBase> = mmm.bind(py); - let mmm_refmut = mmm_cell.borrow_mut(); + let mmm_refmut = mmm_bound.borrow_mut(); // Cannot take any other mutable or immutable borrows whilst the object is borrowed mutably - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_err()); - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_err()); - assert!(mmm_cell.extract::>().is_err()); - assert!(mmm_cell + assert!(mmm_bound.extract::>().is_err()); + assert!(mmm_bound .extract::>() .is_err()); - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_err()); - assert!(mmm_cell.extract::>().is_err()); + assert!(mmm_bound.extract::>().is_err()); // With the borrow dropped, all other borrow attempts will succeed drop(mmm_refmut); - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_ok()); - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_ok()); - assert!(mmm_cell.extract::>().is_ok()); - assert!(mmm_cell + assert!(mmm_bound.extract::>().is_ok()); + assert!(mmm_bound .extract::>() .is_ok()); - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_ok()); - assert!(mmm_cell.extract::>().is_ok()); + assert!(mmm_bound.extract::>().is_ok()); }) } @@ -335,38 +335,38 @@ mod tests { ) .unwrap(); - let mmm_cell: &PyCell = mmm.as_ref(py); + let mmm_bound: &Bound<'_, MutableChildOfMutableChildOfMutableBase> = mmm.bind(py); - let mmm_refmut = mmm_cell.borrow(); + let mmm_refmut = mmm_bound.borrow(); // Further immutable borrows are ok - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_ok()); - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_ok()); - assert!(mmm_cell.extract::>().is_ok()); + assert!(mmm_bound.extract::>().is_ok()); // Further mutable borrows are not ok - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_err()); - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_err()); - assert!(mmm_cell.extract::>().is_err()); + assert!(mmm_bound.extract::>().is_err()); // With the borrow dropped, all mutable borrow attempts will succeed drop(mmm_refmut); - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_ok()); - assert!(mmm_cell + assert!(mmm_bound .extract::>() .is_ok()); - assert!(mmm_cell.extract::>().is_ok()); + assert!(mmm_bound.extract::>().is_ok()); }) } } diff --git a/src/pyclass.rs b/src/pyclass.rs index eb4a5595ca9..ebdc52dc217 100644 --- a/src/pyclass.rs +++ b/src/pyclass.rs @@ -1,7 +1,7 @@ //! `PyClass` and related traits. use crate::{ - callback::IntoPyCallbackOutput, ffi, impl_::pyclass::PyClassImpl, IntoPy, PyCell, PyObject, - PyResult, PyTypeInfo, Python, + callback::IntoPyCallbackOutput, ffi, impl_::pyclass::PyClassImpl, Bound, IntoPy, PyCell, + PyObject, PyResult, PyTypeInfo, Python, }; use std::{cmp::Ordering, os::raw::c_int}; @@ -216,6 +216,18 @@ pub trait Frozen: boolean_struct::private::Boolean {} impl Frozen for boolean_struct::True {} impl Frozen for boolean_struct::False {} +impl<'py, T: PyClass> Bound<'py, T> { + #[cfg(feature = "macros")] + pub(crate) fn release_ref(&self) { + self.get_cell().release_ref(); + } + + #[cfg(feature = "macros")] + pub(crate) fn release_mut(&self) { + self.get_cell().release_mut(); + } +} + mod tests { #[test] fn test_compare_op_matches() { diff --git a/tests/ui/invalid_frozen_pyclass_borrow.rs b/tests/ui/invalid_frozen_pyclass_borrow.rs index 1f18eab6170..aa8969ab1a6 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.rs +++ b/tests/ui/invalid_frozen_pyclass_borrow.rs @@ -12,7 +12,7 @@ impl Foo { } fn borrow_mut_fails(foo: Py, py: Python) { - let borrow = foo.as_ref(py).borrow_mut(); + let borrow = foo.bind(py).borrow_mut(); } #[pyclass(subclass)] @@ -22,7 +22,7 @@ struct MutableBase; struct ImmutableChild; fn borrow_mut_of_child_fails(child: Py, py: Python) { - let borrow = child.as_ref(py).borrow_mut(); + let borrow = child.bind(py).borrow_mut(); } fn py_get_of_mutable_class_fails(class: Py) { diff --git a/tests/ui/invalid_frozen_pyclass_borrow.stderr b/tests/ui/invalid_frozen_pyclass_borrow.stderr index 5e09d512ae7..1a8e45964ca 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.stderr +++ b/tests/ui/invalid_frozen_pyclass_borrow.stderr @@ -17,34 +17,34 @@ note: required by a bound in `extract_pyclass_ref_mut` | ^^^^^^^^^^^^^^ required by this bound in `extract_pyclass_ref_mut` error[E0271]: type mismatch resolving `::Frozen == False` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:15:33 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:15:31 | -15 | let borrow = foo.as_ref(py).borrow_mut(); - | ^^^^^^^^^^ expected `False`, found `True` +15 | let borrow = foo.bind(py).borrow_mut(); + | ^^^^^^^^^^ expected `False`, found `True` | -note: required by a bound in `pyo3::PyCell::::borrow_mut` - --> src/pycell.rs +note: required by a bound in `pyo3::Bound::<'py, T>::borrow_mut` + --> src/instance.rs | - | pub fn borrow_mut(&self) -> PyRefMut<'_, T> + | pub fn borrow_mut(&self) -> PyRefMut<'py, T> | ---------- required by a bound in this associated function | where | T: PyClass, - | ^^^^^^^^^^^^^^ required by this bound in `PyCell::::borrow_mut` + | ^^^^^^^^^^^^^^ required by this bound in `Bound::<'py, T>::borrow_mut` error[E0271]: type mismatch resolving `::Frozen == False` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:25:35 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:25:33 | -25 | let borrow = child.as_ref(py).borrow_mut(); - | ^^^^^^^^^^ expected `False`, found `True` +25 | let borrow = child.bind(py).borrow_mut(); + | ^^^^^^^^^^ expected `False`, found `True` | -note: required by a bound in `pyo3::PyCell::::borrow_mut` - --> src/pycell.rs +note: required by a bound in `pyo3::Bound::<'py, T>::borrow_mut` + --> src/instance.rs | - | pub fn borrow_mut(&self) -> PyRefMut<'_, T> + | pub fn borrow_mut(&self) -> PyRefMut<'py, T> | ---------- required by a bound in this associated function | where | T: PyClass, - | ^^^^^^^^^^^^^^ required by this bound in `PyCell::::borrow_mut` + | ^^^^^^^^^^^^^^ required by this bound in `Bound::<'py, T>::borrow_mut` error[E0271]: type mismatch resolving `::Frozen == True` --> tests/ui/invalid_frozen_pyclass_borrow.rs:29:11 From 1d224610c30b86f5125a849fcba4ad4eb969f9e7 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:21:47 +0100 Subject: [PATCH 190/349] docs: update `Python classes` section of the guide (#3914) * docs: update `Python classes` section of the guide * review feedback davidhewitt * migration guide entry --- examples/decorator/src/lib.rs | 6 +-- guide/src/class.md | 69 +++++++++++++++++------------------ guide/src/class/call.md | 4 +- guide/src/class/numeric.md | 14 +++---- guide/src/class/object.md | 6 +-- guide/src/class/protocols.md | 2 +- guide/src/migration.md | 4 ++ src/types/tuple.rs | 6 +++ 8 files changed, 60 insertions(+), 51 deletions(-) diff --git a/examples/decorator/src/lib.rs b/examples/decorator/src/lib.rs index 9dccabc7341..cfb09c112d5 100644 --- a/examples/decorator/src/lib.rs +++ b/examples/decorator/src/lib.rs @@ -40,8 +40,8 @@ impl PyCounter { fn __call__( &self, py: Python<'_>, - args: &PyTuple, - kwargs: Option>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { let old_count = self.count.get(); let new_count = old_count + 1; @@ -51,7 +51,7 @@ impl PyCounter { println!("{} has been called {} time(s).", name, new_count); // After doing something, we finally forward the call to the wrapped function - let ret = self.wraps.call_bound(py, args, kwargs.as_ref())?; + let ret = self.wraps.call_bound(py, args, kwargs)?; // We could do something with the return value of // the function before returning it diff --git a/guide/src/class.md b/guide/src/class.md index cc254973370..3096c9f7842 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -187,26 +187,23 @@ fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { } ``` -## PyCell and interior mutability +## Bound and interior mutability -You sometimes need to convert your `pyclass` into a Python object and access it +You sometimes need to convert your `#[pyclass]` into a Python object and access it from Rust code (e.g., for testing it). -[`PyCell`] is the primary interface for that. +[`Bound`] is the primary interface for that. -`PyCell` is always allocated in the Python heap, so Rust doesn't have ownership of it. -In other words, Rust code can only extract a `&PyCell`, not a `PyCell`. - -Thus, to mutate data behind `&PyCell` safely, PyO3 employs the +To mutate data behind a `Bound<'_, T>` safely, PyO3 employs the [Interior Mutability Pattern](https://doc.rust-lang.org/book/ch15-05-interior-mutability.html) like [`RefCell`]. -Users who are familiar with `RefCell` can use `PyCell` just like `RefCell`. +Users who are familiar with `RefCell` can use `Bound` just like `RefCell`. For users who are not very familiar with `RefCell`, here is a reminder of Rust's rules of borrowing: - At any given time, you can have either (but not both of) one mutable reference or any number of immutable references. - References must always be valid. -`PyCell`, like `RefCell`, ensures these borrowing rules by tracking references at runtime. +`Bound`, like `RefCell`, ensures these borrowing rules by tracking references at runtime. ```rust # use pyo3::prelude::*; @@ -216,8 +213,7 @@ struct MyClass { num: i32, } Python::with_gil(|py| { -# #[allow(deprecated)] - let obj = PyCell::new(py, MyClass { num: 3 }).unwrap(); + let obj = Bound::new(py, MyClass { num: 3 }).unwrap(); { let obj_ref = obj.borrow(); // Get PyRef assert_eq!(obj_ref.num, 3); @@ -232,12 +228,12 @@ Python::with_gil(|py| { assert!(obj.try_borrow_mut().is_err()); } - // You can convert `&PyCell` to a Python object + // You can convert `Bound` to a Python object pyo3::py_run!(py, obj, "assert obj.num == 5"); }); ``` -`&PyCell` is bounded by the same lifetime as a [`GILGuard`]. +A `Bound<'py, T>` is restricted to the GIL lifetime `'py`. To make the object longer lived (for example, to store it in a struct on the Rust side), you can use `Py`, which stores an object longer than the GIL lifetime, and therefore needs a `Python<'_>` token to access. @@ -256,9 +252,8 @@ fn return_myclass() -> Py { let obj = return_myclass(); Python::with_gil(|py| { - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. - let cell = obj.as_ref(py); // Py::as_ref returns &PyCell - let obj_ref = cell.borrow(); // Get PyRef + let bound = obj.bind(py); // Py::bind returns &Bound<'_, MyClass> + let obj_ref = bound.borrow(); // Get PyRef assert_eq!(obj_ref.num, 1); }); ``` @@ -267,7 +262,7 @@ Python::with_gil(|py| { As detailed above, runtime borrow checking is currently enabled by default. But a class can opt of out it by declaring itself `frozen`. It can still use interior mutability via standard Rust types like `RefCell` or `Mutex`, but it is not bound to the implementation provided by PyO3 and can choose the most appropriate strategy on field-by-field basis. -Classes which are `frozen` and also `Sync`, e.g. they do use `Mutex` but not `RefCell`, can be accessed without needing the Python GIL via the `PyCell::get` and `Py::get` methods: +Classes which are `frozen` and also `Sync`, e.g. they do use `Mutex` but not `RefCell`, can be accessed without needing the Python GIL via the `Bound::get` and `Py::get` methods: ```rust use std::sync::atomic::{AtomicUsize, Ordering}; @@ -413,8 +408,9 @@ You can inherit native types such as `PyDict`, if they implement [`PySizedLayout`]({{#PYO3_DOCS_URL}}/pyo3/type_object/trait.PySizedLayout.html). This is not supported when building for the Python limited API (aka the `abi3` feature of PyO3). -However, because of some technical problems, we don't currently provide safe upcasting methods for types -that inherit native types. Even in such cases, you can unsafely get a base class by raw pointer conversion. +To convert between the Rust type and its native base class, you can take +`slf` as a Python object. To access the Rust fields use `slf.borrow()` or +`slf.borrow_mut()`, and to access the base class use `slf.downcast::()`. ```rust # #[cfg(not(Py_LIMITED_API))] { @@ -435,10 +431,9 @@ impl DictWithCounter { Self::default() } - fn set(mut self_: PyRefMut<'_, Self>, key: String, value: &PyAny) -> PyResult<()> { - self_.counter.entry(key.clone()).or_insert(0); - let py = self_.py(); - let dict: &PyDict = unsafe { py.from_borrowed_ptr_or_err(self_.as_ptr())? }; + fn set(slf: &Bound<'_, Self>, key: String, value: &PyAny) -> PyResult<()> { + slf.borrow_mut().counter.entry(key.clone()).or_insert(0); + let dict = slf.downcast::()?; dict.set_item(key, value) } } @@ -492,7 +487,7 @@ struct MyDict { impl MyDict { #[new] #[pyo3(signature = (*args, **kwargs))] - fn new(args: &PyAny, kwargs: Option<&PyAny>) -> Self { + fn new(args: &Bound<'_, PyAny>, kwargs: Option<&Bound<'_, PyAny>>) -> Self { Self { private: 0 } } @@ -703,7 +698,7 @@ Declares a class method callable from Python. * The first parameter is the type object of the class on which the method is called. This may be the type object of a derived class. -* The first parameter implicitly has type `&PyType`. +* The first parameter implicitly has type `&Bound<'_, PyType>`. * For details on `parameter-list`, see the documentation of `Method arguments` section. * The return type must be `PyResult` or `T` for some `T` that implements `IntoPy`. @@ -803,23 +798,23 @@ struct MyClass { my_field: i32, } -// Take a GIL-bound reference when the underlying `PyCell` is irrelevant. +// Take a reference when the underlying `Bound` is irrelevant. #[pyfunction] fn increment_field(my_class: &mut MyClass) { my_class.my_field += 1; } -// Take a GIL-bound reference wrapper when borrowing should be automatic, -// but interaction with the underlying `PyCell` is desired. +// Take a reference wrapper when borrowing should be automatic, +// but interaction with the underlying `Bound` is desired. #[pyfunction] fn print_field(my_class: PyRef<'_, MyClass>) { println!("{}", my_class.my_field); } -// Take a GIL-bound reference to the underlying cell +// Take a reference to the underlying Bound // when borrowing needs to be managed manually. #[pyfunction] -fn increment_then_print_field(my_class: &PyCell) { +fn increment_then_print_field(my_class: &Bound<'_, MyClass>) { my_class.borrow_mut().my_field += 1; println!("{}", my_class.borrow().my_field); @@ -878,9 +873,9 @@ impl MyClass { fn method( &mut self, num: i32, - py_args: &PyTuple, + py_args: &Bound<'_, PyTuple>, name: &str, - py_kwargs: Option<&PyDict>, + py_kwargs: Option<&Bound<'_, PyDict>>, ) -> String { let num_before = self.num; self.num = num; @@ -1232,6 +1227,9 @@ struct MyClass { # #[allow(dead_code)] num: i32, } + +impl pyo3::types::DerefToPyAny for MyClass {} + unsafe impl pyo3::type_object::HasPyGilRef for MyClass { type AsRefTarget = pyo3::PyCell; } @@ -1279,6 +1277,8 @@ impl pyo3::IntoPy for MyClass { impl pyo3::impl_::pyclass::PyClassImpl for MyClass { const IS_BASETYPE: bool = false; const IS_SUBCLASS: bool = false; + const IS_MAPPING: bool = false; + const IS_SEQUENCE: bool = false; type BaseType = PyAny; type ThreadChecker = pyo3::impl_::pyclass::SendablePyClass; type PyClassMutability = <::PyClassMutability as pyo3::impl_::pycell::PyClassMutability>::MutableChild; @@ -1304,7 +1304,7 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { static DOC: pyo3::sync::GILOnceCell<::std::borrow::Cow<'static, ::std::ffi::CStr>> = pyo3::sync::GILOnceCell::new(); DOC.get_or_try_init(py, || { let collector = PyClassImplCollector::::new(); - build_pyclass_doc(::NAME, "", None.or_else(|| collector.new_text_signature())) + build_pyclass_doc(::NAME, "\0", collector.new_text_signature()) }).map(::std::ops::Deref::deref) } } @@ -1317,11 +1317,10 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { ``` -[`GILGuard`]: {{#PYO3_DOCS_URL}}/pyo3/struct.GILGuard.html [`PyTypeInfo`]: {{#PYO3_DOCS_URL}}/pyo3/type_object/trait.PyTypeInfo.html [`Py`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Py.html -[`PyCell`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyCell.html +[`Bound`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html [`PyClass`]: {{#PYO3_DOCS_URL}}/pyo3/pyclass/trait.PyClass.html [`PyRef`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html [`PyRefMut`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRefMut.html diff --git a/guide/src/class/call.md b/guide/src/class/call.md index 3b20986239b..0890df9561a 100644 --- a/guide/src/class/call.md +++ b/guide/src/class/call.md @@ -75,8 +75,8 @@ A [previous implementation] used a normal `u64`, which meant it required a `&mut fn __call__( &mut self, py: Python<'_>, - args: &PyTuple, - kwargs: Option<&PyDict>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { self.count += 1; let name = self.wraps.getattr(py, "__name__")?; diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index 3e6a3cf47e9..9fb609a931d 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -35,7 +35,7 @@ and cast it to an `i32`. # #![allow(dead_code)] use pyo3::prelude::*; -fn wrap(obj: &PyAny) -> Result { +fn wrap(obj: &Bound<'_, PyAny>) -> PyResult { let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?; let val: u32 = val.extract()?; // 👇 This intentionally overflows! @@ -48,7 +48,7 @@ We also add documentation, via `///` comments, which are visible to Python users # #![allow(dead_code)] use pyo3::prelude::*; -fn wrap(obj: &PyAny) -> Result { +fn wrap(obj: &Bound<'_, PyAny>) -> PyResult { let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?; let val: u32 = val.extract()?; Ok(val as i32) @@ -212,7 +212,7 @@ use pyo3::prelude::*; use pyo3::class::basic::CompareOp; use pyo3::types::PyComplex; -fn wrap(obj: &PyAny) -> Result { +fn wrap(obj: &Bound<'_, PyAny>) -> PyResult { let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?; let val: u32 = val.extract()?; Ok(val as i32) @@ -229,7 +229,7 @@ impl Number { Self(value) } - fn __repr__(slf: &PyCell) -> PyResult { + fn __repr__(slf: &Bound<'_, Self>) -> PyResult { // Get the class name dynamically in case `Number` is subclassed let class_name: String = slf.get_type().qualname()?; Ok(format!("{}({})", class_name, slf.borrow().0)) @@ -411,8 +411,8 @@ the contracts of this function. Let's review those contracts: - The GIL must be held. If it's not, calling this function causes a data race. - The pointer must be valid, i.e. it must be properly aligned and point to a valid Python object. -Let's create that helper function. The signature has to be `fn(&PyAny) -> PyResult`. -- `&PyAny` represents a checked borrowed reference, so the pointer derived from it is valid (and not null). +Let's create that helper function. The signature has to be `fn(&Bound<'_, PyAny>) -> PyResult`. +- `&Bound<'_, PyAny>` represents a checked borrowed reference, so the pointer derived from it is valid (and not null). - Whenever we have borrowed references to Python objects in scope, it is guaranteed that the GIL is held. This reference is also where we can get a [`Python`] token to use in our call to [`PyErr::take`]. ```rust @@ -421,7 +421,7 @@ use std::os::raw::c_ulong; use pyo3::prelude::*; use pyo3::ffi; -fn wrap(obj: &PyAny) -> Result { +fn wrap(obj: &Bound<'_, PyAny>) -> Result { let py: Python<'_> = obj.py(); unsafe { diff --git a/guide/src/class/object.md b/guide/src/class/object.md index 729815ade0b..db6cc7d3234 100644 --- a/guide/src/class/object.md +++ b/guide/src/class/object.md @@ -76,7 +76,7 @@ In the `__repr__`, we used a hard-coded class name. This is sometimes not ideal, because if the class is subclassed in Python, we would like the repr to reflect the subclass name. This is typically done in Python code by accessing `self.__class__.__name__`. In order to be able to access the Python type information -*and* the Rust struct, we need to use a `PyCell` as the `self` argument. +*and* the Rust struct, we need to use a `Bound` as the `self` argument. ```rust # use pyo3::prelude::*; @@ -86,7 +86,7 @@ the subclass name. This is typically done in Python code by accessing # #[pymethods] impl Number { - fn __repr__(slf: &PyCell) -> PyResult { + fn __repr__(slf: &Bound<'_, Self>) -> PyResult { // This is the equivalent of `self.__class__.__name__` in Python. let class_name: String = slf.get_type().qualname()?; // To access fields of the Rust struct, we need to borrow the `PyCell`. @@ -263,7 +263,7 @@ impl Number { Self(value) } - fn __repr__(slf: &PyCell) -> PyResult { + fn __repr__(slf: &Bound<'_, Self>) -> PyResult { let class_name: String = slf.get_type().qualname()?; Ok(format!("{}({})", class_name, slf.borrow().0)) } diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 516c051664b..b917c6a3eaf 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -17,7 +17,7 @@ When PyO3 handles a magic method, a couple of changes apply compared to other `# The following sections list of all magic methods PyO3 currently handles. The given signatures should be interpreted as follows: - All methods take a receiver as first argument, shown as ``. It can be - `&self`, `&mut self` or a `PyCell` reference like `self_: PyRef<'_, Self>` and + `&self`, `&mut self` or a `Bound` reference like `self_: PyRef<'_, Self>` and `self_: PyRefMut<'_, Self>`, as described [here](../class.md#inheritance). - An optional `Python<'py>` argument is always allowed as the first argument. - Return values can be optionally wrapped in `PyResult`. diff --git a/guide/src/migration.md b/guide/src/migration.md index 9a958f35458..709e7a70488 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -203,6 +203,10 @@ impl PyClassAsyncIter { `PyType::name` has been renamed to `PyType::qualname` to indicate that it does indeed return the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name), matching the `__qualname__` attribute. The newly added `PyType::name` yields the full name including the module name now which corresponds to `__module__.__name__` on the level of attributes. +### `PyCell` has been deprecated + +Interactions with Python objects implemented in Rust no longer need to go though `PyCell`. Instead iteractions with Python object now consistently go through `Bound` or `Py` independently of whether `T` is native Python object or a `#[pyclass]` implemented in Rust. Use `Bound::new` or `Py::new` respectively to create and `Bound::borrow(_mut)` / `Py::borrow(_mut)` to borrow the Rust object. + ### Migrating from the GIL-Refs API to `Bound` To minimise breakage of code using the GIL-Refs API, the `Bound` smart pointer has been introduced by adding complements to all functions which accept or return GIL Refs. This allows code to migrate by replacing the deprecated APIs with the new ones. diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 3ffaf9c3224..c379b67aa1a 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -631,6 +631,12 @@ impl IntoPy> for Bound<'_, PyTuple> { } } +impl IntoPy> for &'_ Bound<'_, PyTuple> { + fn into_py(self, _: Python<'_>) -> Py { + self.clone().unbind() + } +} + #[cold] fn wrong_tuple_length(t: &Bound<'_, PyTuple>, expected_length: usize) -> PyErr { let msg = format!( From 1c5265e1c263fcb194f7226cabeb93f59f9dbcd7 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 1 Mar 2024 21:51:53 +0100 Subject: [PATCH 191/349] deprecate `from_borrowed_ptr` methods (#3915) * deprecate `from_borrowed_ptr` methods This deprecates the methods on the `Python` marker, aswell as `FromPyPointer` * use `BoundRef` to defer ref cnt inc until after the error case --- pyo3-macros-backend/src/method.rs | 4 +-- src/conversion.rs | 33 ++++++++++++++++++++++++ src/impl_/coroutine.rs | 18 ++++++------- src/instance.rs | 10 ++++++-- src/lib.rs | 4 +-- src/marker.rs | 42 ++++++++++++++++++++++--------- src/pycell.rs | 15 ++++++++--- src/type_object.rs | 5 +++- src/types/boolobject.rs | 5 +++- 9 files changed, 104 insertions(+), 32 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 6ee87fb99a6..42f22204601 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -513,7 +513,7 @@ impl<'a> FnSpec<'a> { holders.pop().unwrap(); // does not actually use holder created by `self_arg` quote! {{ - let __guard = _pyo3::impl_::coroutine::RefGuard::<#cls>::new(py.from_borrowed_ptr::<_pyo3::types::PyAny>(_slf))?; + let __guard = _pyo3::impl_::coroutine::RefGuard::<#cls>::new(&_pyo3::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; async move { function(&__guard, #(#args),*).await } }} } @@ -521,7 +521,7 @@ impl<'a> FnSpec<'a> { holders.pop().unwrap(); // does not actually use holder created by `self_arg` quote! {{ - let mut __guard = _pyo3::impl_::coroutine::RefMutGuard::<#cls>::new(py.from_borrowed_ptr::<_pyo3::types::PyAny>(_slf))?; + let mut __guard = _pyo3::impl_::coroutine::RefMutGuard::<#cls>::new(&_pyo3::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; async move { function(&mut __guard, #(#args),*).await } }} } diff --git a/src/conversion.rs b/src/conversion.rs index fc407fa5bc8..986ac7c537c 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -425,6 +425,7 @@ impl IntoPy> for () { /// # Safety /// /// See safety notes on individual functions. +#[deprecated(since = "0.21.0")] pub unsafe trait FromPyPointer<'p>: Sized { /// Convert from an arbitrary `PyObject`. /// @@ -494,6 +495,13 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Implementations must ensure the object does not get freed during `'p` and avoid type confusion. + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr_or_opt(py, ptr)` or `Bound::from_borrowed_ptr_or_opt(py, ptr)` instead" + ) + )] unsafe fn from_borrowed_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self>; /// Convert from an arbitrary borrowed `PyObject`. @@ -501,7 +509,15 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" + ) + )] unsafe fn from_borrowed_ptr_or_panic(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { + #[allow(deprecated)] Self::from_borrowed_ptr_or_opt(py, ptr).unwrap_or_else(|| err::panic_after_error(py)) } /// Convert from an arbitrary borrowed `PyObject`. @@ -509,7 +525,15 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" + ) + )] unsafe fn from_borrowed_ptr(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { + #[allow(deprecated)] Self::from_borrowed_ptr_or_panic(py, ptr) } /// Convert from an arbitrary borrowed `PyObject`. @@ -517,14 +541,23 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr_or_err(py, ptr)` or `Bound::from_borrowed_ptr_or_err(py, ptr)` instead" + ) + )] unsafe fn from_borrowed_ptr_or_err( py: Python<'p>, ptr: *mut ffi::PyObject, ) -> PyResult<&'p Self> { + #[allow(deprecated)] Self::from_borrowed_ptr_or_opt(py, ptr).ok_or_else(|| err::PyErr::fetch(py)) } } +#[allow(deprecated)] unsafe impl<'p, T> FromPyPointer<'p> for T where T: 'p + crate::PyNativeType, diff --git a/src/impl_/coroutine.rs b/src/impl_/coroutine.rs index a59eddd0e8f..9be95ba3303 100644 --- a/src/impl_/coroutine.rs +++ b/src/impl_/coroutine.rs @@ -8,7 +8,7 @@ use crate::{ coroutine::{cancel::ThrowCallback, Coroutine}, instance::Bound, pyclass::boolean_struct::False, - types::PyString, + types::{PyAnyMethods, PyString}, IntoPy, Py, PyAny, PyCell, PyClass, PyErr, PyObject, PyResult, Python, }; @@ -39,10 +39,10 @@ fn get_ptr(obj: &Py) -> *mut T { pub struct RefGuard(Py); impl RefGuard { - pub fn new(obj: &PyAny) -> PyResult { - let owned: Py = obj.extract()?; - mem::forget(owned.try_borrow(obj.py())?); - Ok(RefGuard(owned)) + pub fn new(obj: &Bound<'_, PyAny>) -> PyResult { + let owned = obj.downcast::()?; + mem::forget(owned.try_borrow()?); + Ok(RefGuard(owned.clone().unbind())) } } @@ -67,10 +67,10 @@ impl Drop for RefGuard { pub struct RefMutGuard>(Py); impl> RefMutGuard { - pub fn new(obj: &PyAny) -> PyResult { - let owned: Py = obj.extract()?; - mem::forget(owned.try_borrow_mut(obj.py())?); - Ok(RefMutGuard(owned)) + pub fn new(obj: &Bound<'_, PyAny>) -> PyResult { + let owned = obj.downcast::()?; + mem::forget(owned.try_borrow_mut()?); + Ok(RefMutGuard(owned.clone().unbind())) } } diff --git a/src/instance.rs b/src/instance.rs index 65c5ee3bc5f..bc655caa70f 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -488,7 +488,10 @@ impl<'py, T> Bound<'py, T> { where T: HasPyGilRef, { - unsafe { self.py().from_borrowed_ptr(self.as_ptr()) } + #[allow(deprecated)] + unsafe { + self.py().from_borrowed_ptr(self.as_ptr()) + } } /// Casts this `Bound` as the corresponding "GIL Ref" type, registering the pointer on the @@ -613,7 +616,10 @@ where { pub(crate) fn into_gil_ref(self) -> &'py T::AsRefTarget { // Safety: self is a borrow over `'py`. - unsafe { self.py().from_borrowed_ptr(self.0.as_ptr()) } + #[allow(deprecated)] + unsafe { + self.py().from_borrowed_ptr(self.0.as_ptr()) + } } } diff --git a/src/lib.rs b/src/lib.rs index c2591cec91e..8c0e6269947 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -304,9 +304,9 @@ //! [Features chapter of the guide]: https://pyo3.rs/latest/features.html#features-reference "Features Reference - PyO3 user guide" //! [`Ungil`]: crate::marker::Ungil pub use crate::class::*; -pub use crate::conversion::{AsPyPointer, FromPyObject, FromPyPointer, IntoPy, ToPyObject}; +pub use crate::conversion::{AsPyPointer, FromPyObject, IntoPy, ToPyObject}; #[allow(deprecated)] -pub use crate::conversion::{PyTryFrom, PyTryInto}; +pub use crate::conversion::{FromPyPointer, PyTryFrom, PyTryInto}; pub use crate::err::{ DowncastError, DowncastIntoError, PyDowncastError, PyErr, PyErrArguments, PyResult, ToPyErr, }; diff --git a/src/marker.rs b/src/marker.rs index 5609601f440..9b3d7329316 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -127,9 +127,9 @@ use crate::types::{ PyAny, PyDict, PyEllipsis, PyModule, PyNone, PyNotImplemented, PyString, PyType, }; use crate::version::PythonVersionInfo; -use crate::{ - ffi, Bound, FromPyPointer, IntoPy, Py, PyNativeType, PyObject, PyTypeCheck, PyTypeInfo, -}; +#[allow(deprecated)] +use crate::FromPyPointer; +use crate::{ffi, Bound, IntoPy, Py, PyNativeType, PyObject, PyTypeCheck, PyTypeInfo}; use std::ffi::{CStr, CString}; use std::marker::PhantomData; use std::os::raw::c_int; @@ -880,7 +880,7 @@ impl<'py> Python<'py> { /// # Safety /// /// Callers must ensure that ensure that the cast is valid. - #[allow(clippy::wrong_self_convention)] + #[allow(clippy::wrong_self_convention, deprecated)] #[cfg_attr( not(feature = "gil-refs"), deprecated( @@ -892,7 +892,6 @@ impl<'py> Python<'py> { where T: FromPyPointer<'py>, { - #[allow(deprecated)] FromPyPointer::from_owned_ptr(self, ptr) } @@ -904,7 +903,7 @@ impl<'py> Python<'py> { /// # Safety /// /// Callers must ensure that ensure that the cast is valid. - #[allow(clippy::wrong_self_convention)] + #[allow(clippy::wrong_self_convention, deprecated)] #[cfg_attr( not(feature = "gil-refs"), deprecated( @@ -916,7 +915,6 @@ impl<'py> Python<'py> { where T: FromPyPointer<'py>, { - #[allow(deprecated)] FromPyPointer::from_owned_ptr_or_err(self, ptr) } @@ -928,7 +926,7 @@ impl<'py> Python<'py> { /// # Safety /// /// Callers must ensure that ensure that the cast is valid. - #[allow(clippy::wrong_self_convention)] + #[allow(clippy::wrong_self_convention, deprecated)] #[cfg_attr( not(feature = "gil-refs"), deprecated( @@ -940,7 +938,6 @@ impl<'py> Python<'py> { where T: FromPyPointer<'py>, { - #[allow(deprecated)] FromPyPointer::from_owned_ptr_or_opt(self, ptr) } @@ -951,7 +948,14 @@ impl<'py> Python<'py> { /// # Safety /// /// Callers must ensure that ensure that the cast is valid. - #[allow(clippy::wrong_self_convention)] + #[allow(clippy::wrong_self_convention, deprecated)] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" + ) + )] pub unsafe fn from_borrowed_ptr(self, ptr: *mut ffi::PyObject) -> &'py T where T: FromPyPointer<'py>, @@ -966,7 +970,14 @@ impl<'py> Python<'py> { /// # Safety /// /// Callers must ensure that ensure that the cast is valid. - #[allow(clippy::wrong_self_convention)] + #[allow(clippy::wrong_self_convention, deprecated)] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr_or_err(py, ptr)` or `Bound::from_borrowed_ptr_or_err(py, ptr)` instead" + ) + )] pub unsafe fn from_borrowed_ptr_or_err(self, ptr: *mut ffi::PyObject) -> PyResult<&'py T> where T: FromPyPointer<'py>, @@ -981,7 +992,14 @@ impl<'py> Python<'py> { /// # Safety /// /// Callers must ensure that ensure that the cast is valid. - #[allow(clippy::wrong_self_convention)] + #[allow(clippy::wrong_self_convention, deprecated)] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr_or_opt(py, ptr)` or `Bound::from_borrowed_ptr_or_opt(py, ptr)` instead" + ) + )] pub unsafe fn from_borrowed_ptr_or_opt(self, ptr: *mut ffi::PyObject) -> Option<&'py T> where T: FromPyPointer<'py>, diff --git a/src/pycell.rs b/src/pycell.rs index 29fd1b37886..9d00de95cbb 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -62,6 +62,7 @@ //! ) -> *mut pyo3::ffi::PyObject { //! use :: pyo3 as _pyo3; //! _pyo3::impl_::trampoline::noargs(_slf, _args, |py, _slf| { +//! # #[allow(deprecated)] //! let _cell = py //! .from_borrowed_ptr::<_pyo3::PyAny>(_slf) //! .downcast::<_pyo3::PyCell>()?; @@ -191,6 +192,8 @@ //! [guide]: https://pyo3.rs/latest/class.html#pycell-and-interior-mutability "PyCell and interior mutability" //! [Interior Mutability]: https://doc.rust-lang.org/book/ch15-05-interior-mutability.html "RefCell and the Interior Mutability Pattern - The Rust Programming Language" +#[allow(deprecated)] +use crate::conversion::FromPyPointer; use crate::exceptions::PyRuntimeError; use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::pyclass::{ @@ -205,7 +208,7 @@ use crate::type_object::{PyLayout, PySizedLayout}; use crate::types::any::PyAnyMethods; use crate::types::PyAny; use crate::{ - conversion::{AsPyPointer, FromPyPointer, ToPyObject}, + conversion::{AsPyPointer, ToPyObject}, type_object::get_tp_free, PyTypeInfo, }; @@ -573,7 +576,10 @@ impl ToPyObject for &PyCell { impl AsRef for PyCell { fn as_ref(&self) -> &PyAny { - unsafe { self.py().from_borrowed_ptr(self.as_ptr()) } + #[allow(deprecated)] + unsafe { + self.py().from_borrowed_ptr(self.as_ptr()) + } } } @@ -581,7 +587,10 @@ impl Deref for PyCell { type Target = PyAny; fn deref(&self) -> &PyAny { - unsafe { self.py().from_borrowed_ptr(self.as_ptr()) } + #[allow(deprecated)] + unsafe { + self.py().from_borrowed_ptr(self.as_ptr()) + } } } diff --git a/src/type_object.rs b/src/type_object.rs index 994781b3fc0..318810b534d 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -76,7 +76,10 @@ pub unsafe trait PyTypeInfo: Sized + HasPyGilRef { fn type_object(py: Python<'_>) -> &PyType { // This isn't implemented in terms of `type_object_bound` because this just borrowed the // object, for legacy reasons. - unsafe { py.from_borrowed_ptr(Self::type_object_raw(py) as _) } + #[allow(deprecated)] + unsafe { + py.from_borrowed_ptr(Self::type_object_raw(py) as _) + } } /// Returns the safe abstraction over the type object. diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 3a2f60f6fa3..52e4c886aab 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -25,7 +25,10 @@ impl PyBool { )] #[inline] pub fn new(py: Python<'_>, val: bool) -> &PyBool { - unsafe { py.from_borrowed_ptr(if val { ffi::Py_True() } else { ffi::Py_False() }) } + #[allow(deprecated)] + unsafe { + py.from_borrowed_ptr(if val { ffi::Py_True() } else { ffi::Py_False() }) + } } /// Depending on `val`, returns `true` or `false`. From d94827720e5335b8281e32451bf5fed1102032cb Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Sat, 2 Mar 2024 22:53:28 +0000 Subject: [PATCH 192/349] Delete duplicate test code (#3926) These used to explicitly call `.iter()`, but that was removed in b65cbb9 to remove lints. --- src/types/frozenset.rs | 6 ------ src/types/set.rs | 6 ------ 2 files changed, 12 deletions(-) diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index cc16a9341c8..34452af0e87 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -351,12 +351,6 @@ mod tests { Python::with_gil(|py| { let set = PyFrozenSet::new(py, &[1]).unwrap(); - // iter method - for el in set { - assert_eq!(1i32, el.extract::().unwrap()); - } - - // intoiterator iteration for el in set { assert_eq!(1i32, el.extract::().unwrap()); } diff --git a/src/types/set.rs b/src/types/set.rs index 8e36ab81f8e..cf10f06e699 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -476,12 +476,6 @@ mod tests { Python::with_gil(|py| { let set = PySet::new(py, &[1]).unwrap(); - // iter method - for el in set { - assert_eq!(1i32, el.extract::<'_, i32>().unwrap()); - } - - // intoiterator iteration for el in set { assert_eq!(1i32, el.extract::<'_, i32>().unwrap()); } From 81be11e67a958f108c50d72d7fea23e837dd1ffb Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 2 Mar 2024 23:55:05 +0100 Subject: [PATCH 193/349] docs: update Python modules section of the guide (#3924) --- guide/src/module.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/guide/src/module.md b/guide/src/module.md index a403a3621e6..0444c750c9c 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -65,22 +65,22 @@ print(my_extension.__doc__) ## Python submodules You can create a module hierarchy within a single extension module by using -[`PyModule.add_submodule()`]({{#PYO3_DOCS_URL}}/pyo3/prelude/struct.PyModule.html#method.add_submodule). +[`Bound<'_, PyModule>::add_submodule()`]({{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyModuleMethods.html#tymethod.add_submodule). For example, you could define the modules `parent_module` and `parent_module.child_module`. ```rust use pyo3::prelude::*; #[pymodule] -fn parent_module(py: Python<'_>, m: &PyModule) -> PyResult<()> { - register_child_module(py, m)?; +fn parent_module(m: &Bound<'_, PyModule>) -> PyResult<()> { + register_child_module(m)?; Ok(()) } -fn register_child_module(py: Python<'_>, parent_module: &PyModule) -> PyResult<()> { - let child_module = PyModule::new_bound(py, "child_module")?; +fn register_child_module(parent_module: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new_bound(parent_module.py(), "child_module")?; child_module.add_function(wrap_pyfunction!(func, &child_module)?)?; - parent_module.add_submodule(child_module.as_gil_ref())?; + parent_module.add_submodule(&child_module)?; Ok(()) } From 2e56f659ed6a08f10f1e56a24251430878e7975f Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 3 Mar 2024 07:00:59 +0000 Subject: [PATCH 194/349] split `PyCell` and `PyClassObject` concepts (#3917) * add test for refguard ref counting * split `PyCell` and `PyClassObject` concepts * rework `create_cell` to `create_class_object` * Apply suggestions from code review Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> * review: Icxolu feedback --------- Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> --- pyo3-macros-backend/src/method.rs | 3 +- src/impl_/coroutine.rs | 35 +++--- src/impl_/pycell.rs | 4 +- src/impl_/pyclass.rs | 21 ++-- src/impl_/pymethods.rs | 14 ++- src/instance.rs | 46 +++---- src/pycell.rs | 197 +++++------------------------- src/pycell/impl_.rs | 170 ++++++++++++++++++++++++-- src/pyclass.rs | 16 +-- src/pyclass/create_type_object.rs | 5 +- src/pyclass_init.rs | 66 +++++----- src/type_object.rs | 2 +- src/types/mod.rs | 2 +- tests/test_coroutine.rs | 14 ++- 14 files changed, 301 insertions(+), 294 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 42f22204601..a9ba960d513 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -645,8 +645,7 @@ impl<'a> FnSpec<'a> { #( #holders )* let result = #call; let initializer: _pyo3::PyClassInitializer::<#cls> = result.convert(py)?; - let cell = initializer.create_cell_from_subtype(py, _slf)?; - ::std::result::Result::Ok(cell as *mut _pyo3::ffi::PyObject) + _pyo3::impl_::pymethods::tp_new_impl(py, initializer, _slf) } } } diff --git a/src/impl_/coroutine.rs b/src/impl_/coroutine.rs index 9be95ba3303..1d3119400a0 100644 --- a/src/impl_/coroutine.rs +++ b/src/impl_/coroutine.rs @@ -1,15 +1,15 @@ use std::{ future::Future, - mem, ops::{Deref, DerefMut}, }; use crate::{ coroutine::{cancel::ThrowCallback, Coroutine}, instance::Bound, + pycell::impl_::PyClassBorrowChecker, pyclass::boolean_struct::False, types::{PyAnyMethods, PyString}, - IntoPy, Py, PyAny, PyCell, PyClass, PyErr, PyObject, PyResult, Python, + IntoPy, Py, PyAny, PyClass, PyErr, PyObject, PyResult, Python, }; pub fn new_coroutine( @@ -32,17 +32,16 @@ where } fn get_ptr(obj: &Py) -> *mut T { - // SAFETY: Py can be casted as *const PyCell - unsafe { &*(obj.as_ptr() as *const PyCell) }.get_ptr() + obj.get_class_object().get_ptr() } pub struct RefGuard(Py); impl RefGuard { pub fn new(obj: &Bound<'_, PyAny>) -> PyResult { - let owned = obj.downcast::()?; - mem::forget(owned.try_borrow()?); - Ok(RefGuard(owned.clone().unbind())) + let bound = obj.downcast::()?; + bound.get_class_object().borrow_checker().try_borrow()?; + Ok(RefGuard(bound.clone().unbind())) } } @@ -57,9 +56,11 @@ impl Deref for RefGuard { impl Drop for RefGuard { fn drop(&mut self) { Python::with_gil(|gil| { - #[allow(deprecated)] - let self_ref = self.0.bind(gil); - self_ref.release_ref() + self.0 + .bind(gil) + .get_class_object() + .borrow_checker() + .release_borrow() }) } } @@ -68,9 +69,9 @@ pub struct RefMutGuard>(Py); impl> RefMutGuard { pub fn new(obj: &Bound<'_, PyAny>) -> PyResult { - let owned = obj.downcast::()?; - mem::forget(owned.try_borrow_mut()?); - Ok(RefMutGuard(owned.clone().unbind())) + let bound = obj.downcast::()?; + bound.get_class_object().borrow_checker().try_borrow_mut()?; + Ok(RefMutGuard(bound.clone().unbind())) } } @@ -92,9 +93,11 @@ impl> DerefMut for RefMutGuard { impl> Drop for RefMutGuard { fn drop(&mut self) { Python::with_gil(|gil| { - #[allow(deprecated)] - let self_ref = self.0.bind(gil); - self_ref.release_mut() + self.0 + .bind(gil) + .get_class_object() + .borrow_checker() + .release_borrow_mut() }) } } diff --git a/src/impl_/pycell.rs b/src/impl_/pycell.rs index 39811aebd29..93514c7bb29 100644 --- a/src/impl_/pycell.rs +++ b/src/impl_/pycell.rs @@ -1,2 +1,4 @@ //! Externally-accessible implementation of pycell -pub use crate::pycell::impl_::{GetBorrowChecker, PyClassMutability}; +pub use crate::pycell::impl_::{ + GetBorrowChecker, PyClassMutability, PyClassObject, PyClassObjectBase, PyClassObjectLayout, +}; diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index e2204fabb02..1a144f736e0 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -2,14 +2,13 @@ use crate::{ exceptions::{PyAttributeError, PyNotImplementedError, PyRuntimeError, PyValueError}, ffi, impl_::freelist::FreeList, - impl_::pycell::{GetBorrowChecker, PyClassMutability}, + impl_::pycell::{GetBorrowChecker, PyClassMutability, PyClassObjectLayout}, internal_tricks::extract_c_string, - pycell::PyCellLayout, pyclass_init::PyObjectInit, types::any::PyAnyMethods, types::PyBool, - Borrowed, Py, PyAny, PyCell, PyClass, PyErr, PyMethodDefType, PyNativeType, PyResult, - PyTypeInfo, Python, + Borrowed, Py, PyAny, PyClass, PyErr, PyMethodDefType, PyNativeType, PyResult, PyTypeInfo, + Python, }; use std::{ borrow::Cow, @@ -26,13 +25,13 @@ pub use lazy_type_object::LazyTypeObject; /// Gets the offset of the dictionary from the start of the object in bytes. #[inline] pub fn dict_offset() -> ffi::Py_ssize_t { - PyCell::::dict_offset() + PyClassObject::::dict_offset() } /// Gets the offset of the weakref list from the start of the object in bytes. #[inline] pub fn weaklist_offset() -> ffi::Py_ssize_t { - PyCell::::weaklist_offset() + PyClassObject::::weaklist_offset() } /// Represents the `__dict__` field for `#[pyclass]`. @@ -883,6 +882,8 @@ macro_rules! generate_pyclass_richcompare_slot { } pub use generate_pyclass_richcompare_slot; +use super::pycell::PyClassObject; + /// Implements a freelist. /// /// Do not implement this trait manually. Instead, use `#[pyclass(freelist = N)]` @@ -1095,7 +1096,7 @@ impl PyClassThreadChecker for ThreadCheckerImpl { /// Trait denoting that this class is suitable to be used as a base type for PyClass. pub trait PyClassBaseType: Sized { - type LayoutAsBase: PyCellLayout; + type LayoutAsBase: PyClassObjectLayout; type BaseNativeType; type Initializer: PyObjectInit; type PyClassMutability: PyClassMutability; @@ -1105,7 +1106,7 @@ pub trait PyClassBaseType: Sized { /// /// In the future this will be extended to immutable PyClasses too. impl PyClassBaseType for T { - type LayoutAsBase = crate::pycell::PyCell; + type LayoutAsBase = crate::impl_::pycell::PyClassObject; type BaseNativeType = T::BaseNativeType; type Initializer = crate::pyclass_init::PyClassInitializer; type PyClassMutability = T::PyClassMutability; @@ -1113,7 +1114,7 @@ impl PyClassBaseType for T { /// Implementation of tp_dealloc for pyclasses without gc pub(crate) unsafe extern "C" fn tp_dealloc(obj: *mut ffi::PyObject) { - crate::impl_::trampoline::dealloc(obj, PyCell::::tp_dealloc) + crate::impl_::trampoline::dealloc(obj, PyClassObject::::tp_dealloc) } /// Implementation of tp_dealloc for pyclasses with gc @@ -1122,7 +1123,7 @@ pub(crate) unsafe extern "C" fn tp_dealloc_with_gc(obj: *mut ffi::Py { ffi::PyObject_GC_UnTrack(obj.cast()); } - crate::impl_::trampoline::dealloc(obj, PyCell::::tp_dealloc) + crate::impl_::trampoline::dealloc(obj, PyClassObject::::tp_dealloc) } pub(crate) unsafe extern "C" fn get_sequence_item_from_mapping( diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 70c95ca0883..bc950bafee4 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -7,8 +7,8 @@ use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; use crate::types::{any::PyAnyMethods, PyModule, PyType}; use crate::{ - ffi, Borrowed, Bound, DowncastError, Py, PyAny, PyCell, PyClass, PyErr, PyObject, PyRef, - PyRefMut, PyResult, PyTraverseError, PyTypeCheck, PyVisit, Python, + ffi, Borrowed, Bound, DowncastError, Py, PyAny, PyCell, PyClass, PyClassInitializer, PyErr, + PyObject, PyRef, PyRefMut, PyResult, PyTraverseError, PyTypeCheck, PyVisit, Python, }; use std::borrow::Cow; use std::ffi::CStr; @@ -569,3 +569,13 @@ impl<'py, T> std::ops::Deref for BoundRef<'_, 'py, T> { self.0 } } + +pub unsafe fn tp_new_impl( + py: Python<'_>, + initializer: PyClassInitializer, + target_type: *mut ffi::PyTypeObject, +) -> PyResult<*mut ffi::PyObject> { + initializer + .create_class_object_of_type(py, target_type) + .map(Bound::into_ptr) +} diff --git a/src/instance.rs b/src/instance.rs index bc655caa70f..66c530d5f53 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,5 +1,5 @@ use crate::err::{self, PyDowncastError, PyErr, PyResult}; -use crate::ffi_ptr_ext::FfiPtrExt; +use crate::impl_::pycell::PyClassObject; use crate::pycell::{PyBorrowError, PyBorrowMutError, PyCell}; use crate::pyclass::boolean_struct::{False, True}; use crate::type_object::HasPyGilRef; @@ -92,14 +92,7 @@ where py: Python<'py>, value: impl Into>, ) -> PyResult> { - let initializer = value.into(); - let obj = initializer.create_cell(py)?; - let ob = unsafe { - obj.cast::() - .assume_owned(py) - .downcast_into_unchecked() - }; - Ok(ob) + value.into().create_class_object(py) } } @@ -332,19 +325,11 @@ where where T: PyClass + Sync, { - let cell = self.get_cell(); - // SAFETY: The class itself is frozen and `Sync` and we do not access anything but `cell.contents.value`. - unsafe { &*cell.get_ptr() } + self.1.get() } - pub(crate) fn get_cell(&'py self) -> &'py PyCell { - let cell = self.as_ptr().cast::>(); - // SAFETY: Bound is known to contain an object which is laid out in memory as a - // PyCell. - // - // Strictly speaking for now `&'py PyCell` is part of the "GIL Ref" API, so this - // could use some further refactoring later to avoid going through this reference. - unsafe { &*cell } + pub(crate) fn get_class_object(&self) -> &PyClassObject { + self.1.get_class_object() } } @@ -887,10 +872,7 @@ where /// # } /// ``` pub fn new(py: Python<'_>, value: impl Into>) -> PyResult> { - let initializer = value.into(); - let obj = initializer.create_cell(py)?; - let ob = unsafe { Py::from_owned_ptr(py, obj as _) }; - Ok(ob) + Bound::new(py, value).map(Bound::unbind) } } @@ -1194,12 +1176,16 @@ where where T: PyClass + Sync, { - let any = self.as_ptr() as *const PyAny; - // SAFETY: The class itself is frozen and `Sync` and we do not access anything but `cell.contents.value`. - unsafe { - let cell: &PyCell = PyNativeType::unchecked_downcast(&*any); - &*cell.get_ptr() - } + // Safety: The class itself is frozen and `Sync` + unsafe { &*self.get_class_object().get_ptr() } + } + + /// Get a view on the underlying `PyClass` contents. + pub(crate) fn get_class_object(&self) -> &PyClassObject { + let class_object = self.as_ptr().cast::>(); + // Safety: Bound is known to contain an object which is laid out in memory as a + // PyClassObject. + unsafe { &*class_object } } } diff --git a/src/pycell.rs b/src/pycell.rs index 9d00de95cbb..43c75314f00 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -192,13 +192,10 @@ //! [guide]: https://pyo3.rs/latest/class.html#pycell-and-interior-mutability "PyCell and interior mutability" //! [Interior Mutability]: https://doc.rust-lang.org/book/ch15-05-interior-mutability.html "RefCell and the Interior Mutability Pattern - The Rust Programming Language" -#[allow(deprecated)] -use crate::conversion::FromPyPointer; +use crate::conversion::{AsPyPointer, ToPyObject}; use crate::exceptions::PyRuntimeError; use crate::ffi_ptr_ext::FfiPtrExt; -use crate::impl_::pyclass::{ - PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, -}; +use crate::impl_::pyclass::PyClassImpl; use crate::pyclass::{ boolean_struct::{False, True}, PyClass, @@ -207,28 +204,15 @@ use crate::pyclass_init::PyClassInitializer; use crate::type_object::{PyLayout, PySizedLayout}; use crate::types::any::PyAnyMethods; use crate::types::PyAny; -use crate::{ - conversion::{AsPyPointer, ToPyObject}, - type_object::get_tp_free, - PyTypeInfo, -}; use crate::{ffi, Bound, IntoPy, PyErr, PyNativeType, PyObject, PyResult, PyTypeCheck, Python}; -use std::cell::UnsafeCell; use std::fmt; use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; pub(crate) mod impl_; -use impl_::{GetBorrowChecker, PyClassBorrowChecker, PyClassMutability}; - -/// Base layout of PyCell. -#[doc(hidden)] -#[repr(C)] -pub struct PyCellBase { - ob_base: T, -} +use impl_::PyClassBorrowChecker; -unsafe impl PyLayout for PyCellBase where U: PySizedLayout {} +use self::impl_::{PyClassObject, PyClassObjectLayout}; /// A container type for (mutably) accessing [`PyClass`] values /// @@ -266,20 +250,8 @@ unsafe impl PyLayout for PyCellBase where U: PySizedLayout {} /// ``` /// For more information on how, when and why (not) to use `PyCell` please see the /// [module-level documentation](self). -#[repr(C)] -pub struct PyCell { - ob_base: ::LayoutAsBase, - contents: PyCellContents, -} - -#[repr(C)] -pub(crate) struct PyCellContents { - pub(crate) value: ManuallyDrop>, - pub(crate) borrow_checker: ::Storage, - pub(crate) thread_checker: T::ThreadChecker, - pub(crate) dict: T::Dict, - pub(crate) weakref: T::WeakRef, -} +#[repr(transparent)] +pub struct PyCell(PyClassObject); unsafe impl PyNativeType for PyCell { type AsRefSource = T; @@ -298,12 +270,7 @@ impl PyCell { ) )] pub fn new(py: Python<'_>, value: impl Into>) -> PyResult<&Self> { - unsafe { - let initializer = value.into(); - let self_ = initializer.create_cell(py)?; - #[allow(deprecated)] - FromPyPointer::from_owned_ptr_or_err(py, self_ as _) - } + Bound::new(py, value).map(Bound::into_gil_ref) } /// Immutably borrows the value `T`. This borrow lasts as long as the returned `PyRef` exists. @@ -423,10 +390,11 @@ impl PyCell { /// }); /// ``` pub unsafe fn try_borrow_unguarded(&self) -> Result<&T, PyBorrowError> { - self.ensure_threadsafe(); - self.borrow_checker() + self.0.ensure_threadsafe(); + self.0 + .borrow_checker() .try_borrow_unguarded() - .map(|_: ()| &*self.contents.value.get()) + .map(|_: ()| &*self.0.get_ptr()) } /// Provide an immutable borrow of the value `T` without acquiring the GIL. @@ -506,45 +474,7 @@ impl PyCell { } pub(crate) fn get_ptr(&self) -> *mut T { - self.contents.value.get() - } - - /// Gets the offset of the dictionary from the start of the struct in bytes. - pub(crate) fn dict_offset() -> ffi::Py_ssize_t { - use memoffset::offset_of; - - let offset = offset_of!(PyCell, contents) + offset_of!(PyCellContents, dict); - - // Py_ssize_t may not be equal to isize on all platforms - #[allow(clippy::useless_conversion)] - offset.try_into().expect("offset should fit in Py_ssize_t") - } - - /// Gets the offset of the weakref list from the start of the struct in bytes. - pub(crate) fn weaklist_offset() -> ffi::Py_ssize_t { - use memoffset::offset_of; - - let offset = offset_of!(PyCell, contents) + offset_of!(PyCellContents, weakref); - - // Py_ssize_t may not be equal to isize on all platforms - #[allow(clippy::useless_conversion)] - offset.try_into().expect("offset should fit in Py_ssize_t") - } - - #[cfg(feature = "macros")] - pub(crate) fn release_ref(&self) { - self.borrow_checker().release_borrow(); - } - - #[cfg(feature = "macros")] - pub(crate) fn release_mut(&self) { - self.borrow_checker().release_borrow_mut(); - } -} - -impl PyCell { - fn borrow_checker(&self) -> &::Checker { - T::PyClassMutability::borrow_checker(self) + self.0.get_ptr() } } @@ -675,7 +605,7 @@ where U: PyClass, { fn as_ref(&self) -> &T::BaseType { - unsafe { &*self.inner.get_cell().ob_base.get_ptr() } + unsafe { &*self.inner.get_class_object().ob_base.get_ptr() } } } @@ -709,7 +639,7 @@ impl<'py, T: PyClass> PyRef<'py, T> { } pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { - let cell = obj.get_cell(); + let cell = obj.get_class_object(); cell.ensure_threadsafe(); cell.borrow_checker() .try_borrow() @@ -717,7 +647,7 @@ impl<'py, T: PyClass> PyRef<'py, T> { } pub(crate) fn try_borrow_threadsafe(obj: &Bound<'py, T>) -> Result { - let cell = obj.get_cell(); + let cell = obj.get_class_object(); cell.check_threadsafe()?; cell.borrow_checker() .try_borrow() @@ -793,13 +723,16 @@ impl<'p, T: PyClass> Deref for PyRef<'p, T> { #[inline] fn deref(&self) -> &T { - unsafe { &*self.inner.get_cell().get_ptr() } + unsafe { &*self.inner.get_class_object().get_ptr() } } } impl<'p, T: PyClass> Drop for PyRef<'p, T> { fn drop(&mut self) { - self.inner.get_cell().borrow_checker().release_borrow() + self.inner + .get_class_object() + .borrow_checker() + .release_borrow() } } @@ -856,7 +789,7 @@ where U: PyClass, { fn as_ref(&self) -> &T::BaseType { - unsafe { &*self.inner.get_cell().ob_base.get_ptr() } + unsafe { &*self.inner.get_class_object().ob_base.get_ptr() } } } @@ -866,7 +799,7 @@ where U: PyClass, { fn as_mut(&mut self) -> &mut T::BaseType { - unsafe { &mut *self.inner.get_cell().ob_base.get_ptr() } + unsafe { &mut *self.inner.get_class_object().ob_base.get_ptr() } } } @@ -900,7 +833,7 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { } pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { - let cell = obj.get_cell(); + let cell = obj.get_class_object(); cell.ensure_threadsafe(); cell.borrow_checker() .try_borrow_mut() @@ -934,20 +867,23 @@ impl<'p, T: PyClass> Deref for PyRefMut<'p, T> { #[inline] fn deref(&self) -> &T { - unsafe { &*self.inner.get_cell().get_ptr() } + unsafe { &*self.inner.get_class_object().get_ptr() } } } impl<'p, T: PyClass> DerefMut for PyRefMut<'p, T> { #[inline] fn deref_mut(&mut self) -> &mut T { - unsafe { &mut *self.inner.get_cell().get_ptr() } + unsafe { &mut *self.inner.get_class_object().get_ptr() } } } impl<'p, T: PyClass> Drop for PyRefMut<'p, T> { fn drop(&mut self) { - self.inner.get_cell().borrow_checker().release_borrow_mut() + self.inner + .get_class_object() + .borrow_checker() + .release_borrow_mut() } } @@ -1034,81 +970,6 @@ impl From for PyErr { } } -#[doc(hidden)] -pub trait PyCellLayout: PyLayout { - fn ensure_threadsafe(&self); - fn check_threadsafe(&self) -> Result<(), PyBorrowError>; - /// Implementation of tp_dealloc. - /// # Safety - /// - slf must be a valid pointer to an instance of a T or a subclass. - /// - slf must not be used after this call (as it will be freed). - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject); -} - -impl PyCellLayout for PyCellBase -where - U: PySizedLayout, - T: PyTypeInfo, -{ - fn ensure_threadsafe(&self) {} - fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - Ok(()) - } - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { - let type_obj = T::type_object_raw(py); - // For `#[pyclass]` types which inherit from PyAny, we can just call tp_free - if type_obj == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type) { - return get_tp_free(ffi::Py_TYPE(slf))(slf as _); - } - - // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. - #[cfg(not(Py_LIMITED_API))] - { - if let Some(dealloc) = (*type_obj).tp_dealloc { - // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which - // assumes the exception is currently GC tracked, so we have to re-track - // before calling the dealloc so that it can safely call Py_GC_UNTRACK. - #[cfg(not(any(Py_3_11, PyPy)))] - if ffi::PyType_FastSubclass(type_obj, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 { - ffi::PyObject_GC_Track(slf.cast()); - } - dealloc(slf as _); - } else { - get_tp_free(ffi::Py_TYPE(slf))(slf as _); - } - } - - #[cfg(Py_LIMITED_API)] - unreachable!("subclassing native types is not possible with the `abi3` feature"); - } -} - -impl PyCellLayout for PyCell -where - ::LayoutAsBase: PyCellLayout, -{ - fn ensure_threadsafe(&self) { - self.contents.thread_checker.ensure(); - self.ob_base.ensure_threadsafe(); - } - fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - if !self.contents.thread_checker.check() { - return Err(PyBorrowError { _private: () }); - } - self.ob_base.check_threadsafe() - } - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { - // Safety: Python only calls tp_dealloc when no references to the object remain. - let cell = &mut *(slf as *mut PyCell); - if cell.contents.thread_checker.can_drop(py) { - ManuallyDrop::drop(&mut cell.contents.value); - } - cell.contents.dict.clear_dict(py); - cell.contents.weakref.clear_weakrefs(slf, py); - ::LayoutAsBase::tp_dealloc(py, slf) - } -} - #[cfg(test)] #[cfg(feature = "macros")] mod tests { diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 29ba7eda7eb..378bec04993 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -1,11 +1,15 @@ #![allow(missing_docs)] -//! Crate-private implementation of pycell +//! Crate-private implementation of PyClassObject -use std::cell::Cell; +use std::cell::{Cell, UnsafeCell}; use std::marker::PhantomData; +use std::mem::ManuallyDrop; -use crate::impl_::pyclass::{PyClassBaseType, PyClassImpl}; -use crate::PyCell; +use crate::impl_::pyclass::{ + PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, +}; +use crate::type_object::{get_tp_free, PyLayout, PySizedLayout}; +use crate::{ffi, PyClass, PyTypeInfo, Python}; use super::{PyBorrowError, PyBorrowMutError}; @@ -156,29 +160,170 @@ impl PyClassBorrowChecker for BorrowChecker { } pub trait GetBorrowChecker { - fn borrow_checker(cell: &PyCell) -> &::Checker; + fn borrow_checker( + class_object: &PyClassObject, + ) -> &::Checker; } impl> GetBorrowChecker for MutableClass { - fn borrow_checker(cell: &PyCell) -> &BorrowChecker { - &cell.contents.borrow_checker + fn borrow_checker(class_object: &PyClassObject) -> &BorrowChecker { + &class_object.contents.borrow_checker } } impl> GetBorrowChecker for ImmutableClass { - fn borrow_checker(cell: &PyCell) -> &EmptySlot { - &cell.contents.borrow_checker + fn borrow_checker(class_object: &PyClassObject) -> &EmptySlot { + &class_object.contents.borrow_checker } } impl, M: PyClassMutability> GetBorrowChecker for ExtendsMutableAncestor where - T::BaseType: PyClassImpl + PyClassBaseType>, + T::BaseType: PyClassImpl + PyClassBaseType>, ::PyClassMutability: PyClassMutability, { - fn borrow_checker(cell: &PyCell) -> &BorrowChecker { - <::PyClassMutability as GetBorrowChecker>::borrow_checker(&cell.ob_base) + fn borrow_checker(class_object: &PyClassObject) -> &BorrowChecker { + <::PyClassMutability as GetBorrowChecker>::borrow_checker(&class_object.ob_base) + } +} + +/// Base layout of PyClassObject. +#[doc(hidden)] +#[repr(C)] +pub struct PyClassObjectBase { + ob_base: T, +} + +unsafe impl PyLayout for PyClassObjectBase where U: PySizedLayout {} + +#[doc(hidden)] +pub trait PyClassObjectLayout: PyLayout { + fn ensure_threadsafe(&self); + fn check_threadsafe(&self) -> Result<(), PyBorrowError>; + /// Implementation of tp_dealloc. + /// # Safety + /// - slf must be a valid pointer to an instance of a T or a subclass. + /// - slf must not be used after this call (as it will be freed). + unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject); +} + +impl PyClassObjectLayout for PyClassObjectBase +where + U: PySizedLayout, + T: PyTypeInfo, +{ + fn ensure_threadsafe(&self) {} + fn check_threadsafe(&self) -> Result<(), PyBorrowError> { + Ok(()) + } + unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { + let type_obj = T::type_object_raw(py); + // For `#[pyclass]` types which inherit from PyAny, we can just call tp_free + if type_obj == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type) { + return get_tp_free(ffi::Py_TYPE(slf))(slf.cast()); + } + + // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. + #[cfg(not(Py_LIMITED_API))] + { + if let Some(dealloc) = (*type_obj).tp_dealloc { + // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which + // assumes the exception is currently GC tracked, so we have to re-track + // before calling the dealloc so that it can safely call Py_GC_UNTRACK. + #[cfg(not(any(Py_3_11, PyPy)))] + if ffi::PyType_FastSubclass(type_obj, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 { + ffi::PyObject_GC_Track(slf.cast()); + } + dealloc(slf); + } else { + get_tp_free(ffi::Py_TYPE(slf))(slf.cast()); + } + } + + #[cfg(Py_LIMITED_API)] + unreachable!("subclassing native types is not possible with the `abi3` feature"); + } +} + +/// The layout of a PyClass as a Python object +#[repr(C)] +pub struct PyClassObject { + pub(crate) ob_base: ::LayoutAsBase, + contents: PyClassObjectContents, +} + +#[repr(C)] +pub(crate) struct PyClassObjectContents { + pub(crate) value: ManuallyDrop>, + pub(crate) borrow_checker: ::Storage, + pub(crate) thread_checker: T::ThreadChecker, + pub(crate) dict: T::Dict, + pub(crate) weakref: T::WeakRef, +} + +impl PyClassObject { + pub(crate) fn get_ptr(&self) -> *mut T { + self.contents.value.get() + } + + /// Gets the offset of the dictionary from the start of the struct in bytes. + pub(crate) fn dict_offset() -> ffi::Py_ssize_t { + use memoffset::offset_of; + + let offset = + offset_of!(PyClassObject, contents) + offset_of!(PyClassObjectContents, dict); + + // Py_ssize_t may not be equal to isize on all platforms + #[allow(clippy::useless_conversion)] + offset.try_into().expect("offset should fit in Py_ssize_t") + } + + /// Gets the offset of the weakref list from the start of the struct in bytes. + pub(crate) fn weaklist_offset() -> ffi::Py_ssize_t { + use memoffset::offset_of; + + let offset = + offset_of!(PyClassObject, contents) + offset_of!(PyClassObjectContents, weakref); + + // Py_ssize_t may not be equal to isize on all platforms + #[allow(clippy::useless_conversion)] + offset.try_into().expect("offset should fit in Py_ssize_t") + } +} + +impl PyClassObject { + pub(crate) fn borrow_checker(&self) -> &::Checker { + T::PyClassMutability::borrow_checker(self) + } +} + +unsafe impl PyLayout for PyClassObject {} +impl PySizedLayout for PyClassObject {} + +impl PyClassObjectLayout for PyClassObject +where + ::LayoutAsBase: PyClassObjectLayout, +{ + fn ensure_threadsafe(&self) { + self.contents.thread_checker.ensure(); + self.ob_base.ensure_threadsafe(); + } + fn check_threadsafe(&self) -> Result<(), PyBorrowError> { + if !self.contents.thread_checker.check() { + return Err(PyBorrowError { _private: () }); + } + self.ob_base.check_threadsafe() + } + unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { + // Safety: Python only calls tp_dealloc when no references to the object remain. + let class_object = &mut *(slf.cast::>()); + if class_object.contents.thread_checker.can_drop(py) { + ManuallyDrop::drop(&mut class_object.contents.value); + } + class_object.contents.dict.clear_dict(py); + class_object.contents.weakref.clear_weakrefs(slf, py); + ::LayoutAsBase::tp_dealloc(py, slf) } } @@ -189,7 +334,6 @@ mod tests { use crate::prelude::*; use crate::pyclass::boolean_struct::{False, True}; - use crate::PyClass; #[pyclass(crate = "crate", subclass)] struct MutableBase; diff --git a/src/pyclass.rs b/src/pyclass.rs index ebdc52dc217..eb4a5595ca9 100644 --- a/src/pyclass.rs +++ b/src/pyclass.rs @@ -1,7 +1,7 @@ //! `PyClass` and related traits. use crate::{ - callback::IntoPyCallbackOutput, ffi, impl_::pyclass::PyClassImpl, Bound, IntoPy, PyCell, - PyObject, PyResult, PyTypeInfo, Python, + callback::IntoPyCallbackOutput, ffi, impl_::pyclass::PyClassImpl, IntoPy, PyCell, PyObject, + PyResult, PyTypeInfo, Python, }; use std::{cmp::Ordering, os::raw::c_int}; @@ -216,18 +216,6 @@ pub trait Frozen: boolean_struct::private::Boolean {} impl Frozen for boolean_struct::True {} impl Frozen for boolean_struct::False {} -impl<'py, T: PyClass> Bound<'py, T> { - #[cfg(feature = "macros")] - pub(crate) fn release_ref(&self) { - self.get_cell().release_ref(); - } - - #[cfg(feature = "macros")] - pub(crate) fn release_mut(&self) { - self.get_cell().release_mut(); - } -} - mod tests { #[test] fn test_compare_op_matches() { diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index d0c271cf31a..52e346212f0 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -3,6 +3,7 @@ use pyo3_ffi::PyType_IS_GC; use crate::{ exceptions::PyTypeError, ffi, + impl_::pycell::PyClassObject, impl_::pyclass::{ assign_sequence_item_from_mapping, get_sequence_item_from_mapping, tp_dealloc, tp_dealloc_with_gc, PyClassItemsIter, @@ -13,7 +14,7 @@ use crate::{ }, types::typeobject::PyTypeMethods, types::PyType, - Py, PyCell, PyClass, PyGetterDef, PyMethodDefType, PyResult, PySetterDef, PyTypeInfo, Python, + Py, PyClass, PyGetterDef, PyMethodDefType, PyResult, PySetterDef, PyTypeInfo, Python, }; use std::{ borrow::Cow, @@ -94,7 +95,7 @@ where T::items_iter(), T::NAME, T::MODULE, - std::mem::size_of::>(), + std::mem::size_of::>(), ) } } diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 94d377a732c..923bc5b7c5a 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -1,13 +1,12 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::callback::IntoPyCallbackOutput; +use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::pyclass::{PyClassBaseType, PyClassDict, PyClassThreadChecker, PyClassWeakRef}; -use crate::{ffi, Py, PyCell, PyClass, PyErr, PyResult, Python}; +use crate::types::PyAnyMethods; +use crate::{ffi, Bound, Py, PyClass, PyErr, PyResult, Python}; use crate::{ ffi::PyTypeObject, - pycell::{ - impl_::{PyClassBorrowChecker, PyClassMutability}, - PyCellContents, - }, + pycell::impl_::{PyClassBorrowChecker, PyClassMutability, PyClassObjectContents}, type_object::{get_tp_alloc, PyTypeInfo}, }; use std::{ @@ -207,57 +206,44 @@ impl PyClassInitializer { } /// Creates a new PyCell and initializes it. - #[doc(hidden)] - pub fn create_cell(self, py: Python<'_>) -> PyResult<*mut PyCell> + pub(crate) fn create_class_object(self, py: Python<'_>) -> PyResult> where T: PyClass, { - unsafe { self.create_cell_from_subtype(py, T::type_object_raw(py)) } + unsafe { self.create_class_object_of_type(py, T::type_object_raw(py)) } } - /// Creates a new PyCell and initializes it given a typeobject `subtype`. - /// Called by the Python `tp_new` implementation generated by a `#[new]` function in a `#[pymethods]` block. + /// Creates a new class object and initializes it given a typeobject `subtype`. /// /// # Safety /// `subtype` must be a valid pointer to the type object of T or a subclass. - #[doc(hidden)] - pub unsafe fn create_cell_from_subtype( + pub(crate) unsafe fn create_class_object_of_type( self, py: Python<'_>, - subtype: *mut crate::ffi::PyTypeObject, - ) -> PyResult<*mut PyCell> + target_type: *mut crate::ffi::PyTypeObject, + ) -> PyResult> where T: PyClass, { - self.into_new_object(py, subtype).map(|obj| obj as _) - } -} - -impl PyObjectInit for PyClassInitializer { - unsafe fn into_new_object( - self, - py: Python<'_>, - subtype: *mut PyTypeObject, - ) -> PyResult<*mut ffi::PyObject> { - /// Layout of a PyCell after base new has been called, but the contents have not yet been + /// Layout of a PyClassObject after base new has been called, but the contents have not yet been /// written. #[repr(C)] - struct PartiallyInitializedPyCell { + struct PartiallyInitializedClassObject { _ob_base: ::LayoutAsBase, - contents: MaybeUninit>, + contents: MaybeUninit>, } let (init, super_init) = match self.0 { - PyClassInitializerImpl::Existing(value) => return Ok(value.into_ptr()), + PyClassInitializerImpl::Existing(value) => return Ok(value.into_bound(py)), PyClassInitializerImpl::New { init, super_init } => (init, super_init), }; - let obj = super_init.into_new_object(py, subtype)?; + let obj = super_init.into_new_object(py, target_type)?; - let cell: *mut PartiallyInitializedPyCell = obj as _; + let part_init: *mut PartiallyInitializedClassObject = obj.cast(); std::ptr::write( - (*cell).contents.as_mut_ptr(), - PyCellContents { + (*part_init).contents.as_mut_ptr(), + PyClassObjectContents { value: ManuallyDrop::new(UnsafeCell::new(init)), borrow_checker: ::Storage::new(), thread_checker: T::ThreadChecker::new(), @@ -265,7 +251,21 @@ impl PyObjectInit for PyClassInitializer { weakref: T::WeakRef::INIT, }, ); - Ok(obj) + + // Safety: obj is a valid pointer to an object of type `target_type`, which` is a known + // subclass of `T` + Ok(obj.assume_owned(py).downcast_into_unchecked()) + } +} + +impl PyObjectInit for PyClassInitializer { + unsafe fn into_new_object( + self, + py: Python<'_>, + subtype: *mut PyTypeObject, + ) -> PyResult<*mut ffi::PyObject> { + self.create_class_object_of_type(py, subtype) + .map(Bound::into_ptr) } private_impl! {} diff --git a/src/type_object.rs b/src/type_object.rs index 318810b534d..84888bee458 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -6,7 +6,7 @@ use crate::types::{PyAny, PyType}; use crate::{ffi, Bound, PyNativeType, Python}; /// `T: PyLayout` represents that `T` is a concrete representation of `U` in the Python heap. -/// E.g., `PyCell` is a concrete representation of all `pyclass`es, and `ffi::PyObject` +/// E.g., `PyClassObject` is a concrete representation of all `pyclass`es, and `ffi::PyObject` /// is of `PyAny`. /// /// This trait is intended to be used internally. diff --git a/src/types/mod.rs b/src/types/mod.rs index 938716f78f4..66312a98a0c 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -290,7 +290,7 @@ macro_rules! pyobject_native_type_sized { unsafe impl $crate::type_object::PyLayout<$name> for $layout {} impl $crate::type_object::PySizedLayout<$name> for $layout {} impl<$($generics,)*> $crate::impl_::pyclass::PyClassBaseType for $name { - type LayoutAsBase = $crate::pycell::PyCellBase<$layout>; + type LayoutAsBase = $crate::impl_::pycell::PyClassObjectBase<$layout>; type BaseNativeType = $name; type Initializer = $crate::pyclass_init::PyNativeTypeInitializer; type PyClassMutability = $crate::pycell::impl_::ImmutableClass; diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index e04aafda24d..db79c72a233 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -3,6 +3,7 @@ use std::{task::Poll, thread, time::Duration}; use futures::{channel::oneshot, future::poll_fn, FutureExt}; +use portable_atomic::{AtomicBool, Ordering}; use pyo3::{ coroutine::CancelHandle, prelude::*, @@ -259,6 +260,15 @@ fn test_async_method_receiver() { self.0 } } + + static IS_DROPPED: AtomicBool = AtomicBool::new(false); + + impl Drop for Counter { + fn drop(&mut self) { + IS_DROPPED.store(true, Ordering::SeqCst); + } + } + Python::with_gil(|gil| { let test = r#" import asyncio @@ -291,5 +301,7 @@ fn test_async_method_receiver() { "#; let locals = [("Counter", gil.get_type_bound::())].into_py_dict_bound(gil); py_run!(gil, *locals, test); - }) + }); + + assert!(IS_DROPPED.load(Ordering::SeqCst)); } From 00eb0146237e16ae902b1f685e1d3fd2173ba104 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sun, 3 Mar 2024 10:15:46 +0100 Subject: [PATCH 195/349] docs: update Python function section of the guide (#3925) * docs: update Python function section of the guide * update `pass_module` types Co-authored-by: David Hewitt --------- Co-authored-by: David Hewitt --- guide/src/function.md | 29 ++++++++++++++-------------- guide/src/function/error_handling.md | 10 ++++------ guide/src/function/signature.md | 22 ++++++++++----------- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/guide/src/function.md b/guide/src/function.md index 2ed38f6256f..1bba52f2235 100644 --- a/guide/src/function.md +++ b/guide/src/function.md @@ -77,7 +77,7 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python - `#[pyo3(pass_module)]` - Set this option to make PyO3 pass the containing module as the first argument to the function. It is then possible to use the module in the function body. The first argument **must** be of type `&PyModule`. + Set this option to make PyO3 pass the containing module as the first argument to the function. It is then possible to use the module in the function body. The first argument **must** be of type `&Bound<'_, PyModule>`, `Bound<'_, PyModule>`, or `Py`. The following example creates a function `pyfunction_with_module` which returns the containing module's name (i.e. `module_with_fn`): @@ -103,14 +103,14 @@ The `#[pyo3]` attribute can be used on individual arguments to modify properties - `#[pyo3(from_py_with = "...")]` - Set this on an option to specify a custom function to convert the function argument from Python to the desired Rust type, instead of using the default `FromPyObject` extraction. The function signature must be `fn(&PyAny) -> PyResult` where `T` is the Rust type of the argument. + Set this on an option to specify a custom function to convert the function argument from Python to the desired Rust type, instead of using the default `FromPyObject` extraction. The function signature must be `fn(&Bound<'_, PyAny>) -> PyResult` where `T` is the Rust type of the argument. The following example uses `from_py_with` to convert the input Python object to its length: ```rust use pyo3::prelude::*; - fn get_length(obj: &PyAny) -> PyResult { + fn get_length(obj: &Bound<'_, PyAny>) -> PyResult { let length = obj.len()?; Ok(length) } @@ -121,7 +121,7 @@ The `#[pyo3]` attribute can be used on individual arguments to modify properties } # Python::with_gil(|py| { - # let f = pyo3::wrap_pyfunction!(object_length)(py).unwrap(); + # let f = pyo3::wrap_pyfunction_bound!(object_length)(py).unwrap(); # assert_eq!(f.call1((vec![1, 2, 3],)).unwrap().extract::().unwrap(), 3); # }); ``` @@ -134,11 +134,11 @@ You can pass Python `def`'d functions and built-in functions to Rust functions [ corresponds to regular Python functions while [`PyCFunction`] describes built-ins such as `repr()`. -You can also use [`PyAny::is_callable`] to check if you have a callable object. `is_callable` will -return `true` for functions (including lambdas), methods and objects with a `__call__` method. -You can call the object with [`PyAny::call`] with the args as first parameter and the kwargs -(or `None`) as second parameter. There are also [`PyAny::call0`] with no args and [`PyAny::call1`] -with only positional args. +You can also use [`Bound<'_, PyAny>::is_callable`] to check if you have a callable object. `is_callable` +will return `true` for functions (including lambdas), methods and objects with a `__call__` method. +You can call the object with [`Bound<'_, PyAny>::call`] with the args as first parameter and the kwargs +(or `None`) as second parameter. There are also [`Bound<'_, PyAny>::call0`] with no args and +[`Bound<'_, PyAny>::call1`] with only positional args. ### Calling Rust functions in Python @@ -149,11 +149,10 @@ The ways to convert a Rust function into a Python object vary depending on the f - use a `#[pyclass]` struct which stores the function as a field and implement `__call__` to call the stored function. - use `PyCFunction::new_closure` to create an object directly from the function. -[`PyAny::is_callable`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#tymethod.is_callable -[`PyAny::call`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#tymethod.call -[`PyAny::call0`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#tymethod.call0 -[`PyAny::call1`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#tymethod.call1 -[`PyObject`]: {{#PYO3_DOCS_URL}}/pyo3/type.PyObject.html +[`Bound<'_, PyAny>::is_callable`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.is_callable +[`Bound<'_, PyAny>::call`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.call +[`Bound<'_, PyAny>::call0`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.call0 +[`Bound<'_, PyAny>::call1`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.call1 [`wrap_pyfunction!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.wrap_pyfunction.html [`PyFunction`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFunction.html [`PyCFunction`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyCFunction.html @@ -180,7 +179,7 @@ An example of `#[pyfn]` is below: use pyo3::prelude::*; #[pymodule] -fn my_extension(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { #[pyfn(m)] fn double(x: usize) -> usize { x * 2 diff --git a/guide/src/function/error_handling.md b/guide/src/function/error_handling.md index b0f63885cdf..09fe5cab27b 100644 --- a/guide/src/function/error_handling.md +++ b/guide/src/function/error_handling.md @@ -44,7 +44,7 @@ fn check_positive(x: i32) -> PyResult<()> { # # fn main(){ # Python::with_gil(|py|{ -# let fun = pyo3::wrap_pyfunction!(check_positive, py).unwrap(); +# let fun = pyo3::wrap_pyfunction_bound!(check_positive, py).unwrap(); # fun.call1((-1,)).unwrap_err(); # fun.call1((1,)).unwrap(); # }); @@ -72,7 +72,7 @@ fn parse_int(x: &str) -> Result { # fn main() { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction!(parse_int, py).unwrap(); +# let fun = pyo3::wrap_pyfunction_bound!(parse_int, py).unwrap(); # let value: usize = fun.call1(("5",)).unwrap().extract().unwrap(); # assert_eq!(value, 5); # }); @@ -132,7 +132,7 @@ fn connect(s: String) -> Result<(), CustomIOError> { fn main() { Python::with_gil(|py| { - let fun = pyo3::wrap_pyfunction!(connect, py).unwrap(); + let fun = pyo3::wrap_pyfunction_bound!(connect, py).unwrap(); let err = fun.call1(("0.0.0.0",)).unwrap_err(); assert!(err.is_instance_of::(py)); }); @@ -224,7 +224,7 @@ fn wrapped_get_x() -> Result { # fn main() { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction!(wrapped_get_x, py).unwrap(); +# let fun = pyo3::wrap_pyfunction_bound!(wrapped_get_x, py).unwrap(); # let value: usize = fun.call0().unwrap().extract().unwrap(); # assert_eq!(value, 5); # }); @@ -234,8 +234,6 @@ fn wrapped_get_x() -> Result { [`From`]: https://doc.rust-lang.org/stable/std/convert/trait.From.html [`Result`]: https://doc.rust-lang.org/stable/std/result/enum.Result.html - -[`PyResult`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/type.PyResult.html [`PyResult`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/type.PyResult.html [`PyErr`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html [`pyo3::exceptions`]: {{#PYO3_DOCS_URL}}/pyo3/exceptions/index.html diff --git a/guide/src/function/signature.md b/guide/src/function/signature.md index 4fcfe958b19..b276fc457fb 100644 --- a/guide/src/function/signature.md +++ b/guide/src/function/signature.md @@ -16,7 +16,7 @@ use pyo3::types::PyDict; #[pyfunction] #[pyo3(signature = (**kwds))] -fn num_kwds(kwds: Option<&PyDict>) -> usize { +fn num_kwds(kwds: Option<&Bound<'_, PyDict>>) -> usize { kwds.map_or(0, |dict| dict.len()) } @@ -31,8 +31,8 @@ Just like in Python, the following constructs can be part of the signature:: * `/`: positional-only arguments separator, each parameter defined before `/` is a positional-only parameter. * `*`: var arguments separator, each parameter defined after `*` is a keyword-only parameter. - * `*args`: "args" is var args. Type of the `args` parameter has to be `&PyTuple`. - * `**kwargs`: "kwargs" receives keyword arguments. The type of the `kwargs` parameter has to be `Option<&PyDict>`. + * `*args`: "args" is var args. Type of the `args` parameter has to be `&Bound<'_, PyTuple>`. + * `**kwargs`: "kwargs" receives keyword arguments. The type of the `kwargs` parameter has to be `Option<&Bound<'_, PyDict>>`. * `arg=Value`: arguments with default value. If the `arg` argument is defined after var arguments, it is treated as a keyword-only argument. Note that `Value` has to be valid rust code, PyO3 just inserts it into the generated @@ -59,9 +59,9 @@ impl MyClass { fn method( &mut self, num: i32, - py_args: &PyTuple, + py_args: &Bound<'_, PyTuple>, name: &str, - py_kwargs: Option<&PyDict>, + py_kwargs: Option<&Bound<'_, PyDict>>, ) -> String { let num_before = self.num; self.num = num; @@ -136,7 +136,7 @@ fn increment(x: u64, amount: Option) -> u64 { # # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction!(increment, py)?; +# let fun = pyo3::wrap_pyfunction_bound!(increment, py)?; # # let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; # let sig: String = inspect @@ -164,7 +164,7 @@ fn increment(x: u64, amount: Option) -> u64 { # # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction!(increment, py)?; +# let fun = pyo3::wrap_pyfunction_bound!(increment, py)?; # # let inspect = PyModule::import_bound(py, "inspect")?.getattr("signature")?; # let sig: String = inspect @@ -204,7 +204,7 @@ fn add(a: u64, b: u64) -> u64 { # # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction!(add, py)?; +# let fun = pyo3::wrap_pyfunction_bound!(add, py)?; # # let doc: String = fun.getattr("__doc__")?.extract()?; # assert_eq!(doc, "This function adds two unsigned 64-bit integers."); @@ -252,7 +252,7 @@ fn add(a: u64, b: u64) -> u64 { # # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction!(add, py)?; +# let fun = pyo3::wrap_pyfunction_bound!(add, py)?; # # let doc: String = fun.getattr("__doc__")?.extract()?; # assert_eq!(doc, "This function adds two unsigned 64-bit integers."); @@ -269,7 +269,7 @@ fn add(a: u64, b: u64) -> u64 { # } ``` -PyO3 will include the contents of the annotation unmodified as the `__text_signature`. Below shows how IPython will now present this (see the default value of 0 for b): +PyO3 will include the contents of the annotation unmodified as the `__text_signature__`. Below shows how IPython will now present this (see the default value of 0 for b): ```text >>> pyo3_test.add.__text_signature__ @@ -294,7 +294,7 @@ fn add(a: u64, b: u64) -> u64 { # # fn main() -> PyResult<()> { # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction!(add, py)?; +# let fun = pyo3::wrap_pyfunction_bound!(add, py)?; # # let doc: String = fun.getattr("__doc__")?.extract()?; # assert_eq!(doc, "This function adds two unsigned 64-bit integers."); From 70a7aa808db038ef9c9670f79b50fd67082d4918 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sun, 3 Mar 2024 15:47:25 +0100 Subject: [PATCH 196/349] deprecate the use of `PyCell` in favor of `Bound` and `Py` (#3916) * deprecate the use of `PyCell` in favor of `Bound` and `Py` * update `FromPyObject` for `T: PyClass + Clone` impl * move `PyCell` deprecation to the `gil-refs` feature gate and add a migration note --- guide/src/class.md | 1 + guide/src/types.md | 1 + pyo3-macros-backend/src/pyclass.rs | 1 + src/conversion.rs | 15 ++++++------- src/conversions/std/array.rs | 2 +- src/impl_/pymethods.rs | 7 +++--- src/instance.rs | 7 +++--- src/lib.rs | 4 +++- src/prelude.rs | 4 +++- src/pycell.rs | 22 ++++++++++++++++++- src/pyclass.rs | 7 +++--- src/types/pysuper.rs | 2 +- tests/test_buffer_protocol.rs | 10 ++++----- tests/test_class_new.rs | 4 ++-- tests/test_pyself.rs | 11 ++++++---- tests/test_super.rs | 6 ++--- tests/test_text_signature.rs | 2 +- tests/test_various.rs | 2 +- tests/ui/invalid_frozen_pyclass_borrow.rs | 2 +- tests/ui/invalid_frozen_pyclass_borrow.stderr | 6 ++--- tests/ui/pyclass_send.rs | 6 ++--- 21 files changed, 77 insertions(+), 45 deletions(-) diff --git a/guide/src/class.md b/guide/src/class.md index 3096c9f7842..b9b47420ccb 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -1230,6 +1230,7 @@ struct MyClass { impl pyo3::types::DerefToPyAny for MyClass {} +# #[allow(deprecated)] unsafe impl pyo3::type_object::HasPyGilRef for MyClass { type AsRefTarget = pyo3::PyCell; } diff --git a/guide/src/types.md b/guide/src/types.md index 51ea10f4545..372c6c8632f 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -96,6 +96,7 @@ For a `&PyAny` object reference `any` where the underlying object is a `#[pyclas let obj: &PyAny = Py::new(py, MyClass {})?.into_ref(py); // To &PyCell with PyAny::downcast +# #[allow(deprecated)] let _: &PyCell = obj.downcast()?; // To Py (aka PyObject) with .into() diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 5ec6ac172e6..734b9565a66 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1281,6 +1281,7 @@ fn impl_pytypeinfo( }; quote! { + #[allow(deprecated)] unsafe impl _pyo3::type_object::HasPyGilRef for #cls { type AsRefTarget = _pyo3::PyCell; } diff --git a/src/conversion.rs b/src/conversion.rs index 986ac7c537c..68633d196be 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -6,9 +6,7 @@ use crate::pyclass::boolean_struct::False; use crate::type_object::PyTypeInfo; use crate::types::any::PyAnyMethods; use crate::types::PyTuple; -use crate::{ - ffi, gil, Bound, Py, PyAny, PyCell, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, -}; +use crate::{ffi, gil, Bound, Py, PyAny, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python}; use std::ptr::NonNull; /// Returns a borrowed pointer to a Python object. @@ -265,7 +263,8 @@ where } } -impl<'py, T> FromPyObject<'py> for &'py PyCell +#[allow(deprecated)] +impl<'py, T> FromPyObject<'py> for &'py crate::PyCell where T: PyClass, { @@ -278,9 +277,9 @@ impl FromPyObject<'_> for T where T: PyClass + Clone, { - fn extract(obj: &PyAny) -> PyResult { - let cell: &PyCell = obj.downcast()?; - Ok(unsafe { cell.try_borrow_unguarded()?.clone() }) + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { + let bound = obj.downcast::()?; + Ok(bound.try_borrow()?.clone()) } } @@ -389,7 +388,7 @@ mod implementations { } } - impl<'v, T> PyTryFrom<'v> for PyCell + impl<'v, T> PyTryFrom<'v> for crate::PyCell where T: 'v + PyClass, { diff --git a/src/conversions/std/array.rs b/src/conversions/std/array.rs index d901ff4e59d..bf21244e3b2 100644 --- a/src/conversions/std/array.rs +++ b/src/conversions/std/array.rs @@ -240,7 +240,7 @@ mod tests { let array: [Foo; 8] = [Foo, Foo, Foo, Foo, Foo, Foo, Foo, Foo]; let pyobject = array.into_py(py); let list = pyobject.downcast_bound::(py).unwrap(); - let _cell: &crate::PyCell = list.get_item(4).unwrap().extract().unwrap(); + let _bound = list.get_item(4).unwrap().downcast::().unwrap(); }); } diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index bc950bafee4..df89dba7dbd 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -7,8 +7,8 @@ use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; use crate::types::{any::PyAnyMethods, PyModule, PyType}; use crate::{ - ffi, Borrowed, Bound, DowncastError, Py, PyAny, PyCell, PyClass, PyClassInitializer, PyErr, - PyObject, PyRef, PyRefMut, PyResult, PyTraverseError, PyTypeCheck, PyVisit, Python, + ffi, Borrowed, Bound, DowncastError, Py, PyAny, PyClass, PyClassInitializer, PyErr, PyObject, + PyRef, PyRefMut, PyResult, PyTraverseError, PyTypeCheck, PyVisit, Python, }; use std::borrow::Cow; use std::ffi::CStr; @@ -518,7 +518,8 @@ impl<'a> From> for &'a PyModule { } } -impl<'a, 'py, T: PyClass> From> for &'a PyCell { +#[allow(deprecated)] +impl<'a, 'py, T: PyClass> From> for &'a crate::PyCell { #[inline] fn from(bound: BoundRef<'a, 'py, T>) -> Self { bound.0.as_gil_ref() diff --git a/src/instance.rs b/src/instance.rs index 66c530d5f53..cef06062430 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,6 +1,6 @@ use crate::err::{self, PyDowncastError, PyErr, PyResult}; use crate::impl_::pycell::PyClassObject; -use crate::pycell::{PyBorrowError, PyBorrowMutError, PyCell}; +use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; use crate::type_object::HasPyGilRef; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; @@ -1698,11 +1698,12 @@ impl std::convert::From> for Py { } // `&PyCell` can be converted to `Py` -impl std::convert::From<&PyCell> for Py +#[allow(deprecated)] +impl std::convert::From<&crate::PyCell> for Py where T: PyClass, { - fn from(cell: &PyCell) -> Self { + fn from(cell: &crate::PyCell) -> Self { cell.as_borrowed().to_owned().unbind() } } diff --git a/src/lib.rs b/src/lib.rs index 8c0e6269947..26d2ec55da1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -315,7 +315,9 @@ pub use crate::gil::GILPool; pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter}; pub use crate::instance::{Borrowed, Bound, Py, PyNativeType, PyObject}; pub use crate::marker::Python; -pub use crate::pycell::{PyCell, PyRef, PyRefMut}; +#[allow(deprecated)] +pub use crate::pycell::PyCell; +pub use crate::pycell::{PyRef, PyRefMut}; pub use crate::pyclass::PyClass; pub use crate::pyclass_init::PyClassInitializer; pub use crate::type_object::{PyTypeCheck, PyTypeInfo}; diff --git a/src/prelude.rs b/src/prelude.rs index dcf4fe71cdd..ef42b2706e9 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -14,7 +14,9 @@ pub use crate::conversion::{PyTryFrom, PyTryInto}; pub use crate::err::{PyErr, PyResult}; pub use crate::instance::{Borrowed, Bound, Py, PyObject}; pub use crate::marker::Python; -pub use crate::pycell::{PyCell, PyRef, PyRefMut}; +#[allow(deprecated)] +pub use crate::pycell::PyCell; +pub use crate::pycell::{PyRef, PyRefMut}; pub use crate::pyclass_init::PyClassInitializer; pub use crate::types::{PyAny, PyModule}; pub use crate::PyNativeType; diff --git a/src/pycell.rs b/src/pycell.rs index 43c75314f00..397e7b6a22a 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -62,7 +62,7 @@ //! ) -> *mut pyo3::ffi::PyObject { //! use :: pyo3 as _pyo3; //! _pyo3::impl_::trampoline::noargs(_slf, _args, |py, _slf| { -//! # #[allow(deprecated)] +//! # #[allow(deprecated)] //! let _cell = py //! .from_borrowed_ptr::<_pyo3::PyAny>(_slf) //! .downcast::<_pyo3::PyCell>()?; @@ -154,6 +154,7 @@ //! # pub struct Number { //! # inner: u32, //! # } +//! # #[allow(deprecated)] //! #[pyfunction] //! fn swap_numbers(a: &PyCell, b: &PyCell) { //! // Check that the pointers are unequal @@ -250,13 +251,22 @@ use self::impl_::{PyClassObject, PyClassObjectLayout}; /// ``` /// For more information on how, when and why (not) to use `PyCell` please see the /// [module-level documentation](self). +#[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`PyCell` was merged into `Bound`, use that instead; see the migration guide for more info" + ) +)] #[repr(transparent)] pub struct PyCell(PyClassObject); +#[allow(deprecated)] unsafe impl PyNativeType for PyCell { type AsRefSource = T; } +#[allow(deprecated)] impl PyCell { /// Makes a new `PyCell` on the Python heap and return the reference to it. /// @@ -478,9 +488,12 @@ impl PyCell { } } +#[allow(deprecated)] unsafe impl PyLayout for PyCell {} +#[allow(deprecated)] impl PySizedLayout for PyCell {} +#[allow(deprecated)] impl PyTypeCheck for PyCell where T: PyClass, @@ -492,18 +505,21 @@ where } } +#[allow(deprecated)] unsafe impl AsPyPointer for PyCell { fn as_ptr(&self) -> *mut ffi::PyObject { (self as *const _) as *mut _ } } +#[allow(deprecated)] impl ToPyObject for &PyCell { fn to_object(&self, py: Python<'_>) -> PyObject { unsafe { PyObject::from_borrowed_ptr(py, self.as_ptr()) } } } +#[allow(deprecated)] impl AsRef for PyCell { fn as_ref(&self) -> &PyAny { #[allow(deprecated)] @@ -513,6 +529,7 @@ impl AsRef for PyCell { } } +#[allow(deprecated)] impl Deref for PyCell { type Target = PyAny; @@ -524,6 +541,7 @@ impl Deref for PyCell { } } +#[allow(deprecated)] impl fmt::Debug for PyCell { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.try_borrow() { @@ -748,6 +766,7 @@ impl IntoPy for &'_ PyRef<'_, T> { } } +#[allow(deprecated)] impl<'a, T: PyClass> std::convert::TryFrom<&'a PyCell> for crate::PyRef<'a, T> { type Error = PyBorrowError; fn try_from(cell: &'a crate::PyCell) -> Result { @@ -905,6 +924,7 @@ unsafe impl<'a, T: PyClass> AsPyPointer for PyRefMut<'a, T> { } } +#[allow(deprecated)] impl<'a, T: PyClass> std::convert::TryFrom<&'a PyCell> for crate::PyRefMut<'a, T> { diff --git a/src/pyclass.rs b/src/pyclass.rs index eb4a5595ca9..b9b01cac26a 100644 --- a/src/pyclass.rs +++ b/src/pyclass.rs @@ -1,7 +1,7 @@ //! `PyClass` and related traits. use crate::{ - callback::IntoPyCallbackOutput, ffi, impl_::pyclass::PyClassImpl, IntoPy, PyCell, PyObject, - PyResult, PyTypeInfo, Python, + callback::IntoPyCallbackOutput, ffi, impl_::pyclass::PyClassImpl, IntoPy, PyObject, PyResult, + PyTypeInfo, Python, }; use std::{cmp::Ordering, os::raw::c_int}; @@ -15,7 +15,8 @@ pub use self::gc::{PyTraverseError, PyVisit}; /// /// The `#[pyclass]` attribute implements this trait for your Rust struct - /// you shouldn't implement this trait directly. -pub trait PyClass: PyTypeInfo> + PyClassImpl { +#[allow(deprecated)] +pub trait PyClass: PyTypeInfo> + PyClassImpl { /// Whether the pyclass is frozen. /// /// This can be enabled via `#[pyclass(frozen)]`. diff --git a/src/types/pysuper.rs b/src/types/pysuper.rs index 261a91f289b..0f1a47444d6 100644 --- a/src/types/pysuper.rs +++ b/src/types/pysuper.rs @@ -62,7 +62,7 @@ impl PySuper { /// (SubClass {}, BaseClass::new()) /// } /// - /// fn method(self_: &PyCell) -> PyResult<&PyAny> { + /// fn method<'py>(self_: &Bound<'py, Self>) -> PyResult> { /// let super_ = self_.py_super()?; /// super_.call_method("method", (), None) /// } diff --git a/tests/test_buffer_protocol.rs b/tests/test_buffer_protocol.rs index 85ff6a4004e..dca900808a8 100644 --- a/tests/test_buffer_protocol.rs +++ b/tests/test_buffer_protocol.rs @@ -24,11 +24,11 @@ struct TestBufferClass { #[pymethods] impl TestBufferClass { unsafe fn __getbuffer__( - slf: &PyCell, + slf: Bound<'_, Self>, view: *mut ffi::Py_buffer, flags: c_int, ) -> PyResult<()> { - fill_view_from_readonly_data(view, flags, &slf.borrow().vec, slf) + fill_view_from_readonly_data(view, flags, &slf.borrow().vec, slf.into_any()) } unsafe fn __releasebuffer__(&self, view: *mut ffi::Py_buffer) { @@ -105,12 +105,12 @@ fn test_releasebuffer_unraisable_error() { #[pymethods] impl ReleaseBufferError { unsafe fn __getbuffer__( - slf: &PyCell, + slf: Bound<'_, Self>, view: *mut ffi::Py_buffer, flags: c_int, ) -> PyResult<()> { static BUF_BYTES: &[u8] = b"hello world"; - fill_view_from_readonly_data(view, flags, BUF_BYTES, slf) + fill_view_from_readonly_data(view, flags, BUF_BYTES, slf.into_any()) } unsafe fn __releasebuffer__(&self, _view: *mut ffi::Py_buffer) -> PyResult<()> { @@ -145,7 +145,7 @@ unsafe fn fill_view_from_readonly_data( view: *mut ffi::Py_buffer, flags: c_int, data: &[u8], - owner: &PyAny, + owner: Bound<'_, PyAny>, ) -> PyResult<()> { if view.is_null() { return Err(PyBufferError::new_err("View is null")); diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index 9e16631bb83..161c60e9489 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -23,7 +23,7 @@ fn empty_class_with_new() { assert!(typeobj .call((), None) .unwrap() - .downcast::>() + .downcast::() .is_ok()); // Calling with arbitrary args or kwargs is not ok @@ -52,7 +52,7 @@ fn unit_class_with_new() { assert!(typeobj .call((), None) .unwrap() - .downcast::>() + .downcast::() .is_ok()); }); } diff --git a/tests/test_pyself.rs b/tests/test_pyself.rs index fa736f68455..901f52de530 100644 --- a/tests/test_pyself.rs +++ b/tests/test_pyself.rs @@ -18,15 +18,18 @@ struct Reader { #[pymethods] impl Reader { - fn clone_ref(slf: &PyCell) -> &PyCell { + fn clone_ref<'a, 'py>(slf: &'a Bound<'py, Self>) -> &'a Bound<'py, Self> { slf } - fn clone_ref_with_py<'py>(slf: &'py PyCell, _py: Python<'py>) -> &'py PyCell { + fn clone_ref_with_py<'a, 'py>( + slf: &'a Bound<'py, Self>, + _py: Python<'py>, + ) -> &'a Bound<'py, Self> { slf } - fn get_iter(slf: &PyCell, keys: Py) -> Iter { + fn get_iter(slf: &Bound<'_, Self>, keys: Py) -> Iter { Iter { - reader: slf.into(), + reader: slf.clone().unbind(), keys, idx: 0, } diff --git a/tests/test_super.rs b/tests/test_super.rs index 8dcedf808a5..3647e7d5b23 100644 --- a/tests/test_super.rs +++ b/tests/test_super.rs @@ -29,14 +29,14 @@ impl SubClass { (SubClass {}, BaseClass::new()) } - fn method(self_: &PyCell) -> PyResult<&PyAny> { + fn method<'py>(self_: &Bound<'py, Self>) -> PyResult> { let super_ = self_.py_super()?; super_.call_method("method", (), None) } - fn method_super_new(self_: &PyCell) -> PyResult<&PyAny> { + fn method_super_new<'py>(self_: &Bound<'py, Self>) -> PyResult> { #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] - let super_ = PySuper::new(self_.get_type(), self_)?; + let super_ = PySuper::new_bound(&self_.get_type(), self_)?; super_.call_method("method", (), None) } } diff --git a/tests/test_text_signature.rs b/tests/test_text_signature.rs index 6b56999c62f..0b93500db7e 100644 --- a/tests/test_text_signature.rs +++ b/tests/test_text_signature.rs @@ -367,7 +367,7 @@ fn test_methods() { let _ = a; } #[pyo3(text_signature = "($self, b)")] - fn pyself_method(_this: &PyCell, b: i32) { + fn pyself_method(_this: &Bound<'_, Self>, b: i32) { let _ = b; } #[classmethod] diff --git a/tests/test_various.rs b/tests/test_various.rs index 6f1bfd908a7..250f39834d1 100644 --- a/tests/test_various.rs +++ b/tests/test_various.rs @@ -124,7 +124,7 @@ impl PickleSupport { } pub fn __reduce__<'py>( - slf: &'py PyCell, + slf: &Bound<'py, Self>, py: Python<'py>, ) -> PyResult<(PyObject, Bound<'py, PyTuple>, PyObject)> { let cls = slf.to_object(py).getattr(py, "__class__")?; diff --git a/tests/ui/invalid_frozen_pyclass_borrow.rs b/tests/ui/invalid_frozen_pyclass_borrow.rs index aa8969ab1a6..6379a8707c5 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.rs +++ b/tests/ui/invalid_frozen_pyclass_borrow.rs @@ -29,7 +29,7 @@ fn py_get_of_mutable_class_fails(class: Py) { class.get(); } -fn pyclass_get_of_mutable_class_fails(class: &PyCell) { +fn pyclass_get_of_mutable_class_fails(class: &Bound<'_, MutableBase>) { class.get(); } diff --git a/tests/ui/invalid_frozen_pyclass_borrow.stderr b/tests/ui/invalid_frozen_pyclass_borrow.stderr index 1a8e45964ca..3acfbeb1823 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.stderr +++ b/tests/ui/invalid_frozen_pyclass_borrow.stderr @@ -67,11 +67,11 @@ error[E0271]: type mismatch resolving `::Frozen == True` 33 | class.get(); | ^^^ expected `True`, found `False` | -note: required by a bound in `pyo3::PyCell::::get` - --> src/pycell.rs +note: required by a bound in `pyo3::Bound::<'py, T>::get` + --> src/instance.rs | | pub fn get(&self) -> &T | --- required by a bound in this associated function | where | T: PyClass + Sync, - | ^^^^^^^^^^^^^ required by this bound in `PyCell::::get` + | ^^^^^^^^^^^^^ required by this bound in `Bound::<'py, T>::get` diff --git a/tests/ui/pyclass_send.rs b/tests/ui/pyclass_send.rs index 533302740d7..2747c2cb3bb 100644 --- a/tests/ui/pyclass_send.rs +++ b/tests/ui/pyclass_send.rs @@ -8,15 +8,15 @@ struct NotThreadSafe { fn main() { let obj = Python::with_gil(|py| { - PyCell::new(py, NotThreadSafe { data: Rc::new(5) }) + Bound::new(py, NotThreadSafe { data: Rc::new(5) }) .unwrap() - .to_object(py) + .unbind(py) }); std::thread::spawn(move || { Python::with_gil(|py| { // Uh oh, moved Rc to a new thread! - let c: &PyCell = obj.as_ref(py).downcast().unwrap(); + let c = obj.bind(py).downcast::().unwrap(); assert_eq!(*c.borrow().data, 5); }) From 4114dcb1a04160ef98fa17da52f83cce89d1497c Mon Sep 17 00:00:00 2001 From: Bruno Kolenbrander <59372212+mejrs@users.noreply.github.com> Date: Mon, 4 Mar 2024 08:54:04 +0100 Subject: [PATCH 197/349] Thread pyo3's path through the builder functions (#3907) * Thread pyo3's path through the builder functions * preserve span of pyo3_path --------- Co-authored-by: David Hewitt --- pyo3-macros-backend/src/deprecations.rs | 19 +- pyo3-macros-backend/src/frompyobject.rs | 72 ++--- pyo3-macros-backend/src/konst.rs | 17 +- pyo3-macros-backend/src/method.rs | 179 +++++++----- pyo3-macros-backend/src/module.rs | 43 +-- pyo3-macros-backend/src/params.rs | 54 ++-- pyo3-macros-backend/src/pyclass.rs | 326 +++++++++++---------- pyo3-macros-backend/src/pyfunction.rs | 25 +- pyo3-macros-backend/src/pyimpl.rs | 55 ++-- pyo3-macros-backend/src/pymethod.rs | 367 ++++++++++++++---------- pyo3-macros-backend/src/quotes.rs | 18 +- pyo3-macros-backend/src/utils.rs | 40 ++- tests/ui/invalid_proto_pymethods.stderr | 2 +- 13 files changed, 706 insertions(+), 511 deletions(-) diff --git a/pyo3-macros-backend/src/deprecations.rs b/pyo3-macros-backend/src/deprecations.rs index ea2922737b9..3f1f34144f6 100644 --- a/pyo3-macros-backend/src/deprecations.rs +++ b/pyo3-macros-backend/src/deprecations.rs @@ -1,3 +1,4 @@ +use crate::utils::Ctx; use proc_macro2::{Span, TokenStream}; use quote::{quote_spanned, ToTokens}; @@ -14,12 +15,11 @@ impl Deprecation { } } -#[derive(Default)] -pub struct Deprecations(Vec<(Deprecation, Span)>); +pub struct Deprecations<'ctx>(Vec<(Deprecation, Span)>, &'ctx Ctx); -impl Deprecations { - pub fn new() -> Self { - Deprecations(Vec::new()) +impl<'ctx> Deprecations<'ctx> { + pub fn new(ctx: &'ctx Ctx) -> Self { + Deprecations(Vec::new(), ctx) } pub fn push(&mut self, deprecation: Deprecation, span: Span) { @@ -27,15 +27,18 @@ impl Deprecations { } } -impl ToTokens for Deprecations { +impl<'ctx> ToTokens for Deprecations<'ctx> { fn to_tokens(&self, tokens: &mut TokenStream) { - for (deprecation, span) in &self.0 { + let Self(deprecations, Ctx { pyo3_path }) = self; + + for (deprecation, span) in deprecations { + let pyo3_path = pyo3_path.to_tokens_spanned(*span); let ident = deprecation.ident(*span); quote_spanned!( *span => #[allow(clippy::let_unit_value)] { - let _ = _pyo3::impl_::deprecations::#ident; + let _ = #pyo3_path::impl_::deprecations::#ident; } ) .to_tokens(tokens) diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index c1410180d05..24471c1aae8 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -1,7 +1,5 @@ -use crate::{ - attributes::{self, get_pyo3_options, CrateAttribute, FromPyWithAttribute}, - utils::get_pyo3_crate, -}; +use crate::attributes::{self, get_pyo3_options, CrateAttribute, FromPyWithAttribute}; +use crate::utils::Ctx; use proc_macro2::TokenStream; use quote::{format_ident, quote}; use syn::{ @@ -46,14 +44,15 @@ impl<'a> Enum<'a> { } /// Build derivation body for enums. - fn build(&self) -> TokenStream { + fn build(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; let mut var_extracts = Vec::new(); let mut variant_names = Vec::new(); let mut error_names = Vec::new(); for var in &self.variants { - let struct_derive = var.build(); + let struct_derive = var.build(ctx); let ext = quote!({ - let maybe_ret = || -> _pyo3::PyResult { + let maybe_ret = || -> #pyo3_path::PyResult { #struct_derive }(); @@ -73,7 +72,7 @@ impl<'a> Enum<'a> { #(#var_extracts),* ]; ::std::result::Result::Err( - _pyo3::impl_::frompyobject::failed_to_extract_enum( + #pyo3_path::impl_::frompyobject::failed_to_extract_enum( obj.py(), #ty_name, &[#(#variant_names),*], @@ -239,16 +238,16 @@ impl<'a> Container<'a> { } /// Build derivation body for a struct. - fn build(&self) -> TokenStream { + fn build(&self, ctx: &Ctx) -> TokenStream { match &self.ty { ContainerType::StructNewtype(ident, from_py_with) => { - self.build_newtype_struct(Some(ident), from_py_with) + self.build_newtype_struct(Some(ident), from_py_with, ctx) } ContainerType::TupleNewtype(from_py_with) => { - self.build_newtype_struct(None, from_py_with) + self.build_newtype_struct(None, from_py_with, ctx) } - ContainerType::Tuple(tups) => self.build_tuple_struct(tups), - ContainerType::Struct(tups) => self.build_struct(tups), + ContainerType::Tuple(tups) => self.build_tuple_struct(tups, ctx), + ContainerType::Struct(tups) => self.build_struct(tups, ctx), } } @@ -256,7 +255,9 @@ impl<'a> Container<'a> { &self, field_ident: Option<&Ident>, from_py_with: &Option, + ctx: &Ctx, ) -> TokenStream { + let Ctx { pyo3_path } = ctx; let self_ty = &self.path; let struct_name = self.name(); if let Some(ident) = field_ident { @@ -264,32 +265,33 @@ impl<'a> Container<'a> { match from_py_with { None => quote! { Ok(#self_ty { - #ident: _pyo3::impl_::frompyobject::extract_struct_field(obj, #struct_name, #field_name)? + #ident: #pyo3_path::impl_::frompyobject::extract_struct_field(obj, #struct_name, #field_name)? }) }, Some(FromPyWithAttribute { value: expr_path, .. }) => quote! { Ok(#self_ty { - #ident: _pyo3::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, #field_name)? + #ident: #pyo3_path::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, #field_name)? }) }, } } else { match from_py_with { None => quote!( - _pyo3::impl_::frompyobject::extract_tuple_struct_field(obj, #struct_name, 0).map(#self_ty) + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field(obj, #struct_name, 0).map(#self_ty) ), Some(FromPyWithAttribute { value: expr_path, .. }) => quote! ( - _pyo3::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, 0).map(#self_ty) + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, 0).map(#self_ty) ), } } } - fn build_tuple_struct(&self, struct_fields: &[TupleStructField]) -> TokenStream { + fn build_tuple_struct(&self, struct_fields: &[TupleStructField], ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; let self_ty = &self.path; let struct_name = &self.name(); let field_idents: Vec<_> = (0..struct_fields.len()) @@ -298,12 +300,12 @@ impl<'a> Container<'a> { let fields = struct_fields.iter().zip(&field_idents).enumerate().map(|(index, (field, ident))| { match &field.from_py_with { None => quote!( - _pyo3::impl_::frompyobject::extract_tuple_struct_field(&#ident, #struct_name, #index)? + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field(&#ident, #struct_name, #index)? ), Some(FromPyWithAttribute { value: expr_path, .. }) => quote! ( - _pyo3::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, &#ident, #struct_name, #index)? + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, &#ident, #struct_name, #index)? ), } }); @@ -315,7 +317,8 @@ impl<'a> Container<'a> { ) } - fn build_struct(&self, struct_fields: &[NamedStructField<'_>]) -> TokenStream { + fn build_struct(&self, struct_fields: &[NamedStructField<'_>], ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; let self_ty = &self.path; let struct_name = &self.name(); let mut fields: Punctuated = Punctuated::new(); @@ -324,27 +327,27 @@ impl<'a> Container<'a> { let field_name = ident.to_string(); let getter = match field.getter.as_ref().unwrap_or(&FieldGetter::GetAttr(None)) { FieldGetter::GetAttr(Some(name)) => { - quote!(getattr(_pyo3::intern!(obj.py(), #name))) + quote!(getattr(#pyo3_path::intern!(obj.py(), #name))) } FieldGetter::GetAttr(None) => { - quote!(getattr(_pyo3::intern!(obj.py(), #field_name))) + quote!(getattr(#pyo3_path::intern!(obj.py(), #field_name))) } FieldGetter::GetItem(Some(syn::Lit::Str(key))) => { - quote!(get_item(_pyo3::intern!(obj.py(), #key))) + quote!(get_item(#pyo3_path::intern!(obj.py(), #key))) } FieldGetter::GetItem(Some(key)) => quote!(get_item(#key)), FieldGetter::GetItem(None) => { - quote!(get_item(_pyo3::intern!(obj.py(), #field_name))) + quote!(get_item(#pyo3_path::intern!(obj.py(), #field_name))) } }; let extractor = match &field.from_py_with { None => { - quote!(_pyo3::impl_::frompyobject::extract_struct_field(&obj.#getter?, #struct_name, #field_name)?) + quote!(#pyo3_path::impl_::frompyobject::extract_struct_field(&obj.#getter?, #struct_name, #field_name)?) } Some(FromPyWithAttribute { value: expr_path, .. }) => { - quote! (_pyo3::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, &obj.#getter?, #struct_name, #field_name)?) + quote! (#pyo3_path::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, &obj.#getter?, #struct_name, #field_name)?) } }; @@ -579,7 +582,9 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { .push(parse_quote!(#gen_ident: FromPyObject<#lt_param>)) } let options = ContainerOptions::from_attrs(&tokens.attrs)?; - let krate = get_pyo3_crate(&options.krate); + let ctx = &Ctx::new(&options.krate); + let Ctx { pyo3_path } = &ctx; + let derives = match &tokens.data { syn::Data::Enum(en) => { if options.transparent || options.annotation.is_some() { @@ -587,7 +592,7 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { at top level for enums"); } let en = Enum::new(en, &tokens.ident)?; - en.build() + en.build(ctx) } syn::Data::Struct(st) => { if let Some(lit_str) = &options.annotation { @@ -595,7 +600,7 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { } let ident = &tokens.ident; let st = Container::new(&st.fields, parse_quote!(#ident), options)?; - st.build() + st.build(ctx) } syn::Data::Union(_) => bail_spanned!( tokens.span() => "#[derive(FromPyObject)] is not supported for unions" @@ -607,12 +612,11 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] const _: () = { - use #krate as _pyo3; - use _pyo3::prelude::PyAnyMethods; + use #pyo3_path::prelude::PyAnyMethods; #[automatically_derived] - impl #trait_generics _pyo3::FromPyObject<#lt_param> for #ident #generics #where_clause { - fn extract_bound(obj: &_pyo3::Bound<#lt_param, _pyo3::PyAny>) -> _pyo3::PyResult { + impl #trait_generics #pyo3_path::FromPyObject<#lt_param> for #ident #generics #where_clause { + fn extract_bound(obj: &#pyo3_path::Bound<#lt_param, #pyo3_path::PyAny>) -> #pyo3_path::PyResult { #derives } } diff --git a/pyo3-macros-backend/src/konst.rs b/pyo3-macros-backend/src/konst.rs index 935c9d4a302..9a41a2b7178 100644 --- a/pyo3-macros-backend/src/konst.rs +++ b/pyo3-macros-backend/src/konst.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; +use crate::utils::Ctx; use crate::{ attributes::{self, get_pyo3_options, take_attributes, NameAttribute}, deprecations::Deprecations, @@ -13,12 +14,12 @@ use syn::{ Result, }; -pub struct ConstSpec { +pub struct ConstSpec<'ctx> { pub rust_ident: syn::Ident, - pub attributes: ConstAttributes, + pub attributes: ConstAttributes<'ctx>, } -impl ConstSpec { +impl ConstSpec<'_> { pub fn python_name(&self) -> Cow<'_, Ident> { if let Some(name) = &self.attributes.name { Cow::Borrowed(&name.value.0) @@ -34,10 +35,10 @@ impl ConstSpec { } } -pub struct ConstAttributes { +pub struct ConstAttributes<'ctx> { pub is_class_attr: bool, pub name: Option, - pub deprecations: Deprecations, + pub deprecations: Deprecations<'ctx>, } pub enum PyO3ConstAttribute { @@ -55,12 +56,12 @@ impl Parse for PyO3ConstAttribute { } } -impl ConstAttributes { - pub fn from_attrs(attrs: &mut Vec) -> syn::Result { +impl<'ctx> ConstAttributes<'ctx> { + pub fn from_attrs(attrs: &mut Vec, ctx: &'ctx Ctx) -> syn::Result { let mut attributes = ConstAttributes { is_class_attr: false, name: None, - deprecations: Deprecations::new(), + deprecations: Deprecations::new(ctx), }; take_attributes(attrs, |attr| { diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index a9ba960d513..1d2d22b236b 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -4,6 +4,7 @@ use proc_macro2::{Span, TokenStream}; use quote::{quote, quote_spanned, ToTokens}; use syn::{ext::IdentExt, spanned::Spanned, Ident, Result}; +use crate::utils::Ctx; use crate::{ attributes::{TextSignatureAttribute, TextSignatureAttributeValue}, deprecations::{Deprecation, Deprecations}, @@ -108,13 +109,16 @@ impl FnType { cls: Option<&syn::Type>, error_mode: ExtractErrorMode, holders: &mut Vec, + ctx: &Ctx, ) -> TokenStream { + let Ctx { pyo3_path } = ctx; match self { FnType::Getter(st) | FnType::Setter(st) | FnType::Fn(st) => { let mut receiver = st.receiver( cls.expect("no class given for Fn with a \"self\" receiver"), error_mode, holders, + ctx, ); syn::Token![,](Span::call_site()).to_tokens(&mut receiver); receiver @@ -125,22 +129,24 @@ impl FnType { FnType::FnClass(span) | FnType::FnNewClass(span) => { let py = syn::Ident::new("py", Span::call_site()); let slf: Ident = syn::Ident::new("_slf", Span::call_site()); + let pyo3_path = pyo3_path.to_tokens_spanned(*span); quote_spanned! { *span => #[allow(clippy::useless_conversion)] ::std::convert::Into::into( - _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf.cast()) - .downcast_unchecked::<_pyo3::types::PyType>() + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf.cast()) + .downcast_unchecked::<#pyo3_path::types::PyType>() ), } } FnType::FnModule(span) => { let py = syn::Ident::new("py", Span::call_site()); let slf: Ident = syn::Ident::new("_slf", Span::call_site()); + let pyo3_path = pyo3_path.to_tokens_spanned(*span); quote_spanned! { *span => #[allow(clippy::useless_conversion)] ::std::convert::Into::into( - _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf.cast()) - .downcast_unchecked::<_pyo3::types::PyModule>() + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf.cast()) + .downcast_unchecked::<#pyo3_path::types::PyModule>() ), } } @@ -161,13 +167,14 @@ pub enum ExtractErrorMode { } impl ExtractErrorMode { - pub fn handle_error(self, extract: TokenStream) -> TokenStream { + pub fn handle_error(self, extract: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; match self { ExtractErrorMode::Raise => quote! { #extract? }, ExtractErrorMode::NotImplemented => quote! { match #extract { ::std::result::Result::Ok(value) => value, - ::std::result::Result::Err(_) => { return _pyo3::callback::convert(py, py.NotImplemented()); }, + ::std::result::Result::Err(_) => { return #pyo3_path::callback::convert(py, py.NotImplemented()); }, } }, } @@ -180,11 +187,13 @@ impl SelfType { cls: &syn::Type, error_mode: ExtractErrorMode, holders: &mut Vec, + ctx: &Ctx, ) -> TokenStream { // Due to use of quote_spanned in this function, need to bind these idents to the // main macro callsite. let py = syn::Ident::new("py", Span::call_site()); let slf = syn::Ident::new("_slf", Span::call_site()); + let Ctx { pyo3_path } = ctx; match self { SelfType::Receiver { span, mutable } => { let method = if *mutable { @@ -193,29 +202,35 @@ impl SelfType { syn::Ident::new("extract_pyclass_ref", *span) }; let holder = syn::Ident::new(&format!("holder_{}", holders.len()), *span); + let pyo3_path = pyo3_path.to_tokens_spanned(*span); holders.push(quote_spanned! { *span => #[allow(clippy::let_unit_value)] - let mut #holder = _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT; - let mut #slf = _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf); + let mut #holder = #pyo3_path::impl_::extract_argument::FunctionArgumentHolder::INIT; + let mut #slf = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf); }); - error_mode.handle_error(quote_spanned! { *span => - _pyo3::impl_::extract_argument::#method::<#cls>( - &#slf, - &mut #holder, - ) - }) + error_mode.handle_error( + quote_spanned! { *span => + #pyo3_path::impl_::extract_argument::#method::<#cls>( + &#slf, + &mut #holder, + ) + }, + ctx, + ) } SelfType::TryFromBoundRef(span) => { + let pyo3_path = pyo3_path.to_tokens_spanned(*span); error_mode.handle_error( quote_spanned! { *span => - _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf).downcast::<#cls>() - .map_err(::std::convert::Into::<_pyo3::PyErr>::into) + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf).downcast::<#cls>() + .map_err(::std::convert::Into::<#pyo3_path::PyErr>::into) .and_then( #[allow(unknown_lints, clippy::unnecessary_fallible_conversions)] // In case slf is Py (unknown_lints can be removed when MSRV is 1.75+) |bound| ::std::convert::TryFrom::try_from(bound).map_err(::std::convert::Into::into) ) - } + }, + ctx ) } } @@ -264,7 +279,7 @@ pub struct FnSpec<'a> { pub text_signature: Option, pub asyncness: Option, pub unsafety: Option, - pub deprecations: Deprecations, + pub deprecations: Deprecations<'a>, } pub fn get_return_info(output: &syn::ReturnType) -> syn::Type { @@ -303,6 +318,7 @@ impl<'a> FnSpec<'a> { sig: &'a mut syn::Signature, meth_attrs: &mut Vec, options: PyFunctionOptions, + ctx: &'a Ctx, ) -> Result> { let PyFunctionOptions { text_signature, @@ -312,7 +328,7 @@ impl<'a> FnSpec<'a> { } = options; let mut python_name = name.map(|name| name.value.0); - let mut deprecations = Deprecations::new(); + let mut deprecations = Deprecations::new(ctx); let fn_type = Self::parse_fn_type(sig, meth_attrs, &mut python_name, &mut deprecations)?; ensure_signatures_on_valid_method(&fn_type, signature.as_ref(), text_signature.as_ref())?; @@ -366,7 +382,7 @@ impl<'a> FnSpec<'a> { sig: &syn::Signature, meth_attrs: &mut Vec, python_name: &mut Option, - deprecations: &mut Deprecations, + deprecations: &mut Deprecations<'_>, ) -> Result { let mut method_attributes = parse_method_attributes(meth_attrs, deprecations)?; @@ -480,7 +496,9 @@ impl<'a> FnSpec<'a> { &self, ident: &proc_macro2::Ident, cls: Option<&syn::Type>, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path } = ctx; let mut cancel_handle_iter = self .signature .arguments @@ -495,7 +513,7 @@ impl<'a> FnSpec<'a> { } let rust_call = |args: Vec, holders: &mut Vec| { - let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise, holders); + let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise, holders, ctx); let call = if self.asyncness.is_some() { let throw_callback = if cancel_handle.is_some() { @@ -505,7 +523,7 @@ impl<'a> FnSpec<'a> { }; let python_name = &self.python_name; let qualname_prefix = match cls { - Some(cls) => quote!(Some(<#cls as _pyo3::PyTypeInfo>::NAME)), + Some(cls) => quote!(Some(<#cls as #pyo3_path::PyTypeInfo>::NAME)), None => quote!(None), }; let future = match self.tp { @@ -513,7 +531,7 @@ impl<'a> FnSpec<'a> { holders.pop().unwrap(); // does not actually use holder created by `self_arg` quote! {{ - let __guard = _pyo3::impl_::coroutine::RefGuard::<#cls>::new(&_pyo3::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; + let __guard = #pyo3_path::impl_::coroutine::RefGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; async move { function(&__guard, #(#args),*).await } }} } @@ -521,7 +539,7 @@ impl<'a> FnSpec<'a> { holders.pop().unwrap(); // does not actually use holder created by `self_arg` quote! {{ - let mut __guard = _pyo3::impl_::coroutine::RefMutGuard::<#cls>::new(&_pyo3::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; + let mut __guard = #pyo3_path::impl_::coroutine::RefMutGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; async move { function(&mut __guard, #(#args),*).await } }} } @@ -529,16 +547,16 @@ impl<'a> FnSpec<'a> { }; let mut call = quote! {{ let future = #future; - _pyo3::impl_::coroutine::new_coroutine( - _pyo3::intern!(py, stringify!(#python_name)), + #pyo3_path::impl_::coroutine::new_coroutine( + #pyo3_path::intern!(py, stringify!(#python_name)), #qualname_prefix, #throw_callback, - async move { _pyo3::impl_::wrap::OkWrap::wrap(future.await) }, + async move { #pyo3_path::impl_::wrap::OkWrap::wrap(future.await) }, ) }}; if cancel_handle.is_some() { call = quote! {{ - let __cancel_handle = _pyo3::coroutine::CancelHandle::new(); + let __cancel_handle = #pyo3_path::coroutine::CancelHandle::new(); let __throw_callback = __cancel_handle.throw_callback(); #call }}; @@ -547,7 +565,7 @@ impl<'a> FnSpec<'a> { } else { quote! { function(#self_arg #(#args),*) } }; - quotes::map_result_into_ptr(quotes::ok_wrap(call)) + quotes::map_result_into_ptr(quotes::ok_wrap(call, ctx), ctx) }; let func_name = &self.name; @@ -578,9 +596,9 @@ impl<'a> FnSpec<'a> { quote! { unsafe fn #ident<'py>( - py: _pyo3::Python<'py>, - _slf: *mut _pyo3::ffi::PyObject, - ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { + py: #pyo3_path::Python<'py>, + _slf: *mut #pyo3_path::ffi::PyObject, + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #( #holders )* let result = #call; @@ -590,16 +608,16 @@ impl<'a> FnSpec<'a> { } CallingConvention::Fastcall => { let mut holders = Vec::new(); - let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders)?; + let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders, ctx)?; let call = rust_call(args, &mut holders); quote! { unsafe fn #ident<'py>( - py: _pyo3::Python<'py>, - _slf: *mut _pyo3::ffi::PyObject, - _args: *const *mut _pyo3::ffi::PyObject, - _nargs: _pyo3::ffi::Py_ssize_t, - _kwnames: *mut _pyo3::ffi::PyObject - ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { + py: #pyo3_path::Python<'py>, + _slf: *mut #pyo3_path::ffi::PyObject, + _args: *const *mut #pyo3_path::ffi::PyObject, + _nargs: #pyo3_path::ffi::Py_ssize_t, + _kwnames: *mut #pyo3_path::ffi::PyObject + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert #( #holders )* @@ -610,15 +628,15 @@ impl<'a> FnSpec<'a> { } CallingConvention::Varargs => { let mut holders = Vec::new(); - let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders)?; + let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx)?; let call = rust_call(args, &mut holders); quote! { unsafe fn #ident<'py>( - py: _pyo3::Python<'py>, - _slf: *mut _pyo3::ffi::PyObject, - _args: *mut _pyo3::ffi::PyObject, - _kwargs: *mut _pyo3::ffi::PyObject - ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { + py: #pyo3_path::Python<'py>, + _slf: *mut #pyo3_path::ffi::PyObject, + _args: *mut #pyo3_path::ffi::PyObject, + _kwargs: *mut #pyo3_path::ffi::PyObject + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert #( #holders )* @@ -629,23 +647,25 @@ impl<'a> FnSpec<'a> { } CallingConvention::TpNew => { let mut holders = Vec::new(); - let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders)?; - let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise, &mut holders); + let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx)?; + let self_arg = self + .tp + .self_arg(cls, ExtractErrorMode::Raise, &mut holders, ctx); let call = quote! { #rust_name(#self_arg #(#args),*) }; quote! { unsafe fn #ident( - py: _pyo3::Python<'_>, - _slf: *mut _pyo3::ffi::PyTypeObject, - _args: *mut _pyo3::ffi::PyObject, - _kwargs: *mut _pyo3::ffi::PyObject - ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { - use _pyo3::callback::IntoPyCallbackOutput; + py: #pyo3_path::Python<'_>, + _slf: *mut #pyo3_path::ffi::PyTypeObject, + _args: *mut #pyo3_path::ffi::PyObject, + _kwargs: *mut #pyo3_path::ffi::PyObject + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { + use #pyo3_path::callback::IntoPyCallbackOutput; let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert #( #holders )* let result = #call; - let initializer: _pyo3::PyClassInitializer::<#cls> = result.convert(py)?; - _pyo3::impl_::pymethods::tp_new_impl(py, initializer, _slf) + let initializer: #pyo3_path::PyClassInitializer::<#cls> = result.convert(py)?; + #pyo3_path::impl_::pymethods::tp_new_impl(py, initializer, _slf) } } } @@ -654,19 +674,20 @@ impl<'a> FnSpec<'a> { /// Return a `PyMethodDef` constructor for this function, matching the selected /// calling convention. - pub fn get_methoddef(&self, wrapper: impl ToTokens, doc: &PythonDoc) -> TokenStream { + pub fn get_methoddef(&self, wrapper: impl ToTokens, doc: &PythonDoc, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; let python_name = self.null_terminated_python_name(); match self.convention { CallingConvention::Noargs => quote! { - _pyo3::impl_::pymethods::PyMethodDef::noargs( + #pyo3_path::impl_::pymethods::PyMethodDef::noargs( #python_name, - _pyo3::impl_::pymethods::PyCFunction({ + #pyo3_path::impl_::pymethods::PyCFunction({ unsafe extern "C" fn trampoline( - _slf: *mut _pyo3::ffi::PyObject, - _args: *mut _pyo3::ffi::PyObject, - ) -> *mut _pyo3::ffi::PyObject + _slf: *mut #pyo3_path::ffi::PyObject, + _args: *mut #pyo3_path::ffi::PyObject, + ) -> *mut #pyo3_path::ffi::PyObject { - _pyo3::impl_::trampoline::noargs( + #pyo3_path::impl_::trampoline::noargs( _slf, _args, #wrapper @@ -678,17 +699,17 @@ impl<'a> FnSpec<'a> { ) }, CallingConvention::Fastcall => quote! { - _pyo3::impl_::pymethods::PyMethodDef::fastcall_cfunction_with_keywords( + #pyo3_path::impl_::pymethods::PyMethodDef::fastcall_cfunction_with_keywords( #python_name, - _pyo3::impl_::pymethods::PyCFunctionFastWithKeywords({ + #pyo3_path::impl_::pymethods::PyCFunctionFastWithKeywords({ unsafe extern "C" fn trampoline( - _slf: *mut _pyo3::ffi::PyObject, - _args: *const *mut _pyo3::ffi::PyObject, - _nargs: _pyo3::ffi::Py_ssize_t, - _kwnames: *mut _pyo3::ffi::PyObject - ) -> *mut _pyo3::ffi::PyObject + _slf: *mut #pyo3_path::ffi::PyObject, + _args: *const *mut #pyo3_path::ffi::PyObject, + _nargs: #pyo3_path::ffi::Py_ssize_t, + _kwnames: *mut #pyo3_path::ffi::PyObject + ) -> *mut #pyo3_path::ffi::PyObject { - _pyo3::impl_::trampoline::fastcall_with_keywords( + #pyo3_path::impl_::trampoline::fastcall_with_keywords( _slf, _args, _nargs, @@ -702,16 +723,16 @@ impl<'a> FnSpec<'a> { ) }, CallingConvention::Varargs => quote! { - _pyo3::impl_::pymethods::PyMethodDef::cfunction_with_keywords( + #pyo3_path::impl_::pymethods::PyMethodDef::cfunction_with_keywords( #python_name, - _pyo3::impl_::pymethods::PyCFunctionWithKeywords({ + #pyo3_path::impl_::pymethods::PyCFunctionWithKeywords({ unsafe extern "C" fn trampoline( - _slf: *mut _pyo3::ffi::PyObject, - _args: *mut _pyo3::ffi::PyObject, - _kwargs: *mut _pyo3::ffi::PyObject, - ) -> *mut _pyo3::ffi::PyObject + _slf: *mut #pyo3_path::ffi::PyObject, + _args: *mut #pyo3_path::ffi::PyObject, + _kwargs: *mut #pyo3_path::ffi::PyObject, + ) -> *mut #pyo3_path::ffi::PyObject { - _pyo3::impl_::trampoline::cfunction_with_keywords( + #pyo3_path::impl_::trampoline::cfunction_with_keywords( _slf, _args, _kwargs, @@ -783,7 +804,7 @@ impl MethodTypeAttribute { /// Otherwise will either return a parse error or the attribute. fn parse_if_matching_attribute( attr: &syn::Attribute, - deprecations: &mut Deprecations, + deprecations: &mut Deprecations<'_>, ) -> Result> { fn ensure_no_arguments(meta: &syn::Meta, ident: &str) -> syn::Result<()> { match meta { @@ -869,7 +890,7 @@ impl Display for MethodTypeAttribute { fn parse_method_attributes( attrs: &mut Vec, - deprecations: &mut Deprecations, + deprecations: &mut Deprecations<'_>, ) -> Result> { let mut new_attrs = Vec::new(); let mut found_attrs = Vec::new(); diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 0bdb1cbc50c..7528ef8102f 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -1,10 +1,10 @@ //! Code generation for the function that initializes a python module and adds classes and function. +use crate::utils::Ctx; use crate::{ attributes::{self, take_attributes, take_pyo3_options, CrateAttribute, NameAttribute}, get_doc, pyfunction::{impl_wrap_pyfunction, PyFunctionOptions}, - utils::get_pyo3_crate, }; use proc_macro2::TokenStream; use quote::quote; @@ -73,7 +73,8 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { bail_spanned!(module.span() => "`#[pymodule]` can only be used on inline modules") }; let options = PyModuleOptions::from_attrs(attrs)?; - let krate = get_pyo3_crate(&options.krate); + let ctx = &Ctx::new(&options.krate); + let Ctx { pyo3_path } = ctx; let doc = get_doc(attrs, None); let mut module_items = Vec::new(); @@ -164,8 +165,8 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { #initialization impl MakeDef { - const fn make_def() -> #krate::impl_::pymodule::ModuleDef { - use #krate::impl_::pymodule as impl_; + const fn make_def() -> #pyo3_path::impl_::pymodule::ModuleDef { + use #pyo3_path::impl_::pymodule as impl_; const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule); unsafe { impl_::ModuleDef::new( @@ -177,8 +178,8 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { } } - fn __pyo3_pymodule(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { - use #krate::impl_::pymodule::PyAddToModule; + fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { + use #pyo3_path::impl_::pymodule::PyAddToModule; #( #(#module_items_cfg_attrs)* #module_items::add_to_module(module)?; @@ -195,7 +196,8 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result { let options = PyModuleOptions::from_attrs(&mut function.attrs)?; process_functions_in_module(&options, &mut function)?; - let krate = get_pyo3_crate(&options.krate); + let ctx = &Ctx::new(&options.krate); + let Ctx { pyo3_path } = ctx; let ident = &function.sig.ident; let vis = &function.vis; let doc = get_doc(&function.attrs, None); @@ -222,10 +224,10 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] const _: () = { - use #krate::impl_::pymodule as impl_; - use #krate::impl_::pymethods::BoundRef; + use #pyo3_path::impl_::pymodule as impl_; + use #pyo3_path::impl_::pymethods::BoundRef; - fn __pyo3_pymodule(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { + fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { #ident(#(#module_args),*) } @@ -247,36 +249,37 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result fn module_initialization(options: PyModuleOptions, ident: &syn::Ident) -> TokenStream { let name = options.name.unwrap_or_else(|| ident.unraw()); - let krate = get_pyo3_crate(&options.krate); + let ctx = &Ctx::new(&options.krate); + let Ctx { pyo3_path } = ctx; let pyinit_symbol = format!("PyInit_{}", name); quote! { pub const __PYO3_NAME: &'static str = concat!(stringify!(#name), "\0"); pub(super) struct MakeDef; - pub static DEF: #krate::impl_::pymodule::ModuleDef = MakeDef::make_def(); + pub static DEF: #pyo3_path::impl_::pymodule::ModuleDef = MakeDef::make_def(); - pub fn add_to_module(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { - use #krate::prelude::PyModuleMethods; + pub fn add_to_module(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { + use #pyo3_path::prelude::PyModuleMethods; module.add_submodule(DEF.make_module(module.py())?.bind(module.py())) } /// This autogenerated function is called by the python interpreter when importing /// the module. #[export_name = #pyinit_symbol] - pub unsafe extern "C" fn __pyo3_init() -> *mut #krate::ffi::PyObject { - #krate::impl_::trampoline::module_init(|py| DEF.make_module(py)) + pub unsafe extern "C" fn __pyo3_init() -> *mut #pyo3_path::ffi::PyObject { + #pyo3_path::impl_::trampoline::module_init(|py| DEF.make_module(py)) } } } /// Finds and takes care of the #[pyfn(...)] in `#[pymodule]` fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn) -> Result<()> { - let krate = get_pyo3_crate(&options.krate); - + let ctx = &Ctx::new(&options.krate); + let Ctx { pyo3_path } = ctx; let mut stmts: Vec = vec![syn::parse_quote!( #[allow(unknown_lints, unused_imports, redundant_imports)] - use #krate::{PyNativeType, types::PyModuleMethods}; + use #pyo3_path::{PyNativeType, types::PyModuleMethods}; )]; for mut stmt in func.block.stmts.drain(..) { @@ -287,7 +290,7 @@ fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn let name = &func.sig.ident; let statements: Vec = syn::parse_quote! { #wrapped_function - #module_name.as_borrowed().add_function(#krate::wrap_pyfunction!(#name, #module_name.as_borrowed())?)?; + #module_name.as_borrowed().add_function(#pyo3_path::wrap_pyfunction!(#name, #module_name.as_borrowed())?)?; }; stmts.extend(statements); } diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 679d3e4260a..7260362fa43 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -1,3 +1,4 @@ +use crate::utils::Ctx; use crate::{ method::{FnArg, FnSpec}, pyfunction::FunctionSignature, @@ -30,8 +31,10 @@ pub fn impl_arg_params( self_: Option<&syn::Type>, fastcall: bool, holders: &mut Vec, + ctx: &Ctx, ) -> Result<(TokenStream, Vec)> { let args_array = syn::Ident::new("output", Span::call_site()); + let Ctx { pyo3_path } = ctx; if !fastcall && is_forwarded_args(&spec.signature) { // In the varargs convention, we can just pass though if the signature @@ -40,12 +43,12 @@ pub fn impl_arg_params( .signature .arguments .iter() - .map(|arg| impl_arg_param(arg, &mut 0, &args_array, holders)) + .map(|arg| impl_arg_param(arg, &mut 0, &args_array, holders, ctx)) .collect::>()?; return Ok(( quote! { - let _args = _pyo3::impl_::pymethods::BoundRef::ref_from_ptr(py, &_args); - let _kwargs = _pyo3::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_kwargs); + let _args = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_args); + let _kwargs = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_kwargs); }, arg_convert, )); @@ -64,7 +67,7 @@ pub fn impl_arg_params( .iter() .map(|(name, required)| { quote! { - _pyo3::impl_::extract_argument::KeywordOnlyParameterDescription { + #pyo3_path::impl_::extract_argument::KeywordOnlyParameterDescription { name: #name, required: #required, } @@ -78,22 +81,22 @@ pub fn impl_arg_params( .signature .arguments .iter() - .map(|arg| impl_arg_param(arg, &mut option_pos, &args_array, holders)) + .map(|arg| impl_arg_param(arg, &mut option_pos, &args_array, holders, ctx)) .collect::>()?; let args_handler = if spec.signature.python_signature.varargs.is_some() { - quote! { _pyo3::impl_::extract_argument::TupleVarargs } + quote! { #pyo3_path::impl_::extract_argument::TupleVarargs } } else { - quote! { _pyo3::impl_::extract_argument::NoVarargs } + quote! { #pyo3_path::impl_::extract_argument::NoVarargs } }; let kwargs_handler = if spec.signature.python_signature.kwargs.is_some() { - quote! { _pyo3::impl_::extract_argument::DictVarkeywords } + quote! { #pyo3_path::impl_::extract_argument::DictVarkeywords } } else { - quote! { _pyo3::impl_::extract_argument::NoVarkeywords } + quote! { #pyo3_path::impl_::extract_argument::NoVarkeywords } }; let cls_name = if let Some(cls) = self_ { - quote! { ::std::option::Option::Some(<#cls as _pyo3::type_object::PyTypeInfo>::NAME) } + quote! { ::std::option::Option::Some(<#cls as #pyo3_path::type_object::PyTypeInfo>::NAME) } } else { quote! { ::std::option::Option::None } }; @@ -123,7 +126,7 @@ pub fn impl_arg_params( // create array of arguments, and then parse Ok(( quote! { - const DESCRIPTION: _pyo3::impl_::extract_argument::FunctionDescription = _pyo3::impl_::extract_argument::FunctionDescription { + const DESCRIPTION: #pyo3_path::impl_::extract_argument::FunctionDescription = #pyo3_path::impl_::extract_argument::FunctionDescription { cls_name: #cls_name, func_name: stringify!(#python_name), positional_parameter_names: &[#(#positional_parameter_names),*], @@ -145,7 +148,11 @@ fn impl_arg_param( option_pos: &mut usize, args_array: &syn::Ident, holders: &mut Vec, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path } = ctx; + let pyo3_path = pyo3_path.to_tokens_spanned(arg.ty.span()); + // Use this macro inside this function, to ensure that all code generated here is associated // with the function argument macro_rules! quote_arg_span { @@ -167,7 +174,7 @@ fn impl_arg_param( let holder = syn::Ident::new(&format!("holder_{}", holders.len()), arg.ty.span()); holders.push(quote_arg_span! { #[allow(clippy::let_unit_value)] - let mut #holder = _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT; + let mut #holder = #pyo3_path::impl_::extract_argument::FunctionArgumentHolder::INIT; }); holder }; @@ -179,7 +186,7 @@ fn impl_arg_param( ); let holder = push_holder(); return Ok(quote_arg_span! { - _pyo3::impl_::extract_argument::extract_argument( + #pyo3_path::impl_::extract_argument::extract_argument( &_args, &mut #holder, #name_str @@ -192,7 +199,7 @@ fn impl_arg_param( ); let holder = push_holder(); return Ok(quote_arg_span! { - _pyo3::impl_::extract_argument::extract_optional_argument( + #pyo3_path::impl_::extract_argument::extract_optional_argument( _kwargs.as_deref(), &mut #holder, #name_str, @@ -209,14 +216,17 @@ fn impl_arg_param( // Option arguments have special treatment: the default should be specified _without_ the // Some() wrapper. Maybe this should be changed in future?! if arg.optional.is_some() { - default = Some(default.map_or_else(|| quote!(::std::option::Option::None), some_wrap)); + default = Some(default.map_or_else( + || quote!(::std::option::Option::None), + |tokens| some_wrap(tokens, ctx), + )); } let tokens = if let Some(expr_path) = arg.attrs.from_py_with.as_ref().map(|attr| &attr.value) { if let Some(default) = default { quote_arg_span! { #[allow(clippy::redundant_closure)] - _pyo3::impl_::extract_argument::from_py_with_with_default( + #pyo3_path::impl_::extract_argument::from_py_with_with_default( #arg_value.as_deref(), #name_str, #expr_path as fn(_) -> _, @@ -225,8 +235,8 @@ fn impl_arg_param( } } else { quote_arg_span! { - _pyo3::impl_::extract_argument::from_py_with( - &_pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value), + #pyo3_path::impl_::extract_argument::from_py_with( + &#pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value), #name_str, #expr_path as fn(_) -> _, )? @@ -236,7 +246,7 @@ fn impl_arg_param( let holder = push_holder(); quote_arg_span! { #[allow(clippy::redundant_closure)] - _pyo3::impl_::extract_argument::extract_optional_argument( + #pyo3_path::impl_::extract_argument::extract_optional_argument( #arg_value.as_deref(), &mut #holder, #name_str, @@ -247,7 +257,7 @@ fn impl_arg_param( let holder = push_holder(); quote_arg_span! { #[allow(clippy::redundant_closure)] - _pyo3::impl_::extract_argument::extract_argument_with_default( + #pyo3_path::impl_::extract_argument::extract_argument_with_default( #arg_value.as_deref(), &mut #holder, #name_str, @@ -257,8 +267,8 @@ fn impl_arg_param( } else { let holder = push_holder(); quote_arg_span! { - _pyo3::impl_::extract_argument::extract_argument( - &_pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value), + #pyo3_path::impl_::extract_argument::extract_argument( + &#pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value), &mut #holder, #name_str )? diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 734b9565a66..9470045733c 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -13,7 +13,8 @@ use crate::pymethod::{ impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, MethodAndSlotDef, PropertyType, SlotDef, __INT__, __REPR__, __RICHCMP__, }; -use crate::utils::{self, apply_renaming_rule, get_pyo3_crate, PythonDoc}; +use crate::utils::Ctx; +use crate::utils::{self, apply_renaming_rule, PythonDoc}; use crate::PyFunctionOptions; use proc_macro2::{Ident, Span, TokenStream}; use quote::{format_ident, quote}; @@ -189,7 +190,8 @@ pub fn build_py_class( ) -> syn::Result { args.options.take_pyo3_options(&mut class.attrs)?; let doc = utils::get_doc(&class.attrs, None); - let krate = get_pyo3_crate(&args.options.krate); + + let ctx = &Ctx::new(&args.options.krate); if let Some(lt) = class.generics.lifetimes().next() { bail_spanned!( @@ -251,7 +253,7 @@ pub fn build_py_class( } } - impl_class(&class.ident, &args, doc, field_options, methods_type, krate) + impl_class(&class.ident, &args, doc, field_options, methods_type, ctx) } enum Annotated { @@ -342,9 +344,10 @@ fn impl_class( doc: PythonDoc, field_options: Vec<(&syn::Field, FieldPyO3Options)>, methods_type: PyClassMethodsType, - krate: syn::Path, + ctx: &Ctx, ) -> syn::Result { - let pytypeinfo_impl = impl_pytypeinfo(cls, args, None); + let Ctx { pyo3_path } = ctx; + let pytypeinfo_impl = impl_pytypeinfo(cls, args, None, ctx); let py_class_impl = PyClassImplsBuilder::new( cls, @@ -355,19 +358,18 @@ fn impl_class( args.options.rename_all.as_ref(), args.options.frozen, field_options, + ctx, )?, vec![], ) .doc(doc) - .impl_all()?; + .impl_all(ctx)?; Ok(quote! { // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] const _: () = { - use #krate as _pyo3; - - impl _pyo3::types::DerefToPyAny for #cls {} + impl #pyo3_path::types::DerefToPyAny for #cls {} #pytypeinfo_impl @@ -405,6 +407,7 @@ pub fn build_py_enum( ) -> syn::Result { args.options.take_pyo3_options(&mut enum_.attrs)?; + let ctx = &Ctx::new(&args.options.krate); if let Some(extends) = &args.options.extends { bail_spanned!(extends.span() => "enums can't extend from other classes"); } else if let Some(subclass) = &args.options.subclass { @@ -415,7 +418,7 @@ pub fn build_py_enum( let doc = utils::get_doc(&enum_.attrs, None); let enum_ = PyClassEnum::new(enum_)?; - impl_enum(enum_, &args, doc, method_type) + impl_enum(enum_, &args, doc, method_type, ctx) } struct PyClassSimpleEnum<'a> { @@ -665,11 +668,14 @@ fn impl_enum( args: &PyClassArgs, doc: PythonDoc, methods_type: PyClassMethodsType, + ctx: &Ctx, ) -> Result { match enum_ { - PyClassEnum::Simple(simple_enum) => impl_simple_enum(simple_enum, args, doc, methods_type), + PyClassEnum::Simple(simple_enum) => { + impl_simple_enum(simple_enum, args, doc, methods_type, ctx) + } PyClassEnum::Complex(complex_enum) => { - impl_complex_enum(complex_enum, args, doc, methods_type) + impl_complex_enum(complex_enum, args, doc, methods_type, ctx) } } } @@ -679,12 +685,13 @@ fn impl_simple_enum( args: &PyClassArgs, doc: PythonDoc, methods_type: PyClassMethodsType, + ctx: &Ctx, ) -> Result { - let krate = get_pyo3_crate(&args.options.krate); + let Ctx { pyo3_path } = ctx; let cls = simple_enum.ident; let ty: syn::Type = syn::parse_quote!(#cls); let variants = simple_enum.variants; - let pytypeinfo = impl_pytypeinfo(cls, args, None); + let pytypeinfo = impl_pytypeinfo(cls, args, None, ctx); let (default_repr, default_repr_slot) = { let variants_repr = variants.iter().map(|variant| { @@ -704,7 +711,8 @@ fn impl_simple_enum( } } }; - let repr_slot = generate_default_protocol_slot(&ty, &mut repr_impl, &__REPR__).unwrap(); + let repr_slot = + generate_default_protocol_slot(&ty, &mut repr_impl, &__REPR__, ctx).unwrap(); (repr_impl, repr_slot) }; @@ -723,7 +731,7 @@ fn impl_simple_enum( } } }; - let int_slot = generate_default_protocol_slot(&ty, &mut int_impl, &__INT__).unwrap(); + let int_slot = generate_default_protocol_slot(&ty, &mut int_impl, &__INT__, ctx).unwrap(); (int_impl, int_slot) }; @@ -731,30 +739,30 @@ fn impl_simple_enum( let mut richcmp_impl: syn::ImplItemFn = syn::parse_quote! { fn __pyo3__richcmp__( &self, - py: _pyo3::Python, - other: &_pyo3::PyAny, - op: _pyo3::basic::CompareOp - ) -> _pyo3::PyResult<_pyo3::PyObject> { - use _pyo3::conversion::ToPyObject; + py: #pyo3_path::Python, + other: &#pyo3_path::PyAny, + op: #pyo3_path::basic::CompareOp + ) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { + use #pyo3_path::conversion::ToPyObject; use ::core::result::Result::*; match op { - _pyo3::basic::CompareOp::Eq => { + #pyo3_path::basic::CompareOp::Eq => { let self_val = self.__pyo3__int__(); if let Ok(i) = other.extract::<#repr_type>() { return Ok((self_val == i).to_object(py)); } - if let Ok(other) = other.extract::<_pyo3::PyRef>() { + if let Ok(other) = other.extract::<#pyo3_path::PyRef>() { return Ok((self_val == other.__pyo3__int__()).to_object(py)); } return Ok(py.NotImplemented()); } - _pyo3::basic::CompareOp::Ne => { + #pyo3_path::basic::CompareOp::Ne => { let self_val = self.__pyo3__int__(); if let Ok(i) = other.extract::<#repr_type>() { return Ok((self_val != i).to_object(py)); } - if let Ok(other) = other.extract::<_pyo3::PyRef>() { + if let Ok(other) = other.extract::<#pyo3_path::PyRef>() { return Ok((self_val != other.__pyo3__int__()).to_object(py)); } @@ -765,7 +773,7 @@ fn impl_simple_enum( } }; let richcmp_slot = - generate_default_protocol_slot(&ty, &mut richcmp_impl, &__RICHCMP__).unwrap(); + generate_default_protocol_slot(&ty, &mut richcmp_impl, &__RICHCMP__, ctx).unwrap(); (richcmp_impl, richcmp_slot) }; @@ -778,18 +786,17 @@ fn impl_simple_enum( simple_enum_default_methods( cls, variants.iter().map(|v| (v.ident, v.get_python_name(args))), + ctx, ), default_slots, ) .doc(doc) - .impl_all()?; + .impl_all(ctx)?; Ok(quote! { // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] const _: () = { - use #krate as _pyo3; - #pytypeinfo #pyclass_impls @@ -810,7 +817,10 @@ fn impl_complex_enum( args: &PyClassArgs, doc: PythonDoc, methods_type: PyClassMethodsType, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path } = ctx; + // Need to rig the enum PyClass options let args = { let mut rigged_args = args.clone(); @@ -821,10 +831,10 @@ fn impl_complex_enum( rigged_args }; - let krate = get_pyo3_crate(&args.options.krate); + let ctx = &Ctx::new(&args.options.krate); let cls = complex_enum.ident; let variants = complex_enum.variants; - let pytypeinfo = impl_pytypeinfo(cls, &args, None); + let pytypeinfo = impl_pytypeinfo(cls, &args, None, ctx); let default_slots = vec![]; @@ -837,6 +847,7 @@ fn impl_complex_enum( variants .iter() .map(|v| (v.get_ident(), v.get_python_name(&args))), + ctx, ), default_slots, ) @@ -851,17 +862,17 @@ fn impl_complex_enum( let variant_cls = gen_complex_enum_variant_class_ident(cls, variant.get_ident()); quote! { #cls::#variant_ident { .. } => { - let pyclass_init = _pyo3::PyClassInitializer::from(self).add_subclass(#variant_cls); - let variant_value = _pyo3::Py::new(py, pyclass_init).unwrap(); - _pyo3::IntoPy::into_py(variant_value, py) + let pyclass_init = #pyo3_path::PyClassInitializer::from(self).add_subclass(#variant_cls); + let variant_value = #pyo3_path::Py::new(py, pyclass_init).unwrap(); + #pyo3_path::IntoPy::into_py(variant_value, py) } } }) .collect(); quote! { - impl _pyo3::IntoPy<_pyo3::PyObject> for #cls { - fn into_py(self, py: _pyo3::Python) -> _pyo3::PyObject { + impl #pyo3_path::IntoPy<#pyo3_path::PyObject> for #cls { + fn into_py(self, py: #pyo3_path::Python) -> #pyo3_path::PyObject { match self { #(#match_arms)* } @@ -871,11 +882,11 @@ fn impl_complex_enum( }; let pyclass_impls: TokenStream = vec![ - impl_builder.impl_pyclass(), - impl_builder.impl_extractext(), + impl_builder.impl_pyclass(ctx), + impl_builder.impl_extractext(ctx), enum_into_py_impl, - impl_builder.impl_pyclassimpl()?, - impl_builder.impl_freelist(), + impl_builder.impl_pyclassimpl(ctx)?, + impl_builder.impl_freelist(ctx), ] .into_iter() .collect(); @@ -900,12 +911,12 @@ fn impl_complex_enum( options: parse_quote!(extends = #cls, frozen), }; - let variant_cls_pytypeinfo = impl_pytypeinfo(&variant_cls, &variant_args, None); + let variant_cls_pytypeinfo = impl_pytypeinfo(&variant_cls, &variant_args, None, ctx); variant_cls_pytypeinfos.push(variant_cls_pytypeinfo); - let variant_new = complex_enum_variant_new(cls, variant)?; + let variant_new = complex_enum_variant_new(cls, variant, ctx)?; - let (variant_cls_impl, field_getters) = impl_complex_enum_variant_cls(cls, variant)?; + let (variant_cls_impl, field_getters) = impl_complex_enum_variant_cls(cls, variant, ctx)?; variant_cls_impls.push(variant_cls_impl); let pyclass_impl = PyClassImplsBuilder::new( @@ -915,7 +926,7 @@ fn impl_complex_enum( field_getters, vec![variant_new], ) - .impl_all()?; + .impl_all(ctx)?; variant_cls_pyclass_impls.push(pyclass_impl); } @@ -924,8 +935,6 @@ fn impl_complex_enum( // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] const _: () = { - use #krate as _pyo3; - #pytypeinfo #pyclass_impls @@ -948,10 +957,11 @@ fn impl_complex_enum( fn impl_complex_enum_variant_cls( enum_name: &syn::Ident, variant: &PyClassEnumVariant<'_>, + ctx: &Ctx, ) -> Result<(TokenStream, Vec)> { match variant { PyClassEnumVariant::Struct(struct_variant) => { - impl_complex_enum_struct_variant_cls(enum_name, struct_variant) + impl_complex_enum_struct_variant_cls(enum_name, struct_variant, ctx) } } } @@ -959,7 +969,9 @@ fn impl_complex_enum_variant_cls( fn impl_complex_enum_struct_variant_cls( enum_name: &syn::Ident, variant: &PyClassEnumStructVariant<'_>, + ctx: &Ctx, ) -> Result<(TokenStream, Vec)> { + let Ctx { pyo3_path } = ctx; let variant_ident = &variant.ident; let variant_cls = gen_complex_enum_variant_class_ident(enum_name, variant.ident); let variant_cls_type = parse_quote!(#variant_cls); @@ -978,10 +990,11 @@ fn impl_complex_enum_struct_variant_cls( field_name, field_type, field.span, + ctx, )?; let field_getter_impl = quote! { - fn #field_name(slf: _pyo3::PyRef) -> _pyo3::PyResult<#field_type> { + fn #field_name(slf: #pyo3_path::PyRef) -> #pyo3_path::PyResult<#field_type> { match &*slf.into_super() { #enum_name::#variant_ident { #field_name, .. } => Ok(#field_name.clone()), _ => unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), @@ -999,9 +1012,9 @@ fn impl_complex_enum_struct_variant_cls( #[doc(hidden)] #[allow(non_snake_case)] impl #variant_cls { - fn __pymethod_constructor__(py: _pyo3::Python<'_>, #(#fields_with_types,)*) -> _pyo3::PyClassInitializer<#variant_cls> { + fn __pymethod_constructor__(py: #pyo3_path::Python<'_>, #(#fields_with_types,)*) -> #pyo3_path::PyClassInitializer<#variant_cls> { let base_value = #enum_name::#variant_ident { #(#field_names,)* }; - _pyo3::PyClassInitializer::from(base_value).add_subclass(#variant_cls) + #pyo3_path::PyClassInitializer::from(base_value).add_subclass(#variant_cls) } #(#field_getter_impls)* @@ -1019,11 +1032,13 @@ fn generate_default_protocol_slot( cls: &syn::Type, method: &mut syn::ImplItemFn, slot: &SlotDef, + ctx: &Ctx, ) -> syn::Result { let spec = FnSpec::parse( &mut method.sig, &mut Vec::new(), PyFunctionOptions::default(), + ctx, ) .unwrap(); let name = spec.name.to_string(); @@ -1031,12 +1046,14 @@ fn generate_default_protocol_slot( &syn::parse_quote!(#cls), &spec, &format!("__default_{}__", name), + ctx, ) } fn simple_enum_default_methods<'a>( cls: &'a syn::Ident, unit_variant_names: impl IntoIterator)>, + ctx: &Ctx, ) -> Vec { let cls_type = syn::parse_quote!(#cls); let variant_to_attribute = |var_ident: &syn::Ident, py_ident: &syn::Ident| ConstSpec { @@ -1047,18 +1064,19 @@ fn simple_enum_default_methods<'a>( kw: syn::parse_quote! { name }, value: NameLitStr(py_ident.clone()), }), - deprecations: Default::default(), + deprecations: Deprecations::new(ctx), }, }; unit_variant_names .into_iter() - .map(|(var, py_name)| gen_py_const(&cls_type, &variant_to_attribute(var, &py_name))) + .map(|(var, py_name)| gen_py_const(&cls_type, &variant_to_attribute(var, &py_name), ctx)) .collect() } fn complex_enum_default_methods<'a>( cls: &'a syn::Ident, variant_names: impl IntoIterator)>, + ctx: &Ctx, ) -> Vec { let cls_type = syn::parse_quote!(#cls); let variant_to_attribute = |var_ident: &syn::Ident, py_ident: &syn::Ident| ConstSpec { @@ -1069,13 +1087,13 @@ fn complex_enum_default_methods<'a>( kw: syn::parse_quote! { name }, value: NameLitStr(py_ident.clone()), }), - deprecations: Default::default(), + deprecations: Deprecations::new(ctx), }, }; variant_names .into_iter() .map(|(var, py_name)| { - gen_complex_enum_variant_attr(cls, &cls_type, &variant_to_attribute(var, &py_name)) + gen_complex_enum_variant_attr(cls, &cls_type, &variant_to_attribute(var, &py_name), ctx) }) .collect() } @@ -1083,8 +1101,10 @@ fn complex_enum_default_methods<'a>( pub fn gen_complex_enum_variant_attr( cls: &syn::Ident, cls_type: &syn::Type, - spec: &ConstSpec, + spec: &ConstSpec<'_>, + ctx: &Ctx, ) -> MethodAndMethodDef { + let Ctx { pyo3_path } = ctx; let member = &spec.rust_ident; let wrapper_ident = format_ident!("__pymethod_variant_cls_{}__", member); let deprecations = &spec.attributes.deprecations; @@ -1092,17 +1112,17 @@ pub fn gen_complex_enum_variant_attr( let variant_cls = format_ident!("{}_{}", cls, member); let associated_method = quote! { - fn #wrapper_ident(py: _pyo3::Python<'_>) -> _pyo3::PyResult<_pyo3::PyObject> { + fn #wrapper_ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { #deprecations ::std::result::Result::Ok(py.get_type_bound::<#variant_cls>().into_any().unbind()) } }; let method_def = quote! { - _pyo3::class::PyMethodDefType::ClassAttribute({ - _pyo3::class::PyClassAttributeDef::new( + #pyo3_path::class::PyMethodDefType::ClassAttribute({ + #pyo3_path::class::PyClassAttributeDef::new( #python_name, - _pyo3::impl_::pymethods::PyClassAttributeFactory(#cls_type::#wrapper_ident) + #pyo3_path::impl_::pymethods::PyClassAttributeFactory(#cls_type::#wrapper_ident) ) }) }; @@ -1116,10 +1136,11 @@ pub fn gen_complex_enum_variant_attr( fn complex_enum_variant_new<'a>( cls: &'a syn::Ident, variant: &'a PyClassEnumVariant<'a>, + ctx: &Ctx, ) -> Result { match variant { PyClassEnumVariant::Struct(struct_variant) => { - complex_enum_struct_variant_new(cls, struct_variant) + complex_enum_struct_variant_new(cls, struct_variant, ctx) } } } @@ -1127,12 +1148,14 @@ fn complex_enum_variant_new<'a>( fn complex_enum_struct_variant_new<'a>( cls: &'a syn::Ident, variant: &'a PyClassEnumStructVariant<'a>, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path } = ctx; let variant_cls = format_ident!("{}_{}", cls, variant.ident); let variant_cls_type: syn::Type = parse_quote!(#variant_cls); let arg_py_ident: syn::Ident = parse_quote!(py); - let arg_py_type: syn::Type = parse_quote!(_pyo3::Python<'_>); + let arg_py_type: syn::Type = parse_quote!(#pyo3_path::Python<'_>); let args = { let mut no_pyo3_attrs = vec![]; @@ -1180,10 +1203,10 @@ fn complex_enum_struct_variant_new<'a>( text_signature: None, asyncness: None, unsafety: None, - deprecations: Deprecations::default(), + deprecations: Deprecations::new(ctx), }; - crate::pymethod::impl_py_method_def_new(&variant_cls_type, &spec) + crate::pymethod::impl_py_method_def_new(&variant_cls_type, &spec, ctx) } fn complex_enum_variant_field_getter<'a>( @@ -1191,6 +1214,7 @@ fn complex_enum_variant_field_getter<'a>( field_name: &'a syn::Ident, field_type: &'a syn::Type, field_span: Span, + ctx: &Ctx, ) -> Result { let signature = crate::pyfunction::FunctionSignature::from_arguments(vec![])?; @@ -1206,7 +1230,7 @@ fn complex_enum_variant_field_getter<'a>( text_signature: None, asyncness: None, unsafety: None, - deprecations: Deprecations::default(), + deprecations: Deprecations::new(ctx), }; let property_type = crate::pymethod::PropertyType::Function { @@ -1215,7 +1239,7 @@ fn complex_enum_variant_field_getter<'a>( doc: crate::get_doc(&[], None), }; - let getter = crate::pymethod::impl_py_getter_def(variant_cls_type, property_type)?; + let getter = crate::pymethod::impl_py_getter_def(variant_cls_type, property_type, ctx)?; Ok(getter) } @@ -1224,6 +1248,7 @@ fn descriptors_to_items( rename_all: Option<&RenameAllAttribute>, frozen: Option, field_options: Vec<(&syn::Field, FieldPyO3Options)>, + ctx: &Ctx, ) -> syn::Result> { let ty = syn::parse_quote!(#cls); let mut items = Vec::new(); @@ -1246,6 +1271,7 @@ fn descriptors_to_items( python_name: options.name.as_ref(), renaming_rule: rename_all.map(|rename_all| rename_all.value.rule), }, + ctx, )?; items.push(getter); } @@ -1260,6 +1286,7 @@ fn descriptors_to_items( python_name: options.name.as_ref(), renaming_rule: rename_all.map(|rename_all| rename_all.value.rule), }, + ctx, )?; items.push(setter); }; @@ -1270,8 +1297,10 @@ fn descriptors_to_items( fn impl_pytypeinfo( cls: &syn::Ident, attr: &PyClassArgs, - deprecations: Option<&Deprecations>, + deprecations: Option<&Deprecations<'_>>, + ctx: &Ctx, ) -> TokenStream { + let Ctx { pyo3_path } = ctx; let cls_name = get_class_python_name(cls, attr).to_string(); let module = if let Some(ModuleAttribute { value, .. }) = &attr.options.module { @@ -1282,20 +1311,20 @@ fn impl_pytypeinfo( quote! { #[allow(deprecated)] - unsafe impl _pyo3::type_object::HasPyGilRef for #cls { - type AsRefTarget = _pyo3::PyCell; + unsafe impl #pyo3_path::type_object::HasPyGilRef for #cls { + type AsRefTarget = #pyo3_path::PyCell; } - unsafe impl _pyo3::type_object::PyTypeInfo for #cls { + unsafe impl #pyo3_path::type_object::PyTypeInfo for #cls { const NAME: &'static str = #cls_name; const MODULE: ::std::option::Option<&'static str> = #module; #[inline] - fn type_object_raw(py: _pyo3::Python<'_>) -> *mut _pyo3::ffi::PyTypeObject { - use _pyo3::prelude::PyTypeMethods; + fn type_object_raw(py: #pyo3_path::Python<'_>) -> *mut #pyo3_path::ffi::PyTypeObject { + use #pyo3_path::prelude::PyTypeMethods; #deprecations - <#cls as _pyo3::impl_::pyclass::PyClassImpl>::lazy_type_object() + <#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::lazy_type_object() .get_or_init(py) .as_type_ptr() } @@ -1342,82 +1371,85 @@ impl<'a> PyClassImplsBuilder<'a> { } } - fn impl_all(&self) -> Result { + fn impl_all(&self, ctx: &Ctx) -> Result { let tokens = vec![ - self.impl_pyclass(), - self.impl_extractext(), - self.impl_into_py(), - self.impl_pyclassimpl()?, - self.impl_freelist(), + self.impl_pyclass(ctx), + self.impl_extractext(ctx), + self.impl_into_py(ctx), + self.impl_pyclassimpl(ctx)?, + self.impl_freelist(ctx), ] .into_iter() .collect(); Ok(tokens) } - fn impl_pyclass(&self) -> TokenStream { + fn impl_pyclass(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; let cls = self.cls; let frozen = if self.attr.options.frozen.is_some() { - quote! { _pyo3::pyclass::boolean_struct::True } + quote! { #pyo3_path::pyclass::boolean_struct::True } } else { - quote! { _pyo3::pyclass::boolean_struct::False } + quote! { #pyo3_path::pyclass::boolean_struct::False } }; quote! { - impl _pyo3::PyClass for #cls { + impl #pyo3_path::PyClass for #cls { type Frozen = #frozen; } } } - fn impl_extractext(&self) -> TokenStream { + fn impl_extractext(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; let cls = self.cls; if self.attr.options.frozen.is_some() { quote! { - impl<'a, 'py> _pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a #cls + impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a #cls { - type Holder = ::std::option::Option<_pyo3::PyRef<'py, #cls>>; + type Holder = ::std::option::Option<#pyo3_path::PyRef<'py, #cls>>; #[inline] - fn extract(obj: &'a _pyo3::Bound<'py, _pyo3::PyAny>, holder: &'a mut Self::Holder) -> _pyo3::PyResult { - _pyo3::impl_::extract_argument::extract_pyclass_ref(obj, holder) + fn extract(obj: &'a #pyo3_path::Bound<'py, #pyo3_path::PyAny>, holder: &'a mut Self::Holder) -> #pyo3_path::PyResult { + #pyo3_path::impl_::extract_argument::extract_pyclass_ref(obj, holder) } } } } else { quote! { - impl<'a, 'py> _pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a #cls + impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a #cls { - type Holder = ::std::option::Option<_pyo3::PyRef<'py, #cls>>; + type Holder = ::std::option::Option<#pyo3_path::PyRef<'py, #cls>>; #[inline] - fn extract(obj: &'a _pyo3::Bound<'py, _pyo3::PyAny>, holder: &'a mut Self::Holder) -> _pyo3::PyResult { - _pyo3::impl_::extract_argument::extract_pyclass_ref(obj, holder) + fn extract(obj: &'a #pyo3_path::Bound<'py, #pyo3_path::PyAny>, holder: &'a mut Self::Holder) -> #pyo3_path::PyResult { + #pyo3_path::impl_::extract_argument::extract_pyclass_ref(obj, holder) } } - impl<'a, 'py> _pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a mut #cls + impl<'a, 'py> #pyo3_path::impl_::extract_argument::PyFunctionArgument<'a, 'py> for &'a mut #cls { - type Holder = ::std::option::Option<_pyo3::PyRefMut<'py, #cls>>; + type Holder = ::std::option::Option<#pyo3_path::PyRefMut<'py, #cls>>; #[inline] - fn extract(obj: &'a _pyo3::Bound<'py, _pyo3::PyAny>, holder: &'a mut Self::Holder) -> _pyo3::PyResult { - _pyo3::impl_::extract_argument::extract_pyclass_ref_mut(obj, holder) + fn extract(obj: &'a #pyo3_path::Bound<'py, #pyo3_path::PyAny>, holder: &'a mut Self::Holder) -> #pyo3_path::PyResult { + #pyo3_path::impl_::extract_argument::extract_pyclass_ref_mut(obj, holder) } } } } } - fn impl_into_py(&self) -> TokenStream { + fn impl_into_py(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; let cls = self.cls; let attr = self.attr; // If #cls is not extended type, we allow Self->PyObject conversion if attr.options.extends.is_none() { quote! { - impl _pyo3::IntoPy<_pyo3::PyObject> for #cls { - fn into_py(self, py: _pyo3::Python) -> _pyo3::PyObject { - _pyo3::IntoPy::into_py(_pyo3::Py::new(py, self).unwrap(), py) + impl #pyo3_path::IntoPy<#pyo3_path::PyObject> for #cls { + fn into_py(self, py: #pyo3_path::Python) -> #pyo3_path::PyObject { + #pyo3_path::IntoPy::into_py(#pyo3_path::Py::new(py, self).unwrap(), py) } } } @@ -1425,13 +1457,14 @@ impl<'a> PyClassImplsBuilder<'a> { quote! {} } } - fn impl_pyclassimpl(&self) -> Result { + fn impl_pyclassimpl(&self, ctx: &Ctx) -> Result { + let Ctx { pyo3_path } = ctx; let cls = self.cls; let doc = self.doc.as_ref().map_or(quote! {"\0"}, |doc| quote! {#doc}); let is_basetype = self.attr.options.subclass.is_some(); let base = match &self.attr.options.extends { Some(extends_attr) => extends_attr.value.clone(), - None => parse_quote! { _pyo3::PyAny }, + None => parse_quote! { #pyo3_path::PyAny }, }; let is_subclass = self.attr.options.extends.is_some(); let is_mapping: bool = self.attr.options.mapping.is_some(); @@ -1444,8 +1477,8 @@ impl<'a> PyClassImplsBuilder<'a> { let dict_offset = if self.attr.options.dict.is_some() { quote! { - fn dict_offset() -> ::std::option::Option<_pyo3::ffi::Py_ssize_t> { - ::std::option::Option::Some(_pyo3::impl_::pyclass::dict_offset::()) + fn dict_offset() -> ::std::option::Option<#pyo3_path::ffi::Py_ssize_t> { + ::std::option::Option::Some(#pyo3_path::impl_::pyclass::dict_offset::()) } } } else { @@ -1455,8 +1488,8 @@ impl<'a> PyClassImplsBuilder<'a> { // insert space for weak ref let weaklist_offset = if self.attr.options.weakref.is_some() { quote! { - fn weaklist_offset() -> ::std::option::Option<_pyo3::ffi::Py_ssize_t> { - ::std::option::Option::Some(_pyo3::impl_::pyclass::weaklist_offset::()) + fn weaklist_offset() -> ::std::option::Option<#pyo3_path::ffi::Py_ssize_t> { + ::std::option::Option::Some(#pyo3_path::impl_::pyclass::weaklist_offset::()) } } } else { @@ -1464,9 +1497,9 @@ impl<'a> PyClassImplsBuilder<'a> { }; let thread_checker = if self.attr.options.unsendable.is_some() { - quote! { _pyo3::impl_::pyclass::ThreadCheckerImpl } + quote! { #pyo3_path::impl_::pyclass::ThreadCheckerImpl } } else { - quote! { _pyo3::impl_::pyclass::SendablePyClass<#cls> } + quote! { #pyo3_path::impl_::pyclass::SendablePyClass<#cls> } }; let (pymethods_items, inventory, inventory_class) = match self.methods_type { @@ -1481,13 +1514,13 @@ impl<'a> PyClassImplsBuilder<'a> { quote! { ::std::boxed::Box::new( ::std::iter::Iterator::map( - _pyo3::inventory::iter::<::Inventory>(), - _pyo3::impl_::pyclass::PyClassInventory::items + #pyo3_path::inventory::iter::<::Inventory>(), + #pyo3_path::impl_::pyclass::PyClassInventory::items ) ) }, Some(quote! { type Inventory = #inventory_class_name; }), - Some(define_inventory_class(&inventory_class_name)), + Some(define_inventory_class(&inventory_class_name, ctx)), ) } }; @@ -1504,7 +1537,7 @@ impl<'a> PyClassImplsBuilder<'a> { let default_method_defs = self.default_methods.iter().map(|meth| &meth.method_def); let default_slot_defs = self.default_slots.iter().map(|slot| &slot.slot_def); - let freelist_slots = self.freelist_slots(); + let freelist_slots = self.freelist_slots(ctx); let class_mutability = if self.attr.options.frozen.is_some() { quote! { @@ -1519,26 +1552,26 @@ impl<'a> PyClassImplsBuilder<'a> { let cls = self.cls; let attr = self.attr; let dict = if attr.options.dict.is_some() { - quote! { _pyo3::impl_::pyclass::PyClassDictSlot } + quote! { #pyo3_path::impl_::pyclass::PyClassDictSlot } } else { - quote! { _pyo3::impl_::pyclass::PyClassDummySlot } + quote! { #pyo3_path::impl_::pyclass::PyClassDummySlot } }; // insert space for weak ref let weakref = if attr.options.weakref.is_some() { - quote! { _pyo3::impl_::pyclass::PyClassWeakRefSlot } + quote! { #pyo3_path::impl_::pyclass::PyClassWeakRefSlot } } else { - quote! { _pyo3::impl_::pyclass::PyClassDummySlot } + quote! { #pyo3_path::impl_::pyclass::PyClassDummySlot } }; let base_nativetype = if attr.options.extends.is_some() { - quote! { ::BaseNativeType } + quote! { ::BaseNativeType } } else { - quote! { _pyo3::PyAny } + quote! { #pyo3_path::PyAny } }; Ok(quote! { - impl _pyo3::impl_::pyclass::PyClassImpl for #cls { + impl #pyo3_path::impl_::pyclass::PyClassImpl for #cls { const IS_BASETYPE: bool = #is_basetype; const IS_SUBCLASS: bool = #is_subclass; const IS_MAPPING: bool = #is_mapping; @@ -1547,13 +1580,13 @@ impl<'a> PyClassImplsBuilder<'a> { type BaseType = #base; type ThreadChecker = #thread_checker; #inventory - type PyClassMutability = <<#base as _pyo3::impl_::pyclass::PyClassBaseType>::PyClassMutability as _pyo3::impl_::pycell::PyClassMutability>::#class_mutability; + type PyClassMutability = <<#base as #pyo3_path::impl_::pyclass::PyClassBaseType>::PyClassMutability as #pyo3_path::impl_::pycell::PyClassMutability>::#class_mutability; type Dict = #dict; type WeakRef = #weakref; type BaseNativeType = #base_nativetype; - fn items_iter() -> _pyo3::impl_::pyclass::PyClassItemsIter { - use _pyo3::impl_::pyclass::*; + fn items_iter() -> #pyo3_path::impl_::pyclass::PyClassItemsIter { + use #pyo3_path::impl_::pyclass::*; let collector = PyClassImplCollector::::new(); static INTRINSIC_ITEMS: PyClassItems = PyClassItems { methods: &[#(#default_method_defs),*], @@ -1562,12 +1595,12 @@ impl<'a> PyClassImplsBuilder<'a> { PyClassItemsIter::new(&INTRINSIC_ITEMS, #pymethods_items) } - fn doc(py: _pyo3::Python<'_>) -> _pyo3::PyResult<&'static ::std::ffi::CStr> { - use _pyo3::impl_::pyclass::*; - static DOC: _pyo3::sync::GILOnceCell<::std::borrow::Cow<'static, ::std::ffi::CStr>> = _pyo3::sync::GILOnceCell::new(); + fn doc(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<&'static ::std::ffi::CStr> { + use #pyo3_path::impl_::pyclass::*; + static DOC: #pyo3_path::sync::GILOnceCell<::std::borrow::Cow<'static, ::std::ffi::CStr>> = #pyo3_path::sync::GILOnceCell::new(); DOC.get_or_try_init(py, || { let collector = PyClassImplCollector::::new(); - build_pyclass_doc(<#cls as _pyo3::PyTypeInfo>::NAME, #doc, collector.new_text_signature()) + build_pyclass_doc(<#cls as #pyo3_path::PyTypeInfo>::NAME, #doc, collector.new_text_signature()) }).map(::std::ops::Deref::deref) } @@ -1575,8 +1608,8 @@ impl<'a> PyClassImplsBuilder<'a> { #weaklist_offset - fn lazy_type_object() -> &'static _pyo3::impl_::pyclass::LazyTypeObject { - use _pyo3::impl_::pyclass::LazyTypeObject; + fn lazy_type_object() -> &'static #pyo3_path::impl_::pyclass::LazyTypeObject { + use #pyo3_path::impl_::pyclass::LazyTypeObject; static TYPE_OBJECT: LazyTypeObject<#cls> = LazyTypeObject::new(); &TYPE_OBJECT } @@ -1592,20 +1625,21 @@ impl<'a> PyClassImplsBuilder<'a> { }) } - fn impl_freelist(&self) -> TokenStream { + fn impl_freelist(&self, ctx: &Ctx) -> TokenStream { let cls = self.cls; + let Ctx { pyo3_path } = ctx; self.attr.options.freelist.as_ref().map_or(quote!{}, |freelist| { let freelist = &freelist.value; quote! { - impl _pyo3::impl_::pyclass::PyClassWithFreeList for #cls { + impl #pyo3_path::impl_::pyclass::PyClassWithFreeList for #cls { #[inline] - fn get_free_list(py: _pyo3::Python<'_>) -> &mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> { - static mut FREELIST: *mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> = 0 as *mut _; + fn get_free_list(py: #pyo3_path::Python<'_>) -> &mut #pyo3_path::impl_::freelist::FreeList<*mut #pyo3_path::ffi::PyObject> { + static mut FREELIST: *mut #pyo3_path::impl_::freelist::FreeList<*mut #pyo3_path::ffi::PyObject> = 0 as *mut _; unsafe { if FREELIST.is_null() { FREELIST = ::std::boxed::Box::into_raw(::std::boxed::Box::new( - _pyo3::impl_::freelist::FreeList::with_capacity(#freelist))); + #pyo3_path::impl_::freelist::FreeList::with_capacity(#freelist))); } &mut *FREELIST } @@ -1615,21 +1649,22 @@ impl<'a> PyClassImplsBuilder<'a> { }) } - fn freelist_slots(&self) -> Vec { + fn freelist_slots(&self, ctx: &Ctx) -> Vec { + let Ctx { pyo3_path } = ctx; let cls = self.cls; if self.attr.options.freelist.is_some() { vec![ quote! { - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::Py_tp_alloc, - pfunc: _pyo3::impl_::pyclass::alloc_with_freelist::<#cls> as *mut _, + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_alloc, + pfunc: #pyo3_path::impl_::pyclass::alloc_with_freelist::<#cls> as *mut _, } }, quote! { - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::Py_tp_free, - pfunc: _pyo3::impl_::pyclass::free_with_freelist::<#cls> as *mut _, + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_free, + pfunc: #pyo3_path::impl_::pyclass::free_with_freelist::<#cls> as *mut _, } }, ] @@ -1639,25 +1674,26 @@ impl<'a> PyClassImplsBuilder<'a> { } } -fn define_inventory_class(inventory_class_name: &syn::Ident) -> TokenStream { +fn define_inventory_class(inventory_class_name: &syn::Ident, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; quote! { #[doc(hidden)] pub struct #inventory_class_name { - items: _pyo3::impl_::pyclass::PyClassItems, + items: #pyo3_path::impl_::pyclass::PyClassItems, } impl #inventory_class_name { - pub const fn new(items: _pyo3::impl_::pyclass::PyClassItems) -> Self { + pub const fn new(items: #pyo3_path::impl_::pyclass::PyClassItems) -> Self { Self { items } } } - impl _pyo3::impl_::pyclass::PyClassInventory for #inventory_class_name { - fn items(&self) -> &_pyo3::impl_::pyclass::PyClassItems { + impl #pyo3_path::impl_::pyclass::PyClassInventory for #inventory_class_name { + fn items(&self) -> &#pyo3_path::impl_::pyclass::PyClassItems { &self.items } } - _pyo3::inventory::collect!(#inventory_class_name); + #pyo3_path::inventory::collect!(#inventory_class_name); } } diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 45de94e4a10..4b1e0eadeba 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -1,3 +1,4 @@ +use crate::utils::Ctx; use crate::{ attributes::{ self, get_pyo3_options, take_attributes, take_pyo3_options, CrateAttribute, @@ -6,7 +7,6 @@ use crate::{ deprecations::Deprecations, method::{self, CallingConvention, FnArg}, pymethod::check_generic, - utils::get_pyo3_crate, }; use proc_macro2::TokenStream; use quote::{format_ident, quote}; @@ -205,6 +205,9 @@ pub fn impl_wrap_pyfunction( krate, } = options; + let ctx = &Ctx::new(&krate); + let Ctx { pyo3_path } = &ctx; + let python_name = name.map_or_else(|| func.sig.ident.unraw(), |name| name.value.0); let tp = if pass_module.is_some() { @@ -249,17 +252,15 @@ pub fn impl_wrap_pyfunction( text_signature, asyncness: func.sig.asyncness, unsafety: func.sig.unsafety, - deprecations: Deprecations::new(), + deprecations: Deprecations::new(ctx), }; - let krate = get_pyo3_crate(&krate); - let vis = &func.vis; let name = &func.sig.ident; let wrapper_ident = format_ident!("__pyfunction_{}", spec.name); - let wrapper = spec.get_wrapper_function(&wrapper_ident, None)?; - let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&func.attrs)); + let wrapper = spec.get_wrapper_function(&wrapper_ident, None, ctx)?; + let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&func.attrs), ctx); let wrapped_pyfunction = quote! { @@ -268,12 +269,12 @@ pub fn impl_wrap_pyfunction( #[doc(hidden)] #vis mod #name { pub(crate) struct MakeDef; - pub const DEF: #krate::impl_::pymethods::PyMethodDef = MakeDef::DEF; + pub const DEF: #pyo3_path::impl_::pymethods::PyMethodDef = MakeDef::DEF; - pub fn add_to_module(module: &#krate::Bound<'_, #krate::types::PyModule>) -> #krate::PyResult<()> { - use #krate::prelude::PyModuleMethods; + pub fn add_to_module(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { + use #pyo3_path::prelude::PyModuleMethods; use ::std::convert::Into; - module.add_function(#krate::types::PyCFunction::internal_new(module.py(), &DEF, module.into())?) + module.add_function(#pyo3_path::types::PyCFunction::internal_new(module.py(), &DEF, module.into())?) } } @@ -284,10 +285,8 @@ pub fn impl_wrap_pyfunction( // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] const _: () = { - use #krate as _pyo3; - impl #name::MakeDef { - const DEF: #krate::impl_::pymethods::PyMethodDef = #methoddef; + const DEF: #pyo3_path::impl_::pymethods::PyMethodDef = #methoddef; } #[allow(non_snake_case)] diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index 5802638e340..30a6d6dd17e 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -1,11 +1,11 @@ use std::collections::HashSet; +use crate::utils::Ctx; use crate::{ attributes::{take_pyo3_options, CrateAttribute}, konst::{ConstAttributes, ConstSpec}, pyfunction::PyFunctionOptions, pymethod::{self, is_proto_method, MethodAndMethodDef, MethodAndSlotDef}, - utils::get_pyo3_crate, }; use proc_macro2::TokenStream; use pymethod::GeneratedPyMethod; @@ -90,6 +90,7 @@ pub fn impl_methods( methods_type: PyClassMethodsType, options: PyImplOptions, ) -> syn::Result { + let ctx = &Ctx::new(&options.krate); let mut trait_impls = Vec::new(); let mut proto_impls = Vec::new(); let mut methods = Vec::new(); @@ -102,7 +103,8 @@ pub fn impl_methods( syn::ImplItem::Fn(meth) => { let mut fun_options = PyFunctionOptions::from_attrs(&mut meth.attrs)?; fun_options.krate = fun_options.krate.or_else(|| options.krate.clone()); - match pymethod::gen_py_method(ty, &mut meth.sig, &mut meth.attrs, fun_options)? { + match pymethod::gen_py_method(ty, &mut meth.sig, &mut meth.attrs, fun_options, ctx)? + { GeneratedPyMethod::Method(MethodAndMethodDef { associated_method, method_def, @@ -127,7 +129,7 @@ pub fn impl_methods( } } syn::ImplItem::Const(konst) => { - let attributes = ConstAttributes::from_attrs(&mut konst.attrs)?; + let attributes = ConstAttributes::from_attrs(&mut konst.attrs, ctx)?; if attributes.is_class_attr { let spec = ConstSpec { rust_ident: konst.ident.clone(), @@ -137,7 +139,7 @@ pub fn impl_methods( let MethodAndMethodDef { associated_method, method_def, - } = gen_py_const(ty, &spec); + } = gen_py_const(ty, &spec, ctx); methods.push(quote!(#(#attrs)* #method_def)); associated_methods.push(quote!(#(#attrs)* #associated_method)); if is_proto_method(&spec.python_name().to_string()) { @@ -158,21 +160,19 @@ pub fn impl_methods( } } - add_shared_proto_slots(ty, &mut proto_impls, implemented_proto_fragments); + add_shared_proto_slots(ty, &mut proto_impls, implemented_proto_fragments, ctx); - let krate = get_pyo3_crate(&options.krate); + let ctx = &Ctx::new(&options.krate); let items = match methods_type { - PyClassMethodsType::Specialization => impl_py_methods(ty, methods, proto_impls), - PyClassMethodsType::Inventory => submit_methods_inventory(ty, methods, proto_impls), + PyClassMethodsType::Specialization => impl_py_methods(ty, methods, proto_impls, ctx), + PyClassMethodsType::Inventory => submit_methods_inventory(ty, methods, proto_impls, ctx), }; Ok(quote! { // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] const _: () = { - use #krate as _pyo3; - #(#trait_impls)* #items @@ -186,24 +186,25 @@ pub fn impl_methods( }) } -pub fn gen_py_const(cls: &syn::Type, spec: &ConstSpec) -> MethodAndMethodDef { +pub fn gen_py_const(cls: &syn::Type, spec: &ConstSpec<'_>, ctx: &Ctx) -> MethodAndMethodDef { let member = &spec.rust_ident; let wrapper_ident = format_ident!("__pymethod_{}__", member); let deprecations = &spec.attributes.deprecations; let python_name = &spec.null_terminated_python_name(); + let Ctx { pyo3_path } = ctx; let associated_method = quote! { - fn #wrapper_ident(py: _pyo3::Python<'_>) -> _pyo3::PyResult<_pyo3::PyObject> { + fn #wrapper_ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { #deprecations - ::std::result::Result::Ok(_pyo3::IntoPy::into_py(#cls::#member, py)) + ::std::result::Result::Ok(#pyo3_path::IntoPy::into_py(#cls::#member, py)) } }; let method_def = quote! { - _pyo3::class::PyMethodDefType::ClassAttribute({ - _pyo3::class::PyClassAttributeDef::new( + #pyo3_path::class::PyMethodDefType::ClassAttribute({ + #pyo3_path::class::PyClassAttributeDef::new( #python_name, - _pyo3::impl_::pymethods::PyClassAttributeFactory(#cls::#wrapper_ident) + #pyo3_path::impl_::pymethods::PyClassAttributeFactory(#cls::#wrapper_ident) ) }) }; @@ -218,13 +219,15 @@ fn impl_py_methods( ty: &syn::Type, methods: Vec, proto_impls: Vec, + ctx: &Ctx, ) -> TokenStream { + let Ctx { pyo3_path } = ctx; quote! { - impl _pyo3::impl_::pyclass::PyMethods<#ty> - for _pyo3::impl_::pyclass::PyClassImplCollector<#ty> + impl #pyo3_path::impl_::pyclass::PyMethods<#ty> + for #pyo3_path::impl_::pyclass::PyClassImplCollector<#ty> { - fn py_methods(self) -> &'static _pyo3::impl_::pyclass::PyClassItems { - static ITEMS: _pyo3::impl_::pyclass::PyClassItems = _pyo3::impl_::pyclass::PyClassItems { + fn py_methods(self) -> &'static #pyo3_path::impl_::pyclass::PyClassItems { + static ITEMS: #pyo3_path::impl_::pyclass::PyClassItems = #pyo3_path::impl_::pyclass::PyClassItems { methods: &[#(#methods),*], slots: &[#(#proto_impls),*] }; @@ -238,13 +241,15 @@ fn add_shared_proto_slots( ty: &syn::Type, proto_impls: &mut Vec, mut implemented_proto_fragments: HashSet, + ctx: &Ctx, ) { + let Ctx { pyo3_path } = ctx; macro_rules! try_add_shared_slot { ($slot:ident, $($fragments:literal),*) => {{ let mut implemented = false; $(implemented |= implemented_proto_fragments.remove($fragments));*; if implemented { - proto_impls.push(quote! { _pyo3::impl_::pyclass::$slot!(#ty) }) + proto_impls.push(quote! { #pyo3_path::impl_::pyclass::$slot!(#ty) }) } }}; } @@ -294,11 +299,13 @@ fn submit_methods_inventory( ty: &syn::Type, methods: Vec, proto_impls: Vec, + ctx: &Ctx, ) -> TokenStream { + let Ctx { pyo3_path } = ctx; quote! { - _pyo3::inventory::submit! { - type Inventory = <#ty as _pyo3::impl_::pyclass::PyClassImpl>::Inventory; - Inventory::new(_pyo3::impl_::pyclass::PyClassItems { methods: &[#(#methods),*], slots: &[#(#proto_impls),*] }) + #pyo3_path::inventory::submit! { + type Inventory = <#ty as #pyo3_path::impl_::pyclass::PyClassImpl>::Inventory; + Inventory::new(#pyo3_path::impl_::pyclass::PyClassItems { methods: &[#(#methods),*], slots: &[#(#proto_impls),*] }) } } } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 4cb07a9ad2a..1c19112d183 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; use crate::attributes::{NameAttribute, RenamingRule}; use crate::method::{CallingConvention, ExtractErrorMode}; +use crate::utils::Ctx; use crate::utils::PythonDoc; use crate::{ method::{FnArg, FnSpec, FnType, SelfType}, @@ -160,8 +161,9 @@ impl<'a> PyMethod<'a> { sig: &'a mut syn::Signature, meth_attrs: &mut Vec, options: PyFunctionOptions, + ctx: &'a Ctx, ) -> Result { - let spec = FnSpec::parse(sig, meth_attrs, options)?; + let spec = FnSpec::parse(sig, meth_attrs, options, ctx)?; let method_name = spec.python_name.to_string(); let kind = PyMethodKind::from_name(&method_name); @@ -186,33 +188,35 @@ pub fn gen_py_method( sig: &mut syn::Signature, meth_attrs: &mut Vec, options: PyFunctionOptions, + ctx: &Ctx, ) -> Result { check_generic(sig)?; ensure_function_options_valid(&options)?; - let method = PyMethod::parse(sig, meth_attrs, options)?; + let method = PyMethod::parse(sig, meth_attrs, options, ctx)?; let spec = &method.spec; + let Ctx { pyo3_path } = ctx; Ok(match (method.kind, &spec.tp) { // Class attributes go before protos so that class attributes can be used to set proto // method to None. (_, FnType::ClassAttribute) => { - GeneratedPyMethod::Method(impl_py_class_attribute(cls, spec)?) + GeneratedPyMethod::Method(impl_py_class_attribute(cls, spec, ctx)?) } (PyMethodKind::Proto(proto_kind), _) => { ensure_no_forbidden_protocol_attributes(&proto_kind, spec, &method.method_name)?; match proto_kind { PyMethodProtoKind::Slot(slot_def) => { - let slot = slot_def.generate_type_slot(cls, spec, &method.method_name)?; + let slot = slot_def.generate_type_slot(cls, spec, &method.method_name, ctx)?; GeneratedPyMethod::Proto(slot) } PyMethodProtoKind::Call => { - GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec)?) + GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec, ctx)?) } PyMethodProtoKind::Traverse => { - GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec)?) + GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec, ctx)?) } PyMethodProtoKind::SlotFragment(slot_fragment_def) => { - let proto = slot_fragment_def.generate_pyproto_fragment(cls, spec)?; + let proto = slot_fragment_def.generate_pyproto_fragment(cls, spec, ctx)?; GeneratedPyMethod::SlotTraitImpl(method.method_name, proto) } } @@ -223,22 +227,25 @@ pub fn gen_py_method( spec, &spec.get_doc(meth_attrs), None, + ctx, )?), (_, FnType::FnClass(_)) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, &spec.get_doc(meth_attrs), - Some(quote!(_pyo3::ffi::METH_CLASS)), + Some(quote!(#pyo3_path::ffi::METH_CLASS)), + ctx, )?), (_, FnType::FnStatic) => GeneratedPyMethod::Method(impl_py_method_def( cls, spec, &spec.get_doc(meth_attrs), - Some(quote!(_pyo3::ffi::METH_STATIC)), + Some(quote!(#pyo3_path::ffi::METH_STATIC)), + ctx, )?), // special prototypes (_, FnType::FnNew) | (_, FnType::FnNewClass(_)) => { - GeneratedPyMethod::Proto(impl_py_method_def_new(cls, spec)?) + GeneratedPyMethod::Proto(impl_py_method_def_new(cls, spec, ctx)?) } (_, FnType::Getter(self_type)) => GeneratedPyMethod::Method(impl_py_getter_def( @@ -248,6 +255,7 @@ pub fn gen_py_method( spec, doc: spec.get_doc(meth_attrs), }, + ctx, )?), (_, FnType::Setter(self_type)) => GeneratedPyMethod::Method(impl_py_setter_def( cls, @@ -256,6 +264,7 @@ pub fn gen_py_method( spec, doc: spec.get_doc(meth_attrs), }, + ctx, )?), (_, FnType::FnModule(_)) => { unreachable!("methods cannot be FnModule") @@ -305,18 +314,20 @@ pub fn impl_py_method_def( spec: &FnSpec<'_>, doc: &PythonDoc, flags: Option, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path } = ctx; let wrapper_ident = format_ident!("__pymethod_{}__", spec.python_name); - let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls))?; + let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?; let add_flags = flags.map(|flags| quote!(.flags(#flags))); let methoddef_type = match spec.tp { FnType::FnStatic => quote!(Static), FnType::FnClass(_) => quote!(Class), _ => quote!(Method), }; - let methoddef = spec.get_methoddef(quote! { #cls::#wrapper_ident }, doc); + let methoddef = spec.get_methoddef(quote! { #cls::#wrapper_ident }, doc, ctx); let method_def = quote! { - _pyo3::class::PyMethodDefType::#methoddef_type(#methoddef #add_flags) + #pyo3_path::class::PyMethodDefType::#methoddef_type(#methoddef #add_flags) }; Ok(MethodAndMethodDef { associated_method, @@ -325,9 +336,14 @@ pub fn impl_py_method_def( } /// Also used by pyclass. -pub fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec<'_>) -> Result { +pub fn impl_py_method_def_new( + cls: &syn::Type, + spec: &FnSpec<'_>, + ctx: &Ctx, +) -> Result { + let Ctx { pyo3_path } = ctx; let wrapper_ident = syn::Ident::new("__pymethod___new____", Span::call_site()); - let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls))?; + let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?; // Use just the text_signature_call_signature() because the class' Python name // isn't known to `#[pymethods]` - that has to be attached at runtime from the PyClassImpl // trait implementation created by `#[pyclass]`. @@ -337,18 +353,18 @@ pub fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec<'_>) -> Result *mut _pyo3::ffi::PyObject + subtype: *mut #pyo3_path::ffi::PyTypeObject, + args: *mut #pyo3_path::ffi::PyObject, + kwargs: *mut #pyo3_path::ffi::PyObject, + ) -> *mut #pyo3_path::ffi::PyObject { #deprecations - use _pyo3::impl_::pyclass::*; + use #pyo3_path::impl_::pyclass::*; impl PyClassNewTextSignature<#cls> for PyClassImplCollector<#cls> { #[inline] fn new_text_signature(self) -> ::std::option::Option<&'static str> { @@ -356,7 +372,7 @@ pub fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec<'_>) -> Result) -> Result) -> Result) -> Result { +fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result { + let Ctx { pyo3_path } = ctx; + // HACK: __call__ proto slot must always use varargs calling convention, so change the spec. // Probably indicates there's a refactoring opportunity somewhere. spec.convention = CallingConvention::Varargs; let wrapper_ident = syn::Ident::new("__pymethod___call____", Span::call_site()); - let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls))?; + let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?; let slot_def = quote! { - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::Py_tp_call, + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_call, pfunc: { unsafe extern "C" fn trampoline( - slf: *mut _pyo3::ffi::PyObject, - args: *mut _pyo3::ffi::PyObject, - kwargs: *mut _pyo3::ffi::PyObject, - ) -> *mut _pyo3::ffi::PyObject + slf: *mut #pyo3_path::ffi::PyObject, + args: *mut #pyo3_path::ffi::PyObject, + kwargs: *mut #pyo3_path::ffi::PyObject, + ) -> *mut #pyo3_path::ffi::PyObject { - _pyo3::impl_::trampoline::ternaryfunc( + #pyo3_path::impl_::trampoline::ternaryfunc( slf, args, kwargs, @@ -398,7 +416,7 @@ fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>) -> Result) -> Result) -> syn::Result { +fn impl_traverse_slot( + cls: &syn::Type, + spec: &FnSpec<'_>, + ctx: &Ctx, +) -> syn::Result { + let Ctx { pyo3_path } = ctx; if let (Some(py_arg), _) = split_off_python_arg(&spec.signature.arguments) { return Err(syn::Error::new_spanned(py_arg.ty, "__traverse__ may not take `Python`. \ Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. \ @@ -419,17 +442,17 @@ fn impl_traverse_slot(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result ::std::os::raw::c_int { - _pyo3::impl_::pymethods::_call_traverse::<#cls>(slf, #cls::#rust_fn_ident, visit, arg) + #pyo3_path::impl_::pymethods::_call_traverse::<#cls>(slf, #cls::#rust_fn_ident, visit, arg) } }; let slot_def = quote! { - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::Py_tp_traverse, - pfunc: #cls::__pymethod_traverse__ as _pyo3::ffi::traverseproc as _ + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_traverse, + pfunc: #cls::__pymethod_traverse__ as #pyo3_path::ffi::traverseproc as _ } }; Ok(MethodAndSlotDef { @@ -438,7 +461,12 @@ fn impl_traverse_slot(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result) -> syn::Result { +fn impl_py_class_attribute( + cls: &syn::Type, + spec: &FnSpec<'_>, + ctx: &Ctx, +) -> syn::Result { + let Ctx { pyo3_path } = ctx; let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); ensure_spanned!( args.is_empty(), @@ -454,20 +482,20 @@ fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result) -> _pyo3::PyResult<_pyo3::PyObject> { + fn #wrapper_ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { let function = #cls::#name; // Shadow the method name to avoid #3017 - _pyo3::impl_::wrap::map_result_into_py(py, #body) + #pyo3_path::impl_::wrap::map_result_into_py(py, #body) } }; let method_def = quote! { - _pyo3::class::PyMethodDefType::ClassAttribute({ - _pyo3::class::PyClassAttributeDef::new( + #pyo3_path::class::PyMethodDefType::ClassAttribute({ + #pyo3_path::class::PyClassAttributeDef::new( #python_name, - _pyo3::impl_::pymethods::PyClassAttributeFactory(#cls::#wrapper_ident) + #pyo3_path::impl_::pymethods::PyClassAttributeFactory(#cls::#wrapper_ident) ) }) }; @@ -483,9 +511,10 @@ fn impl_call_setter( spec: &FnSpec<'_>, self_type: &SelfType, holders: &mut Vec, + ctx: &Ctx, ) -> syn::Result { let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); - let slf = self_type.receiver(cls, ExtractErrorMode::Raise, holders); + let slf = self_type.receiver(cls, ExtractErrorMode::Raise, holders, ctx); if args.is_empty() { bail_spanned!(spec.name.span() => "setter function expected to have one argument"); @@ -510,7 +539,9 @@ fn impl_call_setter( pub fn impl_py_setter_def( cls: &syn::Type, property_type: PropertyType<'_>, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path } = ctx; let python_name = property_type.null_terminated_python_name()?; let doc = property_type.doc(); let mut holders = Vec::new(); @@ -522,7 +553,7 @@ pub fn impl_py_setter_def( mutable: true, span: Span::call_site(), } - .receiver(cls, ExtractErrorMode::Raise, &mut holders); + .receiver(cls, ExtractErrorMode::Raise, &mut holders, ctx); if let Some(ident) = &field.ident { // named struct field quote!({ #slf.#ident = _val; }) @@ -534,7 +565,7 @@ pub fn impl_py_setter_def( } PropertyType::Function { spec, self_type, .. - } => impl_call_setter(cls, spec, self_type, &mut holders)?, + } => impl_call_setter(cls, spec, self_type, &mut holders, ctx)?, }; let wrapper_ident = match property_type { @@ -568,27 +599,27 @@ pub fn impl_py_setter_def( let associated_method = quote! { #cfg_attrs unsafe fn #wrapper_ident( - py: _pyo3::Python<'_>, - _slf: *mut _pyo3::ffi::PyObject, - _value: *mut _pyo3::ffi::PyObject, - ) -> _pyo3::PyResult<::std::os::raw::c_int> { + py: #pyo3_path::Python<'_>, + _slf: *mut #pyo3_path::ffi::PyObject, + _value: *mut #pyo3_path::ffi::PyObject, + ) -> #pyo3_path::PyResult<::std::os::raw::c_int> { use ::std::convert::Into; - let _value = _pyo3::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_value) + let _value = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_value) .ok_or_else(|| { - _pyo3::exceptions::PyAttributeError::new_err("can't delete attribute") + #pyo3_path::exceptions::PyAttributeError::new_err("can't delete attribute") })?; - let _val = _pyo3::FromPyObject::extract_bound(_value.into())?; + let _val = #pyo3_path::FromPyObject::extract_bound(_value.into())?; #( #holders )* - _pyo3::callback::convert(py, #setter_impl) + #pyo3_path::callback::convert(py, #setter_impl) } }; let method_def = quote! { #cfg_attrs - _pyo3::class::PyMethodDefType::Setter( - _pyo3::class::PySetterDef::new( + #pyo3_path::class::PyMethodDefType::Setter( + #pyo3_path::class::PySetterDef::new( #python_name, - _pyo3::impl_::pymethods::PySetter(#cls::#wrapper_ident), + #pyo3_path::impl_::pymethods::PySetter(#cls::#wrapper_ident), #doc ) ) @@ -605,9 +636,10 @@ fn impl_call_getter( spec: &FnSpec<'_>, self_type: &SelfType, holders: &mut Vec, + ctx: &Ctx, ) -> syn::Result { let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); - let slf = self_type.receiver(cls, ExtractErrorMode::Raise, holders); + let slf = self_type.receiver(cls, ExtractErrorMode::Raise, holders, ctx); ensure_spanned!( args.is_empty(), args[0].ty.span() => "getter function can only have one argument (of type pyo3::Python)" @@ -627,7 +659,9 @@ fn impl_call_getter( pub fn impl_py_getter_def( cls: &syn::Type, property_type: PropertyType<'_>, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path } = ctx; let python_name = property_type.null_terminated_python_name()?; let doc = property_type.doc(); @@ -640,7 +674,7 @@ pub fn impl_py_getter_def( mutable: false, span: Span::call_site(), } - .receiver(cls, ExtractErrorMode::Raise, &mut holders); + .receiver(cls, ExtractErrorMode::Raise, &mut holders, ctx); let field_token = if let Some(ident) = &field.ident { // named struct field ident.to_token_stream() @@ -648,17 +682,23 @@ pub fn impl_py_getter_def( // tuple struct field syn::Index::from(field_index).to_token_stream() }; - quotes::map_result_into_ptr(quotes::ok_wrap(quote! { - ::std::clone::Clone::clone(&(#slf.#field_token)) - })) + quotes::map_result_into_ptr( + quotes::ok_wrap( + quote! { + ::std::clone::Clone::clone(&(#slf.#field_token)) + }, + ctx, + ), + ctx, + ) } // Forward to `IntoPyCallbackOutput`, to handle `#[getter]`s returning results. PropertyType::Function { spec, self_type, .. } => { - let call = impl_call_getter(cls, spec, self_type, &mut holders)?; + let call = impl_call_getter(cls, spec, self_type, &mut holders, ctx)?; quote! { - _pyo3::callback::convert(py, #call) + #pyo3_path::callback::convert(py, #call) } } }; @@ -694,9 +734,9 @@ pub fn impl_py_getter_def( let associated_method = quote! { #cfg_attrs unsafe fn #wrapper_ident( - py: _pyo3::Python<'_>, - _slf: *mut _pyo3::ffi::PyObject - ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { + py: #pyo3_path::Python<'_>, + _slf: *mut #pyo3_path::ffi::PyObject + ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { #( #holders )* let result = #body; result @@ -705,10 +745,10 @@ pub fn impl_py_getter_def( let method_def = quote! { #cfg_attrs - _pyo3::class::PyMethodDefType::Getter( - _pyo3::class::PyGetterDef::new( + #pyo3_path::class::PyMethodDefType::Getter( + #pyo3_path::class::PyGetterDef::new( #python_name, - _pyo3::impl_::pymethods::PyGetter(#cls::#wrapper_ident), + #pyo3_path::impl_::pymethods::PyGetter(#cls::#wrapper_ident), #doc ) ) @@ -786,7 +826,7 @@ pub const __REPR__: SlotDef = SlotDef::new("Py_tp_repr", "reprfunc"); const __HASH__: SlotDef = SlotDef::new("Py_tp_hash", "hashfunc") .ret_ty(Ty::PyHashT) .return_conversion(TokenGenerator( - || quote! { _pyo3::callback::HashCallbackOutput }, + |Ctx { pyo3_path }: &Ctx| quote! { #pyo3_path::callback::HashCallbackOutput }, )); pub const __RICHCMP__: SlotDef = SlotDef::new("Py_tp_richcompare", "richcmpfunc") .extract_error_mode(ExtractErrorMode::NotImplemented) @@ -796,14 +836,16 @@ const __GET__: SlotDef = SlotDef::new("Py_tp_descr_get", "descrgetfunc") const __ITER__: SlotDef = SlotDef::new("Py_tp_iter", "getiterfunc"); const __NEXT__: SlotDef = SlotDef::new("Py_tp_iternext", "iternextfunc") .return_specialized_conversion( - TokenGenerator(|| quote! { IterBaseKind, IterOptionKind, IterResultOptionKind }), - TokenGenerator(|| quote! { iter_tag }), + TokenGenerator(|_| quote! { IterBaseKind, IterOptionKind, IterResultOptionKind }), + TokenGenerator(|_| quote! { iter_tag }), ); const __AWAIT__: SlotDef = SlotDef::new("Py_am_await", "unaryfunc"); const __AITER__: SlotDef = SlotDef::new("Py_am_aiter", "unaryfunc"); const __ANEXT__: SlotDef = SlotDef::new("Py_am_anext", "unaryfunc").return_specialized_conversion( - TokenGenerator(|| quote! { AsyncIterBaseKind, AsyncIterOptionKind, AsyncIterResultOptionKind }), - TokenGenerator(|| quote! { async_iter_tag }), + TokenGenerator( + |_| quote! { AsyncIterBaseKind, AsyncIterOptionKind, AsyncIterResultOptionKind }, + ), + TokenGenerator(|_| quote! { async_iter_tag }), ); const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc").ret_ty(Ty::PySsizeT); const __CONTAINS__: SlotDef = SlotDef::new("Py_sq_contains", "objobjproc") @@ -905,16 +947,17 @@ enum Ty { } impl Ty { - fn ffi_type(self) -> TokenStream { + fn ffi_type(self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; match self { - Ty::Object | Ty::MaybeNullObject => quote! { *mut _pyo3::ffi::PyObject }, - Ty::NonNullObject => quote! { ::std::ptr::NonNull<_pyo3::ffi::PyObject> }, - Ty::IPowModulo => quote! { _pyo3::impl_::pymethods::IPowModulo }, + Ty::Object | Ty::MaybeNullObject => quote! { *mut #pyo3_path::ffi::PyObject }, + Ty::NonNullObject => quote! { ::std::ptr::NonNull<#pyo3_path::ffi::PyObject> }, + Ty::IPowModulo => quote! { #pyo3_path::impl_::pymethods::IPowModulo }, Ty::Int | Ty::CompareOp => quote! { ::std::os::raw::c_int }, - Ty::PyHashT => quote! { _pyo3::ffi::Py_hash_t }, - Ty::PySsizeT => quote! { _pyo3::ffi::Py_ssize_t }, + Ty::PyHashT => quote! { #pyo3_path::ffi::Py_hash_t }, + Ty::PySsizeT => quote! { #pyo3_path::ffi::Py_ssize_t }, Ty::Void => quote! { () }, - Ty::PyBuffer => quote! { *mut _pyo3::ffi::Py_buffer }, + Ty::PyBuffer => quote! { *mut #pyo3_path::ffi::Py_buffer }, } } @@ -924,14 +967,16 @@ impl Ty { arg: &FnArg<'_>, extract_error_mode: ExtractErrorMode, holders: &mut Vec, + ctx: &Ctx, ) -> TokenStream { + let Ctx { pyo3_path } = ctx; let name_str = arg.name.unraw().to_string(); match self { Ty::Object => extract_object( extract_error_mode, holders, &name_str, - quote! { #ident }, + quote! { #ident },ctx ), Ty::MaybeNullObject => extract_object( extract_error_mode, @@ -939,36 +984,36 @@ impl Ty { &name_str, quote! { if #ident.is_null() { - _pyo3::ffi::Py_None() + #pyo3_path::ffi::Py_None() } else { #ident } - }, + },ctx ), Ty::NonNullObject => extract_object( extract_error_mode, holders, &name_str, - quote! { #ident.as_ptr() }, + quote! { #ident.as_ptr() },ctx ), Ty::IPowModulo => extract_object( extract_error_mode, holders, &name_str, - quote! { #ident.as_ptr() }, + quote! { #ident.as_ptr() },ctx ), Ty::CompareOp => extract_error_mode.handle_error( quote! { - _pyo3::class::basic::CompareOp::from_raw(#ident) - .ok_or_else(|| _pyo3::exceptions::PyValueError::new_err("invalid comparison operator")) - }, + #pyo3_path::class::basic::CompareOp::from_raw(#ident) + .ok_or_else(|| #pyo3_path::exceptions::PyValueError::new_err("invalid comparison operator")) + },ctx ), Ty::PySsizeT => { let ty = arg.ty; extract_error_mode.handle_error( quote! { - ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| _pyo3::exceptions::PyValueError::new_err(e.to_string())) - }, + ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| #pyo3_path::exceptions::PyValueError::new_err(e.to_string())) + },ctx ) } // Just pass other types through unmodified @@ -982,19 +1027,24 @@ fn extract_object( holders: &mut Vec, name: &str, source_ptr: TokenStream, + ctx: &Ctx, ) -> TokenStream { + let Ctx { pyo3_path } = ctx; let holder = syn::Ident::new(&format!("holder_{}", holders.len()), Span::call_site()); holders.push(quote! { #[allow(clippy::let_unit_value)] - let mut #holder = _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT; + let mut #holder = #pyo3_path::impl_::extract_argument::FunctionArgumentHolder::INIT; }); - extract_error_mode.handle_error(quote! { - _pyo3::impl_::extract_argument::extract_argument( - &_pyo3::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr), - &mut #holder, - #name - ) - }) + extract_error_mode.handle_error( + quote! { + #pyo3_path::impl_::extract_argument::extract_argument( + &#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr), + &mut #holder, + #name + ) + }, + ctx, + ) } enum ReturnMode { @@ -1004,21 +1054,29 @@ enum ReturnMode { } impl ReturnMode { - fn return_call_output(&self, call: TokenStream) -> TokenStream { + fn return_call_output(&self, call: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; match self { - ReturnMode::Conversion(conversion) => quote! { - let _result: _pyo3::PyResult<#conversion> = _pyo3::callback::convert(py, #call); - _pyo3::callback::convert(py, _result) - }, - ReturnMode::SpecializedConversion(traits, tag) => quote! { - let _result = #call; - use _pyo3::impl_::pymethods::{#traits}; - (&_result).#tag().convert(py, _result) - }, + ReturnMode::Conversion(conversion) => { + let conversion = TokenGeneratorCtx(*conversion, ctx); + quote! { + let _result: #pyo3_path::PyResult<#conversion> = #pyo3_path::callback::convert(py, #call); + #pyo3_path::callback::convert(py, _result) + } + } + ReturnMode::SpecializedConversion(traits, tag) => { + let traits = TokenGeneratorCtx(*traits, ctx); + let tag = TokenGeneratorCtx(*tag, ctx); + quote! { + let _result = #call; + use #pyo3_path::impl_::pymethods::{#traits}; + (&_result).#tag().convert(py, _result) + } + } ReturnMode::ReturnSelf => quote! { - let _result: _pyo3::PyResult<()> = _pyo3::callback::convert(py, #call); + let _result: #pyo3_path::PyResult<()> = #pyo3_path::callback::convert(py, #call); _result?; - _pyo3::ffi::Py_XINCREF(_raw_slf); + #pyo3_path::ffi::Py_XINCREF(_raw_slf); ::std::result::Result::Ok(_raw_slf) }, } @@ -1094,7 +1152,9 @@ impl SlotDef { cls: &syn::Type, spec: &FnSpec<'_>, method_name: &str, + ctx: &Ctx, ) -> Result { + let Ctx { pyo3_path } = ctx; let SlotDef { slot, func_ty, @@ -1110,12 +1170,12 @@ impl SlotDef { spec.name.span() => format!("`{}` must be `unsafe fn`", method_name) ); } - let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect(); + let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type(ctx)).collect(); let arg_idents: &Vec<_> = &(0..arguments.len()) .map(|i| format_ident!("arg{}", i)) .collect(); let wrapper_ident = format_ident!("__pymethod_{}__", method_name); - let ret_ty = ret_ty.ffi_type(); + let ret_ty = ret_ty.ffi_type(ctx); let mut holders = Vec::new(); let body = generate_method_body( cls, @@ -1124,14 +1184,15 @@ impl SlotDef { *extract_error_mode, &mut holders, return_mode.as_ref(), + ctx, )?; let name = spec.name; let associated_method = quote! { unsafe fn #wrapper_ident( - py: _pyo3::Python<'_>, - _raw_slf: *mut _pyo3::ffi::PyObject, + py: #pyo3_path::Python<'_>, + _raw_slf: *mut #pyo3_path::ffi::PyObject, #(#arg_idents: #arg_types),* - ) -> _pyo3::PyResult<#ret_ty> { + ) -> #pyo3_path::PyResult<#ret_ty> { let function = #cls::#name; // Shadow the method name to avoid #3017 let _slf = _raw_slf; #( #holders )* @@ -1140,20 +1201,20 @@ impl SlotDef { }; let slot_def = quote! {{ unsafe extern "C" fn trampoline( - _slf: *mut _pyo3::ffi::PyObject, + _slf: *mut #pyo3_path::ffi::PyObject, #(#arg_idents: #arg_types),* ) -> #ret_ty { - _pyo3::impl_::trampoline:: #func_ty ( + #pyo3_path::impl_::trampoline:: #func_ty ( _slf, #(#arg_idents,)* #cls::#wrapper_ident ) } - _pyo3::ffi::PyType_Slot { - slot: _pyo3::ffi::#slot, - pfunc: trampoline as _pyo3::ffi::#func_ty as _ + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::#slot, + pfunc: trampoline as #pyo3_path::ffi::#func_ty as _ } }}; Ok(MethodAndSlotDef { @@ -1170,15 +1231,19 @@ fn generate_method_body( extract_error_mode: ExtractErrorMode, holders: &mut Vec, return_mode: Option<&ReturnMode>, + ctx: &Ctx, ) -> Result { - let self_arg = spec.tp.self_arg(Some(cls), extract_error_mode, holders); + let Ctx { pyo3_path } = ctx; + let self_arg = spec + .tp + .self_arg(Some(cls), extract_error_mode, holders, ctx); let rust_name = spec.name; - let args = extract_proto_arguments(spec, arguments, extract_error_mode, holders)?; + let args = extract_proto_arguments(spec, arguments, extract_error_mode, holders, ctx)?; let call = quote! { #cls::#rust_name(#self_arg #(#args),*) }; Ok(if let Some(return_mode) = return_mode { - return_mode.return_call_output(call) + return_mode.return_call_output(call, ctx) } else { - quote! { _pyo3::callback::convert(py, #call) } + quote! { #pyo3_path::callback::convert(py, #call) } }) } @@ -1209,7 +1274,13 @@ impl SlotFragmentDef { self } - fn generate_pyproto_fragment(&self, cls: &syn::Type, spec: &FnSpec<'_>) -> Result { + fn generate_pyproto_fragment( + &self, + cls: &syn::Type, + spec: &FnSpec<'_>, + ctx: &Ctx, + ) -> Result { + let Ctx { pyo3_path } = ctx; let SlotFragmentDef { fragment, arguments, @@ -1219,7 +1290,7 @@ impl SlotFragmentDef { let fragment_trait = format_ident!("PyClass{}SlotFragment", fragment); let method = syn::Ident::new(fragment, Span::call_site()); let wrapper_ident = format_ident!("__pymethod_{}__", fragment); - let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect(); + let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type(ctx)).collect(); let arg_idents: &Vec<_> = &(0..arguments.len()) .map(|i| format_ident!("arg{}", i)) .collect(); @@ -1231,30 +1302,31 @@ impl SlotFragmentDef { *extract_error_mode, &mut holders, None, + ctx, )?; - let ret_ty = ret_ty.ffi_type(); + let ret_ty = ret_ty.ffi_type(ctx); Ok(quote! { impl #cls { unsafe fn #wrapper_ident( - py: _pyo3::Python, - _raw_slf: *mut _pyo3::ffi::PyObject, + py: #pyo3_path::Python, + _raw_slf: *mut #pyo3_path::ffi::PyObject, #(#arg_idents: #arg_types),* - ) -> _pyo3::PyResult<#ret_ty> { + ) -> #pyo3_path::PyResult<#ret_ty> { let _slf = _raw_slf; #( #holders )* #body } } - impl _pyo3::impl_::pyclass::#fragment_trait<#cls> for _pyo3::impl_::pyclass::PyClassImplCollector<#cls> { + impl #pyo3_path::impl_::pyclass::#fragment_trait<#cls> for #pyo3_path::impl_::pyclass::PyClassImplCollector<#cls> { #[inline] unsafe fn #method( self, - py: _pyo3::Python, - _raw_slf: *mut _pyo3::ffi::PyObject, + py: #pyo3_path::Python, + _raw_slf: *mut #pyo3_path::ffi::PyObject, #(#arg_idents: #arg_types),* - ) -> _pyo3::PyResult<#ret_ty> { + ) -> #pyo3_path::PyResult<#ret_ty> { #cls::#wrapper_ident(py, _raw_slf, #(#arg_idents),*) } } @@ -1341,6 +1413,7 @@ fn extract_proto_arguments( proto_args: &[Ty], extract_error_mode: ExtractErrorMode, holders: &mut Vec, + ctx: &Ctx, ) -> Result> { let mut args = Vec::with_capacity(spec.signature.arguments.len()); let mut non_python_args = 0; @@ -1352,7 +1425,7 @@ fn extract_proto_arguments( let ident = syn::Ident::new(&format!("arg{}", non_python_args), Span::call_site()); let conversions = proto_args.get(non_python_args) .ok_or_else(|| err_spanned!(arg.ty.span() => format!("Expected at most {} non-python arguments", proto_args.len())))? - .extract(&ident, arg, extract_error_mode, holders); + .extract(&ident, arg, extract_error_mode, holders, ctx); non_python_args += 1; args.push(conversions); } @@ -1372,10 +1445,14 @@ impl ToTokens for StaticIdent { } } -struct TokenGenerator(fn() -> TokenStream); +#[derive(Clone, Copy)] +struct TokenGenerator(fn(&Ctx) -> TokenStream); + +struct TokenGeneratorCtx<'ctx>(TokenGenerator, &'ctx Ctx); -impl ToTokens for TokenGenerator { +impl ToTokens for TokenGeneratorCtx<'_> { fn to_tokens(&self, tokens: &mut TokenStream) { - self.0().to_tokens(tokens) + let Self(TokenGenerator(gen), ctx) = self; + (gen)(ctx).to_tokens(tokens) } } diff --git a/pyo3-macros-backend/src/quotes.rs b/pyo3-macros-backend/src/quotes.rs index 239036ef3ca..0219cb9ae7f 100644 --- a/pyo3-macros-backend/src/quotes.rs +++ b/pyo3-macros-backend/src/quotes.rs @@ -1,21 +1,25 @@ +use crate::utils::Ctx; use proc_macro2::TokenStream; use quote::quote; -pub(crate) fn some_wrap(obj: TokenStream) -> TokenStream { +pub(crate) fn some_wrap(obj: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; quote! { - _pyo3::impl_::wrap::SomeWrap::wrap(#obj) + #pyo3_path::impl_::wrap::SomeWrap::wrap(#obj) } } -pub(crate) fn ok_wrap(obj: TokenStream) -> TokenStream { +pub(crate) fn ok_wrap(obj: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; quote! { - _pyo3::impl_::wrap::OkWrap::wrap(#obj) - .map_err(::core::convert::Into::<_pyo3::PyErr>::into) + #pyo3_path::impl_::wrap::OkWrap::wrap(#obj) + .map_err(::core::convert::Into::<#pyo3_path::PyErr>::into) } } -pub(crate) fn map_result_into_ptr(result: TokenStream) -> TokenStream { +pub(crate) fn map_result_into_ptr(result: TokenStream, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; quote! { - _pyo3::impl_::wrap::map_result_into_ptr(py, #result) + #pyo3_path::impl_::wrap::map_result_into_ptr(py, #result) } } diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index 9f0f2678476..ca32abb42b3 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -144,11 +144,41 @@ pub fn unwrap_ty_group(mut ty: &syn::Type) -> &syn::Type { ty } -/// Extract the path to the pyo3 crate, or use the default (`::pyo3`). -pub(crate) fn get_pyo3_crate(attr: &Option) -> syn::Path { - match attr { - Some(attr) => attr.value.0.clone(), - None => syn::parse_str("::pyo3").unwrap(), +pub struct Ctx { + pub pyo3_path: PyO3CratePath, +} + +impl Ctx { + pub(crate) fn new(attr: &Option) -> Self { + let pyo3_path = match attr { + Some(attr) => PyO3CratePath::Given(attr.value.0.clone()), + None => PyO3CratePath::Default, + }; + + Self { pyo3_path } + } +} + +pub enum PyO3CratePath { + Given(syn::Path), + Default, +} + +impl PyO3CratePath { + pub fn to_tokens_spanned(&self, span: Span) -> TokenStream { + match self { + Self::Given(path) => quote::quote_spanned! { span => #path }, + Self::Default => quote::quote_spanned! { span => ::pyo3 }, + } + } +} + +impl quote::ToTokens for PyO3CratePath { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Given(path) => path.to_tokens(tokens), + Self::Default => quote::quote! { ::pyo3 }.to_tokens(tokens), + } } } diff --git a/tests/ui/invalid_proto_pymethods.stderr b/tests/ui/invalid_proto_pymethods.stderr index 275a6b93c46..c9f6adff3a1 100644 --- a/tests/ui/invalid_proto_pymethods.stderr +++ b/tests/ui/invalid_proto_pymethods.stderr @@ -31,4 +31,4 @@ error[E0592]: duplicate definitions with name `__pymethod___richcmp____` | duplicate definitions for `__pymethod___richcmp____` | other definition for `__pymethod___richcmp____` | - = note: this error originates in the macro `_pyo3::impl_::pyclass::generate_pyclass_richcompare_slot` which comes from the expansion of the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) + = note: this error originates in the macro `::pyo3::impl_::pyclass::generate_pyclass_richcompare_slot` which comes from the expansion of the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) From 811a3e5d009e72eb41e0594b3f65e3228a2d4902 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Mon, 4 Mar 2024 21:24:38 +0000 Subject: [PATCH 198/349] Add IntoIterator for &Bound types (#3923) * Add IntoIterator for &Bound<'py, PyList> * Add a test for Bound<'_, PyList>.into_iter * Implement IntoIterator for more &Bound types * Remove some explicit .iter() calls * Implement IntoIterator for &Bound<'py, PyIterator> --- pyo3-benches/benches/bench_dict.rs | 2 +- pyo3-benches/benches/bench_list.rs | 2 +- pyo3-benches/benches/bench_set.rs | 2 +- src/conversions/hashbrown.rs | 2 +- src/conversions/indexmap.rs | 2 +- src/conversions/std/map.rs | 4 +-- src/types/dict.rs | 29 +++++++++++++++++++++ src/types/frozenset.rs | 21 +++++++++++++++ src/types/iterator.rs | 42 ++++++++++++++++++++++++++++++ src/types/list.rs | 23 ++++++++++++++++ src/types/mod.rs | 2 +- src/types/set.rs | 27 +++++++++++++++++++ src/types/tuple.rs | 26 ++++++++++++++++++ 13 files changed, 176 insertions(+), 8 deletions(-) diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs index b63e010c1ff..e6a1e2e6b0b 100644 --- a/pyo3-benches/benches/bench_dict.rs +++ b/pyo3-benches/benches/bench_dict.rs @@ -11,7 +11,7 @@ fn iter_dict(b: &mut Bencher<'_>) { let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict_bound(py); let mut sum = 0; b.iter(|| { - for (k, _v) in dict.iter() { + for (k, _v) in &dict { let i: u64 = k.extract().unwrap(); sum += i; } diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs index dddd04d81ee..774dddcea38 100644 --- a/pyo3-benches/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -11,7 +11,7 @@ fn iter_list(b: &mut Bencher<'_>) { let list = PyList::new_bound(py, 0..LEN); let mut sum = 0; b.iter(|| { - for x in list.iter() { + for x in &list { let i: u64 = x.extract().unwrap(); sum += i; } diff --git a/pyo3-benches/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs index 49243a63fd4..b04cb491304 100644 --- a/pyo3-benches/benches/bench_set.rs +++ b/pyo3-benches/benches/bench_set.rs @@ -20,7 +20,7 @@ fn iter_set(b: &mut Bencher<'_>) { let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); let mut sum = 0; b.iter(|| { - for x in set.iter() { + for x in &set { let i: u64 = x.extract().unwrap(); sum += i; } diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index 8ac95d87f76..9eea7734bfc 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -60,7 +60,7 @@ where fn extract_bound(ob: &Bound<'py, PyAny>) -> Result { let dict = ob.downcast::()?; let mut ret = hashbrown::HashMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(k.extract()?, v.extract()?); } Ok(ret) diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index bb9ae0126d0..fdbe057f32d 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -125,7 +125,7 @@ where fn extract_bound(ob: &Bound<'py, PyAny>) -> Result { let dict = ob.downcast::()?; let mut ret = indexmap::IndexMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(k.extract()?, v.extract()?); } Ok(ret) diff --git a/src/conversions/std/map.rs b/src/conversions/std/map.rs index c17caa8b252..49eff4c1e8d 100644 --- a/src/conversions/std/map.rs +++ b/src/conversions/std/map.rs @@ -76,7 +76,7 @@ where fn extract_bound(ob: &Bound<'py, PyAny>) -> Result { let dict = ob.downcast::()?; let mut ret = collections::HashMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(k.extract()?, v.extract()?); } Ok(ret) @@ -96,7 +96,7 @@ where fn extract_bound(ob: &Bound<'py, PyAny>) -> Result { let dict = ob.downcast::()?; let mut ret = collections::BTreeMap::new(); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(k.extract()?, v.extract()?); } Ok(ret) diff --git a/src/types/dict.rs b/src/types/dict.rs index 5d1243463f2..2302004d238 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -672,6 +672,15 @@ impl<'py> IntoIterator for Bound<'py, PyDict> { } } +impl<'py> IntoIterator for &Bound<'py, PyDict> { + type Item = (Bound<'py, PyAny>, Bound<'py, PyAny>); + type IntoIter = BoundDictIterator<'py>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + mod borrowed_iter { use super::*; @@ -1123,6 +1132,26 @@ mod tests { }); } + #[test] + fn test_iter_bound() { + Python::with_gil(|py| { + let mut v = HashMap::new(); + v.insert(7, 32); + v.insert(8, 42); + v.insert(9, 123); + let ob = v.to_object(py); + let dict: &Bound<'_, PyDict> = ob.downcast_bound(py).unwrap(); + let mut key_sum = 0; + let mut value_sum = 0; + for (key, value) in dict { + key_sum += key.extract::().unwrap(); + value_sum += value.extract::().unwrap(); + } + assert_eq!(7 + 8 + 9, key_sum); + assert_eq!(32 + 42 + 123, value_sum); + }); + } + #[test] fn test_iter_value_mutated() { Python::with_gil(|py| { diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 34452af0e87..8a60efb8caa 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -249,6 +249,16 @@ impl<'py> IntoIterator for Bound<'py, PyFrozenSet> { } } +impl<'py> IntoIterator for &Bound<'py, PyFrozenSet> { + type Item = Bound<'py, PyAny>; + type IntoIter = BoundFrozenSetIterator<'py>; + + /// Returns an iterator of values in this set. + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + /// PyO3 implementation of an iterator for a Python `frozenset` object. pub struct BoundFrozenSetIterator<'p> { it: Bound<'p, PyIterator>, @@ -357,6 +367,17 @@ mod tests { }); } + #[test] + fn test_frozenset_iter_bound() { + Python::with_gil(|py| { + let set = PyFrozenSet::new_bound(py, &[1]).unwrap(); + + for el in &set { + assert_eq!(1i32, el.extract::().unwrap()); + } + }); + } + #[test] fn test_frozenset_iter_size_hint() { Python::with_gil(|py| { diff --git a/src/types/iterator.rs b/src/types/iterator.rs index 848a8e692ca..bd01fe81a78 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -111,6 +111,15 @@ impl<'py> Borrowed<'_, 'py, PyIterator> { } } +impl<'py> IntoIterator for &Bound<'py, PyIterator> { + type Item = PyResult>; + type IntoIter = Bound<'py, PyIterator>; + + fn into_iter(self) -> Self::IntoIter { + self.clone() + } +} + impl PyTypeCheck for PyIterator { const NAME: &'static str = "Iterator"; @@ -246,6 +255,39 @@ def fibonacci(target): }); } + #[test] + fn fibonacci_generator_bound() { + use crate::types::any::PyAnyMethods; + use crate::Bound; + + let fibonacci_generator = r#" +def fibonacci(target): + a = 1 + b = 1 + for _ in range(target): + yield a + a, b = b, a + b +"#; + + Python::with_gil(|py| { + let context = PyDict::new_bound(py); + py.run_bound(fibonacci_generator, None, Some(&context)) + .unwrap(); + + let generator: Bound<'_, PyIterator> = py + .eval_bound("fibonacci(5)", None, Some(&context)) + .unwrap() + .downcast_into() + .unwrap(); + let mut items = vec![]; + for actual in &generator { + let actual = actual.unwrap().extract::().unwrap(); + items.push(actual); + } + assert_eq!(items, [1, 1, 2, 3, 5]); + }); + } + #[test] fn int_not_iterable() { Python::with_gil(|py| { diff --git a/src/types/list.rs b/src/types/list.rs index a8df26ddbc5..3aa7ddca3f0 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -703,6 +703,15 @@ impl<'py> IntoIterator for Bound<'py, PyList> { } } +impl<'py> IntoIterator for &Bound<'py, PyList> { + type Item = Bound<'py, PyAny>; + type IntoIter = BoundListIterator<'py>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + #[cfg(test)] #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { @@ -911,6 +920,20 @@ mod tests { }); } + #[test] + fn test_into_iter_bound() { + use crate::types::any::PyAnyMethods; + + Python::with_gil(|py| { + let list = PyList::new_bound(py, [1, 2, 3, 4]); + let mut items = vec![]; + for item in &list { + items.push(item.extract::().unwrap()); + } + assert_eq!(items, vec![1, 2, 3, 4]); + }); + } + #[test] fn test_extract() { Python::with_gil(|py| { diff --git a/src/types/mod.rs b/src/types/mod.rs index 66312a98a0c..f0f025ee8e1 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -64,7 +64,7 @@ pub use self::typeobject::{PyType, PyTypeMethods}; /// Python::with_gil(|py| { /// let dict = py.eval_bound("{'a':'b', 'c':'d'}", None, None)?.downcast_into::()?; /// -/// for (key, value) in dict.iter() { +/// for (key, value) in &dict { /// println!("key: {}, value: {}", key, value); /// } /// diff --git a/src/types/set.rs b/src/types/set.rs index cf10f06e699..c043fa5ba4f 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -309,6 +309,20 @@ impl<'py> IntoIterator for Bound<'py, PySet> { } } +impl<'py> IntoIterator for &Bound<'py, PySet> { + type Item = Bound<'py, PyAny>; + type IntoIter = BoundSetIterator<'py>; + + /// Returns an iterator of values in this set. + /// + /// # Panics + /// + /// If PyO3 detects that the set is mutated during iteration, it will panic. + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + /// PyO3 implementation of an iterator for a Python `set` object. pub struct BoundSetIterator<'p> { it: Bound<'p, PyIterator>, @@ -482,6 +496,19 @@ mod tests { }); } + #[test] + fn test_set_iter_bound() { + use crate::types::any::PyAnyMethods; + + Python::with_gil(|py| { + let set = PySet::new_bound(py, &[1]).unwrap(); + + for el in &set { + assert_eq!(1i32, el.extract::().unwrap()); + } + }); + } + #[test] #[should_panic] fn test_set_iter_mutation() { diff --git a/src/types/tuple.rs b/src/types/tuple.rs index c379b67aa1a..d8053e1441a 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -554,6 +554,15 @@ impl<'py> IntoIterator for Bound<'py, PyTuple> { } } +impl<'py> IntoIterator for &Bound<'py, PyTuple> { + type Item = Bound<'py, PyAny>; + type IntoIter = BoundTupleIterator<'py>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + /// Used by `PyTuple::iter_borrowed()`. pub struct BorrowedTupleIterator<'a, 'py> { tuple: Borrowed<'a, 'py, PyTuple>, @@ -976,6 +985,23 @@ mod tests { }); } + #[test] + fn test_into_iter_bound() { + use crate::Bound; + + Python::with_gil(|py| { + let ob = (1, 2, 3).to_object(py); + let tuple: &Bound<'_, PyTuple> = ob.downcast_bound(py).unwrap(); + assert_eq!(3, tuple.len()); + + let mut items = vec![]; + for item in tuple { + items.push(item.extract::().unwrap()); + } + assert_eq!(items, vec![1, 2, 3]); + }); + } + #[test] #[cfg(not(Py_LIMITED_API))] fn test_as_slice() { From b08ee4b7e17b88988f2d2ab94e24596c580222ac Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 4 Mar 2024 22:23:35 +0000 Subject: [PATCH 199/349] remove `gil-refs` feature from `full` feature (#3930) --- Cargo.toml | 3 +-- noxfile.py | 18 +++++++++++++----- src/conversions/chrono.rs | 4 ++-- src/conversions/num_bigint.rs | 8 ++++---- src/conversions/rust_decimal.rs | 14 +++----------- 5 files changed, 23 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e7364a7c9f5..32fa5a006b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,7 +110,6 @@ nightly = [] # This is mostly intended for testing purposes - activating *all* of these isn't particularly useful. full = [ "macros", - "gil-refs", # "multiple-pymethods", # TODO re-add this when MSRV is greater than 1.62 "anyhow", "chrono", @@ -140,7 +139,7 @@ members = [ [package.metadata.docs.rs] no-default-features = true -features = ["full"] +features = ["full", "gil-refs"] rustdoc-args = ["--cfg", "docsrs"] [workspace.lints.clippy] diff --git a/noxfile.py b/noxfile.py index add737e5f15..2aa0f464b13 100644 --- a/noxfile.py +++ b/noxfile.py @@ -46,6 +46,7 @@ def test_rust(session: nox.Session): _run_cargo_test(session, features="abi3") if "skip-full" not in session.posargs: _run_cargo_test(session, features="full") + _run_cargo_test(session, features="full gil-refs") _run_cargo_test(session, features="abi3 full") @@ -617,6 +618,7 @@ def check_feature_powerset(session: nox.Session): EXCLUDED_FROM_FULL = { "nightly", + "gil-refs", "extension-module", "full", "default", @@ -658,10 +660,15 @@ def check_feature_powerset(session: nox.Session): session.error("no experimental features exist; please simplify the noxfile") features_to_skip = [ - *EXCLUDED_FROM_FULL, + *(EXCLUDED_FROM_FULL - {"gil-refs"}), *abi3_version_features, ] + # deny warnings + env = os.environ.copy() + rust_flags = env.get("RUSTFLAGS", "") + env["RUSTFLAGS"] = f"{rust_flags} -Dwarnings" + comma_join = ",".join _run_cargo( session, @@ -672,6 +679,7 @@ def check_feature_powerset(session: nox.Session): *(f"--group-features={comma_join(group)}" for group in features_to_group), "check", "--all-targets", + env=env, ) @@ -715,8 +723,8 @@ def _get_feature_sets() -> Tuple[Tuple[str, ...], ...]: "--no-default-features", "--features=abi3", ), - ("--features=full multiple-pymethods",), - ("--features=abi3 full multiple-pymethods",), + ("--features=full gil-refs multiple-pymethods",), + ("--features=abi3 full gil-refs multiple-pymethods",), ) else: return ( @@ -725,8 +733,8 @@ def _get_feature_sets() -> Tuple[Tuple[str, ...], ...]: "--no-default-features", "--features=abi3", ), - ("--features=full",), - ("--features=abi3 full",), + ("--features=full gil-refs",), + ("--features=abi3 full gil-refs",), ) diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 3aa6dbeb4ca..134724b2249 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -980,13 +980,13 @@ mod tests { let td = new_py_datetime_ob(py, "timedelta", (0, 3600, 0)); let py_timedelta = new_py_datetime_ob(py, "timezone", (td,)); // Should be equal - assert!(offset.as_ref(py).eq(py_timedelta).unwrap()); + assert!(offset.bind(py).eq(py_timedelta).unwrap()); // Same but with negative values let offset = FixedOffset::east_opt(-3600).unwrap().to_object(py); let td = new_py_datetime_ob(py, "timedelta", (0, -3600, 0)); let py_timedelta = new_py_datetime_ob(py, "timezone", (td,)); - assert!(offset.as_ref(py).eq(py_timedelta).unwrap()); + assert!(offset.bind(py).eq(py_timedelta).unwrap()); }) } diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 69bc5493366..743df8a9923 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -282,7 +282,7 @@ mod tests { let mut f0 = 1.to_object(py); let mut f1 = 1.to_object(py); std::iter::from_fn(move || { - let f2 = f0.call_method1(py, "__add__", (f1.as_ref(py),)).unwrap(); + let f2 = f0.call_method1(py, "__add__", (f1.bind(py),)).unwrap(); Some(std::mem::replace(&mut f0, std::mem::replace(&mut f1, f2))) }) } @@ -295,7 +295,7 @@ mod tests { // Python -> Rust assert_eq!(py_result.extract::(py).unwrap(), rs_result); // Rust -> Python - assert!(py_result.as_ref(py).eq(rs_result).unwrap()); + assert!(py_result.bind(py).eq(rs_result).unwrap()); } }); } @@ -308,7 +308,7 @@ mod tests { // Python -> Rust assert_eq!(py_result.extract::(py).unwrap(), rs_result); // Rust -> Python - assert!(py_result.as_ref(py).eq(&rs_result).unwrap()); + assert!(py_result.bind(py).eq(&rs_result).unwrap()); // negate @@ -318,7 +318,7 @@ mod tests { // Python -> Rust assert_eq!(py_result.extract::(py).unwrap(), rs_result); // Rust -> Python - assert!(py_result.as_ref(py).eq(rs_result).unwrap()); + assert!(py_result.bind(py).eq(rs_result).unwrap()); } }); } diff --git a/src/conversions/rust_decimal.rs b/src/conversions/rust_decimal.rs index 8bf2e33752d..b75cd875128 100644 --- a/src/conversions/rust_decimal.rs +++ b/src/conversions/rust_decimal.rs @@ -54,9 +54,7 @@ use crate::sync::GILOnceCell; use crate::types::any::PyAnyMethods; use crate::types::string::PyStringMethods; use crate::types::PyType; -use crate::{ - intern, Bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, -}; +use crate::{Bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; use rust_decimal::Decimal; use std::str::FromStr; @@ -74,14 +72,8 @@ impl FromPyObject<'_> for Decimal { static DECIMAL_CLS: GILOnceCell> = GILOnceCell::new(); -fn get_decimal_cls(py: Python<'_>) -> PyResult<&PyType> { - DECIMAL_CLS - .get_or_try_init(py, || { - py.import_bound(intern!(py, "decimal"))? - .getattr(intern!(py, "Decimal"))? - .extract() - }) - .map(|ty| ty.as_ref(py)) +fn get_decimal_cls(py: Python<'_>) -> PyResult<&Bound<'_, PyType>> { + DECIMAL_CLS.get_or_try_init_type_ref(py, "decimal", "Decimal") } impl ToPyObject for Decimal { From fe84fed966e7217963dbee9ad1189ebc3df5c2db Mon Sep 17 00:00:00 2001 From: Thomas Tanon Date: Wed, 6 Mar 2024 01:31:56 +0100 Subject: [PATCH 200/349] Allow inline struct, enum, fn and mod inside of declarative modules (#3902) * Inline struct, enum, fn and mod inside of declarative modules * remove news fragment --------- Co-authored-by: David Hewitt --- pyo3-macros-backend/src/module.rs | 161 +++++++++++++++--- tests/test_declarative_module.rs | 36 +++- tests/ui/invalid_pymodule_trait.stderr | 2 +- .../invalid_pymodule_two_pymodule_init.stderr | 2 +- 4 files changed, 169 insertions(+), 32 deletions(-) diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 7528ef8102f..7173fa8647d 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -115,19 +115,10 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { for item in &mut *items { match item { Item::Use(item_use) => { - let mut is_pyo3 = false; - item_use.attrs.retain(|attr| { - let found = attr.path().is_ident("pymodule_export"); - is_pyo3 |= found; - !found - }); - if is_pyo3 { - let cfg_attrs = item_use - .attrs - .iter() - .filter(|attr| attr.path().is_ident("cfg")) - .cloned() - .collect::>(); + let is_pymodule_export = + find_and_remove_attribute(&mut item_use.attrs, "pymodule_export"); + if is_pymodule_export { + let cfg_attrs = get_cfg_attributes(&item_use.attrs); extract_use_items( &item_use.tree, &cfg_attrs, @@ -137,23 +128,116 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { } } Item::Fn(item_fn) => { - let mut is_module_init = false; - item_fn.attrs.retain(|attr| { - let found = attr.path().is_ident("pymodule_init"); - is_module_init |= found; - !found - }); - if is_module_init { - ensure_spanned!(pymodule_init.is_none(), item_fn.span() => "only one pymodule_init may be specified"); - let ident = &item_fn.sig.ident; + ensure_spanned!( + !has_attribute(&item_fn.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + let is_pymodule_init = + find_and_remove_attribute(&mut item_fn.attrs, "pymodule_init"); + let ident = &item_fn.sig.ident; + if is_pymodule_init { + ensure_spanned!( + !has_attribute(&item_fn.attrs, "pyfunction"), + item_fn.span() => "`#[pyfunction]` cannot be used alongside `#[pymodule_init]`" + ); + ensure_spanned!(pymodule_init.is_none(), item_fn.span() => "only one `#[pymodule_init]` may be specified"); pymodule_init = Some(quote! { #ident(module)?; }); - } else { - bail_spanned!(item.span() => "only 'use' statements and and pymodule_init functions are allowed in #[pymodule]") + } else if has_attribute(&item_fn.attrs, "pyfunction") { + module_items.push(ident.clone()); + module_items_cfg_attrs.push(get_cfg_attributes(&item_fn.attrs)); } } - item => { - bail_spanned!(item.span() => "only 'use' statements and and pymodule_init functions are allowed in #[pymodule]") + Item::Struct(item_struct) => { + ensure_spanned!( + !has_attribute(&item_struct.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + if has_attribute(&item_struct.attrs, "pyclass") { + module_items.push(item_struct.ident.clone()); + module_items_cfg_attrs.push(get_cfg_attributes(&item_struct.attrs)); + } + } + Item::Enum(item_enum) => { + ensure_spanned!( + !has_attribute(&item_enum.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + if has_attribute(&item_enum.attrs, "pyclass") { + module_items.push(item_enum.ident.clone()); + module_items_cfg_attrs.push(get_cfg_attributes(&item_enum.attrs)); + } + } + Item::Mod(item_mod) => { + ensure_spanned!( + !has_attribute(&item_mod.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + if has_attribute(&item_mod.attrs, "pymodule") { + module_items.push(item_mod.ident.clone()); + module_items_cfg_attrs.push(get_cfg_attributes(&item_mod.attrs)); + } + } + Item::ForeignMod(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + } + Item::Trait(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + } + Item::Const(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + } + Item::Static(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + } + Item::Macro(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + } + Item::ExternCrate(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + } + Item::Impl(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); } + Item::TraitAlias(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + } + Item::Type(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + } + Item::Union(item) => { + ensure_spanned!( + !has_attribute(&item.attrs, "pymodule_export"), + item.span() => "`#[pymodule_export]` may only be used on `use` statements" + ); + } + _ => (), } } @@ -355,6 +439,31 @@ fn get_pyfn_attr(attrs: &mut Vec) -> syn::Result Vec { + attrs + .iter() + .filter(|attr| attr.path().is_ident("cfg")) + .cloned() + .collect() +} + +fn find_and_remove_attribute(attrs: &mut Vec, ident: &str) -> bool { + let mut found = false; + attrs.retain(|attr| { + if attr.path().is_ident(ident) { + found = true; + false + } else { + true + } + }); + found +} + +fn has_attribute(attrs: &[syn::Attribute], ident: &str) -> bool { + attrs.iter().any(|attr| attr.path().is_ident(ident)) +} + enum PyModulePyO3Option { Crate(CrateAttribute), Name(NameAttribute), diff --git a/tests/test_declarative_module.rs b/tests/test_declarative_module.rs index 86913d9b800..3cb93765173 100644 --- a/tests/test_declarative_module.rs +++ b/tests/test_declarative_module.rs @@ -15,8 +15,8 @@ struct ValueClass { #[pymethods] impl ValueClass { #[new] - fn new(value: usize) -> ValueClass { - ValueClass { value } + fn new(value: usize) -> Self { + Self { value } } } @@ -48,6 +48,33 @@ mod declarative_module { #[pymodule_export] use super::{declarative_module2, double, MyError, ValueClass as Value}; + #[pymodule] + mod inner { + use super::*; + + #[pyfunction] + fn triple(x: usize) -> usize { + x * 3 + } + + #[pyclass] + struct Struct; + + #[pymethods] + impl Struct { + #[new] + fn new() -> Self { + Self + } + } + + #[pyclass] + enum Enum { + A, + B, + } + } + #[pymodule_init] fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("double2", m.getattr("double")?) @@ -65,7 +92,6 @@ mod declarative_submodule { use super::{double, double_value}; } -/// A module written using declarative syntax. #[pymodule] #[pyo3(name = "declarative_module_renamed")] mod declarative_module2 { @@ -84,7 +110,7 @@ fn test_declarative_module() { ); py_assert!(py, m, "m.double(2) == 4"); - py_assert!(py, m, "m.double2(3) == 6"); + py_assert!(py, m, "m.inner.triple(3) == 9"); py_assert!(py, m, "m.declarative_submodule.double(4) == 8"); py_assert!( py, @@ -97,5 +123,7 @@ fn test_declarative_module() { py_assert!(py, m, "not hasattr(m, 'LocatedClass')"); #[cfg(not(Py_LIMITED_API))] py_assert!(py, m, "hasattr(m, 'LocatedClass')"); + py_assert!(py, m, "isinstance(m.inner.Struct(), m.inner.Struct)"); + py_assert!(py, m, "isinstance(m.inner.Enum.A, m.inner.Enum)"); }) } diff --git a/tests/ui/invalid_pymodule_trait.stderr b/tests/ui/invalid_pymodule_trait.stderr index 3ed128617f5..4b02f14a540 100644 --- a/tests/ui/invalid_pymodule_trait.stderr +++ b/tests/ui/invalid_pymodule_trait.stderr @@ -1,4 +1,4 @@ -error: only 'use' statements and and pymodule_init functions are allowed in #[pymodule] +error: `#[pymodule_export]` may only be used on `use` statements --> tests/ui/invalid_pymodule_trait.rs:5:5 | 5 | #[pymodule_export] diff --git a/tests/ui/invalid_pymodule_two_pymodule_init.stderr b/tests/ui/invalid_pymodule_two_pymodule_init.stderr index 9f0900f9348..c117ebd573f 100644 --- a/tests/ui/invalid_pymodule_two_pymodule_init.stderr +++ b/tests/ui/invalid_pymodule_two_pymodule_init.stderr @@ -1,4 +1,4 @@ -error: only one pymodule_init may be specified +error: only one `#[pymodule_init]` may be specified --> tests/ui/invalid_pymodule_two_pymodule_init.rs:11:5 | 11 | fn init2(m: &PyModule) -> PyResult<()> { From 57bbc32e7c98a8266a81ca7db517d714832ad916 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 6 Mar 2024 00:54:45 +0000 Subject: [PATCH 201/349] add `experimental-async` feature (#3931) * add `experimental-async` feature * gate async doctests on feature --- Cargo.toml | 6 +- guide/src/async-await.md | 6 +- guide/src/features.md | 6 ++ newsfragments/3931.added.md | 1 + pyo3-macros-backend/Cargo.toml | 3 + pyo3-macros-backend/src/method.rs | 7 ++ pyo3-macros/Cargo.toml | 1 + src/impl_.rs | 2 +- src/lib.rs | 2 +- tests/test_compile_error.rs | 5 +- tests/test_coroutine.rs | 2 +- tests/ui/invalid_argument_attributes.rs | 25 ------- tests/ui/invalid_argument_attributes.stderr | 79 --------------------- tests/ui/invalid_cancel_handle.rs | 22 ++++++ tests/ui/invalid_cancel_handle.stderr | 72 +++++++++++++++++++ 15 files changed, 129 insertions(+), 110 deletions(-) create mode 100644 newsfragments/3931.added.md create mode 100644 tests/ui/invalid_cancel_handle.rs create mode 100644 tests/ui/invalid_cancel_handle.stderr diff --git a/Cargo.toml b/Cargo.toml index 32fa5a006b6..32dc0ba75ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,9 @@ pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.0-dev", featu [features] default = ["macros"] +# Enables support for `async fn` for `#[pyfunction]` and `#[pymethods]`. +experimental-async = ["macros", "pyo3-macros/experimental-async"] + # Enables pyo3::inspect module and additional type information on FromPyObject # and IntoPy traits experimental-inspect = [] @@ -115,8 +118,9 @@ full = [ "chrono", "chrono-tz", "either", - "experimental-inspect", + "experimental-async", "experimental-declarative-modules", + "experimental-inspect", "eyre", "hashbrown", "indexmap", diff --git a/guide/src/async-await.md b/guide/src/async-await.md index c14b5d93d84..688f0a65bc4 100644 --- a/guide/src/async-await.md +++ b/guide/src/async-await.md @@ -6,6 +6,7 @@ ```rust # #![allow(dead_code)] +# #[cfg(feature = "experimental-async")] { use std::{thread, time::Duration}; use futures::channel::oneshot; use pyo3::prelude::*; @@ -20,6 +21,7 @@ async fn sleep(seconds: f64, result: Option) -> Option { rx.await.unwrap(); result } +# } ``` *Python awaitables instantiated with this method can only be awaited in *asyncio* context. Other Python async runtime may be supported in the future.* @@ -72,6 +74,7 @@ Cancellation on the Python side can be caught using [`CancelHandle`]({{#PYO3_DOC ```rust # #![allow(dead_code)] +# #[cfg(feature = "experimental-async")] { use futures::FutureExt; use pyo3::prelude::*; use pyo3::coroutine::CancelHandle; @@ -83,11 +86,12 @@ async fn cancellable(#[pyo3(cancel_handle)] mut cancel: CancelHandle) { _ = cancel.cancelled().fuse() => println!("cancelled"), } } +# } ``` ## The `Coroutine` type -To make a Rust future awaitable in Python, PyO3 defines a [`Coroutine`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.Coroutine.html) type, which implements the Python [coroutine protocol](https://docs.python.org/3/library/collections.abc.html#collections.abc.Coroutine). +To make a Rust future awaitable in Python, PyO3 defines a [`Coroutine`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.Coroutine.html) type, which implements the Python [coroutine protocol](https://docs.python.org/3/library/collections.abc.html#collections.abc.Coroutine). Each `coroutine.send` call is translated to a `Future::poll` call. If a [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) parameter is declared, the exception passed to `coroutine.throw` call is stored in it and can be retrieved with [`CancelHandle::cancelled`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html#method.cancelled); otherwise, it cancels the Rust future, and the exception is reraised; diff --git a/guide/src/features.md b/guide/src/features.md index 43124e0076e..118284959d6 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -51,6 +51,12 @@ If you do not enable this feature, you should call `pyo3::prepare_freethreaded_p ## Advanced Features +### `experimental-async` + +This feature adds support for `async fn` in `#[pyfunction]` and `#[pymethods]`. + +The feature has some unfinished refinements and performance improvements. To help finish this off, see [issue #1632](https://github.com/PyO3/pyo3/issues/1632) and its associated draft PRs. + ### `experimental-inspect` This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types. diff --git a/newsfragments/3931.added.md b/newsfragments/3931.added.md new file mode 100644 index 00000000000..b532adeeae5 --- /dev/null +++ b/newsfragments/3931.added.md @@ -0,0 +1 @@ +Add `experimental-async` feature. diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 0e83cc29fd4..665c8c3510d 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -26,3 +26,6 @@ features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-trait [lints] workspace = true + +[features] +experimental-async = [] diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 1d2d22b236b..9a8cb828b22 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -512,6 +512,13 @@ impl<'a> FnSpec<'a> { } } + if self.asyncness.is_some() { + ensure_spanned!( + cfg!(feature = "experimental-async"), + self.asyncness.span() => "async functions are only supported with the `experimental-async` feature" + ); + } + let rust_call = |args: Vec, holders: &mut Vec| { let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise, holders, ctx); diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index a0368a5f364..97d2de07cba 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -15,6 +15,7 @@ proc-macro = true [features] multiple-pymethods = [] +experimental-async = ["pyo3-macros-backend/experimental-async"] experimental-declarative-modules = [] [dependencies] diff --git a/src/impl_.rs b/src/impl_.rs index 77f9ff4ea1f..ea71b257c0e 100644 --- a/src/impl_.rs +++ b/src/impl_.rs @@ -6,7 +6,7 @@ //! APIs may may change at any time without documentation in the CHANGELOG and without //! breaking semver guarantees. -#[cfg(feature = "macros")] +#[cfg(feature = "experimental-async")] pub mod coroutine; pub mod deprecations; pub mod extract_argument; diff --git a/src/lib.rs b/src/lib.rs index 26d2ec55da1..c2c7d96a40a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -418,7 +418,7 @@ pub mod buffer; pub mod callback; pub mod conversion; mod conversions; -#[cfg(feature = "macros")] +#[cfg(feature = "experimental-async")] pub mod coroutine; #[macro_use] #[doc(hidden)] diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 5f2d25db92f..f5fbec7a533 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -27,7 +27,8 @@ fn test_compile_errors() { t.compile_fail("tests/ui/wrong_aspyref_lifetimes.rs"); t.compile_fail("tests/ui/invalid_pyfunctions.rs"); t.compile_fail("tests/ui/invalid_pymethods.rs"); - #[cfg(Py_LIMITED_API)] + // output changes with async feature + #[cfg(all(Py_LIMITED_API, feature = "experimental-async"))] t.compile_fail("tests/ui/abi3_nativetype_inheritance.rs"); t.compile_fail("tests/ui/invalid_intern_arg.rs"); t.compile_fail("tests/ui/invalid_frozen_pyclass_borrow.rs"); @@ -48,4 +49,6 @@ fn test_compile_errors() { t.compile_fail("tests/ui/invalid_pymodule_trait.rs"); #[cfg(feature = "experimental-declarative-modules")] t.compile_fail("tests/ui/invalid_pymodule_two_pymodule_init.rs"); + #[cfg(feature = "experimental-async")] + t.compile_fail("tests/ui/invalid_cancel_handle.rs"); } diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index db79c72a233..17539fa113e 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -1,4 +1,4 @@ -#![cfg(feature = "macros")] +#![cfg(feature = "experimental-async")] #![cfg(not(target_arch = "wasm32"))] use std::{task::Poll, thread, time::Duration}; diff --git a/tests/ui/invalid_argument_attributes.rs b/tests/ui/invalid_argument_attributes.rs index ed9d6ce6971..311c6c03e0e 100644 --- a/tests/ui/invalid_argument_attributes.rs +++ b/tests/ui/invalid_argument_attributes.rs @@ -15,29 +15,4 @@ fn from_py_with_value_not_a_string(#[pyo3(from_py_with = func)] param: String) { #[pyfunction] fn from_py_with_repeated(#[pyo3(from_py_with = "func", from_py_with = "func")] param: String) {} -#[pyfunction] -async fn from_py_with_value_and_cancel_handle( - #[pyo3(from_py_with = "func", cancel_handle)] _param: String, -) { -} - -#[pyfunction] -async fn cancel_handle_repeated(#[pyo3(cancel_handle, cancel_handle)] _param: String) {} - -#[pyfunction] -async fn cancel_handle_repeated2( - #[pyo3(cancel_handle)] _param: String, - #[pyo3(cancel_handle)] _param2: String, -) { -} - -#[pyfunction] -fn cancel_handle_synchronous(#[pyo3(cancel_handle)] _param: String) {} - -#[pyfunction] -async fn cancel_handle_wrong_type(#[pyo3(cancel_handle)] _param: String) {} - -#[pyfunction] -async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} - fn main() {} diff --git a/tests/ui/invalid_argument_attributes.stderr b/tests/ui/invalid_argument_attributes.stderr index c122dd25f8c..d27c25fd179 100644 --- a/tests/ui/invalid_argument_attributes.stderr +++ b/tests/ui/invalid_argument_attributes.stderr @@ -27,82 +27,3 @@ error: `from_py_with` may only be specified once per argument | 16 | fn from_py_with_repeated(#[pyo3(from_py_with = "func", from_py_with = "func")] param: String) {} | ^^^^^^^^^^^^ - -error: `from_py_with` and `cancel_handle` cannot be specified together - --> tests/ui/invalid_argument_attributes.rs:20:35 - | -20 | #[pyo3(from_py_with = "func", cancel_handle)] _param: String, - | ^^^^^^^^^^^^^ - -error: `cancel_handle` may only be specified once per argument - --> tests/ui/invalid_argument_attributes.rs:25:55 - | -25 | async fn cancel_handle_repeated(#[pyo3(cancel_handle, cancel_handle)] _param: String) {} - | ^^^^^^^^^^^^^ - -error: `cancel_handle` may only be specified once - --> tests/ui/invalid_argument_attributes.rs:30:28 - | -30 | #[pyo3(cancel_handle)] _param2: String, - | ^^^^^^^ - -error: `cancel_handle` attribute can only be used with `async fn` - --> tests/ui/invalid_argument_attributes.rs:35:53 - | -35 | fn cancel_handle_synchronous(#[pyo3(cancel_handle)] _param: String) {} - | ^^^^^^ - -error[E0308]: mismatched types - --> tests/ui/invalid_argument_attributes.rs:37:1 - | -37 | #[pyfunction] - | ^^^^^^^^^^^^^ - | | - | expected `String`, found `CancelHandle` - | arguments to this function are incorrect - | -note: function defined here - --> tests/ui/invalid_argument_attributes.rs:38:10 - | -38 | async fn cancel_handle_wrong_type(#[pyo3(cancel_handle)] _param: String) {} - | ^^^^^^^^^^^^^^^^^^^^^^^^ -------------- - = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) - -error[E0277]: the trait bound `CancelHandle: PyClass` is not satisfied - --> tests/ui/invalid_argument_attributes.rs:41:50 - | -41 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} - | ^^^^ the trait `PyClass` is not implemented for `CancelHandle` - | - = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` - = note: required for `CancelHandle` to implement `FromPyObject<'_>` - = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>` -note: required by a bound in `extract_argument` - --> src/impl_/extract_argument.rs - | - | pub fn extract_argument<'a, 'py, T>( - | ---------------- required by a bound in this function -... - | T: PyFunctionArgument<'a, 'py>, - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `extract_argument` - -error[E0277]: the trait bound `CancelHandle: Clone` is not satisfied - --> tests/ui/invalid_argument_attributes.rs:41:50 - | -41 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} - | ^^^^ the trait `Clone` is not implemented for `CancelHandle` - | - = help: the following other types implement trait `PyFunctionArgument<'a, 'py>`: - &'a pyo3::Bound<'py, T> - &'a pyo3::coroutine::Coroutine - &'a mut pyo3::coroutine::Coroutine - = note: required for `CancelHandle` to implement `FromPyObject<'_>` - = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>` -note: required by a bound in `extract_argument` - --> src/impl_/extract_argument.rs - | - | pub fn extract_argument<'a, 'py, T>( - | ---------------- required by a bound in this function -... - | T: PyFunctionArgument<'a, 'py>, - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `extract_argument` diff --git a/tests/ui/invalid_cancel_handle.rs b/tests/ui/invalid_cancel_handle.rs new file mode 100644 index 00000000000..59076b14418 --- /dev/null +++ b/tests/ui/invalid_cancel_handle.rs @@ -0,0 +1,22 @@ +use pyo3::prelude::*; + +#[pyfunction] +async fn cancel_handle_repeated(#[pyo3(cancel_handle, cancel_handle)] _param: String) {} + +#[pyfunction] +async fn cancel_handle_repeated2( + #[pyo3(cancel_handle)] _param: String, + #[pyo3(cancel_handle)] _param2: String, +) { +} + +#[pyfunction] +fn cancel_handle_synchronous(#[pyo3(cancel_handle)] _param: String) {} + +#[pyfunction] +async fn cancel_handle_wrong_type(#[pyo3(cancel_handle)] _param: String) {} + +#[pyfunction] +async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} + +fn main() {} diff --git a/tests/ui/invalid_cancel_handle.stderr b/tests/ui/invalid_cancel_handle.stderr new file mode 100644 index 00000000000..6dc3ff3ccab --- /dev/null +++ b/tests/ui/invalid_cancel_handle.stderr @@ -0,0 +1,72 @@ +error: `cancel_handle` may only be specified once per argument + --> tests/ui/invalid_cancel_handle.rs:4:55 + | +4 | async fn cancel_handle_repeated(#[pyo3(cancel_handle, cancel_handle)] _param: String) {} + | ^^^^^^^^^^^^^ + +error: `cancel_handle` may only be specified once + --> tests/ui/invalid_cancel_handle.rs:9:28 + | +9 | #[pyo3(cancel_handle)] _param2: String, + | ^^^^^^^ + +error: `cancel_handle` attribute can only be used with `async fn` + --> tests/ui/invalid_cancel_handle.rs:14:53 + | +14 | fn cancel_handle_synchronous(#[pyo3(cancel_handle)] _param: String) {} + | ^^^^^^ + +error[E0308]: mismatched types + --> tests/ui/invalid_cancel_handle.rs:16:1 + | +16 | #[pyfunction] + | ^^^^^^^^^^^^^ + | | + | expected `String`, found `CancelHandle` + | arguments to this function are incorrect + | +note: function defined here + --> tests/ui/invalid_cancel_handle.rs:17:10 + | +17 | async fn cancel_handle_wrong_type(#[pyo3(cancel_handle)] _param: String) {} + | ^^^^^^^^^^^^^^^^^^^^^^^^ -------------- + = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `CancelHandle: PyClass` is not satisfied + --> tests/ui/invalid_cancel_handle.rs:20:50 + | +20 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} + | ^^^^ the trait `PyClass` is not implemented for `CancelHandle` + | + = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` + = note: required for `CancelHandle` to implement `FromPyObject<'_>` + = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>` +note: required by a bound in `extract_argument` + --> src/impl_/extract_argument.rs + | + | pub fn extract_argument<'a, 'py, T>( + | ---------------- required by a bound in this function +... + | T: PyFunctionArgument<'a, 'py>, + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `extract_argument` + +error[E0277]: the trait bound `CancelHandle: Clone` is not satisfied + --> tests/ui/invalid_cancel_handle.rs:20:50 + | +20 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} + | ^^^^ the trait `Clone` is not implemented for `CancelHandle` + | + = help: the following other types implement trait `PyFunctionArgument<'a, 'py>`: + &'a pyo3::Bound<'py, T> + &'a pyo3::coroutine::Coroutine + &'a mut pyo3::coroutine::Coroutine + = note: required for `CancelHandle` to implement `FromPyObject<'_>` + = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>` +note: required by a bound in `extract_argument` + --> src/impl_/extract_argument.rs + | + | pub fn extract_argument<'a, 'py, T>( + | ---------------- required by a bound in this function +... + | T: PyFunctionArgument<'a, 'py>, + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `extract_argument` From d35f41e0bf158911efc844652683ee615c0f1d16 Mon Sep 17 00:00:00 2001 From: Thomas Tanon Date: Wed, 6 Mar 2024 18:49:43 +0100 Subject: [PATCH 202/349] Chrono: allow deprecation warning until upgrading to 0.4.35+ (#3935) * Chrono: allow deprecation warning until upgrading to 0.4.35+ Requiring 0.4.35 is blocked on MSRV (Chrono: 1.61, PyO3: 1.56) * also allow deprecated chrono in example --------- Co-authored-by: David Hewitt --- src/conversions/chrono.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 134724b2249..2e46a9e54ff 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -19,6 +19,9 @@ //! # Example: Convert a `datetime.datetime` to chrono's `DateTime` //! //! ```rust +//! # // `chrono::Duration` has been renamed to `chrono::TimeDelta` and its constructors changed +//! # // TODO: upgrade to Chrono 0.4.35+ after upgrading our MSRV to 1.61+ +//! # #![allow(deprecated)] //! use chrono::{DateTime, Duration, TimeZone, Utc}; //! use pyo3::{Python, ToPyObject}; //! @@ -39,6 +42,11 @@ //! }); //! } //! ``` + +// `chrono::Duration` has been renamed to `chrono::TimeDelta` and its constructors changed +// TODO: upgrade to Chrono 0.4.35+ after upgrading our MSRV to 1.61+ +#![allow(deprecated)] + use crate::exceptions::{PyTypeError, PyUserWarning, PyValueError}; #[cfg(Py_LIMITED_API)] use crate::sync::GILOnceCell; From 0f7ddb19a5a67eb9266319987a39b5efb07d6e05 Mon Sep 17 00:00:00 2001 From: Thomas Tanon Date: Wed, 6 Mar 2024 19:19:43 +0100 Subject: [PATCH 203/349] PyPy: remove PyCode (PyCode_Type does not exist) (#3934) --- newsfragments/3934.removed.md | 1 + pyo3-ffi/src/cpython/code.rs | 1 + src/types/code.rs | 14 ++++++++++++++ src/types/mod.rs | 4 ++-- 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 newsfragments/3934.removed.md diff --git a/newsfragments/3934.removed.md b/newsfragments/3934.removed.md new file mode 100644 index 00000000000..66741d3f6b3 --- /dev/null +++ b/newsfragments/3934.removed.md @@ -0,0 +1 @@ +Remove `PyCode` and `PyCode_Type` on PyPy: `PyCode_Type` is not exposed by PyPy. \ No newline at end of file diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index 05f21a137b5..498eab59cce 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -230,6 +230,7 @@ pub const CO_FUTURE_GENERATOR_STOP: c_int = 0x8_0000; pub const CO_MAXBLOCKS: usize = 20; +#[cfg(not(PyPy))] #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { pub static mut PyCode_Type: PyTypeObject; diff --git a/src/types/code.rs b/src/types/code.rs index 8956deb1a71..f60e7783aa4 100644 --- a/src/types/code.rs +++ b/src/types/code.rs @@ -10,3 +10,17 @@ pyobject_native_type_core!( pyobject_native_static_type_object!(ffi::PyCode_Type), #checkfunction=ffi::PyCode_Check ); + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::PyTypeMethods; + use crate::{PyTypeInfo, Python}; + + #[test] + fn test_type_object() { + Python::with_gil(|py| { + assert_eq!(PyCode::type_object_bound(py).name().unwrap(), "code"); + }) + } +} diff --git a/src/types/mod.rs b/src/types/mod.rs index f0f025ee8e1..fc74b03de5b 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -5,7 +5,7 @@ pub use self::boolobject::{PyBool, PyBoolMethods}; pub use self::bytearray::{PyByteArray, PyByteArrayMethods}; pub use self::bytes::{PyBytes, PyBytesMethods}; pub use self::capsule::{PyCapsule, PyCapsuleMethods}; -#[cfg(not(Py_LIMITED_API))] +#[cfg(all(not(Py_LIMITED_API), not(PyPy)))] pub use self::code::PyCode; pub use self::complex::{PyComplex, PyComplexMethods}; #[allow(deprecated)] @@ -316,7 +316,7 @@ pub(crate) mod boolobject; pub(crate) mod bytearray; pub(crate) mod bytes; pub(crate) mod capsule; -#[cfg(not(Py_LIMITED_API))] +#[cfg(all(not(Py_LIMITED_API), not(PyPy)))] mod code; pub(crate) mod complex; #[cfg(not(Py_LIMITED_API))] From fbd531195aeb18eb23a679bc30946285120376ce Mon Sep 17 00:00:00 2001 From: Thomas Tanon Date: Wed, 6 Mar 2024 19:20:02 +0100 Subject: [PATCH 204/349] PyAddToModule: Properly propagate initialization error (#3919) Better than panics --- pyo3-macros-backend/src/pyclass.rs | 19 ++++++++++++-- src/impl_/pymodule.rs | 9 +------ src/types/mod.rs | 12 +++++++++ tests/test_declarative_module.rs | 41 ++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 9470045733c..3eca80861b7 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -881,11 +881,12 @@ fn impl_complex_enum( } }; - let pyclass_impls: TokenStream = vec![ + let pyclass_impls: TokenStream = [ impl_builder.impl_pyclass(ctx), impl_builder.impl_extractext(ctx), enum_into_py_impl, impl_builder.impl_pyclassimpl(ctx)?, + impl_builder.impl_add_to_module(ctx), impl_builder.impl_freelist(ctx), ] .into_iter() @@ -1372,11 +1373,12 @@ impl<'a> PyClassImplsBuilder<'a> { } fn impl_all(&self, ctx: &Ctx) -> Result { - let tokens = vec![ + let tokens = [ self.impl_pyclass(ctx), self.impl_extractext(ctx), self.impl_into_py(ctx), self.impl_pyclassimpl(ctx)?, + self.impl_add_to_module(ctx), self.impl_freelist(ctx), ] .into_iter() @@ -1625,6 +1627,19 @@ impl<'a> PyClassImplsBuilder<'a> { }) } + fn impl_add_to_module(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; + let cls = self.cls; + quote! { + impl #pyo3_path::impl_::pymodule::PyAddToModule for #cls { + fn add_to_module(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { + use #pyo3_path::types::PyModuleMethods; + module.add_class::() + } + } + } + } + fn impl_freelist(&self, ctx: &Ctx) -> TokenStream { let cls = self.cls; let Ctx { pyo3_path } = ctx; diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index 9fff799c37b..a19cbda526f 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -7,8 +7,7 @@ use portable_atomic::{AtomicI64, Ordering}; #[cfg(not(PyPy))] use crate::exceptions::PyImportError; -use crate::types::module::PyModuleMethods; -use crate::{ffi, sync::GILOnceCell, types::PyModule, Bound, Py, PyResult, PyTypeInfo, Python}; +use crate::{ffi, sync::GILOnceCell, types::PyModule, Bound, Py, PyResult, Python}; /// `Sync` wrapper of `ffi::PyModuleDef`. pub struct ModuleDef { @@ -141,12 +140,6 @@ pub trait PyAddToModule { fn add_to_module(module: &Bound<'_, PyModule>) -> PyResult<()>; } -impl PyAddToModule for T { - fn add_to_module(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add(Self::NAME, Self::type_object_bound(module.py())) - } -} - #[cfg(test)] mod tests { use std::sync::atomic::{AtomicBool, Ordering}; diff --git a/src/types/mod.rs b/src/types/mod.rs index fc74b03de5b..cee45e8678d 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -249,6 +249,18 @@ macro_rules! pyobject_native_type_info( } )? } + + impl<$($generics,)*> $crate::impl_::pymodule::PyAddToModule for $name { + fn add_to_module( + module: &$crate::Bound<'_, $crate::types::PyModule>, + ) -> $crate::PyResult<()> { + use $crate::types::PyModuleMethods; + module.add( + ::NAME, + ::type_object_bound(module.py()), + ) + } + } }; ); diff --git a/tests/test_declarative_module.rs b/tests/test_declarative_module.rs index 3cb93765173..9eea8e2df07 100644 --- a/tests/test_declarative_module.rs +++ b/tests/test_declarative_module.rs @@ -3,6 +3,8 @@ use pyo3::create_exception; use pyo3::exceptions::PyException; use pyo3::prelude::*; +#[cfg(not(Py_LIMITED_API))] +use pyo3::types::PyBool; #[path = "../src/tests/common.rs"] mod common; @@ -127,3 +129,42 @@ fn test_declarative_module() { py_assert!(py, m, "isinstance(m.inner.Enum.A, m.inner.Enum)"); }) } + +#[cfg(not(Py_LIMITED_API))] +#[pyclass(extends = PyBool)] +struct ExtendsBool; + +#[cfg(not(Py_LIMITED_API))] +#[pymodule] +mod class_initialization_module { + #[pymodule_export] + use super::ExtendsBool; +} + +#[test] +#[cfg(not(Py_LIMITED_API))] +fn test_class_initialization_fails() { + Python::with_gil(|py| { + let err = class_initialization_module::DEF + .make_module(py) + .unwrap_err(); + assert_eq!( + err.to_string(), + "RuntimeError: An error occurred while initializing class ExtendsBool" + ); + }) +} + +#[pymodule] +mod r#type { + #[pymodule_export] + use super::double; +} + +#[test] +fn test_raw_ident_module() { + Python::with_gil(|py| { + let m = pyo3::wrap_pymodule!(r#type)(py).into_bound(py); + py_assert!(py, m, "m.double(2) == 4"); + }) +} From 31c4820010cc1b5e65ea27d024e5a330496ccece Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 8 Mar 2024 01:28:11 +0100 Subject: [PATCH 205/349] deprecate `&PyModule` as `#[pymodule]` argument type (#3936) * deprecate `&PyModule` as `#[pymodule]` argument type * cleanup * add ui tests * fix deprecations in tests * fix maturin and setuptools-rust starters * run `deprecated` ui test only when `gil-refs` as disabled --- examples/maturin-starter/src/lib.rs | 2 +- examples/setuptools-rust-starter/src/lib.rs | 2 +- guide/src/exception.md | 2 +- guide/src/migration.md | 2 +- pyo3-macros-backend/src/module.rs | 25 +++++++++++++ pytests/src/lib.rs | 2 +- src/impl_/pymethods.rs | 40 +++++++++++++++++++++ src/tests/hygiene/pymodule.rs | 2 ++ src/types/module.rs | 8 ++--- tests/test_compile_error.rs | 1 + tests/test_module.rs | 2 +- tests/test_no_imports.rs | 1 + tests/ui/deprecations.rs | 35 ++++++++++++++++++ tests/ui/deprecations.stderr | 12 +++++++ 14 files changed, 126 insertions(+), 10 deletions(-) diff --git a/examples/maturin-starter/src/lib.rs b/examples/maturin-starter/src/lib.rs index 8b0748f8311..faa147b2a10 100644 --- a/examples/maturin-starter/src/lib.rs +++ b/examples/maturin-starter/src/lib.rs @@ -20,7 +20,7 @@ impl ExampleClass { /// An example module implemented in Rust using PyO3. #[pymodule] -fn maturin_starter(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn maturin_starter(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_wrapped(wrap_pymodule!(submodule::submodule))?; diff --git a/examples/setuptools-rust-starter/src/lib.rs b/examples/setuptools-rust-starter/src/lib.rs index 2bcd411c238..d31284be7a3 100644 --- a/examples/setuptools-rust-starter/src/lib.rs +++ b/examples/setuptools-rust-starter/src/lib.rs @@ -20,7 +20,7 @@ impl ExampleClass { /// An example module implemented in Rust using PyO3. #[pymodule] -fn _setuptools_rust_starter(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn _setuptools_rust_starter(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_wrapped(wrap_pymodule!(submodule::submodule))?; diff --git a/guide/src/exception.md b/guide/src/exception.md index 0be44167760..04550bd4b92 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -44,7 +44,7 @@ use pyo3::exceptions::PyException; pyo3::create_exception!(mymodule, CustomError, PyException); #[pymodule] -fn mymodule(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn mymodule(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { // ... other elements added to module ... m.add("CustomError", py.get_type_bound::())?; diff --git a/guide/src/migration.md b/guide/src/migration.md index 709e7a70488..8537f9a1a69 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -847,7 +847,7 @@ fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { To fix it, make the private submodule visible, e.g. with `pub` or `pub(crate)`. -```rust +```rust,ignore mod foo { use pyo3::prelude::*; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 7173fa8647d..fb02c074996 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -11,6 +11,7 @@ use quote::quote; use syn::{ ext::IdentExt, parse::{Parse, ParseStream}, + parse_quote, parse_quote_spanned, spanned::Spanned, token::Comma, Item, Path, Result, @@ -281,6 +282,7 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result let options = PyModuleOptions::from_attrs(&mut function.attrs)?; process_functions_in_module(&options, &mut function)?; let ctx = &Ctx::new(&options.krate); + let stmts = std::mem::take(&mut function.block.stmts); let Ctx { pyo3_path } = ctx; let ident = &function.sig.ident; let vis = &function.vis; @@ -295,6 +297,29 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result } module_args.push(quote!(::std::convert::Into::into(BoundRef(module)))); + let extractors = function + .sig + .inputs + .iter() + .filter_map(|param| { + if let syn::FnArg::Typed(pat_type) = param { + if let syn::Pat::Ident(pat_ident) = &*pat_type.pat { + let ident = &pat_ident.ident; + return Some([ + parse_quote! { let (#ident, e) = #pyo3_path::impl_::pymethods::inspect_type(#ident); }, + parse_quote_spanned! { pat_type.span() => e.extract_gil_ref(); }, + ]); + } + } + None + }) + .flatten(); + + function.block.stmts = extractors.chain(stmts).collect(); + function + .attrs + .push(parse_quote!(#[allow(clippy::used_underscore_binding)])); + Ok(quote! { #function #vis mod #ident { diff --git a/pytests/src/lib.rs b/pytests/src/lib.rs index bfd80edb719..cbd65c8012c 100644 --- a/pytests/src/lib.rs +++ b/pytests/src/lib.rs @@ -18,7 +18,7 @@ pub mod sequence; pub mod subclassing; #[pymodule] -fn pyo3_pytests(py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn pyo3_pytests(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pymodule!(awaitable::awaitable))?; #[cfg(not(Py_LIMITED_API))] m.add_wrapped(wrap_pymodule!(buf_and_str::buf_and_str))?; diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index df89dba7dbd..a7df90b572d 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -580,3 +580,43 @@ pub unsafe fn tp_new_impl( .create_class_object_of_type(py, target_type) .map(Bound::into_ptr) } + +pub fn inspect_type(t: T) -> (T, Extractor) { + (t, Extractor::new()) +} + +pub struct Extractor(NotAGilRef); +pub struct NotAGilRef(std::marker::PhantomData); + +pub trait IsGilRef {} + +impl IsGilRef for &'_ T {} + +impl Extractor { + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + Extractor(NotAGilRef(std::marker::PhantomData)) + } +} + +impl Extractor { + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `&Bound<'_, T>` instead for this function argument" + ) + )] + pub fn extract_gil_ref(&self) {} +} + +impl NotAGilRef { + pub fn extract_gil_ref(&self) {} +} + +impl std::ops::Deref for Extractor { + type Target = NotAGilRef; + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/src/tests/hygiene/pymodule.rs b/src/tests/hygiene/pymodule.rs index bb49d3823c1..6229708dd06 100644 --- a/src/tests/hygiene/pymodule.rs +++ b/src/tests/hygiene/pymodule.rs @@ -7,12 +7,14 @@ fn do_something(x: i32) -> crate::PyResult { ::std::result::Result::Ok(x) } +#[allow(deprecated)] #[crate::pymodule] #[pyo3(crate = "crate")] fn foo(_py: crate::Python<'_>, _m: &crate::types::PyModule) -> crate::PyResult<()> { ::std::result::Result::Ok(()) } +#[allow(deprecated)] #[crate::pymodule] #[pyo3(crate = "crate")] fn my_module(_py: crate::Python<'_>, m: &crate::types::PyModule) -> crate::PyResult<()> { diff --git a/src/types/module.rs b/src/types/module.rs index d6f3acb2567..993060703e6 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -335,11 +335,11 @@ impl PyModule { /// use pyo3::prelude::*; /// /// #[pymodule] - /// fn my_module(py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// fn my_module(py: Python<'_>, module: &Bound<'_, PyModule>) -> PyResult<()> { /// let submodule = PyModule::new_bound(py, "submodule")?; /// submodule.add("super_useful_constant", "important")?; /// - /// module.add_submodule(submodule.as_gil_ref())?; + /// module.add_submodule(&submodule)?; /// Ok(()) /// } /// ``` @@ -530,11 +530,11 @@ pub trait PyModuleMethods<'py>: crate::sealed::Sealed { /// use pyo3::prelude::*; /// /// #[pymodule] - /// fn my_module(py: Python<'_>, module: &PyModule) -> PyResult<()> { + /// fn my_module(py: Python<'_>, module: &Bound<'_, PyModule>) -> PyResult<()> { /// let submodule = PyModule::new_bound(py, "submodule")?; /// submodule.add("super_useful_constant", "important")?; /// - /// module.add_submodule(submodule.as_gil_ref())?; + /// module.add_submodule(&submodule)?; /// Ok(()) /// } /// ``` diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index f5fbec7a533..3f1052c3077 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -18,6 +18,7 @@ fn test_compile_errors() { t.compile_fail("tests/ui/invalid_pymethod_names.rs"); t.compile_fail("tests/ui/invalid_pymodule_args.rs"); t.compile_fail("tests/ui/reject_generics.rs"); + #[cfg(not(feature = "gil-refs"))] t.compile_fail("tests/ui/deprecations.rs"); t.compile_fail("tests/ui/invalid_closure.rs"); t.compile_fail("tests/ui/pyclass_send.rs"); diff --git a/tests/test_module.rs b/tests/test_module.rs index d4c4acca90f..0eef3c0e2d1 100644 --- a/tests/test_module.rs +++ b/tests/test_module.rs @@ -118,7 +118,7 @@ fn test_module_with_functions() { /// This module uses a legacy two-argument module function. #[pymodule] -fn module_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +fn module_with_explicit_py_arg(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(double, m)?)?; Ok(()) } diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index 69f4b6e4651..4c77cc8e2a0 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -10,6 +10,7 @@ fn basic_function(py: pyo3::Python<'_>, x: Option) -> pyo3::PyOb x.unwrap_or_else(|| py.None()) } +#[allow(deprecated)] #[pyo3::pymodule] fn basic_module(_py: pyo3::Python<'_>, m: &pyo3::types::PyModule) -> pyo3::PyResult<()> { #[pyfn(m)] diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index 4177dd6da22..06bcf8197a7 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -14,3 +14,38 @@ impl MyClass { } fn main() {} + +#[pyfunction] +fn double(x: usize) -> usize { + x * 2 +} + +#[pymodule] +fn module_gil_ref(m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(double, m)?)?; + Ok(()) +} + +#[pymodule] +fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(double, m)?)?; + Ok(()) +} + +#[pymodule] +fn module_bound(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(double, m)?)?; + Ok(()) +} + +#[pymodule] +fn module_bound_with_explicit_py_arg(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(double, m)?)?; + Ok(()) +} + +#[pymodule] +fn module_bound_by_value(m: Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(double, &m)?)?; + Ok(()) +} diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index a857b5ee420..2440f909019 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -9,3 +9,15 @@ note: the lint level is defined here | 1 | #![deny(deprecated)] | ^^^^^^^^^^ + +error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument + --> tests/ui/deprecations.rs:24:19 + | +24 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { + | ^ + +error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument + --> tests/ui/deprecations.rs:30:57 + | +30 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + | ^ From 770d9b7f014cc70213950065b9b60bb3679464cd Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 8 Mar 2024 07:43:48 +0000 Subject: [PATCH 206/349] add `FromPyObjectBound` trait for extracting `&str` without GIL Refs (#3928) * add `FromPyObjectBound` adjustment for `&str` without GIL Refs * review: alex, Icxolu feedback * add newsfragment * add newsfragment for `FromPyObject` trait change * make some examples compatible with abi3 < 3.10 * seal `FromPyObjectBound` * fixup chrono_tz conversion --- guide/src/migration.md | 10 ++++ newsfragments/3928.added.md | 1 + newsfragments/3928.changed.md | 1 + noxfile.py | 3 +- src/conversion.rs | 83 ++++++++++++++++++++++++++- src/conversions/chrono_tz.rs | 8 ++- src/conversions/either.rs | 4 +- src/conversions/std/slice.rs | 43 ++++++++++++-- src/conversions/std/string.rs | 55 +++++++++++++++--- src/impl_/extract_argument.rs | 19 +++++- src/impl_/pymodule.rs | 9 ++- src/inspect/types.rs | 5 +- src/instance.rs | 7 ++- src/types/any.rs | 32 +++++------ src/types/bytes.rs | 9 ++- src/types/module.rs | 10 +++- tests/test_class_conversion.rs | 2 +- tests/test_compile_error.rs | 1 + tests/test_pyfunction.rs | 2 +- tests/ui/invalid_cancel_handle.stderr | 2 + 20 files changed, 251 insertions(+), 55 deletions(-) create mode 100644 newsfragments/3928.added.md create mode 100644 newsfragments/3928.changed.md diff --git a/guide/src/migration.md b/guide/src/migration.md index 8537f9a1a69..fa4c23cc969 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -211,6 +211,8 @@ Interactions with Python objects implemented in Rust no longer need to go though To minimise breakage of code using the GIL-Refs API, the `Bound` smart pointer has been introduced by adding complements to all functions which accept or return GIL Refs. This allows code to migrate by replacing the deprecated APIs with the new ones. +To identify what to migrate, temporarily switch off the `gil-refs` feature to see deprecation warnings on all uses of APIs accepting and producing GIL Refs. Over one or more PRs it should be possible to follow the deprecation hints to update code. Depending on your development environment, switching off the `gil-refs` feature may introduce [some very targeted breakages](#deactivating-the-gil-refs-feature), so you may need to fixup those first. + For example, the following APIs have gained updated variants: - `PyList::new`, `PyTyple::new` and similar constructors have replacements `PyList::new_bound`, `PyTuple::new_bound` etc. - `FromPyObject::extract` has a new `FromPyObject::extract_bound` (see the section below) @@ -272,7 +274,15 @@ impl<'py> FromPyObject<'py> for MyType { The expectation is that in 0.22 `extract_bound` will have the default implementation removed and in 0.23 `extract` will be removed. +### Deactivating the `gil-refs` feature + +As a final step of migration, deactivating the `gil-refs` feature will set up code for best performance and is intended to set up a forward-compatible API for PyO3 0.22. + +There is one notable API removed when this feature is disabled. `FromPyObject` trait implementations for types which borrow directly from the input data cannot be implemented by PyO3 without GIL Refs (while the migration is ongoing). These types are `&str`, `Cow<'_, str>`, `&[u8]`, `Cow<'_, u8>`. + +To ease pain during migration, these types instead implement a new temporary trait `FromPyObjectBound` which is the expected future form of `FromPyObject`. The new temporary trait ensures is that `obj.extract::<&str>()` continues to work (with the new constraint that the extracted value now depends on the input `obj` lifetime), as well for these types in `#[pyfunction]` arguments. +An unfortunate final point here is that PyO3 cannot offer this new implementation for `&str` on `abi3` builds for Python older than 3.10. On code which needs `abi3` builds for these older Python versions, many cases of `.extract::<&str>()` may need to be replaced with `.extract::()`, which is string data which borrows from the Python `str` object. Alternatively, use `.extract::>()`, `.extract::()` to copy the data into Rust for these versions. ## from 0.19.* to 0.20 diff --git a/newsfragments/3928.added.md b/newsfragments/3928.added.md new file mode 100644 index 00000000000..e768e6b0163 --- /dev/null +++ b/newsfragments/3928.added.md @@ -0,0 +1 @@ +Implement `FromPyObject` for `Cow`. diff --git a/newsfragments/3928.changed.md b/newsfragments/3928.changed.md new file mode 100644 index 00000000000..a4734c41ed3 --- /dev/null +++ b/newsfragments/3928.changed.md @@ -0,0 +1 @@ +Move implementations of `FromPyObject` for `&str`, `Cow`, `&[u8]` and `Cow<[u8]>` onto a temporary trait `FromPyObjectBound` when `gil-refs` feature is deactivated. diff --git a/noxfile.py b/noxfile.py index 2aa0f464b13..3bba2574885 100644 --- a/noxfile.py +++ b/noxfile.py @@ -613,8 +613,7 @@ def check_feature_powerset(session: nox.Session): if toml is None: session.error("requires Python 3.11 or `toml` to be installed") - with (PYO3_DIR / "Cargo.toml").open("rb") as cargo_toml_file: - cargo_toml = toml.load(cargo_toml_file) + cargo_toml = toml.loads((PYO3_DIR / "Cargo.toml").read_text()) EXCLUDED_FROM_FULL = { "nightly", diff --git a/src/conversion.rs b/src/conversion.rs index 68633d196be..efb09b6b341 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -178,12 +178,12 @@ pub trait IntoPy: Sized { /// Python::with_gil(|py| { /// // Calling `.extract()` on a `Bound` smart pointer /// let obj: Bound<'_, PyString> = PyString::new_bound(py, "blah"); -/// let s: &str = obj.extract()?; +/// let s: String = obj.extract()?; /// # assert_eq!(s, "blah"); /// /// // Calling `.extract(py)` on a `Py` smart pointer /// let obj: Py = obj.unbind(); -/// let s: &str = obj.extract(py)?; +/// let s: String = obj.extract(py)?; /// # assert_eq!(s, "blah"); /// # Ok(()) /// }) @@ -237,6 +237,85 @@ pub trait FromPyObject<'py>: Sized { } } +mod from_py_object_bound_sealed { + /// Private seal for the `FromPyObjectBound` trait. + /// + /// This prevents downstream types from implementing the trait before + /// PyO3 is ready to declare the trait as public API. + pub trait Sealed {} + + // This generic implementation is why the seal is separate from + // `crate::sealed::Sealed`. + impl<'py, T> Sealed for T where T: super::FromPyObject<'py> {} + #[cfg(not(feature = "gil-refs"))] + impl Sealed for &'_ str {} + #[cfg(not(feature = "gil-refs"))] + impl Sealed for std::borrow::Cow<'_, str> {} + #[cfg(not(feature = "gil-refs"))] + impl Sealed for &'_ [u8] {} + #[cfg(not(feature = "gil-refs"))] + impl Sealed for std::borrow::Cow<'_, [u8]> {} +} + +/// Expected form of [`FromPyObject`] to be used in a future PyO3 release. +/// +/// The difference between this and `FromPyObject` is that this trait takes an +/// additional lifetime `'a`, which is the lifetime of the input `Bound`. +/// +/// This allows implementations for `&'a str` and `&'a [u8]`, which could not +/// be expressed by the existing `FromPyObject` trait once the GIL Refs API was +/// removed. +/// +/// # Usage +/// +/// Users are prevented from implementing this trait, instead they should implement +/// the normal `FromPyObject` trait. This trait has a blanket implementation +/// for `T: FromPyObject`. +/// +/// The only case where this trait may have a use case to be implemented is when the +/// lifetime of the extracted value is tied to the lifetime `'a` of the input `Bound` +/// instead of the GIL lifetime `py`, as is the case for the `&'a str` implementation. +/// +/// Please contact the PyO3 maintainers if you believe you have a use case for implementing +/// this trait before PyO3 is ready to change the main `FromPyObject` trait to take an +/// additional lifetime. +/// +/// Similarly, users should typically not call these trait methods and should instead +/// use this via the `extract` method on `Bound` and `Py`. +pub trait FromPyObjectBound<'a, 'py>: Sized + from_py_object_bound_sealed::Sealed { + /// Extracts `Self` from the bound smart pointer `obj`. + /// + /// Users are advised against calling this method directly: instead, use this via + /// [`Bound<'_, PyAny>::extract`] or [`Py::extract`]. + fn from_py_object_bound(ob: &'a Bound<'py, PyAny>) -> PyResult; + + /// Extracts the type hint information for this type when it appears as an argument. + /// + /// For example, `Vec` would return `Sequence[int]`. + /// The default implementation returns `Any`, which is correct for any type. + /// + /// For most types, the return value for this method will be identical to that of [`IntoPy::type_output`]. + /// It may be different for some types, such as `Dict`, to allow duck-typing: functions return `Dict` but take `Mapping` as argument. + #[cfg(feature = "experimental-inspect")] + fn type_input() -> TypeInfo { + TypeInfo::Any + } +} + +impl<'py, T> FromPyObjectBound<'_, 'py> for T +where + T: FromPyObject<'py>, +{ + fn from_py_object_bound(ob: &Bound<'py, PyAny>) -> PyResult { + Self::extract_bound(ob) + } + + #[cfg(feature = "experimental-inspect")] + fn type_input() -> TypeInfo { + ::type_input() + } +} + /// Identity conversion: allows using existing `PyObject` instances where /// `T: ToPyObject` is expected. impl ToPyObject for &'_ T { diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index b4d0e7edf8c..845814c4dab 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -34,6 +34,7 @@ //! } //! ``` use crate::exceptions::PyValueError; +use crate::pybacked::PyBackedStr; use crate::sync::GILOnceCell; use crate::types::{any::PyAnyMethods, PyType}; use crate::{ @@ -62,8 +63,11 @@ impl IntoPy for Tz { impl FromPyObject<'_> for Tz { fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - Tz::from_str(ob.getattr(intern!(ob.py(), "key"))?.extract()?) - .map_err(|e| PyValueError::new_err(e.to_string())) + Tz::from_str( + &ob.getattr(intern!(ob.py(), "key"))? + .extract::()?, + ) + .map_err(|e| PyValueError::new_err(e.to_string())) } } diff --git a/src/conversions/either.rs b/src/conversions/either.rs index 16d5ad491eb..84ec88ea009 100644 --- a/src/conversions/either.rs +++ b/src/conversions/either.rs @@ -113,6 +113,8 @@ where #[cfg(test)] mod tests { + use std::borrow::Cow; + use crate::exceptions::PyTypeError; use crate::{Python, ToPyObject}; @@ -132,7 +134,7 @@ mod tests { let r = E::Right("foo".to_owned()); let obj_r = r.to_object(py); - assert_eq!(obj_r.extract::<&str>(py).unwrap(), "foo"); + assert_eq!(obj_r.extract::>(py).unwrap(), "foo"); assert_eq!(obj_r.extract::(py).unwrap(), r); let obj_s = "foo".to_object(py); diff --git a/src/conversions/std/slice.rs b/src/conversions/std/slice.rs index dad27f081cf..18bb52cdd1a 100644 --- a/src/conversions/std/slice.rs +++ b/src/conversions/std/slice.rs @@ -2,9 +2,11 @@ use std::borrow::Cow; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; +#[cfg(not(feature = "gil-refs"))] +use crate::types::PyBytesMethods; use crate::{ - types::{PyByteArray, PyBytes}, - FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, + types::{PyAnyMethods, PyByteArray, PyByteArrayMethods, PyBytes}, + Bound, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, }; impl<'a> IntoPy for &'a [u8] { @@ -18,7 +20,8 @@ impl<'a> IntoPy for &'a [u8] { } } -impl<'py> FromPyObject<'py> for &'py [u8] { +#[cfg(feature = "gil-refs")] +impl<'py> crate::FromPyObject<'py> for &'py [u8] { fn extract(obj: &'py PyAny) -> PyResult { Ok(obj.downcast::()?.as_bytes()) } @@ -29,13 +32,38 @@ impl<'py> FromPyObject<'py> for &'py [u8] { } } +#[cfg(not(feature = "gil-refs"))] +impl<'a> crate::conversion::FromPyObjectBound<'a, '_> for &'a [u8] { + fn from_py_object_bound(obj: &'a Bound<'_, PyAny>) -> PyResult { + Ok(obj.downcast::()?.as_bytes()) + } + + #[cfg(feature = "experimental-inspect")] + fn type_input() -> TypeInfo { + Self::type_output() + } +} + /// Special-purpose trait impl to efficiently handle both `bytes` and `bytearray` /// /// If the source object is a `bytes` object, the `Cow` will be borrowed and /// pointing into the source object, and no copying or heap allocations will happen. /// If it is a `bytearray`, its contents will be copied to an owned `Cow`. -impl<'py> FromPyObject<'py> for Cow<'py, [u8]> { - fn extract(ob: &'py PyAny) -> PyResult { +#[cfg(feature = "gil-refs")] +impl<'py> crate::FromPyObject<'py> for Cow<'py, [u8]> { + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { + if let Ok(bytes) = ob.downcast::() { + return Ok(Cow::Borrowed(bytes.clone().into_gil_ref().as_bytes())); + } + + let byte_array = ob.downcast::()?; + Ok(Cow::Owned(byte_array.to_vec())) + } +} + +#[cfg(not(feature = "gil-refs"))] +impl<'a> crate::conversion::FromPyObjectBound<'a, '_> for Cow<'a, [u8]> { + fn from_py_object_bound(ob: &'a Bound<'_, PyAny>) -> PyResult { if let Ok(bytes) = ob.downcast::() { return Ok(Cow::Borrowed(bytes.as_bytes())); } @@ -43,6 +71,11 @@ impl<'py> FromPyObject<'py> for Cow<'py, [u8]> { let byte_array = ob.downcast::()?; Ok(Cow::Owned(byte_array.to_vec())) } + + #[cfg(feature = "experimental-inspect")] + fn type_input() -> TypeInfo { + Self::type_output() + } } impl ToPyObject for Cow<'_, [u8]> { diff --git a/src/conversions/std/string.rs b/src/conversions/std/string.rs index 60e8ff7eb53..d013890a77e 100644 --- a/src/conversions/std/string.rs +++ b/src/conversions/std/string.rs @@ -113,7 +113,8 @@ impl<'a> IntoPy for &'a String { } /// Allows extracting strings from Python objects. -/// Accepts Python `str` and `unicode` objects. +/// Accepts Python `str` objects. +#[cfg(feature = "gil-refs")] impl<'py> FromPyObject<'py> for &'py str { fn extract(ob: &'py PyAny) -> PyResult { ob.downcast::()?.to_str() @@ -125,6 +126,42 @@ impl<'py> FromPyObject<'py> for &'py str { } } +#[cfg(all(not(feature = "gil-refs"), any(Py_3_10, not(Py_LIMITED_API))))] +impl<'a> crate::conversion::FromPyObjectBound<'a, '_> for &'a str { + fn from_py_object_bound(ob: &'a Bound<'_, PyAny>) -> PyResult { + ob.downcast::()?.to_str() + } + + #[cfg(feature = "experimental-inspect")] + fn type_input() -> TypeInfo { + ::type_input() + } +} + +#[cfg(feature = "gil-refs")] +impl<'py> FromPyObject<'py> for Cow<'py, str> { + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { + ob.extract().map(Cow::Owned) + } + + #[cfg(feature = "experimental-inspect")] + fn type_input() -> TypeInfo { + ::type_input() + } +} + +#[cfg(not(feature = "gil-refs"))] +impl<'a> crate::conversion::FromPyObjectBound<'a, '_> for Cow<'a, str> { + fn from_py_object_bound(ob: &'a Bound<'_, PyAny>) -> PyResult { + ob.downcast::()?.to_cow() + } + + #[cfg(feature = "experimental-inspect")] + fn type_input() -> TypeInfo { + ::type_input() + } +} + /// Allows extracting strings from Python objects. /// Accepts Python `str` and `unicode` objects. impl FromPyObject<'_> for String { @@ -169,9 +206,9 @@ mod tests { Python::with_gil(|py| { let s = "Hello Python"; let py_string: PyObject = Cow::Borrowed(s).into_py(py); - assert_eq!(s, py_string.extract::<&str>(py).unwrap()); + assert_eq!(s, py_string.extract::>(py).unwrap()); let py_string: PyObject = Cow::::Owned(s.into()).into_py(py); - assert_eq!(s, py_string.extract::<&str>(py).unwrap()); + assert_eq!(s, py_string.extract::>(py).unwrap()); }) } @@ -180,9 +217,9 @@ mod tests { Python::with_gil(|py| { let s = "Hello Python"; let py_string = Cow::Borrowed(s).to_object(py); - assert_eq!(s, py_string.extract::<&str>(py).unwrap()); + assert_eq!(s, py_string.extract::>(py).unwrap()); let py_string = Cow::::Owned(s.into()).to_object(py); - assert_eq!(s, py_string.extract::<&str>(py).unwrap()); + assert_eq!(s, py_string.extract::>(py).unwrap()); }) } @@ -201,7 +238,7 @@ mod tests { let s = "Hello Python"; let py_string = s.to_object(py); - let s2: &str = py_string.bind(py).extract().unwrap(); + let s2: Cow<'_, str> = py_string.bind(py).extract().unwrap(); assert_eq!(s, s2); }) } @@ -238,19 +275,19 @@ mod tests { assert_eq!( s, IntoPy::::into_py(s3, py) - .extract::<&str>(py) + .extract::>(py) .unwrap() ); assert_eq!( s, IntoPy::::into_py(s2, py) - .extract::<&str>(py) + .extract::>(py) .unwrap() ); assert_eq!( s, IntoPy::::into_py(s, py) - .extract::<&str>(py) + .extract::>(py) .unwrap() ); }) diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index d2e2b18cf05..82285b3d50c 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -1,10 +1,10 @@ use crate::{ + conversion::FromPyObjectBound, exceptions::PyTypeError, ffi, pyclass::boolean_struct::False, types::{any::PyAnyMethods, dict::PyDictMethods, tuple::PyTupleMethods, PyDict, PyTuple}, - Borrowed, Bound, FromPyObject, PyAny, PyClass, PyErr, PyRef, PyRefMut, PyResult, PyTypeCheck, - Python, + Borrowed, Bound, PyAny, PyClass, PyErr, PyRef, PyRefMut, PyResult, PyTypeCheck, Python, }; /// Helper type used to keep implementation more concise. @@ -27,7 +27,7 @@ pub trait PyFunctionArgument<'a, 'py>: Sized + 'a { impl<'a, 'py, T> PyFunctionArgument<'a, 'py> for T where - T: FromPyObject<'py> + 'a, + T: FromPyObjectBound<'a, 'py> + 'a, { type Holder = (); @@ -52,6 +52,19 @@ where } } +#[cfg(all(Py_LIMITED_API, not(any(feature = "gil-refs", Py_3_10))))] +impl<'a> PyFunctionArgument<'a, '_> for &'a str { + type Holder = Option>; + + #[inline] + fn extract( + obj: &'a Bound<'_, PyAny>, + holder: &'a mut Option>, + ) -> PyResult { + Ok(holder.insert(obj.extract()?)) + } +} + /// Trait for types which can be a function argument holder - they should /// to be able to const-initialize to an empty value. pub trait FunctionArgumentHolder: Sized { diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index a19cbda526f..9ca80e3918c 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -142,7 +142,10 @@ pub trait PyAddToModule { #[cfg(test)] mod tests { - use std::sync::atomic::{AtomicBool, Ordering}; + use std::{ + borrow::Cow, + sync::atomic::{AtomicBool, Ordering}, + }; use crate::{ types::{any::PyAnyMethods, module::PyModuleMethods, PyModule}, @@ -169,7 +172,7 @@ mod tests { module .getattr("__name__") .unwrap() - .extract::<&str>() + .extract::>() .unwrap(), "test_module", ); @@ -177,7 +180,7 @@ mod tests { module .getattr("__doc__") .unwrap() - .extract::<&str>() + .extract::>() .unwrap(), "some doc", ); diff --git a/src/inspect/types.rs b/src/inspect/types.rs index d8f8e6a19ea..13ce62585b1 100644 --- a/src/inspect/types.rs +++ b/src/inspect/types.rs @@ -464,7 +464,10 @@ mod conversion { assert_display(&String::type_input(), "str"); assert_display(&<&[u8]>::type_output(), "bytes"); - assert_display(&<&[u8]>::type_input(), "bytes"); + assert_display( + &<&[u8] as crate::conversion::FromPyObjectBound>::type_input(), + "bytes", + ); } #[test] diff --git a/src/instance.rs b/src/instance.rs index cef06062430..069bea69f8c 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1315,9 +1315,12 @@ impl Py { /// Extracts some type from the Python object. /// /// This is a wrapper function around `FromPyObject::extract()`. - pub fn extract<'py, D>(&self, py: Python<'py>) -> PyResult + pub fn extract<'a, 'py, D>(&'a self, py: Python<'py>) -> PyResult where - D: FromPyObject<'py>, + D: crate::conversion::FromPyObjectBound<'a, 'py>, + // TODO it might be possible to relax this bound in future, to allow + // e.g. `.extract::<&str>(py)` where `py` is short-lived. + 'py: 'a, { self.bind(py).as_any().extract() } diff --git a/src/types/any.rs b/src/types/any.rs index 0207217ba96..e4dbaf80200 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1,5 +1,5 @@ use crate::class::basic::CompareOp; -use crate::conversion::{AsPyPointer, FromPyObject, IntoPy, ToPyObject}; +use crate::conversion::{AsPyPointer, FromPyObject, FromPyObjectBound, IntoPy, ToPyObject}; use crate::err::{DowncastError, DowncastIntoError, PyDowncastError, PyErr, PyResult}; use crate::exceptions::{PyAttributeError, PyTypeError}; use crate::ffi_ptr_ext::FfiPtrExt; @@ -391,7 +391,7 @@ impl PyAny { /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; /// let result = fun.call(args, Some(&kwargs))?; - /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); + /// assert_eq!(result.extract::()?, "called with args and kwargs"); /// Ok(()) /// }) /// # } @@ -452,7 +452,7 @@ impl PyAny { /// let fun = module.getattr("function")?; /// let args = ("hello",); /// let result = fun.call1(args)?; - /// assert_eq!(result.extract::<&str>()?, "called with args"); + /// assert_eq!(result.extract::()?, "called with args"); /// Ok(()) /// }) /// # } @@ -491,7 +491,7 @@ impl PyAny { /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; /// let result = instance.call_method("method", args, Some(&kwargs))?; - /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); + /// assert_eq!(result.extract::()?, "called with args and kwargs"); /// Ok(()) /// }) /// # } @@ -532,7 +532,7 @@ impl PyAny { /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let instance = module.getattr("a")?; /// let result = instance.call_method0("method")?; - /// assert_eq!(result.extract::<&str>()?, "called with no arguments"); + /// assert_eq!(result.extract::()?, "called with no arguments"); /// Ok(()) /// }) /// # } @@ -573,7 +573,7 @@ impl PyAny { /// let instance = module.getattr("a")?; /// let args = ("hello",); /// let result = instance.call_method1("method", args)?; - /// assert_eq!(result.extract::<&str>()?, "called with args"); + /// assert_eq!(result.extract::()?, "called with args"); /// Ok(()) /// }) /// # } @@ -1271,7 +1271,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; /// let result = fun.call(args, Some(&kwargs))?; - /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); + /// assert_eq!(result.extract::()?, "called with args and kwargs"); /// Ok(()) /// }) /// # } @@ -1326,7 +1326,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// let fun = module.getattr("function")?; /// let args = ("hello",); /// let result = fun.call1(args)?; - /// assert_eq!(result.extract::<&str>()?, "called with args"); + /// assert_eq!(result.extract::()?, "called with args"); /// Ok(()) /// }) /// # } @@ -1363,7 +1363,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// let kwargs = PyDict::new_bound(py); /// kwargs.set_item("cruel", "world")?; /// let result = instance.call_method("method", args, Some(&kwargs))?; - /// assert_eq!(result.extract::<&str>()?, "called with args and kwargs"); + /// assert_eq!(result.extract::()?, "called with args and kwargs"); /// Ok(()) /// }) /// # } @@ -1404,7 +1404,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// let module = PyModule::from_code_bound(py, CODE, "", "")?; /// let instance = module.getattr("a")?; /// let result = instance.call_method0("method")?; - /// assert_eq!(result.extract::<&str>()?, "called with no arguments"); + /// assert_eq!(result.extract::()?, "called with no arguments"); /// Ok(()) /// }) /// # } @@ -1440,7 +1440,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// let instance = module.getattr("a")?; /// let args = ("hello",); /// let result = instance.call_method1("method", args)?; - /// assert_eq!(result.extract::<&str>()?, "called with args"); + /// assert_eq!(result.extract::()?, "called with args"); /// Ok(()) /// }) /// # } @@ -1642,9 +1642,9 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// Extracts some type from the Python object. /// /// This is a wrapper function around [`FromPyObject::extract()`]. - fn extract(&self) -> PyResult + fn extract<'a, T>(&'a self) -> PyResult where - D: FromPyObject<'py>; + T: FromPyObjectBound<'a, 'py>; /// Returns the reference count for the Python object. fn get_refcnt(&self) -> isize; @@ -2178,11 +2178,11 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { std::mem::transmute(self) } - fn extract(&self) -> PyResult + fn extract<'a, T>(&'a self) -> PyResult where - D: FromPyObject<'py>, + T: FromPyObjectBound<'a, 'py>, { - FromPyObject::extract_bound(self) + FromPyObjectBound::from_py_object_bound(self) } fn get_refcnt(&self) -> isize { diff --git a/src/types/bytes.rs b/src/types/bytes.rs index 2a54c172d1f..55c295c416f 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -196,14 +196,13 @@ impl> Index for Bound<'_, PyBytes> { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; #[test] fn test_bytes_index() { Python::with_gil(|py| { - let bytes = PyBytes::new(py, b"Hello World"); + let bytes = PyBytes::new_bound(py, b"Hello World"); assert_eq!(bytes[1], b'e'); }); } @@ -222,7 +221,7 @@ mod tests { #[test] fn test_bytes_new_with() -> super::PyResult<()> { Python::with_gil(|py| -> super::PyResult<()> { - let py_bytes = PyBytes::new_with(py, 10, |b: &mut [u8]| { + let py_bytes = PyBytes::new_bound_with(py, 10, |b: &mut [u8]| { b.copy_from_slice(b"Hello Rust"); Ok(()) })?; @@ -235,7 +234,7 @@ mod tests { #[test] fn test_bytes_new_with_zero_initialised() -> super::PyResult<()> { Python::with_gil(|py| -> super::PyResult<()> { - let py_bytes = PyBytes::new_with(py, 10, |_b: &mut [u8]| Ok(()))?; + let py_bytes = PyBytes::new_bound_with(py, 10, |_b: &mut [u8]| Ok(()))?; let bytes: &[u8] = py_bytes.extract()?; assert_eq!(bytes, &[0; 10]); Ok(()) @@ -246,7 +245,7 @@ mod tests { fn test_bytes_new_with_error() { use crate::exceptions::PyValueError; Python::with_gil(|py| { - let py_bytes_result = PyBytes::new_with(py, 10, |_b: &mut [u8]| { + let py_bytes_result = PyBytes::new_bound_with(py, 10, |_b: &mut [u8]| { Err(PyValueError::new_err("Hello Crustaceans!")) }); assert!(py_bytes_result.is_err()); diff --git a/src/types/module.rs b/src/types/module.rs index 993060703e6..11afcf76c83 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -10,6 +10,8 @@ use crate::{exceptions, ffi, Bound, IntoPy, Py, PyNativeType, PyObject, Python}; use std::ffi::CString; use std::str; +use super::PyStringMethods; + /// Represents a Python [`module`][1] object. /// /// As with all other Python objects, modules are first class citizens. @@ -399,8 +401,12 @@ impl PyModule { /// [1]: crate::prelude::pyfunction /// [2]: crate::wrap_pyfunction pub fn add_function<'a>(&'a self, fun: &'a PyCFunction) -> PyResult<()> { - let name = fun.getattr(__name__(self.py()))?.extract()?; - self.add(name, fun) + let name = fun + .as_borrowed() + .getattr(__name__(self.py()))? + .downcast_into::()?; + let name = name.to_cow()?; + self.add(&name, fun) } } diff --git a/tests/test_class_conversion.rs b/tests/test_class_conversion.rs index a09195278b0..ede8928f865 100644 --- a/tests/test_class_conversion.rs +++ b/tests/test_class_conversion.rs @@ -151,7 +151,7 @@ fn test_pycell_deref() { // Should be able to deref as PyAny assert_eq!( obj.call_method0("foo") - .and_then(|e| e.extract::<&str>()) + .and_then(|e| e.extract::()) .unwrap(), "SubClass" ); diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 3f1052c3077..38b5c4727c6 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -51,5 +51,6 @@ fn test_compile_errors() { #[cfg(feature = "experimental-declarative-modules")] t.compile_fail("tests/ui/invalid_pymodule_two_pymodule_init.rs"); #[cfg(feature = "experimental-async")] + #[cfg(any(not(Py_LIMITED_API), Py_3_10))] // to avoid PyFunctionArgument for &str t.compile_fail("tests/ui/invalid_cancel_handle.rs"); } diff --git a/tests/test_pyfunction.rs b/tests/test_pyfunction.rs index 838f7353737..32c3ede6309 100644 --- a/tests/test_pyfunction.rs +++ b/tests/test_pyfunction.rs @@ -215,7 +215,7 @@ struct ValueClass { fn conversion_error( str_arg: &str, int_arg: i64, - tuple_arg: (&str, f64), + tuple_arg: (String, f64), option_arg: Option, struct_arg: Option, ) { diff --git a/tests/ui/invalid_cancel_handle.stderr b/tests/ui/invalid_cancel_handle.stderr index 6dc3ff3ccab..9e617bca988 100644 --- a/tests/ui/invalid_cancel_handle.stderr +++ b/tests/ui/invalid_cancel_handle.stderr @@ -40,6 +40,7 @@ error[E0277]: the trait bound `CancelHandle: PyClass` is not satisfied | = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` = note: required for `CancelHandle` to implement `FromPyObject<'_>` + = note: required for `CancelHandle` to implement `FromPyObjectBound<'_, '_>` = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>` note: required by a bound in `extract_argument` --> src/impl_/extract_argument.rs @@ -61,6 +62,7 @@ error[E0277]: the trait bound `CancelHandle: Clone` is not satisfied &'a pyo3::coroutine::Coroutine &'a mut pyo3::coroutine::Coroutine = note: required for `CancelHandle` to implement `FromPyObject<'_>` + = note: required for `CancelHandle` to implement `FromPyObjectBound<'_, '_>` = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>` note: required by a bound in `extract_argument` --> src/impl_/extract_argument.rs From 14d1d2a9eec6433fbe6bb552294a99e4bc19e50c Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 8 Mar 2024 14:10:47 +0000 Subject: [PATCH 207/349] docs: use `lychee` to check link URLs (#3941) * guide: install `mdbook-linkcheck` * use `shutil` to copy license files * move from `mdbook-linkcheck` to `lychee` * clean guide & doc build products before build * fix more broken links * review: mejrs --- .github/workflows/gh-pages.yml | 8 +-- .gitignore | 2 + .netlify/build.sh | 10 ++- CHANGELOG.md | 5 +- Contributing.md | 9 ++- guide/book.toml | 2 +- guide/pyclass_parameters.md | 2 +- guide/src/async-await.md | 4 +- guide/src/building_and_distribution.md | 12 ++-- .../multiple_python_versions.md | 6 +- guide/src/class.md | 10 ++- guide/src/class/numeric.md | 2 +- guide/src/class/protocols.md | 2 +- guide/src/conversions/tables.md | 2 +- guide/src/ecosystem/async-await.md | 4 +- guide/src/exception.md | 4 +- guide/src/features.md | 10 +-- guide/src/function.md | 10 +-- guide/src/function/error_handling.md | 2 +- guide/src/memory.md | 2 +- guide/src/migration.md | 4 +- guide/src/module.md | 2 +- guide/src/parallelism.md | 2 +- guide/src/python_from_rust.md | 16 ++--- guide/src/trait_bounds.md | 2 +- noxfile.py | 64 +++++++++++++++++-- src/gil.rs | 2 +- 27 files changed, 135 insertions(+), 65 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index ec8874d5841..a9a7669054a 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -26,9 +26,9 @@ jobs: - uses: dtolnay/rust-toolchain@nightly - name: Setup mdBook - uses: peaceiris/actions-mdbook@v1 + uses: taiki-e/install-action@v2 with: - mdbook-version: "0.4.19" + tool: mdbook,lychee - name: Prepare tag id: prepare_tag @@ -36,11 +36,11 @@ jobs: TAG_NAME="${GITHUB_REF##*/}" echo "::set-output name=tag_name::${TAG_NAME}" - # This builds the book in target/guide. + # This builds the book in target/guide/. - name: Build the guide run: | python -m pip install --upgrade pip && pip install nox - nox -s build-guide + nox -s check-guide env: PYO3_VERSION_TAG: ${{ steps.prepare_tag.outputs.tag_name }} diff --git a/.gitignore b/.gitignore index d27dfa6f9a5..3ad8328cd15 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ dist/ .eggs/ venv* guide/book/ +guide/src/LICENSE-APACHE +guide/src/LICENSE-MIT *.so *.out *.egg-info diff --git a/.netlify/build.sh b/.netlify/build.sh index 10ec241c1d7..dc68c0310cd 100755 --- a/.netlify/build.sh +++ b/.netlify/build.sh @@ -71,9 +71,17 @@ if [ "${INSTALLED_MDBOOK_VERSION}" != "mdbook v${MDBOOK_VERSION}" ]; then cargo install mdbook@${MDBOOK_VERSION} --force fi +# Install latest mdbook-linkcheck. Netlify will cache the cargo bin dir, so this will +# only build mdbook-linkcheck if needed. +MDBOOK_LINKCHECK_VERSION=$(cargo search mdbook-linkcheck --limit 1 | head -1 | tr -s ' ' | cut -d ' ' -f 3 | tr -d '"') +INSTALLED_MDBOOK_LINKCHECK_VERSION=$(mdbook-linkcheck --version || echo "none") +if [ "${INSTALLED_MDBOOK_LINKCHECK_VERSION}" != "mdbook v${MDBOOK_LINKCHECK_VERSION}" ]; then + cargo install mdbook-linkcheck@${MDBOOK_LINKCHECK_VERSION} --force +fi + pip install nox nox -s build-guide -mv target/guide netlify_build/main/ +mv target/guide/ netlify_build/main/ ## Build public docs diff --git a/CHANGELOG.md b/CHANGELOG.md index 295a035370a..fcd4ca4f495 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1687,7 +1687,7 @@ Yanked [0.9.2]: https://github.com/pyo3/pyo3/compare/v0.9.1...v0.9.2 [0.9.1]: https://github.com/pyo3/pyo3/compare/v0.9.0...v0.9.1 [0.9.0]: https://github.com/pyo3/pyo3/compare/v0.8.5...v0.9.0 -[0.8.4]: https://github.com/pyo3/pyo3/compare/v0.8.4...v0.8.5 +[0.8.5]: https://github.com/pyo3/pyo3/compare/v0.8.4...v0.8.5 [0.8.4]: https://github.com/pyo3/pyo3/compare/v0.8.3...v0.8.4 [0.8.3]: https://github.com/pyo3/pyo3/compare/v0.8.2...v0.8.3 [0.8.2]: https://github.com/pyo3/pyo3/compare/v0.8.1...v0.8.2 @@ -1696,7 +1696,8 @@ Yanked [0.7.0]: https://github.com/pyo3/pyo3/compare/v0.6.0...v0.7.0 [0.6.0]: https://github.com/pyo3/pyo3/compare/v0.5.3...v0.6.0 [0.5.3]: https://github.com/pyo3/pyo3/compare/v0.5.2...v0.5.3 -[0.5.2]: https://github.com/pyo3/pyo3/compare/v0.5.0...v0.5.2 +[0.5.2]: https://github.com/pyo3/pyo3/compare/v0.5.1...v0.5.2 +[0.5.1]: https://github.com/pyo3/pyo3/compare/v0.5.0...v0.5.1 [0.5.0]: https://github.com/pyo3/pyo3/compare/v0.4.1...v0.5.0 [0.4.1]: https://github.com/pyo3/pyo3/compare/v0.4.0...v0.4.1 [0.4.0]: https://github.com/pyo3/pyo3/compare/v0.3.2...v0.4.0 diff --git a/Contributing.md b/Contributing.md index b34bf420072..2abdd80d47f 100644 --- a/Contributing.md +++ b/Contributing.md @@ -70,6 +70,12 @@ First, install [`mdbook`][mdbook] and [`nox`][nox]. Then, run nox -s build-guide -- --open ``` +To check all links in the guide are valid, also install [`lychee`][lychee] and use the `check-guide` session instead: + +```shell +nox -s check-guide +``` + ### Help design the next PyO3 Issues which don't yet have a clear solution use the [needs-design](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-design) label. @@ -171,7 +177,7 @@ First, there are Rust-based benchmarks located in the `pyo3-benches` subdirector nox -s bench -Second, there is a Python-based benchmark contained in the `pytests` subdirectory. You can read more about it [here](pytests). +Second, there is a Python-based benchmark contained in the `pytests` subdirectory. You can read more about it [here](https://github.com/PyO3/pyo3/tree/main/pytests). ## Code coverage @@ -211,4 +217,5 @@ In the meanwhile, some of our maintainers have personal GitHub sponsorship pages - [messense](https://github.com/sponsors/messense) [mdbook]: https://rust-lang.github.io/mdBook/cli/index.html +[lychee]: https://github.com/lycheeverse/lychee [nox]: https://github.com/theacodes/nox diff --git a/guide/book.toml b/guide/book.toml index 31fa4bb1587..bccc3506098 100644 --- a/guide/book.toml +++ b/guide/book.toml @@ -9,4 +9,4 @@ command = "python3 guide/pyo3_version.py" [output.html] git-repository-url = "https://github.com/PyO3/pyo3/tree/main/guide" edit-url-template = "https://github.com/PyO3/pyo3/edit/main/guide/{path}" -playground.runnable = false \ No newline at end of file +playground.runnable = false diff --git a/guide/pyclass_parameters.md b/guide/pyclass_parameters.md index 35c54147df5..6951a5b5e15 100644 --- a/guide/pyclass_parameters.md +++ b/guide/pyclass_parameters.md @@ -33,7 +33,7 @@ struct MyClass {} struct MyClass {} ``` -[params-1]: https://docs.rs/pyo3/latest/pyo3/struct.PyAny.html +[params-1]: https://docs.rs/pyo3/latest/pyo3/types/struct.PyAny.html [params-2]: https://en.wikipedia.org/wiki/Free_list [params-3]: https://doc.rust-lang.org/std/marker/trait.Send.html [params-4]: https://doc.rust-lang.org/std/rc/struct.Rc.html diff --git a/guide/src/async-await.md b/guide/src/async-await.md index 688f0a65bc4..c354be7f1b7 100644 --- a/guide/src/async-await.md +++ b/guide/src/async-await.md @@ -38,7 +38,7 @@ However, there is an exception for method receiver, so async methods can accept Even if it is not possible to pass a `py: Python<'_>` parameter to `async fn`, the GIL is still held during the execution of the future – it's also the case for regular `fn` without `Python<'_>`/`&PyAny` parameter, yet the GIL is held. -It is still possible to get a `Python` marker using [`Python::with_gil`]({{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.with_gil); because `with_gil` is reentrant and optimized, the cost will be negligible. +It is still possible to get a `Python` marker using [`Python::with_gil`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.with_gil); because `with_gil` is reentrant and optimized, the cost will be negligible. ## Release the GIL across `.await` @@ -70,7 +70,7 @@ where ## Cancellation -Cancellation on the Python side can be caught using [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) type, by annotating a function parameter with `#[pyo3(cancel_handle)]. +Cancellation on the Python side can be caught using [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) type, by annotating a function parameter with `#[pyo3(cancel_handle)]`. ```rust # #![allow(dead_code)] diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 97e77d24c34..8caa63e688d 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -4,7 +4,7 @@ This chapter of the guide goes into detail on how to build and distribute projec The material in this chapter is intended for users who have already read the PyO3 [README](./index.md). It covers in turn the choices that can be made for Python modules and for Rust binaries. There is also a section at the end about cross-compiling projects using PyO3. -There is an additional sub-chapter dedicated to [supporting multiple Python versions](./building_and_distribution/multiple_python_versions.html). +There is an additional sub-chapter dedicated to [supporting multiple Python versions](./building_and_distribution/multiple_python_versions.md). ## Configuring the Python version @@ -177,7 +177,7 @@ The downside of not linking to `libpython` is that binaries, tests, and examples By default, Python extension modules can only be used with the same Python version they were compiled against. For example, an extension module built for Python 3.5 can't be imported in Python 3.8. [PEP 384](https://www.python.org/dev/peps/pep-0384/) introduced the idea of the limited Python API, which would have a stable ABI enabling extension modules built with it to be used against multiple Python versions. This is also known as `abi3`. -The advantage of building extension modules using the limited Python API is that package vendors only need to build and distribute a single copy (for each OS / architecture), and users can install it on all Python versions from the [minimum version](#minimum-python-version-for-abi3) and up. The downside of this is that PyO3 can't use optimizations which rely on being compiled against a known exact Python version. It's up to you to decide whether this matters for your extension module. It's also possible to design your extension module such that you can distribute `abi3` wheels but allow users compiling from source to benefit from additional optimizations - see the [support for multiple python versions](./building_and_distribution/multiple_python_versions.html) section of this guide, in particular the `#[cfg(Py_LIMITED_API)]` flag. +The advantage of building extension modules using the limited Python API is that package vendors only need to build and distribute a single copy (for each OS / architecture), and users can install it on all Python versions from the [minimum version](#minimum-python-version-for-abi3) and up. The downside of this is that PyO3 can't use optimizations which rely on being compiled against a known exact Python version. It's up to you to decide whether this matters for your extension module. It's also possible to design your extension module such that you can distribute `abi3` wheels but allow users compiling from source to benefit from additional optimizations - see the [support for multiple python versions](./building_and_distribution/multiple_python_versions.md) section of this guide, in particular the `#[cfg(Py_LIMITED_API)]` flag. There are three steps involved in making use of `abi3` when building Python packages as wheels: @@ -198,7 +198,7 @@ See the [corresponding](https://github.com/PyO3/maturin/pull/353) [PRs](https:// Because a single `abi3` wheel can be used with many different Python versions, PyO3 has feature flags `abi3-py37`, `abi3-py38`, `abi3-py39` etc. to set the minimum required Python version for your `abi3` wheel. For example, if you set the `abi3-py37` feature, your extension wheel can be used on all Python 3 versions from Python 3.7 and up. `maturin` and `setuptools-rust` will give the wheel a name like `my-extension-1.0-cp37-abi3-manylinux2020_x86_64.whl`. -As your extension module may be run with multiple different Python versions you may occasionally find you need to check the Python version at runtime to customize behavior. See [the relevant section of this guide](./building_and_distribution/multiple_python_versions.html#checking-the-python-version-at-runtime) on supporting multiple Python versions at runtime. +As your extension module may be run with multiple different Python versions you may occasionally find you need to check the Python version at runtime to customize behavior. See [the relevant section of this guide](./building_and_distribution/multiple_python_versions.md#checking-the-python-version-at-runtime) on supporting multiple Python versions at runtime. PyO3 is only able to link your extension module to abi3 version up to and including your host Python version. E.g., if you set `abi3-py38` and try to compile the crate with a host of Python 3.7, the build will fail. @@ -232,7 +232,7 @@ not work when compiling for `abi3`. These are: If you want to embed the Python interpreter inside a Rust program, there are two modes in which this can be done: dynamically and statically. We'll cover each of these modes in the following sections. Each of them affect how you must distribute your program. Instead of learning how to do this yourself, you might want to consider using a project like [PyOxidizer] to ship your application and all of its dependencies in a single file. -PyO3 automatically switches between the two linking modes depending on whether the Python distribution you have configured PyO3 to use ([see above](#python-version)) contains a shared library or a static library. The static library is most often seen in Python distributions compiled from source without the `--enable-shared` configuration option. For example, this is the default for `pyenv` on macOS. +PyO3 automatically switches between the two linking modes depending on whether the Python distribution you have configured PyO3 to use ([see above](#configuring-the-python-version)) contains a shared library or a static library. The static library is most often seen in Python distributions compiled from source without the `--enable-shared` configuration option. For example, this is the default for `pyenv` on macOS. ### Dynamically embedding the Python interpreter @@ -242,7 +242,7 @@ This mode of embedding works well for Rust tests which need access to the Python For distributing your program to non-technical users, you will have to consider including the Python shared library in your distribution as well as setting up wrapper scripts to set the right environment variables (such as `LD_LIBRARY_PATH` on UNIX, or `PATH` on Windows). -Note that PyPy cannot be embedded in Rust (or any other software). Support for this is tracked on the [PyPy issue tracker](https://foss.heptapod.net/pypy/pypy/-/issues/3286). +Note that PyPy cannot be embedded in Rust (or any other software). Support for this is tracked on the [PyPy issue tracker](https://github.com/pypy/pypy/issues/3836). ### Statically embedding the Python interpreter @@ -285,7 +285,7 @@ Thanks to Rust's great cross-compilation support, cross-compiling using PyO3 is * A toolchain for your target. * The appropriate options in your Cargo `.config` for the platform you're targeting and the toolchain you are using. * A Python interpreter that's already been compiled for your target (optional when building "abi3" extension modules). -* A Python interpreter that is built for your host and available through the `PATH` or setting the [`PYO3_PYTHON`](#python-version) variable (optional when building "abi3" extension modules). +* A Python interpreter that is built for your host and available through the `PATH` or setting the [`PYO3_PYTHON`](#configuring-the-python-version) variable (optional when building "abi3" extension modules). After you've obtained the above, you can build a cross-compiled PyO3 module by using Cargo's `--target` flag. PyO3's build script will detect that you are attempting a cross-compile based on your host machine and the desired target. diff --git a/guide/src/building_and_distribution/multiple_python_versions.md b/guide/src/building_and_distribution/multiple_python_versions.md index 43203686fc8..4e1799a215a 100644 --- a/guide/src/building_and_distribution/multiple_python_versions.md +++ b/guide/src/building_and_distribution/multiple_python_versions.md @@ -85,7 +85,7 @@ This `#[cfg]` marks code which is running on PyPy. ## Checking the Python version at runtime -When building with PyO3's `abi3` feature, your extension module will be compiled against a specific [minimum version](../building_and_distribution.html#minimum-python-version-for-abi3) of Python, but may be running on newer Python versions. +When building with PyO3's `abi3` feature, your extension module will be compiled against a specific [minimum version](../building_and_distribution.md#minimum-python-version-for-abi3) of Python, but may be running on newer Python versions. For example with PyO3's `abi3-py38` feature, your extension will be compiled as if it were for Python 3.8. If you were using `pyo3-build-config`, `#[cfg(Py_3_8)]` would be present. Your user could freely install and run your abi3 extension on Python 3.9. @@ -104,5 +104,5 @@ Python::with_gil(|py| { }); ``` -[`Python::version()`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.version -[`Python::version_info()`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.version_info +[`Python::version()`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.version +[`Python::version_info()`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.version_info diff --git a/guide/src/class.md b/guide/src/class.md index b9b47420ccb..bfcdc108f32 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -16,7 +16,7 @@ This chapter will discuss the functionality and configuration these attributes o - [`#[classmethod]`](#class-methods) - [`#[classattr]`](#class-attributes) - [`#[args]`](#method-arguments) -- [Magic methods and slots](class/protocols.html) +- [Magic methods and slots](class/protocols.md) - [Classes as function arguments](#classes-as-function-arguments) ## Defining a new class @@ -82,7 +82,7 @@ When you need to share ownership of data between Python and Rust, instead of usi A Rust `struct Foo` with a generic parameter `T` generates new compiled implementations each time it is used with a different concrete type for `T`. These new implementations are generated by the compiler at each usage site. This is incompatible with wrapping `Foo` in Python, where there needs to be a single compiled implementation of `Foo` which is integrated with the Python interpreter. -Currently, the best alternative is to write a macro which expands to a new #[pyclass] for each instantiation you want: +Currently, the best alternative is to write a macro which expands to a new `#[pyclass]` for each instantiation you want: ```rust # #![allow(dead_code)] @@ -898,9 +898,7 @@ py_args=('World', 666), py_kwargs=Some({'x': 44, 'y': 55}), name=Hello, num=44, py_args=(), py_kwargs=None, name=World, num=-1, num_before=44 ``` -## Making class method signatures available to Python - -The [`text_signature = "..."`](./function.md#text_signature) option for `#[pyfunction]` also works for `#[pymethods]`: +The [`#[pyo3(text_signature = "...")`](./function/signature.md#overriding-the-generated-signature) option for `#[pyfunction]` also works for `#[pymethods]`. ```rust # #![allow(dead_code)] @@ -1002,7 +1000,7 @@ impl MyClass { Note that `text_signature` on `#[new]` is not compatible with compilation in `abi3` mode until Python 3.10 or greater. -## #[pyclass] enums +## `#[pyclass]` enums Enum support in PyO3 comes in two flavors, depending on what kind of variants the enum has: simple and complex. diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index 9fb609a931d..361d2fb6d36 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -440,6 +440,6 @@ fn wrap(obj: &Bound<'_, PyAny>) -> Result { ``` [`PyErr::take`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/struct.PyErr.html#method.take -[`Python`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html +[`Python`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html [`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html [`pyo3::ffi::PyLong_AsUnsignedLongMask`]: {{#PYO3_DOCS_URL}}/pyo3/ffi/fn.PyLong_AsUnsignedLongMask.html diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index b917c6a3eaf..0a77cd7f2a9 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -450,7 +450,7 @@ Usually, an implementation of `__traverse__` should do nothing but calls to `vis Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. -> Note: these methods are part of the C API, PyPy does not necessarily honor them. If you are building for PyPy you should measure memory consumption to make sure you do not have runaway memory growth. See [this issue on the PyPy bug tracker](https://foss.heptapod.net/pypy/pypy/-/issues/3899). +> Note: these methods are part of the C API, PyPy does not necessarily honor them. If you are building for PyPy you should measure memory consumption to make sure you do not have runaway memory growth. See [this issue on the PyPy bug tracker](https://github.com/pypy/pypy/issues/3848). [`IterNextOutput`]: {{#PYO3_DOCS_URL}}/pyo3/pyclass/enum.IterNextOutput.html [`PySequence`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PySequence.html diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index b0582431156..b6a2d30e55a 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -43,7 +43,7 @@ The table below contains the Python type and the corresponding function argument | `typing.Sequence[T]` | `Vec` | `&PySequence` | | `typing.Mapping[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `&PyMapping` | | `typing.Iterator[Any]` | - | `&PyIterator` | -| `typing.Union[...]` | See [`#[derive(FromPyObject)]`](traits.html#deriving-a-hrefhttpsdocsrspyo3latestpyo3conversiontraitfrompyobjecthtmlfrompyobjecta-for-enums) | - | +| `typing.Union[...]` | See [`#[derive(FromPyObject)]`](traits.md#deriving-frompyobject-for-enums) | - | There are also a few special types related to the GIL and Rust-defined `#[pyclass]`es which may come in useful: diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index ec46e872ba7..1e4ea4ab4b9 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -197,7 +197,7 @@ to do something special with the object that it returns. Normally in Python, that something special is the `await` keyword, but in order to await this coroutine in Rust, we first need to convert it into Rust's version of a `coroutine`: a `Future`. That's where `pyo3-asyncio` comes in. -[`pyo3_asyncio::into_future`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.into_future.html) +[`pyo3_asyncio::async_std::into_future`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.into_future.html) performs this conversion for us. The following example uses `into_future` to call the `py_sleep` function shown above and then await the @@ -349,7 +349,7 @@ implementations _prefer_ control over the main thread, this can still make some Because Python needs to control the main thread, we can't use the convenient proc macros from Rust runtimes to handle the `main` function or `#[test]` functions. Instead, the initialization for PyO3 has to be done from the `main` function and the main -thread must block on [`pyo3_asyncio::run_forever`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/fn.run_forever.html) or [`pyo3_asyncio::async_std::run_until_complete`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.run_until_complete.html). +thread must block on [`pyo3_asyncio::async_std::run_until_complete`](https://docs.rs/pyo3-asyncio/latest/pyo3_asyncio/async_std/fn.run_until_complete.html). Because we have to block on one of those functions, we can't use [`#[async_std::main]`](https://docs.rs/async-std/latest/async_std/attr.main.html) or [`#[tokio::main]`](https://docs.rs/tokio/1.1.0/tokio/attr.main.html) since it's not a good idea to make long blocking calls during an async function. diff --git a/guide/src/exception.md b/guide/src/exception.md index 04550bd4b92..9e8cb780606 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -128,5 +128,5 @@ defines exceptions for several standard library modules. [`PyErr`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html [`PyResult`]: {{#PYO3_DOCS_URL}}/pyo3/type.PyResult.html [`PyErr::from_value`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html#method.from_value -[`PyAny::is_instance`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#method.is_instance -[`PyAny::is_instance_of`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#method.is_instance_of +[`PyAny::is_instance`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.is_instance +[`PyAny::is_instance_of`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.is_instance_of diff --git a/guide/src/features.md b/guide/src/features.md index 118284959d6..3ee620729b3 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -12,7 +12,7 @@ This feature is required when building a Python extension module using PyO3. It tells PyO3's build script to skip linking against `libpython.so` on Unix platforms, where this must not be done. -See the [building and distribution](building_and_distribution.md#linking) section for further detail. +See the [building and distribution](building_and_distribution.md#the-extension-module-feature) section for further detail. ### `abi3` @@ -45,7 +45,7 @@ section for further detail. ### `auto-initialize` -This feature changes [`Python::with_gil`]({{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.with_gil) to automatically initialize a Python interpreter (by calling [`prepare_freethreaded_python`]({{#PYO3_DOCS_URL}}/pyo3/fn.prepare_freethreaded_python.html)) if needed. +This feature changes [`Python::with_gil`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.with_gil) to automatically initialize a Python interpreter (by calling [`prepare_freethreaded_python`]({{#PYO3_DOCS_URL}}/pyo3/fn.prepare_freethreaded_python.html)) if needed. If you do not enable this feature, you should call `pyo3::prepare_freethreaded_python()` before attempting to call any other Python APIs. @@ -114,7 +114,7 @@ Adds a dependency on [anyhow](https://docs.rs/anyhow). Enables a conversion from ### `chrono` Adds a dependency on [chrono](https://docs.rs/chrono). Enables a conversion from [chrono](https://docs.rs/chrono)'s types to python: -- [Duration](https://docs.rs/chrono/latest/chrono/struct.Duration.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) +- [TimeDelta](https://docs.rs/chrono/latest/chrono/struct.TimeDelta.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) - [FixedOffset](https://docs.rs/chrono/latest/chrono/offset/struct.FixedOffset.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) - [Utc](https://docs.rs/chrono/latest/chrono/offset/struct.Utc.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html) - [NaiveDate](https://docs.rs/chrono/latest/chrono/naive/struct.NaiveDate.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html) @@ -129,7 +129,7 @@ It requires at least Python 3.9. ### `either` -Adds a dependency on [either](https://docs.rs/either). Enables a conversions into [either](https://docs.rs/either)’s [`Either`](https://docs.rs/either/latest/either/struct.Report.html) type. +Adds a dependency on [either](https://docs.rs/either). Enables a conversions into [either](https://docs.rs/either)’s [`Either`](https://docs.rs/either/latest/either/enum.Either.html) type. ### `eyre` @@ -145,7 +145,7 @@ Adds a dependency on [indexmap](https://docs.rs/indexmap) and enables conversion ### `num-bigint` -Adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conversions into its [`BigInt`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigInt.html) and [`BigUint`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigUInt.html) types. +Adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conversions into its [`BigInt`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigInt.html) and [`BigUint`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigUint.html) types. ### `num-complex` diff --git a/guide/src/function.md b/guide/src/function.md index 1bba52f2235..cfb1e5ef81e 100644 --- a/guide/src/function.md +++ b/guide/src/function.md @@ -38,7 +38,7 @@ There are also additional sections on the following topics: The `#[pyo3]` attribute can be used to modify properties of the generated Python function. It can take any combination of the following options: - - `#[pyo3(name = "...")]` + - `#[pyo3(name = "...")]` Overrides the name exposed to Python. @@ -67,15 +67,15 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python # }); ``` - - `#[pyo3(signature = (...))]` + - `#[pyo3(signature = (...))]` Defines the function signature in Python. See [Function Signatures](./function/signature.md). - - `#[pyo3(text_signature = "...")]` + - `#[pyo3(text_signature = "...")]` Overrides the PyO3-generated function signature visible in Python tooling (such as via [`inspect.signature`]). See the [corresponding topic in the Function Signatures subchapter](./function/signature.md#making-the-function-signature-available-to-python). - - `#[pyo3(pass_module)]` + - `#[pyo3(pass_module)]` Set this option to make PyO3 pass the containing module as the first argument to the function. It is then possible to use the module in the function body. The first argument **must** be of type `&Bound<'_, PyModule>`, `Bound<'_, PyModule>`, or `Py`. @@ -101,7 +101,7 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python The `#[pyo3]` attribute can be used on individual arguments to modify properties of them in the generated function. It can take any combination of the following options: - - `#[pyo3(from_py_with = "...")]` + - `#[pyo3(from_py_with = "...")]` Set this on an option to specify a custom function to convert the function argument from Python to the desired Rust type, instead of using the default `FromPyObject` extraction. The function signature must be `fn(&Bound<'_, PyAny>) -> PyResult` where `T` is the Rust type of the argument. diff --git a/guide/src/function/error_handling.md b/guide/src/function/error_handling.md index 09fe5cab27b..f55fee90e54 100644 --- a/guide/src/function/error_handling.md +++ b/guide/src/function/error_handling.md @@ -23,7 +23,7 @@ In summary: As indicated in the previous section, when a `PyResult` containing an `Err` crosses from Rust to Python, PyO3 will raise the exception contained within. -Accordingly, to raise an exception from a `#[pyfunction]`, change the return type `T` to `PyResult`. When the function returns an `Err` it will raise a Python exception. (Other `Result` types can be used as long as the error `E` has a `From` conversion for `PyErr`, see [implementing a conversion](#implementing-an-error-conversion) below.) +Accordingly, to raise an exception from a `#[pyfunction]`, change the return type `T` to `PyResult`. When the function returns an `Err` it will raise a Python exception. (Other `Result` types can be used as long as the error `E` has a `From` conversion for `PyErr`, see [custom Rust error types](#custom-rust-error-types) below.) This also works for functions in `#[pymethods]`. diff --git a/guide/src/memory.md b/guide/src/memory.md index 46136e3f1a4..b14ce4496ad 100644 --- a/guide/src/memory.md +++ b/guide/src/memory.md @@ -115,7 +115,7 @@ at the end of each loop iteration, before the `with_gil()` closure ends. When doing this, you must be very careful to ensure that once the `GILPool` is dropped you do not retain access to any owned references created after the `GILPool` was created. Read the -[documentation for `Python::new_pool()`]({{#PYO3_DOCS_URL}}/pyo3/prelude/struct.Python.html#method.new_pool) +[documentation for `Python::new_pool()`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.new_pool) for more information on safety. This memory management can also be applicable when writing extension modules. diff --git a/guide/src/migration.md b/guide/src/migration.md index fa4c23cc969..3ed3e015b3b 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -44,7 +44,7 @@ pyo3 = { version = "0.21", features = ["gil-refs"] } ### `PyTypeInfo` and `PyTryFrom` have been adjusted -The `PyTryFrom` trait has aged poorly, its [`try_from`] method now conflicts with `try_from` in the 2021 edition prelude. A lot of its functionality was also duplicated with `PyTypeInfo`. +The `PyTryFrom` trait has aged poorly, its `try_from` method now conflicts with `TryFrom::try_from` in the 2021 edition prelude. A lot of its functionality was also duplicated with `PyTypeInfo`. To tighten up the PyO3 traits as part of the deprecation of the GIL Refs API the `PyTypeInfo` trait has had a simpler companion `PyTypeCheck`. The methods [`PyAny::downcast`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast) and [`PyAny::downcast_exact`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast_exact) no longer use `PyTryFrom` as a bound, instead using `PyTypeCheck` and `PyTypeInfo` respectively. @@ -1299,7 +1299,7 @@ impl MyClass { ``` Basically you can return `Self` or `Result` directly. -For more, see [the constructor section](class.html#constructor) of this guide. +For more, see [the constructor section](class.md#constructor) of this guide. ### PyCell PyO3 0.9 introduces [`PyCell`], which is a [`RefCell`]-like object wrapper diff --git a/guide/src/module.md b/guide/src/module.md index 0444c750c9c..410716fdd39 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -45,7 +45,7 @@ file. Otherwise, you will get an import error in Python with the following messa `ImportError: dynamic module does not define module export function (PyInit_name_of_your_module)` To import the module, either: - - copy the shared library as described in [Manual builds](building_and_distribution.html#manual-builds), or + - copy the shared library as described in [Manual builds](building_and_distribution.md#manual-builds), or - use a tool, e.g. `maturin develop` with [maturin](https://github.com/PyO3/maturin) or `python setup.py develop` with [setuptools-rust](https://github.com/PyO3/setuptools-rust). diff --git a/guide/src/parallelism.md b/guide/src/parallelism.md index 195106b2420..792e0ed8de4 100644 --- a/guide/src/parallelism.md +++ b/guide/src/parallelism.md @@ -117,4 +117,4 @@ test_word_count_python_sequential 27.3985 (15.82) 45.452 You can see that the Python threaded version is not much slower than the Rust sequential version, which means compared to an execution on a single CPU core the speed has doubled. -[`Python::allow_threads`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.allow_threads +[`Python::allow_threads`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.allow_threads diff --git a/guide/src/python_from_rust.md b/guide/src/python_from_rust.md index d8f3214f58b..4a81d9a668f 100644 --- a/guide/src/python_from_rust.md +++ b/guide/src/python_from_rust.md @@ -10,15 +10,15 @@ Any Python-native object reference (such as `&PyAny`, `&PyList`, or `&PyCell`](types.html#pyt-and-pyobject) smart pointer also exposes these same six API methods, but needs a `Python` token as an additional first argument to prove the GIL is held. +For convenience the [`Py`](types.md#pyt-and-pyobject) smart pointer also exposes these same six API methods, but needs a `Python` token as an additional first argument to prove the GIL is held. The example below calls a Python function behind a `PyObject` (aka `Py`) reference: @@ -113,9 +113,9 @@ fn main() -> PyResult<()> {
-During PyO3's [migration from "GIL Refs" to the `Bound` smart pointer](./migration.md#migrating-from-the-gil-refs-api-to-boundt), [`Py::call`]({{#PYO3_DOCS_URL}}/pyo3/struct.py#method.call) is temporarily named `call_bound` (and `call_method` is temporarily `call_method_bound`). +During PyO3's [migration from "GIL Refs" to the `Bound` smart pointer](./migration.md#migrating-from-the-gil-refs-api-to-boundt), [`Py::call`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html#method.call) is temporarily named `call_bound` (and `call_method` is temporarily `call_method_bound`). -(This temporary naming is only the case for the `Py` smart pointer. The methods on the `&PyAny` GIL Ref such as `call` have not been given replacements, and the methods on the `Bound` smart pointer such as [`Bound::call`]({#PYO3_DOCS_URL}}/pyo3/prelude/trait.pyanymethods#tymethod.call) already use follow the newest API conventions.) +(This temporary naming is only the case for the `Py` smart pointer. The methods on the `&PyAny` GIL Ref such as `call` have not been given replacements, and the methods on the `Bound` smart pointer such as [`Bound::call`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call) already use follow the newest API conventions.)
@@ -428,7 +428,7 @@ fn main() -> PyResult<()> { ``` -[`Python::run`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.run +[`Python::run`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.run [`py_run!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.py_run.html ## Need to use a context manager from Rust? diff --git a/guide/src/trait_bounds.md b/guide/src/trait_bounds.md index 6dfaa2e20aa..b0eee80c80a 100644 --- a/guide/src/trait_bounds.md +++ b/guide/src/trait_bounds.md @@ -1,6 +1,6 @@ # Using in Python a Rust function with trait bounds -PyO3 allows for easy conversion from Rust to Python for certain functions and classes (see the [conversion table](conversions/tables.html). +PyO3 allows for easy conversion from Rust to Python for certain functions and classes (see the [conversion table](conversions/tables.md). However, it is not always straightforward to convert Rust code that requires a given trait implementation as an argument. This tutorial explains how to convert a Rust function that takes a trait as argument for use in Python with classes implementing the same methods as the trait. diff --git a/noxfile.py b/noxfile.py index 3bba2574885..29fe0f916cd 100644 --- a/noxfile.py +++ b/noxfile.py @@ -2,6 +2,7 @@ import json import os import re +import shutil import subprocess import sys import tempfile @@ -25,6 +26,10 @@ PYO3_DIR = Path(__file__).parent +PYO3_TARGET = Path(os.environ.get("CARGO_TARGET_DIR", PYO3_DIR / "target")).absolute() +PYO3_GUIDE_SRC = PYO3_DIR / "guide" / "src" +PYO3_GUIDE_TARGET = PYO3_TARGET / "guide" +PYO3_DOCS_TARGET = PYO3_TARGET / "doc" PY_VERSIONS = ("3.7", "3.8", "3.9", "3.10", "3.11", "3.12") PYPY_VERSIONS = ("3.7", "3.8", "3.9", "3.10") @@ -356,6 +361,7 @@ def docs(session: nox.Session) -> None: rustdoc_flags.append(session.env.get("RUSTDOCFLAGS", "")) session.env["RUSTDOCFLAGS"] = " ".join(rustdoc_flags) + shutil.rmtree(PYO3_DOCS_TARGET, ignore_errors=True) _run_cargo( session, *toolchain_flags, @@ -371,7 +377,49 @@ def docs(session: nox.Session) -> None: @nox.session(name="build-guide", venv_backend="none") def build_guide(session: nox.Session): - _run(session, "mdbook", "build", "-d", "../target/guide", "guide", *session.posargs) + shutil.rmtree(PYO3_GUIDE_TARGET, ignore_errors=True) + _run(session, "mdbook", "build", "-d", PYO3_GUIDE_TARGET, "guide", *session.posargs) + for license in ("LICENSE-APACHE", "LICENSE-MIT"): + target_file = PYO3_GUIDE_TARGET / license + target_file.unlink(missing_ok=True) + shutil.copy(PYO3_DIR / license, target_file) + + +@nox.session(name="check-guide", venv_backend="none") +def check_guide(session: nox.Session): + # reuse other sessions, but with default args + posargs = [*session.posargs] + del session.posargs[:] + build_guide(session) + docs(session) + session.posargs.extend(posargs) + + remaps = { + f"file://{PYO3_GUIDE_SRC}/([^/]*/)*?%7B%7B#PYO3_DOCS_URL\}}\}}": f"file://{PYO3_DOCS_TARGET}", + "%7B%7B#PYO3_DOCS_VERSION\}\}": "latest", + } + remap_args = [] + for key, value in remaps.items(): + remap_args.extend(("--remap", f"{key} {value}")) + # check all links in the guide + _run( + session, + "lychee", + "--include-fragments", + PYO3_GUIDE_SRC, + *remap_args, + *session.posargs, + ) + # check external links in the docs + # (intra-doc links are checked by rustdoc) + _run( + session, + "lychee", + PYO3_DOCS_TARGET, + f"--remap=https://pyo3.rs/main/ file://{PYO3_GUIDE_TARGET}/", + f"--exclude=file://{PYO3_DOCS_TARGET}", + *session.posargs, + ) @nox.session(name="format-guide", venv_backend="none") @@ -451,10 +499,11 @@ def address_sanitizer(session: nox.Session): @nox.session(name="check-changelog") def check_changelog(session: nox.Session): - event_path = os.environ.get("GITHUB_EVENT_PATH") - if event_path is None: + if not _is_github_actions(): session.error("Can only check changelog on github actions") + event_path = os.environ["GITHUB_EVENT_PATH"] + with open(event_path) as event_file: event = json.load(event_file) @@ -762,11 +811,12 @@ def _get_coverage_env() -> Dict[str, str]: def _run(session: nox.Session, *args: str, **kwargs: Any) -> None: """Wrapper for _run(session, which creates nice groups on GitHub Actions.""" - if "GITHUB_ACTIONS" in os.environ: + is_github_actions = _is_github_actions() + if is_github_actions: # Insert ::group:: at the start of nox's command line output print("::group::", end="", flush=True, file=sys.stderr) session.run(*args, **kwargs) - if "GITHUB_ACTIONS" in os.environ: + if is_github_actions: print("::endgroup::", file=sys.stderr) @@ -869,5 +919,9 @@ def _config_file() -> Iterator[_ConfigFile]: yield _ConfigFile(config) +def _is_github_actions() -> bool: + return "GITHUB_ACTIONS" in os.environ + + _BENCHES = "--manifest-path=pyo3-benches/Cargo.toml" _FFI_CHECK = "--manifest-path=pyo3-ffi-check/Cargo.toml" diff --git a/src/gil.rs b/src/gil.rs index 08b1b3f745b..91c3d1cdd27 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -67,7 +67,7 @@ fn gil_is_acquired() -> bool { /// /// This function is unavailable under PyPy because PyPy cannot be embedded in Rust (or any other /// software). Support for this is tracked on the -/// [PyPy issue tracker](https://foss.heptapod.net/pypy/pypy/-/issues/3286). +/// [PyPy issue tracker](https://github.com/pypy/pypy/issues/3836). /// /// # Examples /// ```rust From 908e661237a4bc31939473f80047b67a7d129bda Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 9 Mar 2024 10:52:12 +0100 Subject: [PATCH 208/349] deprecate gil-refs in "self" position (#3943) * deprecate gil-refs in "self" position * feature gate explicit gil-ref tests * fix MSRV * adjust bracketing --------- Co-authored-by: David Hewitt --- pyo3-macros-backend/src/method.rs | 72 ++++++++++++++++++++++++++----- tests/test_class_basics.rs | 8 +++- tests/test_methods.rs | 1 + tests/test_module.rs | 8 +++- tests/ui/deprecations.rs | 23 ++++++++++ tests/ui/deprecations.stderr | 36 +++++++++++++--- 6 files changed, 128 insertions(+), 20 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 9a8cb828b22..3cc2a96e899 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -128,24 +128,24 @@ impl FnType { } FnType::FnClass(span) | FnType::FnNewClass(span) => { let py = syn::Ident::new("py", Span::call_site()); - let slf: Ident = syn::Ident::new("_slf", Span::call_site()); + let slf: Ident = syn::Ident::new("_slf_ref", Span::call_site()); let pyo3_path = pyo3_path.to_tokens_spanned(*span); quote_spanned! { *span => #[allow(clippy::useless_conversion)] ::std::convert::Into::into( - #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf.cast()) + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &*(#slf as *const _ as *const *mut _)) .downcast_unchecked::<#pyo3_path::types::PyType>() ), } } FnType::FnModule(span) => { let py = syn::Ident::new("py", Span::call_site()); - let slf: Ident = syn::Ident::new("_slf", Span::call_site()); + let slf: Ident = syn::Ident::new("_slf_ref", Span::call_site()); let pyo3_path = pyo3_path.to_tokens_spanned(*span); quote_spanned! { *span => #[allow(clippy::useless_conversion)] ::std::convert::Into::into( - #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf.cast()) + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &*(#slf as *const _ as *const *mut _)) .downcast_unchecked::<#pyo3_path::types::PyModule>() ), } @@ -519,7 +519,9 @@ impl<'a> FnSpec<'a> { ); } - let rust_call = |args: Vec, holders: &mut Vec| { + let rust_call = |args: Vec, + self_e: &syn::Ident, + holders: &mut Vec| { let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise, holders, ctx); let call = if self.asyncness.is_some() { @@ -538,6 +540,7 @@ impl<'a> FnSpec<'a> { holders.pop().unwrap(); // does not actually use holder created by `self_arg` quote! {{ + #self_e = #pyo3_path::impl_::pymethods::Extractor::<()>::new(); let __guard = #pyo3_path::impl_::coroutine::RefGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; async move { function(&__guard, #(#args),*).await } }} @@ -546,11 +549,25 @@ impl<'a> FnSpec<'a> { holders.pop().unwrap(); // does not actually use holder created by `self_arg` quote! {{ + #self_e = #pyo3_path::impl_::pymethods::Extractor::<()>::new(); let mut __guard = #pyo3_path::impl_::coroutine::RefMutGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; async move { function(&mut __guard, #(#args),*).await } }} } - _ => quote! { function(#self_arg #(#args),*) }, + _ => { + if self_arg.is_empty() { + quote! {{ + #self_e = #pyo3_path::impl_::pymethods::Extractor::<()>::new(); + function(#(#args),*) + }} + } else { + quote! { function({ + let (self_arg, e) = #pyo3_path::impl_::pymethods::inspect_type(#self_arg); + #self_e = e; + self_arg + }, #(#args),*) } + } + } }; let mut call = quote! {{ let future = #future; @@ -569,10 +586,24 @@ impl<'a> FnSpec<'a> { }}; } call + } else if self_arg.is_empty() { + quote! {{ + #self_e = #pyo3_path::impl_::pymethods::Extractor::<()>::new(); + function(#(#args),*) + }} } else { - quote! { function(#self_arg #(#args),*) } + quote! { + function({ + let (self_arg, e) = #pyo3_path::impl_::pymethods::inspect_type(#self_arg); + #self_e = e; + self_arg + }, #(#args),*) + } }; - quotes::map_result_into_ptr(quotes::ok_wrap(call, ctx), ctx) + ( + quotes::map_result_into_ptr(quotes::ok_wrap(call, ctx), ctx), + self_arg.span(), + ) }; let func_name = &self.name; @@ -582,6 +613,7 @@ impl<'a> FnSpec<'a> { quote!(#func_name) }; + let self_e = syn::Ident::new("self_e", Span::call_site()); Ok(match self.convention { CallingConvention::Noargs => { let mut holders = Vec::new(); @@ -599,16 +631,21 @@ impl<'a> FnSpec<'a> { } }) .collect(); - let call = rust_call(args, &mut holders); + let (call, self_arg_span) = rust_call(args, &self_e, &mut holders); + let extract_gil_ref = + quote_spanned! { self_arg_span => #self_e.extract_gil_ref(); }; quote! { unsafe fn #ident<'py>( py: #pyo3_path::Python<'py>, _slf: *mut #pyo3_path::ffi::PyObject, ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { + let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 + let #self_e; #( #holders )* let result = #call; + #extract_gil_ref result } } @@ -616,7 +653,10 @@ impl<'a> FnSpec<'a> { CallingConvention::Fastcall => { let mut holders = Vec::new(); let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders, ctx)?; - let call = rust_call(args, &mut holders); + let (call, self_arg_span) = rust_call(args, &self_e, &mut holders); + let extract_gil_ref = + quote_spanned! { self_arg_span => #self_e.extract_gil_ref(); }; + quote! { unsafe fn #ident<'py>( py: #pyo3_path::Python<'py>, @@ -625,10 +665,13 @@ impl<'a> FnSpec<'a> { _nargs: #pyo3_path::ffi::Py_ssize_t, _kwnames: *mut #pyo3_path::ffi::PyObject ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { + let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 + let #self_e; #arg_convert #( #holders )* let result = #call; + #extract_gil_ref result } } @@ -636,7 +679,10 @@ impl<'a> FnSpec<'a> { CallingConvention::Varargs => { let mut holders = Vec::new(); let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx)?; - let call = rust_call(args, &mut holders); + let (call, self_arg_span) = rust_call(args, &self_e, &mut holders); + let extract_gil_ref = + quote_spanned! { self_arg_span => #self_e.extract_gil_ref(); }; + quote! { unsafe fn #ident<'py>( py: #pyo3_path::Python<'py>, @@ -644,10 +690,13 @@ impl<'a> FnSpec<'a> { _args: *mut #pyo3_path::ffi::PyObject, _kwargs: *mut #pyo3_path::ffi::PyObject ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { + let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 + let #self_e; #arg_convert #( #holders )* let result = #call; + #extract_gil_ref result } } @@ -667,6 +716,7 @@ impl<'a> FnSpec<'a> { _kwargs: *mut #pyo3_path::ffi::PyObject ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { use #pyo3_path::callback::IntoPyCallbackOutput; + let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert #( #holders )* diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 21ccc4a6bb6..8c6a1c04915 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -307,6 +307,7 @@ impl ClassWithFromPyWithMethods { } #[classmethod] + #[cfg(feature = "gil-refs")] fn classmethod_gil_ref( _cls: &PyType, #[pyo3(from_py_with = "PyAny::len")] argument: usize, @@ -324,16 +325,19 @@ impl ClassWithFromPyWithMethods { fn test_pymethods_from_py_with() { Python::with_gil(|py| { let instance = Py::new(py, ClassWithFromPyWithMethods {}).unwrap(); + let has_gil_refs = cfg!(feature = "gil-refs"); py_run!( py, - instance, + instance + has_gil_refs, r#" arg = {1: 1, 2: 3} assert instance.instance_method(arg) == 2 assert instance.classmethod(arg) == 2 - assert instance.classmethod_gil_ref(arg) == 2 + if has_gil_refs: + assert instance.classmethod_gil_ref(arg) == 2 assert instance.staticmethod(arg) == 2 "# ); diff --git a/tests/test_methods.rs b/tests/test_methods.rs index 68a004bd4bc..a9e7d37d64a 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -78,6 +78,7 @@ impl ClassMethod { #[classmethod] /// Test class method. + #[cfg(feature = "gil-refs")] fn method_gil_ref(cls: &PyType) -> PyResult { Ok(format!("{}.method()!", cls.qualname()?)) } diff --git a/tests/test_module.rs b/tests/test_module.rs index 0eef3c0e2d1..5760c3ebaf3 100644 --- a/tests/test_module.rs +++ b/tests/test_module.rs @@ -45,7 +45,7 @@ fn module_with_functions(m: &Bound<'_, PyModule>) -> PyResult<()> { #[pyfn(m)] #[pyo3(pass_module)] - fn with_module(module: &PyModule) -> PyResult<&str> { + fn with_module<'py>(module: &Bound<'py, PyModule>) -> PyResult> { module.name() } @@ -373,6 +373,7 @@ fn pyfunction_with_module<'py>(module: &Bound<'py, PyModule>) -> PyResult PyResult<&str> { module.name() } @@ -427,6 +428,7 @@ fn pyfunction_with_module_and_args_kwargs<'py>( #[pyfunction] #[pyo3(pass_module)] +#[cfg(feature = "gil-refs")] fn pyfunction_with_pass_module_in_attribute(module: &PyModule) -> PyResult<&str> { module.name() } @@ -434,12 +436,14 @@ fn pyfunction_with_pass_module_in_attribute(module: &PyModule) -> PyResult<&str> #[pymodule] fn module_with_functions_with_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(pyfunction_with_module, m)?)?; + #[cfg(feature = "gil-refs")] m.add_function(wrap_pyfunction!(pyfunction_with_module_gil_ref, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_owned, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_and_py, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_and_arg, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_and_default_arg, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_and_args_kwargs, m)?)?; + #[cfg(feature = "gil-refs")] m.add_function(wrap_pyfunction!( pyfunction_with_pass_module_in_attribute, m @@ -457,6 +461,7 @@ fn test_module_functions_with_module() { m, "m.pyfunction_with_module() == 'module_with_functions_with_module'" ); + #[cfg(feature = "gil-refs")] py_assert!( py, m, @@ -484,6 +489,7 @@ fn test_module_functions_with_module() { "m.pyfunction_with_module_and_args_kwargs(1, x=1, y=2) \ == ('module_with_functions_with_module', 1, 2)" ); + #[cfg(feature = "gil-refs")] py_assert!( py, m, diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index 06bcf8197a7..c062cead3ad 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -1,6 +1,7 @@ #![deny(deprecated)] use pyo3::prelude::*; +use pyo3::types::{PyString, PyType}; #[pyclass] struct MyClass; @@ -11,10 +12,32 @@ impl MyClass { fn new() -> Self { Self } + + #[classmethod] + fn cls_method_gil_ref(_cls: &PyType) {} + + #[classmethod] + fn cls_method_bound(_cls: &Bound<'_, PyType>) {} + + fn method_gil_ref(_slf: &PyCell) {} + + fn method_bound(_slf: &Bound<'_, Self>) {} } fn main() {} +#[pyfunction] +#[pyo3(pass_module)] +fn pyfunction_with_module<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + module.name() +} + +#[pyfunction] +#[pyo3(pass_module)] +fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { + module.name() +} + #[pyfunction] fn double(x: usize) -> usize { x * 2 diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index 2440f909019..edb74f97b74 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -1,7 +1,7 @@ error: use of deprecated constant `pyo3::impl_::deprecations::PYMETHODS_NEW_DEPRECATED_FORM`: use `#[new]` instead of `#[__new__]` - --> tests/ui/deprecations.rs:10:7 + --> tests/ui/deprecations.rs:11:7 | -10 | #[__new__] +11 | #[__new__] | ^^^^^^^ | note: the lint level is defined here @@ -10,14 +10,38 @@ note: the lint level is defined here 1 | #![deny(deprecated)] | ^^^^^^^^^^ +error: use of deprecated struct `pyo3::PyCell`: `PyCell` was merged into `Bound`, use that instead; see the migration guide for more info + --> tests/ui/deprecations.rs:22:30 + | +22 | fn method_gil_ref(_slf: &PyCell) {} + | ^^^^^^ + +error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument + --> tests/ui/deprecations.rs:17:33 + | +17 | fn cls_method_gil_ref(_cls: &PyType) {} + | ^ + +error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument + --> tests/ui/deprecations.rs:22:29 + | +22 | fn method_gil_ref(_slf: &PyCell) {} + | ^ + +error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument + --> tests/ui/deprecations.rs:37:43 + | +37 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { + | ^ + error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:24:19 + --> tests/ui/deprecations.rs:47:19 | -24 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { +47 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:30:57 + --> tests/ui/deprecations.rs:53:57 | -30 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +53 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { | ^ From 75af678f4330db682e9ee99989a1bf7cdd4cbdd9 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 9 Mar 2024 21:10:58 +0100 Subject: [PATCH 209/349] docs: use kebab-case instead of snake_case for guide URLs (#3942) * guide: use kebab-case instead of snake_case * fixup doctest names Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> * review: Icxolu * fix relative url * also remap latest pyo3 * fixup python_from_rust --------- Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> --- .netlify/build.sh | 9 +++++++++ Architecture.md | 2 +- README.md | 4 ++-- build.rs | 2 +- ...ass_parameters.md => pyclass-parameters.md} | 0 guide/src/SUMMARY.md | 14 +++++++------- ...ibution.md => building-and-distribution.md} | 6 +++--- .../multiple-python-versions.md} | 2 +- guide/src/class.md | 2 +- guide/src/exception.md | 2 +- guide/src/features.md | 8 ++++---- .../{error_handling.md => error-handling.md} | 0 .../{getting_started.md => getting-started.md} | 4 ++-- guide/src/module.md | 2 +- ...python_from_rust.md => python-from-rust.md} | 0 ..._typing_hints.md => python-typing-hints.md} | 0 guide/src/{trait_bounds.md => trait-bounds.md} | 0 noxfile.py | 1 + pyo3-build-config/src/impl_.rs | 2 +- pyo3-build-config/src/lib.rs | 2 +- pyo3-ffi/README.md | 2 +- pyo3-ffi/src/lib.rs | 2 +- src/lib.rs | 18 +++++++++--------- 23 files changed, 47 insertions(+), 37 deletions(-) rename guide/{pyclass_parameters.md => pyclass-parameters.md} (100%) rename guide/src/{building_and_distribution.md => building-and-distribution.md} (99%) rename guide/src/{building_and_distribution/multiple_python_versions.md => building-and-distribution/multiple-python-versions.md} (98%) rename guide/src/function/{error_handling.md => error-handling.md} (100%) rename guide/src/{getting_started.md => getting-started.md} (92%) rename guide/src/{python_from_rust.md => python-from-rust.md} (100%) rename guide/src/{python_typing_hints.md => python-typing-hints.md} (100%) rename guide/src/{trait_bounds.md => trait-bounds.md} (100%) diff --git a/.netlify/build.sh b/.netlify/build.sh index dc68c0310cd..e1d86788ca1 100755 --- a/.netlify/build.sh +++ b/.netlify/build.sh @@ -48,6 +48,15 @@ done # Add latest redirect echo "/latest/* /v${PYO3_VERSION}/:splat 302" >> netlify_build/_redirects +# some backwards compatbiility redirects +echo "/latest/building_and_distribution/* /latest/building-and-distribution/:splat 302" >> netlify_build/_redirects +echo "/latest/building-and-distribution/multiple_python_versions/* /latest/building-and-distribution/multiple-python-versions:splat 302" >> netlify_build/_redirects +echo "/latest/function/error_handling/* /latest/function/error-handling/:splat 302" >> netlify_build/_redirects +echo "/latest/getting_started/* /latest/getting-started/:splat 302" >> netlify_build/_redirects +echo "/latest/python_from_rust/* /latest/python-from-rust/:splat 302" >> netlify_build/_redirects +echo "/latest/python_typing_hints/* /latest/python-typing-hints/:splat 302" >> netlify_build/_redirects +echo "/latest/trait_bounds/* /latest/trait-bounds/:splat 302" >> netlify_build/_redirects + ## Add landing page redirect if [ "${CONTEXT}" == "deploy-preview" ]; then echo "/ /main/" >> netlify_build/_redirects diff --git a/Architecture.md b/Architecture.md index 9ec20931c10..60083d71c98 100644 --- a/Architecture.md +++ b/Architecture.md @@ -48,7 +48,7 @@ In the [`pyo3-ffi`] crate, there is lots of conditional compilation such as `#[c `#[cfg(Py_3_7)]`, and `#[cfg(PyPy)]`. `Py_LIMITED_API` corresponds to `#define Py_LIMITED_API` macro in Python/C API. With `Py_LIMITED_API`, we can build a Python-version-agnostic binary called an -[abi3 wheel](https://pyo3.rs/latest/building_and_distribution.html#py_limited_apiabi3). +[abi3 wheel](https://pyo3.rs/latest/building-and-distribution.html#py_limited_apiabi3). `Py_3_7` means that the API is available from Python >= 3.7. There are also `Py_3_8`, `Py_3_9`, and so on. `PyPy` means that the API definition is for PyPy. diff --git a/README.md b/README.md index 21dc125d5ab..c11754c3ca1 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ maturin develop If you want to be able to run `cargo test` or use this project in a Cargo workspace and are running into linker issues, there are some workarounds in [the FAQ](https://pyo3.rs/latest/faq.html#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror). -As well as with `maturin`, it is possible to build using [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](https://pyo3.rs/latest/building_and_distribution.html#manual-builds). Both offer more flexibility than `maturin` but require more configuration to get started. +As well as with `maturin`, it is possible to build using [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](https://pyo3.rs/latest/building-and-distribution.html#manual-builds). Both offer more flexibility than `maturin` but require more configuration to get started. ### Using Python from Rust @@ -162,7 +162,7 @@ fn main() -> PyResult<()> { } ``` -The guide has [a section](https://pyo3.rs/latest/python_from_rust.html) with lots of examples +The guide has [a section](https://pyo3.rs/latest/python-from-rust.html) with lots of examples about this topic. ## Tools and libraries diff --git a/build.rs b/build.rs index e20206310db..7f0ae6e31c8 100644 --- a/build.rs +++ b/build.rs @@ -16,7 +16,7 @@ fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<( \n\ For more information, see \ https://pyo3.rs/v{pyo3_version}/\ - building_and_distribution.html#embedding-python-in-rust", + building-and-distribution.html#embedding-python-in-rust", pyo3_version = env::var("CARGO_PKG_VERSION").unwrap() ); } diff --git a/guide/pyclass_parameters.md b/guide/pyclass-parameters.md similarity index 100% rename from guide/pyclass_parameters.md rename to guide/pyclass-parameters.md diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index 34775a0d185..6aea71b1722 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -4,11 +4,11 @@ --- -- [Getting started](getting_started.md) +- [Getting started](getting-started.md) - [Python modules](module.md) - [Python functions](function.md) - [Function signatures](function/signature.md) - - [Error handling](function/error_handling.md) + - [Error handling](function/error-handling.md) - [Python classes](class.md) - [Class customizations](class/protocols.md) - [Basic object customization](class/object.md) @@ -18,7 +18,7 @@ - [Mapping of Rust types to Python types](conversions/tables.md) - [Conversion traits](conversions/traits.md) - [Python exceptions](exception.md) -- [Calling Python from Rust](python_from_rust.md) +- [Calling Python from Rust](python-from-rust.md) - [Using `async` and `await`](async-await.md) - [GIL, mutability and object types](types.md) - [Parallelism](parallelism.md) @@ -27,8 +27,8 @@ - [Memory management](memory.md) - [Performance](performance.md) - [Advanced topics](advanced.md) -- [Building and distribution](building_and_distribution.md) - - [Supporting multiple Python versions](building_and_distribution/multiple_python_versions.md) +- [Building and distribution](building-and-distribution.md) + - [Supporting multiple Python versions](building-and-distribution/multiple-python-versions.md) - [Useful crates](ecosystem.md) - [Logging](ecosystem/logging.md) - [Using `async` and `await`](ecosystem/async-await.md) @@ -37,8 +37,8 @@ --- [Appendix A: Migration guide](migration.md) -[Appendix B: Trait bounds](trait_bounds.md) -[Appendix C: Python typing hints](python_typing_hints.md) +[Appendix B: Trait bounds](trait-bounds.md) +[Appendix C: Python typing hints](python-typing-hints.md) [CHANGELOG](changelog.md) --- diff --git a/guide/src/building_and_distribution.md b/guide/src/building-and-distribution.md similarity index 99% rename from guide/src/building_and_distribution.md rename to guide/src/building-and-distribution.md index 8caa63e688d..780f135e211 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building-and-distribution.md @@ -4,7 +4,7 @@ This chapter of the guide goes into detail on how to build and distribute projec The material in this chapter is intended for users who have already read the PyO3 [README](./index.md). It covers in turn the choices that can be made for Python modules and for Rust binaries. There is also a section at the end about cross-compiling projects using PyO3. -There is an additional sub-chapter dedicated to [supporting multiple Python versions](./building_and_distribution/multiple_python_versions.md). +There is an additional sub-chapter dedicated to [supporting multiple Python versions](./building-and-distribution/multiple-python-versions.md). ## Configuring the Python version @@ -177,7 +177,7 @@ The downside of not linking to `libpython` is that binaries, tests, and examples By default, Python extension modules can only be used with the same Python version they were compiled against. For example, an extension module built for Python 3.5 can't be imported in Python 3.8. [PEP 384](https://www.python.org/dev/peps/pep-0384/) introduced the idea of the limited Python API, which would have a stable ABI enabling extension modules built with it to be used against multiple Python versions. This is also known as `abi3`. -The advantage of building extension modules using the limited Python API is that package vendors only need to build and distribute a single copy (for each OS / architecture), and users can install it on all Python versions from the [minimum version](#minimum-python-version-for-abi3) and up. The downside of this is that PyO3 can't use optimizations which rely on being compiled against a known exact Python version. It's up to you to decide whether this matters for your extension module. It's also possible to design your extension module such that you can distribute `abi3` wheels but allow users compiling from source to benefit from additional optimizations - see the [support for multiple python versions](./building_and_distribution/multiple_python_versions.md) section of this guide, in particular the `#[cfg(Py_LIMITED_API)]` flag. +The advantage of building extension modules using the limited Python API is that package vendors only need to build and distribute a single copy (for each OS / architecture), and users can install it on all Python versions from the [minimum version](#minimum-python-version-for-abi3) and up. The downside of this is that PyO3 can't use optimizations which rely on being compiled against a known exact Python version. It's up to you to decide whether this matters for your extension module. It's also possible to design your extension module such that you can distribute `abi3` wheels but allow users compiling from source to benefit from additional optimizations - see the [support for multiple python versions](./building-and-distribution/multiple-python-versions.md) section of this guide, in particular the `#[cfg(Py_LIMITED_API)]` flag. There are three steps involved in making use of `abi3` when building Python packages as wheels: @@ -198,7 +198,7 @@ See the [corresponding](https://github.com/PyO3/maturin/pull/353) [PRs](https:// Because a single `abi3` wheel can be used with many different Python versions, PyO3 has feature flags `abi3-py37`, `abi3-py38`, `abi3-py39` etc. to set the minimum required Python version for your `abi3` wheel. For example, if you set the `abi3-py37` feature, your extension wheel can be used on all Python 3 versions from Python 3.7 and up. `maturin` and `setuptools-rust` will give the wheel a name like `my-extension-1.0-cp37-abi3-manylinux2020_x86_64.whl`. -As your extension module may be run with multiple different Python versions you may occasionally find you need to check the Python version at runtime to customize behavior. See [the relevant section of this guide](./building_and_distribution/multiple_python_versions.md#checking-the-python-version-at-runtime) on supporting multiple Python versions at runtime. +As your extension module may be run with multiple different Python versions you may occasionally find you need to check the Python version at runtime to customize behavior. See [the relevant section of this guide](./building-and-distribution/multiple-python-versions.md#checking-the-python-version-at-runtime) on supporting multiple Python versions at runtime. PyO3 is only able to link your extension module to abi3 version up to and including your host Python version. E.g., if you set `abi3-py38` and try to compile the crate with a host of Python 3.7, the build will fail. diff --git a/guide/src/building_and_distribution/multiple_python_versions.md b/guide/src/building-and-distribution/multiple-python-versions.md similarity index 98% rename from guide/src/building_and_distribution/multiple_python_versions.md rename to guide/src/building-and-distribution/multiple-python-versions.md index 4e1799a215a..b328d236c51 100644 --- a/guide/src/building_and_distribution/multiple_python_versions.md +++ b/guide/src/building-and-distribution/multiple-python-versions.md @@ -85,7 +85,7 @@ This `#[cfg]` marks code which is running on PyPy. ## Checking the Python version at runtime -When building with PyO3's `abi3` feature, your extension module will be compiled against a specific [minimum version](../building_and_distribution.md#minimum-python-version-for-abi3) of Python, but may be running on newer Python versions. +When building with PyO3's `abi3` feature, your extension module will be compiled against a specific [minimum version](../building-and-distribution.md#minimum-python-version-for-abi3) of Python, but may be running on newer Python versions. For example with PyO3's `abi3-py38` feature, your extension will be compiled as if it were for Python 3.8. If you were using `pyo3-build-config`, `#[cfg(Py_3_8)]` would be present. Your user could freely install and run your abi3 extension on Python 3.9. diff --git a/guide/src/class.md b/guide/src/class.md index bfcdc108f32..93920a40d88 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -288,7 +288,7 @@ Frozen classes are likely to become the default thereby guiding the PyO3 ecosyst ## Customizing the class -{{#include ../pyclass_parameters.md}} +{{#include ../pyclass-parameters.md}} These parameters are covered in various sections of this guide. diff --git a/guide/src/exception.md b/guide/src/exception.md index 9e8cb780606..ee36f544d74 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -54,7 +54,7 @@ fn mymodule(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { ## Raising an exception -As described in the [function error handling](./function/error_handling.md) chapter, to raise an exception from a `#[pyfunction]` or `#[pymethods]`, return an `Err(PyErr)`. PyO3 will automatically raise this exception for you when returning the result to Python. +As described in the [function error handling](./function/error-handling.md) chapter, to raise an exception from a `#[pyfunction]` or `#[pymethods]`, return an `Err(PyErr)`. PyO3 will automatically raise this exception for you when returning the result to Python. You can also manually write and fetch errors in the Python interpreter's global state: diff --git a/guide/src/features.md b/guide/src/features.md index 3ee620729b3..6e4e5ab70b1 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -12,7 +12,7 @@ This feature is required when building a Python extension module using PyO3. It tells PyO3's build script to skip linking against `libpython.so` on Unix platforms, where this must not be done. -See the [building and distribution](building_and_distribution.md#the-extension-module-feature) section for further detail. +See the [building and distribution](building-and-distribution.md#the-extension-module-feature) section for further detail. ### `abi3` @@ -20,7 +20,7 @@ This feature is used when building Python extension modules to create wheels whi It restricts PyO3's API to a subset of the full Python API which is guaranteed by [PEP 384](https://www.python.org/dev/peps/pep-0384/) to be forwards-compatible with future Python versions. -See the [building and distribution](building_and_distribution.md#py_limited_apiabi3) section for further detail. +See the [building and distribution](building-and-distribution.md#py_limited_apiabi3) section for further detail. ### The `abi3-pyXY` features @@ -28,7 +28,7 @@ See the [building and distribution](building_and_distribution.md#py_limited_apia These features are extensions of the `abi3` feature to specify the exact minimum Python version which the multiple-version-wheel will support. -See the [building and distribution](building_and_distribution.md#minimum-python-version-for-abi3) section for further detail. +See the [building and distribution](building-and-distribution.md#minimum-python-version-for-abi3) section for further detail. ### `generate-import-lib` @@ -38,7 +38,7 @@ for MinGW-w64 and MSVC (cross-)compile targets. Enabling it allows to (cross-)compile extension modules to any Windows targets without having to install the Windows Python distribution files for the target. -See the [building and distribution](building_and_distribution.md#building-abi3-extensions-without-a-python-interpreter) +See the [building and distribution](building-and-distribution.md#building-abi3-extensions-without-a-python-interpreter) section for further detail. ## Features for embedding Python in Rust diff --git a/guide/src/function/error_handling.md b/guide/src/function/error-handling.md similarity index 100% rename from guide/src/function/error_handling.md rename to guide/src/function/error-handling.md diff --git a/guide/src/getting_started.md b/guide/src/getting-started.md similarity index 92% rename from guide/src/getting_started.md rename to guide/src/getting-started.md index 008da8109f6..ef9ca5254b8 100644 --- a/guide/src/getting_started.md +++ b/guide/src/getting-started.md @@ -33,7 +33,7 @@ You can read more about `pyenv`'s configuration options [here](https://github.co ### Building -There are a number of build and Python package management systems such as [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](https://pyo3.rs/latest/building_and_distribution.html#manual-builds). We recommend the use of `maturin`, which you can install [here](https://maturin.rs/installation.html). It is developed to work with PyO3 and provides the most "batteries included" experience, especially if you are aiming to publish to PyPI. `maturin` is just a Python package, so you can add it in the same you already install Python packages. +There are a number of build and Python package management systems such as [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](./building-and-distribution.md#manual-builds). We recommend the use of `maturin`, which you can install [here](https://maturin.rs/installation.html). It is developed to work with PyO3 and provides the most "batteries included" experience, especially if you are aiming to publish to PyPI. `maturin` is just a Python package, so you can add it in the same you already install Python packages. System Python: ```bash @@ -180,4 +180,4 @@ $ python '25' ``` -For more instructions on how to use Python code from Rust, see the [Python from Rust](python_from_rust.md) page. +For more instructions on how to use Python code from Rust, see the [Python from Rust](python-from-rust.md) page. diff --git a/guide/src/module.md b/guide/src/module.md index 410716fdd39..8cac9a5be4c 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -45,7 +45,7 @@ file. Otherwise, you will get an import error in Python with the following messa `ImportError: dynamic module does not define module export function (PyInit_name_of_your_module)` To import the module, either: - - copy the shared library as described in [Manual builds](building_and_distribution.md#manual-builds), or + - copy the shared library as described in [Manual builds](building-and-distribution.md#manual-builds), or - use a tool, e.g. `maturin develop` with [maturin](https://github.com/PyO3/maturin) or `python setup.py develop` with [setuptools-rust](https://github.com/PyO3/setuptools-rust). diff --git a/guide/src/python_from_rust.md b/guide/src/python-from-rust.md similarity index 100% rename from guide/src/python_from_rust.md rename to guide/src/python-from-rust.md diff --git a/guide/src/python_typing_hints.md b/guide/src/python-typing-hints.md similarity index 100% rename from guide/src/python_typing_hints.md rename to guide/src/python-typing-hints.md diff --git a/guide/src/trait_bounds.md b/guide/src/trait-bounds.md similarity index 100% rename from guide/src/trait_bounds.md rename to guide/src/trait-bounds.md diff --git a/noxfile.py b/noxfile.py index 29fe0f916cd..f70fced76d6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -417,6 +417,7 @@ def check_guide(session: nox.Session): "lychee", PYO3_DOCS_TARGET, f"--remap=https://pyo3.rs/main/ file://{PYO3_GUIDE_TARGET}/", + f"--remap=https://pyo3.rs/latest/ file://{PYO3_GUIDE_TARGET}/", f"--exclude=file://{PYO3_DOCS_TARGET}", *session.posargs, ) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 698c58a18ec..7f02f744fc2 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1365,7 +1365,7 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result "Documentation about the `serde` feature." -//! [calling_rust]: https://pyo3.rs/latest/python_from_rust.html "Calling Python from Rust - PyO3 user guide" +//! [calling_rust]: https://pyo3.rs/latest/python-from-rust.html "Calling Python from Rust - PyO3 user guide" //! [examples subdirectory]: https://github.com/PyO3/pyo3/tree/main/examples //! [feature flags]: https://doc.rust-lang.org/cargo/reference/features.html "Features - The Cargo Book" //! [global interpreter lock]: https://docs.python.org/3/glossary.html#term-global-interpreter-lock //! [hashbrown]: https://docs.rs/hashbrown //! [smallvec]: https://docs.rs/smallvec //! [indexmap]: https://docs.rs/indexmap -//! [manual_builds]: https://pyo3.rs/latest/building_and_distribution.html#manual-builds "Manual builds - Building and Distribution - PyO3 user guide" +//! [manual_builds]: https://pyo3.rs/latest/building-and-distribution.html#manual-builds "Manual builds - Building and Distribution - PyO3 user guide" //! [num-bigint]: https://docs.rs/num-bigint //! [num-complex]: https://docs.rs/num-complex //! [serde]: https://docs.rs/serde @@ -453,7 +453,7 @@ pub use pyo3_macros::{pyfunction, pymethods, pymodule, FromPyObject}; /// A proc macro used to expose Rust structs and fieldless enums as Python objects. /// -#[doc = include_str!("../guide/pyclass_parameters.md")] +#[doc = include_str!("../guide/pyclass-parameters.md")] /// /// For more on creating Python classes, /// see the [class section of the guide][1]. @@ -485,8 +485,8 @@ pub mod doc_test { "README.md" => readme_md, "guide/src/advanced.md" => guide_advanced_md, "guide/src/async-await.md" => guide_async_await_md, - "guide/src/building_and_distribution.md" => guide_building_and_distribution_md, - "guide/src/building_and_distribution/multiple_python_versions.md" => guide_bnd_multiple_python_versions_md, + "guide/src/building-and-distribution.md" => guide_building_and_distribution_md, + "guide/src/building-and-distribution/multiple-python-versions.md" => guide_bnd_multiple_python_versions_md, "guide/src/class.md" => guide_class_md, "guide/src/class/call.md" => guide_class_call, "guide/src/class/object.md" => guide_class_object, @@ -504,16 +504,16 @@ pub mod doc_test { "guide/src/faq.md" => guide_faq_md, "guide/src/features.md" => guide_features_md, "guide/src/function.md" => guide_function_md, - "guide/src/function/error_handling.md" => guide_function_error_handling_md, + "guide/src/function/error-handling.md" => guide_function_error_handling_md, "guide/src/function/signature.md" => guide_function_signature_md, "guide/src/memory.md" => guide_memory_md, "guide/src/migration.md" => guide_migration_md, "guide/src/module.md" => guide_module_md, "guide/src/parallelism.md" => guide_parallelism_md, "guide/src/performance.md" => guide_performance_md, - "guide/src/python_from_rust.md" => guide_python_from_rust_md, - "guide/src/python_typing_hints.md" => guide_python_typing_hints_md, - "guide/src/trait_bounds.md" => guide_trait_bounds_md, + "guide/src/python-from-rust.md" => guide_python_from_rust_md, + "guide/src/python-typing-hints.md" => guide_python_typing_hints_md, + "guide/src/trait-bounds.md" => guide_trait_bounds_md, "guide/src/types.md" => guide_types_md, } } From 9145fcfe19ad114d3bd7b4c509d900e6781b7c40 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 10 Mar 2024 15:51:51 +0000 Subject: [PATCH 210/349] docs: major rewrite for Bound API (#3906) * wip bound docs * Update guide/src/python_from_rust/calling-existing-code.md Co-authored-by: Lily Foote * continue to move and tidy up * Apply suggestions from code review Co-authored-by: Lily Foote * update URL * complete python-from-rust.md * progress on types.md; probably more to go * update doctest paths * review: Icxolu * finish updating `types.md` to Bound API * update remainder of the guide to Bound API * Update guide/src/performance.md Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> * Update guide/src/types.md Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> * Update src/lib.rs * review: Icxolu * Update guide/src/python-from-rust.md Co-authored-by: Adam Reichold * Update guide/src/async-await.md Co-authored-by: Adam Reichold * review: adamreichold --------- Co-authored-by: Lily Foote Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> Co-authored-by: Adam Reichold --- Architecture.md | 29 +- guide/src/SUMMARY.md | 27 +- guide/src/advanced.md | 7 - guide/src/async-await.md | 12 +- guide/src/class.md | 40 +- guide/src/conversions/tables.md | 75 +-- guide/src/conversions/traits.md | 2 +- guide/src/ecosystem/async-await.md | 12 +- guide/src/exception.md | 2 +- guide/src/function-calls.md | 1 + guide/src/function.md | 4 +- guide/src/index.md | 16 + guide/src/memory.md | 63 ++- guide/src/migration.md | 16 +- guide/src/performance.md | 44 +- guide/src/python-from-rust.md | 521 +----------------- .../python-from-rust/calling-existing-code.md | 397 +++++++++++++ guide/src/python-from-rust/function-calls.md | 114 ++++ guide/src/rust-from-python.md | 13 + guide/src/trait-bounds.md | 81 +-- guide/src/types.md | 456 +++++++++------ src/lib.rs | 52 +- src/sync.rs | 5 + 23 files changed, 1114 insertions(+), 875 deletions(-) create mode 100644 guide/src/function-calls.md create mode 100644 guide/src/python-from-rust/calling-existing-code.md create mode 100644 guide/src/python-from-rust/function-calls.md create mode 100644 guide/src/rust-from-python.md diff --git a/Architecture.md b/Architecture.md index 60083d71c98..a4218a7f71f 100644 --- a/Architecture.md +++ b/Architecture.md @@ -59,6 +59,7 @@ Those flags are set in [`build.rs`](#6-buildrs-and-pyo3-build-config). [`src/types`] contains bindings to [built-in types](https://docs.python.org/3/library/stdtypes.html) of Python, such as `dict` and `list`. For historical reasons, Python's `object` is called `PyAny` in PyO3 and located in [`src/types/any.rs`]. + Currently, `PyAny` is a straightforward wrapper of `ffi::PyObject`, defined as: ```rust @@ -66,38 +67,16 @@ Currently, `PyAny` is a straightforward wrapper of `ffi::PyObject`, defined as: pub struct PyAny(UnsafeCell); ``` -All built-in types are defined as a C struct. -For example, `dict` is defined as: - -```c -typedef struct { - /* Base object */ - PyObject ob_base; - /* Number of items in the dictionary */ - Py_ssize_t ma_used; - /* Dictionary version */ - uint64_t ma_version_tag; - PyDictKeysObject *ma_keys; - PyObject **ma_values; -} PyDictObject; -``` - -However, we cannot access such a specific data structure with `#[cfg(Py_LIMITED_API)]` set. -Thus, all builtin objects are implemented as opaque types by wrapping `PyAny`, e.g.,: +Concrete Python objects are implemented by wrapping `PyAny`, e.g.,: ```rust #[repr(transparent)] pub struct PyDict(PyAny); ``` -Note that `PyAny` is not a pointer, and it is usually used as a pointer to the object in the -Python heap, as `&PyAny`. -This design choice can be changed -(see the discussion in [#1056](https://github.com/PyO3/pyo3/issues/1056)). +These types are not intended to be accessed directly, and instead are used through the `Py` and `Bound` smart pointers. -Since we need lots of boilerplate for implementing common traits for these types -(e.g., `AsPyPointer`, `AsRef`, and `Debug`), we have some macros in -[`src/types/mod.rs`]. +We have some macros in [`src/types/mod.rs`] which make it easier to implement APIs for concrete Python types. ## 3. `PyClass` and related functionalities diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index 6aea71b1722..4c22c26f587 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -5,22 +5,25 @@ --- - [Getting started](getting-started.md) -- [Python modules](module.md) -- [Python functions](function.md) - - [Function signatures](function/signature.md) - - [Error handling](function/error-handling.md) -- [Python classes](class.md) - - [Class customizations](class/protocols.md) - - [Basic object customization](class/object.md) - - [Emulating numeric types](class/numeric.md) - - [Emulating callable objects](class/call.md) +- [Using Rust from Python](rust-from-python.md) + - [Python modules](module.md) + - [Python functions](function.md) + - [Function signatures](function/signature.md) + - [Error handling](function/error-handling.md) + - [Python classes](class.md) + - [Class customizations](class/protocols.md) + - [Basic object customization](class/object.md) + - [Emulating numeric types](class/numeric.md) + - [Emulating callable objects](class/call.md) +- [Calling Python from Rust](python-from-rust.md) + - [Python object types](types.md) + - [Python exceptions](exception.md) + - [Calling Python functions](python-from-rust/function-calls.md) + - [Executing existing Python code](python-from-rust/calling-existing-code.md) - [Type conversions](conversions.md) - [Mapping of Rust types to Python types](conversions/tables.md) - [Conversion traits](conversions/traits.md) -- [Python exceptions](exception.md) -- [Calling Python from Rust](python-from-rust.md) - [Using `async` and `await`](async-await.md) -- [GIL, mutability and object types](types.md) - [Parallelism](parallelism.md) - [Debugging](debugging.md) - [Features reference](features.md) diff --git a/guide/src/advanced.md b/guide/src/advanced.md index 8264c14dd17..61dc66382f4 100644 --- a/guide/src/advanced.md +++ b/guide/src/advanced.md @@ -5,10 +5,3 @@ PyO3 exposes much of Python's C API through the `ffi` module. The C API is naturally unsafe and requires you to manage reference counts, errors and specific invariants yourself. Please refer to the [C API Reference Manual](https://docs.python.org/3/c-api/) and [The Rustonomicon](https://doc.rust-lang.org/nightly/nomicon/ffi.html) before using any function from that API. - -## Memory management - -PyO3's `&PyAny` "owned references" and `Py` smart pointers are used to -access memory stored in Python's heap. This memory sometimes lives for longer -than expected because of differences in Rust and Python's memory models. See -the chapter on [memory management](./memory.md) for more information. diff --git a/guide/src/async-await.md b/guide/src/async-await.md index c354be7f1b7..06fa1580ad7 100644 --- a/guide/src/async-await.md +++ b/guide/src/async-await.md @@ -30,13 +30,13 @@ async fn sleep(seconds: f64, result: Option) -> Option { Resulting future of an `async fn` decorated by `#[pyfunction]` must be `Send + 'static` to be embedded in a Python object. -As a consequence, `async fn` parameters and return types must also be `Send + 'static`, so it is not possible to have a signature like `async fn does_not_compile(arg: &PyAny, py: Python<'_>) -> &PyAny`. +As a consequence, `async fn` parameters and return types must also be `Send + 'static`, so it is not possible to have a signature like `async fn does_not_compile<'py>(arg: Bound<'py, PyAny>) -> Bound<'py, PyAny>`. -However, there is an exception for method receiver, so async methods can accept `&self`/`&mut self`. Note that this means that the class instance is borrowed for as long as the returned future is not completed, even across yield points and while waiting for I/O operations to complete. Hence, other methods cannot obtain exclusive borrows while the future is still being polled. This is the same as how async methods in Rust generally work but it is more problematic for Rust code interfacing with Python code due to pervasive shared mutability. This strongly suggests to prefer shared borrows `&self` to exclusive ones `&mut self` to avoid racy borrow check failures at runtime. +However, there is an exception for method receivers, so async methods can accept `&self`/`&mut self`. Note that this means that the class instance is borrowed for as long as the returned future is not completed, even across yield points and while waiting for I/O operations to complete. Hence, other methods cannot obtain exclusive borrows while the future is still being polled. This is the same as how async methods in Rust generally work but it is more problematic for Rust code interfacing with Python code due to pervasive shared mutability. This strongly suggests to prefer shared borrows `&self` over exclusive ones `&mut self` to avoid racy borrow check failures at runtime. ## Implicit GIL holding -Even if it is not possible to pass a `py: Python<'_>` parameter to `async fn`, the GIL is still held during the execution of the future – it's also the case for regular `fn` without `Python<'_>`/`&PyAny` parameter, yet the GIL is held. +Even if it is not possible to pass a `py: Python<'py>` parameter to `async fn`, the GIL is still held during the execution of the future – it's also the case for regular `fn` without `Python<'py>`/`Bound<'py, PyAny>` parameter, yet the GIL is held. It is still possible to get a `Python` marker using [`Python::with_gil`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.with_gil); because `with_gil` is reentrant and optimized, the cost will be negligible. @@ -47,7 +47,11 @@ There is currently no simple way to release the GIL when awaiting a future, *but Here is the advised workaround for now: ```rust,ignore -use std::{future::Future, pin::{Pin, pin}, task::{Context, Poll}}; +use std::{ + future::Future, + pin::{Pin, pin}, + task::{Context, Poll}, +}; use pyo3::prelude::*; struct AllowThreads(F); diff --git a/guide/src/class.md b/guide/src/class.md index 93920a40d88..2c6d854ff08 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -60,7 +60,7 @@ enum Shape { Circle { radius: f64 }, Rectangle { width: f64, height: f64 }, RegularPolygon { side_count: u32, radius: f64 }, - Nothing { }, + Nothing {}, } ``` @@ -89,7 +89,7 @@ Currently, the best alternative is to write a macro which expands to a new `#[py use pyo3::prelude::*; struct GenericClass { - data: T + data: T, } macro_rules! create_interface { @@ -102,7 +102,9 @@ macro_rules! create_interface { impl $name { #[new] pub fn new(data: $type) -> Self { - Self { inner: GenericClass { data: data } } + Self { + inner: GenericClass { data: data }, + } } } }; @@ -187,23 +189,21 @@ fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { } ``` -## Bound and interior mutability +## Bound and interior mutability + +Often is is useful to turn a `#[pyclass]` type `T` into a Python object and access it from Rust code. The [`Py`] and [`Bound<'py, T>`] smart pointers are the ways to represent a Python object in PyO3's API. More detail can be found about them [in the Python objects](./types.md#pyo3s-smart-pointers) section of the guide. -You sometimes need to convert your `#[pyclass]` into a Python object and access it -from Rust code (e.g., for testing it). -[`Bound`] is the primary interface for that. +Most Python objects do not offer exclusive (`&mut`) access (see the [section on Python's memory model](./python-from-rust.md#pythons-memory-model)). However, Rust structs wrapped as Python objects (called `pyclass` types) often *do* need `&mut` access. Due to the GIL, PyO3 *can* guarantee exclusive access to them. -To mutate data behind a `Bound<'_, T>` safely, PyO3 employs the -[Interior Mutability Pattern](https://doc.rust-lang.org/book/ch15-05-interior-mutability.html) -like [`RefCell`]. +The Rust borrow checker cannot reason about `&mut` references once an object's ownership has been passed to the Python interpreter. This means that borrow checking is done at runtime using with a scheme very similar to `std::cell::RefCell`. This is known as [interior mutability](https://doc.rust-lang.org/book/ch15-05-interior-mutability.html). -Users who are familiar with `RefCell` can use `Bound` just like `RefCell`. +Users who are familiar with `RefCell` can use `Py` and `Bound<'py, T>` just like `RefCell`. -For users who are not very familiar with `RefCell`, here is a reminder of Rust's rules of borrowing: +For users who are not very familiar with `RefCell`, here is a reminder of Rust's rules of borrowing: - At any given time, you can have either (but not both of) one mutable reference or any number of immutable references. -- References must always be valid. +- References can never outlast the data they refer to. -`Bound`, like `RefCell`, ensures these borrowing rules by tracking references at runtime. +`Py` and `Bound<'py, T>`, like `RefCell`, ensure these borrowing rules by tracking references at runtime. ```rust # use pyo3::prelude::*; @@ -233,10 +233,8 @@ Python::with_gil(|py| { }); ``` -A `Bound<'py, T>` is restricted to the GIL lifetime `'py`. -To make the object longer lived (for example, to store it in a struct on the -Rust side), you can use `Py`, which stores an object longer than the GIL -lifetime, and therefore needs a `Python<'_>` token to access. +A `Bound<'py, T>` is restricted to the GIL lifetime `'py`. To make the object longer lived (for example, to store it in a struct on the +Rust side), use `Py`. `Py` needs a `Python<'_>` token to allow access: ```rust # use pyo3::prelude::*; @@ -252,7 +250,7 @@ fn return_myclass() -> Py { let obj = return_myclass(); Python::with_gil(|py| { - let bound = obj.bind(py); // Py::bind returns &Bound<'_, MyClass> + let bound = obj.bind(py); // Py::bind returns &Bound<'py, MyClass> let obj_ref = bound.borrow(); // Get PyRef assert_eq!(obj_ref.num, 1); }); @@ -431,7 +429,7 @@ impl DictWithCounter { Self::default() } - fn set(slf: &Bound<'_, Self>, key: String, value: &PyAny) -> PyResult<()> { + fn set(slf: &Bound<'_, Self>, key: String, value: Bound<'_, PyAny>) -> PyResult<()> { slf.borrow_mut().counter.entry(key.clone()).or_insert(0); let dict = slf.downcast::()?; dict.set_item(key, value) @@ -1319,7 +1317,7 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { [`PyTypeInfo`]: {{#PYO3_DOCS_URL}}/pyo3/type_object/trait.PyTypeInfo.html [`Py`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Py.html -[`Bound`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html +[`Bound<'_, T>`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html [`PyClass`]: {{#PYO3_DOCS_URL}}/pyo3/pyclass/trait.PyClass.html [`PyRef`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html [`PyRefMut`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRefMut.html diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index b6a2d30e55a..eb33b17acf7 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -12,49 +12,49 @@ The table below contains the Python type and the corresponding function argument | Python | Rust | Rust (Python-native) | | ------------- |:-------------------------------:|:--------------------:| -| `object` | - | `&PyAny` | -| `str` | `String`, `Cow`, `&str`, `char`, `OsString`, `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | -| `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `&PyBytes` | -| `bool` | `bool` | `&PyBool` | -| `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `&PyLong` | -| `float` | `f32`, `f64` | `&PyFloat` | -| `complex` | `num_complex::Complex`[^2] | `&PyComplex` | -| `list[T]` | `Vec` | `&PyList` | -| `dict[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `&PyDict` | -| `tuple[T, U]` | `(T, U)`, `Vec` | `&PyTuple` | -| `set[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `&PySet` | -| `frozenset[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `&PyFrozenSet` | -| `bytearray` | `Vec`, `Cow<[u8]>` | `&PyByteArray` | -| `slice` | - | `&PySlice` | -| `type` | - | `&PyType` | -| `module` | - | `&PyModule` | +| `object` | - | `PyAny` | +| `str` | `String`, `Cow`, `&str`, `char`, `OsString`, `PathBuf`, `Path` | `PyString`, `PyUnicode` | +| `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `PyBytes` | +| `bool` | `bool` | `PyBool` | +| `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `PyLong` | +| `float` | `f32`, `f64` | `PyFloat` | +| `complex` | `num_complex::Complex`[^2] | `PyComplex` | +| `list[T]` | `Vec` | `PyList` | +| `dict[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `PyDict` | +| `tuple[T, U]` | `(T, U)`, `Vec` | `PyTuple` | +| `set[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `PySet` | +| `frozenset[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `PyFrozenSet` | +| `bytearray` | `Vec`, `Cow<[u8]>` | `PyByteArray` | +| `slice` | - | `PySlice` | +| `type` | - | `PyType` | +| `module` | - | `PyModule` | | `collections.abc.Buffer` | - | `PyBuffer` | -| `datetime.datetime` | `SystemTime`, `chrono::DateTime`[^5], `chrono::NaiveDateTime`[^5] | `&PyDateTime` | -| `datetime.date` | `chrono::NaiveDate`[^5] | `&PyDate` | -| `datetime.time` | `chrono::NaiveTime`[^5] | `&PyTime` | -| `datetime.tzinfo` | `chrono::FixedOffset`[^5], `chrono::Utc`[^5], `chrono_tz::TimeZone`[^6] | `&PyTzInfo` | -| `datetime.timedelta` | `Duration`, `chrono::Duration`[^5] | `&PyDelta` | +| `datetime.datetime` | `SystemTime`, `chrono::DateTime`[^5], `chrono::NaiveDateTime`[^5] | `PyDateTime` | +| `datetime.date` | `chrono::NaiveDate`[^5] | `PyDate` | +| `datetime.time` | `chrono::NaiveTime`[^5] | `PyTime` | +| `datetime.tzinfo` | `chrono::FixedOffset`[^5], `chrono::Utc`[^5], `chrono_tz::TimeZone`[^6] | `PyTzInfo` | +| `datetime.timedelta` | `Duration`, `chrono::Duration`[^5] | `PyDelta` | | `decimal.Decimal` | `rust_decimal::Decimal`[^7] | - | | `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::IpV4Addr` | - | | `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::IpV6Addr` | - | -| `os.PathLike ` | `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | -| `pathlib.Path` | `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | +| `os.PathLike ` | `PathBuf`, `Path` | `PyString`, `PyUnicode` | +| `pathlib.Path` | `PathBuf`, `Path` | `PyString`, `PyUnicode` | | `typing.Optional[T]` | `Option` | - | -| `typing.Sequence[T]` | `Vec` | `&PySequence` | +| `typing.Sequence[T]` | `Vec` | `PySequence` | | `typing.Mapping[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `&PyMapping` | -| `typing.Iterator[Any]` | - | `&PyIterator` | +| `typing.Iterator[Any]` | - | `PyIterator` | | `typing.Union[...]` | See [`#[derive(FromPyObject)]`](traits.md#deriving-frompyobject-for-enums) | - | -There are also a few special types related to the GIL and Rust-defined `#[pyclass]`es which may come in useful: +It is also worth remembering the following special types: -| What | Description | -| ------------- | ------------------------------------- | -| `Python` | A GIL token, used to pass to PyO3 constructors to prove ownership of the GIL | -| `Py` | A Python object isolated from the GIL lifetime. This can be sent to other threads. | -| `PyObject` | An alias for `Py` | -| `&PyCell` | A `#[pyclass]` value owned by Python. | -| `PyRef` | A `#[pyclass]` borrowed immutably. | -| `PyRefMut` | A `#[pyclass]` borrowed mutably. | +| What | Description | +| ---------------- | ------------------------------------- | +| `Python<'py>` | A GIL token, used to pass to PyO3 constructors to prove ownership of the GIL. | +| `Bound<'py, T>` | A Python object connected to the GIL lifetime. This provides access to most of PyO3's APIs. | +| `Py` | A Python object isolated from the GIL lifetime. This can be sent to other threads. | +| `PyObject` | An alias for `Py` | +| `PyRef` | A `#[pyclass]` borrowed immutably. | +| `PyRefMut` | A `#[pyclass]` borrowed mutably. | For more detail on accepting `#[pyclass]` values as function arguments, see [the section of this guide on Python Classes](../class.md). @@ -72,9 +72,9 @@ For most PyO3 usage the conversion cost is worth paying to get these benefits. A ### Returning Rust values to Python -When returning values from functions callable from Python, Python-native types (`&PyAny`, `&PyDict` etc.) can be used with zero cost. +When returning values from functions callable from Python, [PyO3's smart pointers](../types.md#pyo3s-smart-pointers) (`Py`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`) can be used with zero cost. -Because these types are references, in some situations the Rust compiler may ask for lifetime annotations. If this is the case, you should use `Py`, `Py` etc. instead - which are also zero-cost. For all of these Python-native types `T`, `Py` can be created from `T` with an `.into()` conversion. +Because `Bound<'py, T>` and `Borrowed<'a, 'py, T>` have lifetime parameters, the Rust compiler may ask for lifetime annotations to be added to your function. See the [section of the guide dedicated to this](../types.md#function-argument-lifetimes). If your function is fallible, it should return `PyResult` or `Result` where `E` implements `From for PyErr`. This will raise a `Python` exception if the `Err` variant is returned. @@ -95,7 +95,8 @@ Finally, the following Rust types are also able to convert to Python as return v | `BTreeMap` | `Dict[K, V]` | | `HashSet` | `Set[T]` | | `BTreeSet` | `Set[T]` | -| `&PyCell` | `T` | +| `Py` | `T` | +| `Bound` | `T` | | `PyRef` | `T` | | `PyRefMut` | `T` | diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index 5cdf2c590b9..3a00a160c65 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -484,7 +484,7 @@ If the input is neither a string nor an integer, the error message will be: - `pyo3(from_py_with = "...")` - apply a custom function to convert the field from Python the desired Rust type. - the argument must be the name of the function as a string. - - the function signature must be `fn(&PyAny) -> PyResult` where `T` is the Rust type of the argument. + - the function signature must be `fn(&Bound) -> PyResult` where `T` is the Rust type of the argument. ### `IntoPy` diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index 1e4ea4ab4b9..0319fa05063 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -120,7 +120,7 @@ Export an async function that makes use of `async-std`: use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] -fn rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { +fn rust_sleep(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>>> { pyo3_asyncio::async_std::future_into_py(py, async { async_std::task::sleep(std::time::Duration::from_secs(1)).await; Ok(Python::with_gil(|py| py.None())) @@ -143,7 +143,7 @@ If you want to use `tokio` instead, here's what your module should look like: use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] -fn rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { +fn rust_sleep(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>>> { pyo3_asyncio::tokio::future_into_py(py, async { tokio::time::sleep(std::time::Duration::from_secs(1)).await; Ok(Python::with_gil(|py| py.None())) @@ -233,7 +233,7 @@ a coroutine argument: ```rust #[pyfunction] -fn await_coro(coro: &PyAny) -> PyResult<()> { +fn await_coro(coro: &Bound<'_, PyAny>>) -> PyResult<()> { // convert the coroutine into a Rust future using the // async_std runtime let f = pyo3_asyncio::async_std::into_future(coro)?; @@ -261,7 +261,7 @@ If for you wanted to pass a callable function to the `#[pyfunction]` instead, (i ```rust #[pyfunction] -fn await_coro(callable: &PyAny) -> PyResult<()> { +fn await_coro(callable: &Bound<'_, PyAny>>) -> PyResult<()> { // get the coroutine by calling the callable let coro = callable.call0()?; @@ -317,7 +317,7 @@ async fn rust_sleep() { } #[pyfunction] -fn call_rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { +fn call_rust_sleep(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>>> { pyo3_asyncio::async_std::future_into_py(py, async move { rust_sleep().await; Ok(Python::with_gil(|py| py.None())) @@ -467,7 +467,7 @@ tokio = "1.4" use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] -fn rust_sleep(py: Python<'_>) -> PyResult<&PyAny> { +fn rust_sleep(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>>> { pyo3_asyncio::tokio::future_into_py(py, async { tokio::time::sleep(std::time::Duration::from_secs(1)).await; Ok(Python::with_gil(|py| py.None())) diff --git a/guide/src/exception.md b/guide/src/exception.md index ee36f544d74..3e2f5034897 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -111,7 +111,7 @@ mod io { pyo3::import_exception!(io, UnsupportedOperation); } -fn tell(file: &PyAny) -> PyResult { +fn tell(file: &Bound<'_, PyAny>) -> PyResult { match file.call_method0("tell") { Err(_) => Err(io::UnsupportedOperation::new_err("not supported: tell")), Ok(x) => x.extract::(), diff --git a/guide/src/function-calls.md b/guide/src/function-calls.md new file mode 100644 index 00000000000..e5c9c1ed9b3 --- /dev/null +++ b/guide/src/function-calls.md @@ -0,0 +1 @@ +# Calling Python functions diff --git a/guide/src/function.md b/guide/src/function.md index cfb1e5ef81e..86ac4c89b46 100644 --- a/guide/src/function.md +++ b/guide/src/function.md @@ -87,7 +87,9 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python #[pyfunction] #[pyo3(pass_module)] - fn pyfunction_with_module<'py>(module: &Bound<'py, PyModule>) -> PyResult> { + fn pyfunction_with_module<'py>( + module: &Bound<'py, PyModule>, + ) -> PyResult> { module.name() } diff --git a/guide/src/index.md b/guide/src/index.md index 80534caea5f..87914975afe 100644 --- a/guide/src/index.md +++ b/guide/src/index.md @@ -2,6 +2,22 @@ Welcome to the PyO3 user guide! This book is a companion to [PyO3's API docs](https://docs.rs/pyo3). It contains examples and documentation to explain all of PyO3's use cases in detail. +The rough order of material in this user guide is as follows: + 1. Getting started + 2. Wrapping Rust code for use from Python + 3. How to use Python code from Rust + 4. Remaining topics which go into advanced concepts in detail + Please choose from the chapters on the left to jump to individual topics, or continue below to start with PyO3's README. +
+⚠️ Warning: API update in progress 🛠️ + +PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the new smart pointer `Bound`. + +While most of this guide has been updated to the new API, it is possible some stray references to the older "GIL Refs" API such as `&PyAny` remain. +
+ +
+ {{#include ../../README.md}} diff --git a/guide/src/memory.md b/guide/src/memory.md index b14ce4496ad..fe98184e3e1 100644 --- a/guide/src/memory.md +++ b/guide/src/memory.md @@ -1,5 +1,15 @@ # Memory management +
+⚠️ Warning: API update in progress 🛠️ + +PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the new smart pointer `Bound`. + +This section on memory management is heavily weighted towards the now-deprecated "GIL Refs" API, which suffered from the drawbacks detailed here as well as CPU overheads. + +See [the smart pointer types](./types.md#pyo3s-smart-pointers) for description on the new, simplified, memory model of the Bound API, which is built as a thin wrapper on Python reference counting. +
+ Rust and Python have very different notions of memory management. Rust has a strict memory model with concepts of ownership, borrowing, and lifetimes, where memory is freed at predictable points in program execution. Python has @@ -10,12 +20,12 @@ Memory in Python is freed eventually by the garbage collector, but not usually in a predictable way. PyO3 bridges the Rust and Python memory models with two different strategies for -accessing memory allocated on Python's heap from inside Rust. These are -GIL-bound, or "owned" references, and GIL-independent `Py` smart pointers. +accessing memory allocated on Python's heap from inside Rust. These are +GIL Refs such as `&'py PyAny`, and GIL-independent `Py` smart pointers. ## GIL-bound memory -PyO3's GIL-bound, "owned references" (`&PyAny` etc.) make PyO3 more ergonomic to +PyO3's GIL Refs such as `&'py PyAny` make PyO3 more ergonomic to use by ensuring that their lifetime can never be longer than the duration the Python GIL is held. This means that most of PyO3's API can assume the GIL is held. (If PyO3 could not assume this, every PyO3 API would need to take a @@ -27,7 +37,10 @@ very simple and easy-to-understand programs like this: # use pyo3::types::PyString; # fn main() -> PyResult<()> { Python::with_gil(|py| -> PyResult<()> { - let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; + #[allow(deprecated)] // py.eval() is part of the GIL Refs API + let hello = py + .eval("\"Hello World!\"", None, None)? + .downcast::()?; println!("Python says: {}", hello); Ok(()) })?; @@ -48,7 +61,10 @@ of the time we don't have to think about this, but consider the following: # fn main() -> PyResult<()> { Python::with_gil(|py| -> PyResult<()> { for _ in 0..10 { - let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; + #[allow(deprecated)] // py.eval() is part of the GIL Refs API + let hello = py + .eval("\"Hello World!\"", None, None)? + .downcast::()?; println!("Python says: {}", hello); } // There are 10 copies of `hello` on Python's heap here. @@ -76,7 +92,10 @@ is to acquire and release the GIL with each iteration of the loop. # fn main() -> PyResult<()> { for _ in 0..10 { Python::with_gil(|py| -> PyResult<()> { - let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; + #[allow(deprecated)] // py.eval() is part of the GIL Refs API + let hello = py + .eval("\"Hello World!\"", None, None)? + .downcast::()?; println!("Python says: {}", hello); Ok(()) })?; // only one copy of `hello` at a time @@ -97,7 +116,10 @@ Python::with_gil(|py| -> PyResult<()> { for _ in 0..10 { let pool = unsafe { py.new_pool() }; let py = pool.python(); - let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; + #[allow(deprecated)] // py.eval() is part of the GIL Refs API + let hello = py + .eval("\"Hello World!\"", None, None)? + .downcast::()?; println!("Python says: {}", hello); } Ok(()) @@ -144,8 +166,12 @@ reference count reaches zero? It depends whether or not we are holding the GIL. # use pyo3::types::PyString; # fn main() -> PyResult<()> { Python::with_gil(|py| -> PyResult<()> { - let hello: Py = py.eval_bound("\"Hello World!\"", None, None)?.extract()?; - println!("Python says: {}", hello.bind(py)); + #[allow(deprecated)] // py.eval() is part of the GIL Refs API + let hello: Py = py.eval("\"Hello World!\"", None, None)?.extract()?; + #[allow(deprecated)] // as_ref is part of the GIL Refs API + { + println!("Python says: {}", hello.as_ref(py)); + } Ok(()) })?; # Ok(()) @@ -166,7 +192,8 @@ we are *not* holding the GIL? # use pyo3::types::PyString; # fn main() -> PyResult<()> { let hello: Py = Python::with_gil(|py| { - py.eval_bound("\"Hello World!\"", None, None)?.extract() + #[allow(deprecated)] // py.eval() is part of the GIL Refs API + py.eval("\"Hello World!\"", None, None)?.extract() })?; // Do some stuff... // Now sometime later in the program we want to access `hello`. @@ -198,12 +225,16 @@ We can avoid the delay in releasing memory if we are careful to drop the # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { +#[allow(deprecated)] // py.eval() is part of the GIL Refs API let hello: Py = - Python::with_gil(|py| py.eval_bound("\"Hello World!\"", None, None)?.extract())?; + Python::with_gil(|py| py.eval("\"Hello World!\"", None, None)?.extract())?; // Do some stuff... // Now sometime later in the program: Python::with_gil(|py| { - println!("Python says: {}", hello.bind(py)); + #[allow(deprecated)] // as_ref is part of the GIL Refs API + { + println!("Python says: {}", hello.as_ref(py)); + } drop(hello); // Memory released here. }); # Ok(()) @@ -220,12 +251,16 @@ until the GIL is dropped. # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { +#[allow(deprecated)] // py.eval() is part of the GIL Refs API let hello: Py = - Python::with_gil(|py| py.eval_bound("\"Hello World!\"", None, None)?.extract())?; + Python::with_gil(|py| py.eval("\"Hello World!\"", None, None)?.extract())?; // Do some stuff... // Now sometime later in the program: Python::with_gil(|py| { - println!("Python says: {}", hello.into_bound(py)); + #[allow(deprecated)] // into_ref is part of the GIL Refs API + { + println!("Python says: {}", hello.into_ref(py)); + } // Memory not released yet. // Do more stuff... // Memory released here at end of `with_gil()` closure. diff --git a/guide/src/migration.md b/guide/src/migration.md index 3ed3e015b3b..5af6eb2336c 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -190,7 +190,9 @@ struct PyClassAsyncIter { impl PyClassAsyncIter { fn __anext__(&mut self) -> PyClassAwaitable { self.number += 1; - PyClassAwaitable { number: self.number } + PyClassAwaitable { + number: self.number, + } } fn __aiter__(slf: Py) -> Py { @@ -312,7 +314,10 @@ Python::with_gil(|py| { // `b` is not in the dictionary assert!(dict.get_item("b").is_none()); // `dict` is not hashable, so this fails with a `TypeError` - assert!(dict.get_item_with_error(dict).unwrap_err().is_instance_of::(py)); + assert!(dict + .get_item_with_error(dict) + .unwrap_err() + .is_instance_of::(py)); }); # } ``` @@ -333,7 +338,10 @@ Python::with_gil(|py| -> PyResult<()> { // `b` is not in the dictionary assert!(dict.get_item("b")?.is_none()); // `dict` is not hashable, so this fails with a `TypeError` - assert!(dict.get_item(dict).unwrap_err().is_instance_of::(py)); + assert!(dict + .get_item(dict) + .unwrap_err() + .is_instance_of::(py)); Ok(()) }); @@ -495,7 +503,7 @@ drop(first); drop(second); ``` -The replacement is [`Python::with_gil`]() which is more cumbersome but enforces the proper nesting by design, e.g. +The replacement is [`Python::with_gil`](https://docs.rs/pyo3/0.18.3/pyo3/marker/struct.Python.html#method.with_gil) which is more cumbersome but enforces the proper nesting by design, e.g. ```rust # #![allow(dead_code)] diff --git a/guide/src/performance.md b/guide/src/performance.md index fe362bed953..c47a91deee5 100644 --- a/guide/src/performance.md +++ b/guide/src/performance.md @@ -4,26 +4,26 @@ To achieve the best possible performance, it is useful to be aware of several tr ## `extract` versus `downcast` -Pythonic API implemented using PyO3 are often polymorphic, i.e. they will accept `&PyAny` and try to turn this into multiple more concrete types to which the requested operation is applied. This often leads to chains of calls to `extract`, e.g. +Pythonic API implemented using PyO3 are often polymorphic, i.e. they will accept `&Bound<'_, PyAny>` and try to turn this into multiple more concrete types to which the requested operation is applied. This often leads to chains of calls to `extract`, e.g. ```rust # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::{exceptions::PyTypeError, types::PyList}; -fn frobnicate_list(list: &PyList) -> PyResult<&PyAny> { +fn frobnicate_list<'py>(list: &Bound<'_, PyList>) -> PyResult> { todo!() } -fn frobnicate_vec(vec: Vec<&PyAny>) -> PyResult<&PyAny> { +fn frobnicate_vec<'py>(vec: Vec>) -> PyResult> { todo!() } #[pyfunction] -fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { - if let Ok(list) = value.extract::<&PyList>() { - frobnicate_list(list) - } else if let Ok(vec) = value.extract::>() { +fn frobnicate<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(list) = value.extract::>() { + frobnicate_list(&list) + } else if let Ok(vec) = value.extract::>>() { frobnicate_vec(vec) } else { Err(PyTypeError::new_err("Cannot frobnicate that type.")) @@ -37,15 +37,15 @@ This suboptimal as the `FromPyObject` trait requires `extract` to have a `Res # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::{exceptions::PyTypeError, types::PyList}; -# fn frobnicate_list(list: &PyList) -> PyResult<&PyAny> { todo!() } -# fn frobnicate_vec(vec: Vec<&PyAny>) -> PyResult<&PyAny> { todo!() } +# fn frobnicate_list<'py>(list: &Bound<'_, PyList>) -> PyResult> { todo!() } +# fn frobnicate_vec<'py>(vec: Vec>) -> PyResult> { todo!() } # #[pyfunction] -fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { +fn frobnicate<'py>(value: &Bound<'py, PyAny>) -> PyResult> { // Use `downcast` instead of `extract` as turning `PyDowncastError` into `PyErr` is quite costly. if let Ok(list) = value.downcast::() { frobnicate_list(list) - } else if let Ok(vec) = value.extract::>() { + } else if let Ok(vec) = value.extract::>>() { frobnicate_vec(vec) } else { Err(PyTypeError::new_err("Cannot frobnicate that type.")) @@ -53,9 +53,9 @@ fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { } ``` -## Access to GIL-bound reference implies access to GIL token +## Access to Bound implies access to GIL token -Calling `Python::with_gil` is effectively a no-op when the GIL is already held, but checking that this is the case still has a cost. If an existing GIL token can not be accessed, for example when implementing a pre-existing trait, but a GIL-bound reference is available, this cost can be avoided by exploiting that access to GIL-bound reference gives zero-cost access to a GIL token via `PyAny::py`. +Calling `Python::with_gil` is effectively a no-op when the GIL is already held, but checking that this is the case still has a cost. If an existing GIL token can not be accessed, for example when implementing a pre-existing trait, but a GIL-bound reference is available, this cost can be avoided by exploiting that access to GIL-bound reference gives zero-cost access to a GIL token via `Bound::py`. For example, instead of writing @@ -66,34 +66,32 @@ For example, instead of writing struct Foo(Py); -struct FooRef<'a>(&'a PyList); +struct FooBound<'py>(Bound<'py, PyList>); -impl PartialEq for FooRef<'_> { +impl PartialEq for FooBound<'_> { fn eq(&self, other: &Foo) -> bool { Python::with_gil(|py| { - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. - let len = other.0.as_ref(py).len(); + let len = other.0.bind(py).len(); self.0.len() == len }) } } ``` -use more efficient +use the more efficient ```rust # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::types::PyList; # struct Foo(Py); -# struct FooRef<'a>(&'a PyList); +# struct FooBound<'py>(Bound<'py, PyList>); # -impl PartialEq for FooRef<'_> { +impl PartialEq for FooBound<'_> { fn eq(&self, other: &Foo) -> bool { - // Access to `&'a PyAny` implies access to `Python<'a>`. + // Access to `&Bound<'py, PyAny>` implies access to `Python<'py>`. let py = self.0.py(); - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. - let len = other.0.as_ref(py).len(); + let len = other.0.bind(py).len(); self.0.len() == len } } diff --git a/guide/src/python-from-rust.md b/guide/src/python-from-rust.md index 4a81d9a668f..ee618f3fa47 100644 --- a/guide/src/python-from-rust.md +++ b/guide/src/python-from-rust.md @@ -1,511 +1,46 @@ # Calling Python in Rust code -This chapter of the guide documents some ways to interact with Python code from Rust: - - How to call Python functions - - How to execute existing Python code - -## Calling Python functions - -Any Python-native object reference (such as `&PyAny`, `&PyList`, or `&PyCell`) can be used to call Python functions. - -PyO3 offers two APIs to make function calls: - -* [`call`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call) - call any callable Python object. -* [`call_method`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method) - call a method on the Python object. - -Both of these APIs take `args` and `kwargs` arguments (for positional and keyword arguments respectively). There are variants for less complex calls: - -* [`call1`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call1) and [`call_method1`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method1) to call only with positional `args`. -* [`call0`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call0) and [`call_method0`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method0) to call with no arguments. - -For convenience the [`Py`](types.md#pyt-and-pyobject) smart pointer also exposes these same six API methods, but needs a `Python` token as an additional first argument to prove the GIL is held. - -The example below calls a Python function behind a `PyObject` (aka `Py`) reference: - -```rust -use pyo3::prelude::*; -use pyo3::types::PyTuple; - -fn main() -> PyResult<()> { - let arg1 = "arg1"; - let arg2 = "arg2"; - let arg3 = "arg3"; - - Python::with_gil(|py| { - let fun: Py = PyModule::from_code_bound( - py, - "def example(*args, **kwargs): - if args != (): - print('called with args', args) - if kwargs != {}: - print('called with kwargs', kwargs) - if args == () and kwargs == {}: - print('called with no arguments')", - "", - "", - )? - .getattr("example")? - .into(); - - // call object without any arguments - fun.call0(py)?; - - // call object with PyTuple - let args = PyTuple::new_bound(py, &[arg1, arg2, arg3]); - fun.call1(py, args)?; - - // pass arguments as rust tuple - let args = (arg1, arg2, arg3); - fun.call1(py, args)?; - Ok(()) - }) -} -``` - -### Creating keyword arguments - -For the `call` and `call_method` APIs, `kwargs` can be `None` or `Some(&PyDict)`. You can use the [`IntoPyDict`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.IntoPyDict.html) trait to convert other dict-like containers, e.g. `HashMap` or `BTreeMap`, as well as tuples with up to 10 elements and `Vec`s where each element is a two-element tuple. - -```rust -use pyo3::prelude::*; -use pyo3::types::IntoPyDict; -use std::collections::HashMap; - -fn main() -> PyResult<()> { - let key1 = "key1"; - let val1 = 1; - let key2 = "key2"; - let val2 = 2; - - Python::with_gil(|py| { - let fun: Py = PyModule::from_code_bound( - py, - "def example(*args, **kwargs): - if args != (): - print('called with args', args) - if kwargs != {}: - print('called with kwargs', kwargs) - if args == () and kwargs == {}: - print('called with no arguments')", - "", - "", - )? - .getattr("example")? - .into(); - - // call object with PyDict - let kwargs = [(key1, val1)].into_py_dict_bound(py); - fun.call_bound(py, (), Some(&kwargs))?; - - // pass arguments as Vec - let kwargs = vec![(key1, val1), (key2, val2)]; - fun.call_bound(py, (), Some(&kwargs.into_py_dict_bound(py)))?; - - // pass arguments as HashMap - let mut kwargs = HashMap::<&str, i32>::new(); - kwargs.insert(key1, 1); - fun.call_bound(py, (), Some(&kwargs.into_py_dict_bound(py)))?; - - Ok(()) - }) -} -``` - -
- -During PyO3's [migration from "GIL Refs" to the `Bound` smart pointer](./migration.md#migrating-from-the-gil-refs-api-to-boundt), [`Py::call`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html#method.call) is temporarily named `call_bound` (and `call_method` is temporarily `call_method_bound`). - -(This temporary naming is only the case for the `Py` smart pointer. The methods on the `&PyAny` GIL Ref such as `call` have not been given replacements, and the methods on the `Bound` smart pointer such as [`Bound::call`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call) already use follow the newest API conventions.) - -
- -## Executing existing Python code - -If you already have some existing Python code that you need to execute from Rust, the following FAQs can help you select the right PyO3 functionality for your situation: - -### Want to access Python APIs? Then use `PyModule::import`. - -[`Pymodule::import`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.import) can -be used to get handle to a Python module from Rust. You can use this to import and use any Python -module available in your environment. - -```rust -use pyo3::prelude::*; - -fn main() -> PyResult<()> { - Python::with_gil(|py| { - let builtins = PyModule::import_bound(py, "builtins")?; - let total: i32 = builtins - .getattr("sum")? - .call1((vec![1, 2, 3],))? - .extract()?; - assert_eq!(total, 6); - Ok(()) - }) -} -``` - -### Want to run just an expression? Then use `eval`. - -[`Python::eval`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval) is -a method to execute a [Python expression](https://docs.python.org/3.7/reference/expressions.html) -and return the evaluated value as a `&PyAny` object. - -```rust -use pyo3::prelude::*; - -# fn main() -> Result<(), ()> { -Python::with_gil(|py| { - let result = py - .eval_bound("[i * 10 for i in range(5)]", None, None) - .map_err(|e| { - e.print_and_set_sys_last_vars(py); - })?; - let res: Vec = result.extract().unwrap(); - assert_eq!(res, vec![0, 10, 20, 30, 40]); - Ok(()) -}) -# } -``` - -### Want to run statements? Then use `run`. - -[`Python::run`] is a method to execute one or more -[Python statements](https://docs.python.org/3.7/reference/simple_stmts.html). -This method returns nothing (like any Python statement), but you can get -access to manipulated objects via the `locals` dict. - -You can also use the [`py_run!`] macro, which is a shorthand for [`Python::run`]. -Since [`py_run!`] panics on exceptions, we recommend you use this macro only for -quickly testing your Python extensions. - -```rust -use pyo3::prelude::*; -use pyo3::py_run; - -# fn main() { -#[pyclass] -struct UserData { - id: u32, - name: String, -} - -#[pymethods] -impl UserData { - fn as_tuple(&self) -> (u32, String) { - (self.id, self.name.clone()) - } - - fn __repr__(&self) -> PyResult { - Ok(format!("User {}(id: {})", self.name, self.id)) - } -} +This chapter of the guide documents some ways to interact with Python code from Rust. -Python::with_gil(|py| { - let userdata = UserData { - id: 34, - name: "Yu".to_string(), - }; - let userdata = Py::new(py, userdata).unwrap(); - let userdata_as_tuple = (34, "Yu"); - py_run!(py, userdata userdata_as_tuple, r#" -assert repr(userdata) == "User Yu(id: 34)" -assert userdata.as_tuple() == userdata_as_tuple - "#); -}) -# } -``` +Below is an introduction to the `'py` lifetime and some general remarks about how PyO3's API reasons about Python code. -## You have a Python file or code snippet? Then use `PyModule::from_code`. - -[`PyModule::from_code`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.from_code) -can be used to generate a Python module which can then be used just as if it was imported with -`PyModule::import`. - -**Warning**: This will compile and execute code. **Never** pass untrusted code -to this function! - -```rust -use pyo3::{ - prelude::*, - types::IntoPyDict, -}; - -# fn main() -> PyResult<()> { -Python::with_gil(|py| { - let activators = PyModule::from_code_bound( - py, - r#" -def relu(x): - """see https://en.wikipedia.org/wiki/Rectifier_(neural_networks)""" - return max(0.0, x) - -def leaky_relu(x, slope=0.01): - return x if x >= 0 else x * slope - "#, - "activators.py", - "activators", - )?; - - let relu_result: f64 = activators.getattr("relu")?.call1((-1.0,))?.extract()?; - assert_eq!(relu_result, 0.0); - - let kwargs = [("slope", 0.2)].into_py_dict_bound(py); - let lrelu_result: f64 = activators - .getattr("leaky_relu")? - .call((-1.0,), Some(&kwargs))? - .extract()?; - assert_eq!(lrelu_result, -0.2); -# Ok(()) -}) -# } -``` - -### Want to embed Python in Rust with additional modules? - -Python maintains the `sys.modules` dict as a cache of all imported modules. -An import in Python will first attempt to lookup the module from this dict, -and if not present will use various strategies to attempt to locate and load -the module. - -The [`append_to_inittab`]({{#PYO3_DOCS_URL}}/pyo3/macro.append_to_inittab.html) -macro can be used to add additional `#[pymodule]` modules to an embedded -Python interpreter. The macro **must** be invoked _before_ initializing Python. - -As an example, the below adds the module `foo` to the embedded interpreter: - -```rust -use pyo3::prelude::*; - -#[pyfunction] -fn add_one(x: i64) -> i64 { - x + 1 -} - -#[pymodule] -fn foo(foo_module: &Bound<'_, PyModule>) -> PyResult<()> { - foo_module.add_function(wrap_pyfunction!(add_one, foo_module)?)?; - Ok(()) -} - -fn main() -> PyResult<()> { - pyo3::append_to_inittab!(foo); - Python::with_gil(|py| Python::run_bound(py, "import foo; foo.add_one(6)", None, None)) -} -``` - -If `append_to_inittab` cannot be used due to constraints in the program, -an alternative is to create a module using [`PyModule::new`] -and insert it manually into `sys.modules`: - -```rust -use pyo3::prelude::*; -use pyo3::types::PyDict; - -#[pyfunction] -pub fn add_one(x: i64) -> i64 { - x + 1 -} - -fn main() -> PyResult<()> { - Python::with_gil(|py| { - // Create new module - let foo_module = PyModule::new_bound(py, "foo")?; - foo_module.add_function(wrap_pyfunction!(add_one, &foo_module)?)?; - - // Import and get sys.modules - let sys = PyModule::import_bound(py, "sys")?; - let py_modules: Bound<'_, PyDict> = sys.getattr("modules")?.downcast_into()?; - - // Insert foo into sys.modules - py_modules.set_item("foo", foo_module)?; - - // Now we can import + run our python code - Python::run_bound(py, "import foo; foo.add_one(6)", None, None) - }) -} -``` - -### Include multiple Python files - -You can include a file at compile time by using -[`std::include_str`](https://doc.rust-lang.org/std/macro.include_str.html) macro. - -Or you can load a file at runtime by using -[`std::fs::read_to_string`](https://doc.rust-lang.org/std/fs/fn.read_to_string.html) function. - -Many Python files can be included and loaded as modules. If one file depends on -another you must preserve correct order while declaring `PyModule`. - -Example directory structure: -```text -. -├── Cargo.lock -├── Cargo.toml -├── python_app -│ ├── app.py -│ └── utils -│ └── foo.py -└── src - └── main.rs -``` - -`python_app/app.py`: -```python -from utils.foo import bar - - -def run(): - return bar() -``` - -`python_app/utils/foo.py`: -```python -def bar(): - return "baz" -``` - -The example below shows: -* how to include content of `app.py` and `utils/foo.py` into your rust binary -* how to call function `run()` (declared in `app.py`) that needs function - imported from `utils/foo.py` - -`src/main.rs`: -```rust,ignore -use pyo3::prelude::*; - -fn main() -> PyResult<()> { - let py_foo = include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/python_app/utils/foo.py" - )); - let py_app = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/python_app/app.py")); - let from_python = Python::with_gil(|py| -> PyResult> { - PyModule::from_code_bound(py, py_foo, "utils.foo", "utils.foo")?; - let app: Py = PyModule::from_code_bound(py, py_app, "", "")? - .getattr("run")? - .into(); - app.call0(py) - }); - - println!("py: {}", from_python?); - Ok(()) -} -``` - -The example below shows: -* how to load content of `app.py` at runtime so that it sees its dependencies - automatically -* how to call function `run()` (declared in `app.py`) that needs function - imported from `utils/foo.py` - -It is recommended to use absolute paths because then your binary can be run -from anywhere as long as your `app.py` is in the expected directory (in this example -that directory is `/usr/share/python_app`). - -`src/main.rs`: -```rust,no_run -use pyo3::prelude::*; -use pyo3::types::PyList; -use std::fs; -use std::path::Path; - -fn main() -> PyResult<()> { - let path = Path::new("/usr/share/python_app"); - let py_app = fs::read_to_string(path.join("app.py"))?; - let from_python = Python::with_gil(|py| -> PyResult> { - let syspath = py.import_bound("sys")?.getattr("path")?.downcast_into::()?; - syspath.insert(0, &path)?; - let app: Py = PyModule::from_code_bound(py, &py_app, "", "")? - .getattr("run")? - .into(); - app.call0(py) - }); - - println!("py: {}", from_python?); - Ok(()) -} -``` - - -[`Python::run`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.run -[`py_run!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.py_run.html - -## Need to use a context manager from Rust? - -Use context managers by directly invoking `__enter__` and `__exit__`. +The subchapters also cover the following topics: + - Python object types available in PyO3's API + - How to work with Python exceptions + - How to call Python functions + - How to execute existing Python code -```rust -use pyo3::prelude::*; +## The `'py` lifetime -fn main() { - Python::with_gil(|py| { - let custom_manager = PyModule::from_code_bound( - py, - r#" -class House(object): - def __init__(self, address): - self.address = address - def __enter__(self): - print(f"Welcome to {self.address}!") - def __exit__(self, type, value, traceback): - if type: - print(f"Sorry you had {type} trouble at {self.address}") - else: - print(f"Thank you for visiting {self.address}, come again soon!") +To safely interact with the Python interpreter a Rust thread must have a corresponding Python thread state and hold the [Global Interpreter Lock (GIL)](#the-global-interpreter-lock). PyO3 has a `Python<'py>` token that is used to prove that these conditions +are met. Its lifetime `'py` is a central part of PyO3's API. - "#, - "house.py", - "house", - ) - .unwrap(); +The `Python<'py>` token serves three purposes: - let house_class = custom_manager.getattr("House").unwrap(); - let house = house_class.call1(("123 Main Street",)).unwrap(); +* It provides global APIs for the Python interpreter, such as [`py.eval_bound()`][eval] and [`py.import_bound()`][import]. +* It can be passed to functions that require a proof of holding the GIL, such as [`Py::clone_ref`][clone_ref]. +* Its lifetime `'py` is used to bind many of PyO3's types to the Python interpreter, such as [`Bound<'py, T>`][Bound]. - house.call_method0("__enter__").unwrap(); +PyO3's types that are bound to the `'py` lifetime, for example `Bound<'py, T>`, all contain a `Python<'py>` token. This means they have full access to the Python interpreter and offer a complete API for interacting with Python objects. - let result = py.eval_bound("undefined_variable + 1", None, None); +Consult [PyO3's API documentation][obtaining-py] to learn how to acquire one of these tokens. - // If the eval threw an exception we'll pass it through to the context manager. - // Otherwise, __exit__ is called with empty arguments (Python "None"). - match result { - Ok(_) => { - let none = py.None(); - house - .call_method1("__exit__", (&none, &none, &none)) - .unwrap(); - } - Err(e) => { - house - .call_method1("__exit__", (e.get_type_bound(py), e.value_bound(py), e.traceback_bound(py))) - .unwrap(); - } - } - }) -} -``` +### The Global Interpreter Lock -## Handling system signals/interrupts (Ctrl-C) +Concurrent programming in Python is aided by the Global Interpreter Lock (GIL), which ensures that only one Python thread can use the Python interpreter and its API at the same time. This allows it to be used to synchronize code. See the [`pyo3::sync`] module for synchronization tools PyO3 offers that are based on the GIL's guarantees. -The best way to handle system signals when running Rust code is to periodically call `Python::check_signals` to handle any signals captured by Python's signal handler. See also [the FAQ entry](./faq.md#ctrl-c-doesnt-do-anything-while-my-rust-code-is-executing). +Non-Python operations (system calls and native Rust code) can unlock the GIL. See [the section on parallelism](parallelism.md) for how to do that using PyO3's API. -Alternatively, set Python's `signal` module to take the default action for a signal: +## Python's memory model -```rust -use pyo3::prelude::*; +Python's memory model differs from Rust's memory model in two key ways: +- There is no concept of ownership; all Python objects are shared and usually implemented via reference counting +- There is no concept of exclusive (`&mut`) references; any reference can mutate a Python object -# fn main() -> PyResult<()> { -Python::with_gil(|py| -> PyResult<()> { - let signal = py.import_bound("signal")?; - // Set SIGINT to have the default action - signal - .getattr("signal")? - .call1((signal.getattr("SIGINT")?, signal.getattr("SIG_DFL")?))?; - Ok(()) -}) -# } -``` +PyO3's API reflects this by providing [smart pointer][smart-pointers] types, `Py`, `Bound<'py, T>`, and (the very rarely used) `Borrowed<'a, 'py, T>`. These smart pointers all use Python reference counting. See the [subchapter on types](./types.md) for more detail on these types. +Because of the lack of exclusive `&mut` references, PyO3's APIs for Python objects, for example [`PyListMethods::append`], use shared references. This is safe because Python objects have internal mechanisms to prevent data races (as of time of writing, the Python GIL). -[`PyModule::new`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.new +[smart-pointers]: https://doc.rust-lang.org/book/ch15-00-smart-pointers.html +[obtaining-py]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#obtaining-a-python-token +[`pyo3::sync`]: {{#PYO3_DOCS_URL}}/pyo3/sync/index.html diff --git a/guide/src/python-from-rust/calling-existing-code.md b/guide/src/python-from-rust/calling-existing-code.md new file mode 100644 index 00000000000..53051d4ce51 --- /dev/null +++ b/guide/src/python-from-rust/calling-existing-code.md @@ -0,0 +1,397 @@ +# Executing existing Python code + +If you already have some existing Python code that you need to execute from Rust, the following FAQs can help you select the right PyO3 functionality for your situation: + +## Want to access Python APIs? Then use `PyModule::import`. + +[`Pymodule::import`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.import) can +be used to get handle to a Python module from Rust. You can use this to import and use any Python +module available in your environment. + +```rust +use pyo3::prelude::*; + +fn main() -> PyResult<()> { + Python::with_gil(|py| { + let builtins = PyModule::import_bound(py, "builtins")?; + let total: i32 = builtins + .getattr("sum")? + .call1((vec![1, 2, 3],))? + .extract()?; + assert_eq!(total, 6); + Ok(()) + }) +} +``` + +## Want to run just an expression? Then use `eval`. + +[`Python::eval`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval) is +a method to execute a [Python expression](https://docs.python.org/3.7/reference/expressions.html) +and return the evaluated value as a `Bound<'py, PyAny>` object. + +```rust +use pyo3::prelude::*; + +# fn main() -> Result<(), ()> { +Python::with_gil(|py| { + let result = py + .eval_bound("[i * 10 for i in range(5)]", None, None) + .map_err(|e| { + e.print_and_set_sys_last_vars(py); + })?; + let res: Vec = result.extract().unwrap(); + assert_eq!(res, vec![0, 10, 20, 30, 40]); + Ok(()) +}) +# } +``` + +## Want to run statements? Then use `run`. + +[`Python::run`] is a method to execute one or more +[Python statements](https://docs.python.org/3.7/reference/simple_stmts.html). +This method returns nothing (like any Python statement), but you can get +access to manipulated objects via the `locals` dict. + +You can also use the [`py_run!`] macro, which is a shorthand for [`Python::run`]. +Since [`py_run!`] panics on exceptions, we recommend you use this macro only for +quickly testing your Python extensions. + +```rust +use pyo3::prelude::*; +use pyo3::py_run; + +# fn main() { +#[pyclass] +struct UserData { + id: u32, + name: String, +} + +#[pymethods] +impl UserData { + fn as_tuple(&self) -> (u32, String) { + (self.id, self.name.clone()) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("User {}(id: {})", self.name, self.id)) + } +} + +Python::with_gil(|py| { + let userdata = UserData { + id: 34, + name: "Yu".to_string(), + }; + let userdata = Py::new(py, userdata).unwrap(); + let userdata_as_tuple = (34, "Yu"); + py_run!(py, userdata userdata_as_tuple, r#" +assert repr(userdata) == "User Yu(id: 34)" +assert userdata.as_tuple() == userdata_as_tuple + "#); +}) +# } +``` + +## You have a Python file or code snippet? Then use `PyModule::from_code`. + +[`PyModule::from_code`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.from_code) +can be used to generate a Python module which can then be used just as if it was imported with +`PyModule::import`. + +**Warning**: This will compile and execute code. **Never** pass untrusted code +to this function! + +```rust +use pyo3::{prelude::*, types::IntoPyDict}; + +# fn main() -> PyResult<()> { +Python::with_gil(|py| { + let activators = PyModule::from_code_bound( + py, + r#" +def relu(x): + """see https://en.wikipedia.org/wiki/Rectifier_(neural_networks)""" + return max(0.0, x) + +def leaky_relu(x, slope=0.01): + return x if x >= 0 else x * slope + "#, + "activators.py", + "activators", + )?; + + let relu_result: f64 = activators.getattr("relu")?.call1((-1.0,))?.extract()?; + assert_eq!(relu_result, 0.0); + + let kwargs = [("slope", 0.2)].into_py_dict_bound(py); + let lrelu_result: f64 = activators + .getattr("leaky_relu")? + .call((-1.0,), Some(&kwargs))? + .extract()?; + assert_eq!(lrelu_result, -0.2); +# Ok(()) +}) +# } +``` + +## Want to embed Python in Rust with additional modules? + +Python maintains the `sys.modules` dict as a cache of all imported modules. +An import in Python will first attempt to lookup the module from this dict, +and if not present will use various strategies to attempt to locate and load +the module. + +The [`append_to_inittab`]({{#PYO3_DOCS_URL}}/pyo3/macro.append_to_inittab.html) +macro can be used to add additional `#[pymodule]` modules to an embedded +Python interpreter. The macro **must** be invoked _before_ initializing Python. + +As an example, the below adds the module `foo` to the embedded interpreter: + +```rust +use pyo3::prelude::*; + +#[pyfunction] +fn add_one(x: i64) -> i64 { + x + 1 +} + +#[pymodule] +fn foo(foo_module: &Bound<'_, PyModule>) -> PyResult<()> { + foo_module.add_function(wrap_pyfunction!(add_one, foo_module)?)?; + Ok(()) +} + +fn main() -> PyResult<()> { + pyo3::append_to_inittab!(foo); + Python::with_gil(|py| Python::run_bound(py, "import foo; foo.add_one(6)", None, None)) +} +``` + +If `append_to_inittab` cannot be used due to constraints in the program, +an alternative is to create a module using [`PyModule::new`] +and insert it manually into `sys.modules`: + +```rust +use pyo3::prelude::*; +use pyo3::types::PyDict; + +#[pyfunction] +pub fn add_one(x: i64) -> i64 { + x + 1 +} + +fn main() -> PyResult<()> { + Python::with_gil(|py| { + // Create new module + let foo_module = PyModule::new_bound(py, "foo")?; + foo_module.add_function(wrap_pyfunction!(add_one, &foo_module)?)?; + + // Import and get sys.modules + let sys = PyModule::import_bound(py, "sys")?; + let py_modules: Bound<'_, PyDict> = sys.getattr("modules")?.downcast_into()?; + + // Insert foo into sys.modules + py_modules.set_item("foo", foo_module)?; + + // Now we can import + run our python code + Python::run_bound(py, "import foo; foo.add_one(6)", None, None) + }) +} +``` + +## Include multiple Python files + +You can include a file at compile time by using +[`std::include_str`](https://doc.rust-lang.org/std/macro.include_str.html) macro. + +Or you can load a file at runtime by using +[`std::fs::read_to_string`](https://doc.rust-lang.org/std/fs/fn.read_to_string.html) function. + +Many Python files can be included and loaded as modules. If one file depends on +another you must preserve correct order while declaring `PyModule`. + +Example directory structure: +```text +. +├── Cargo.lock +├── Cargo.toml +├── python_app +│ ├── app.py +│ └── utils +│ └── foo.py +└── src + └── main.rs +``` + +`python_app/app.py`: +```python +from utils.foo import bar + + +def run(): + return bar() +``` + +`python_app/utils/foo.py`: +```python +def bar(): + return "baz" +``` + +The example below shows: +* how to include content of `app.py` and `utils/foo.py` into your rust binary +* how to call function `run()` (declared in `app.py`) that needs function + imported from `utils/foo.py` + +`src/main.rs`: +```rust,ignore +use pyo3::prelude::*; + +fn main() -> PyResult<()> { + let py_foo = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/python_app/utils/foo.py" + )); + let py_app = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/python_app/app.py")); + let from_python = Python::with_gil(|py| -> PyResult> { + PyModule::from_code_bound(py, py_foo, "utils.foo", "utils.foo")?; + let app: Py = PyModule::from_code_bound(py, py_app, "", "")? + .getattr("run")? + .into(); + app.call0(py) + }); + + println!("py: {}", from_python?); + Ok(()) +} +``` + +The example below shows: +* how to load content of `app.py` at runtime so that it sees its dependencies + automatically +* how to call function `run()` (declared in `app.py`) that needs function + imported from `utils/foo.py` + +It is recommended to use absolute paths because then your binary can be run +from anywhere as long as your `app.py` is in the expected directory (in this example +that directory is `/usr/share/python_app`). + +`src/main.rs`: +```rust,no_run +use pyo3::prelude::*; +use pyo3::types::PyList; +use std::fs; +use std::path::Path; + +fn main() -> PyResult<()> { + let path = Path::new("/usr/share/python_app"); + let py_app = fs::read_to_string(path.join("app.py"))?; + let from_python = Python::with_gil(|py| -> PyResult> { + let syspath = py + .import_bound("sys")? + .getattr("path")? + .downcast_into::()?; + syspath.insert(0, &path)?; + let app: Py = PyModule::from_code_bound(py, &py_app, "", "")? + .getattr("run")? + .into(); + app.call0(py) + }); + + println!("py: {}", from_python?); + Ok(()) +} +``` + + +[`Python::run`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.run +[`py_run!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.py_run.html + +## Need to use a context manager from Rust? + +Use context managers by directly invoking `__enter__` and `__exit__`. + +```rust +use pyo3::prelude::*; + +fn main() { + Python::with_gil(|py| { + let custom_manager = PyModule::from_code_bound( + py, + r#" +class House(object): + def __init__(self, address): + self.address = address + def __enter__(self): + print(f"Welcome to {self.address}!") + def __exit__(self, type, value, traceback): + if type: + print(f"Sorry you had {type} trouble at {self.address}") + else: + print(f"Thank you for visiting {self.address}, come again soon!") + + "#, + "house.py", + "house", + ) + .unwrap(); + + let house_class = custom_manager.getattr("House").unwrap(); + let house = house_class.call1(("123 Main Street",)).unwrap(); + + house.call_method0("__enter__").unwrap(); + + let result = py.eval_bound("undefined_variable + 1", None, None); + + // If the eval threw an exception we'll pass it through to the context manager. + // Otherwise, __exit__ is called with empty arguments (Python "None"). + match result { + Ok(_) => { + let none = py.None(); + house + .call_method1("__exit__", (&none, &none, &none)) + .unwrap(); + } + Err(e) => { + house + .call_method1( + "__exit__", + ( + e.get_type_bound(py), + e.value_bound(py), + e.traceback_bound(py), + ), + ) + .unwrap(); + } + } + }) +} +``` + +## Handling system signals/interrupts (Ctrl-C) + +The best way to handle system signals when running Rust code is to periodically call `Python::check_signals` to handle any signals captured by Python's signal handler. See also [the FAQ entry](../faq.md#ctrl-c-doesnt-do-anything-while-my-rust-code-is-executing). + +Alternatively, set Python's `signal` module to take the default action for a signal: + +```rust +use pyo3::prelude::*; + +# fn main() -> PyResult<()> { +Python::with_gil(|py| -> PyResult<()> { + let signal = py.import_bound("signal")?; + // Set SIGINT to have the default action + signal + .getattr("signal")? + .call1((signal.getattr("SIGINT")?, signal.getattr("SIG_DFL")?))?; + Ok(()) +}) +# } +``` + + +[`PyModule::new`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.new diff --git a/guide/src/python-from-rust/function-calls.md b/guide/src/python-from-rust/function-calls.md new file mode 100644 index 00000000000..f97de1f24ce --- /dev/null +++ b/guide/src/python-from-rust/function-calls.md @@ -0,0 +1,114 @@ +# Calling Python functions + +The `Bound<'py, T>` smart pointer (such as `Bound<'py, PyAny>`, `Bound<'py, PyList>`, or `Bound<'py, MyClass>`) can be used to call Python functions. + +PyO3 offers two APIs to make function calls: + +* [`call`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call) - call any callable Python object. +* [`call_method`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method) - call a method on the Python object. + +Both of these APIs take `args` and `kwargs` arguments (for positional and keyword arguments respectively). There are variants for less complex calls: + +* [`call1`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call1) and [`call_method1`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method1) to call only with positional `args`. +* [`call0`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call0) and [`call_method0`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method0) to call with no arguments. + +For convenience the [`Py`](../types.md#pyt-and-pyobject) smart pointer also exposes these same six API methods, but needs a `Python` token as an additional first argument to prove the GIL is held. + +The example below calls a Python function behind a `PyObject` (aka `Py`) reference: + +```rust +use pyo3::prelude::*; +use pyo3::types::PyTuple; + +fn main() -> PyResult<()> { + let arg1 = "arg1"; + let arg2 = "arg2"; + let arg3 = "arg3"; + + Python::with_gil(|py| { + let fun: Py = PyModule::from_code_bound( + py, + "def example(*args, **kwargs): + if args != (): + print('called with args', args) + if kwargs != {}: + print('called with kwargs', kwargs) + if args == () and kwargs == {}: + print('called with no arguments')", + "", + "", + )? + .getattr("example")? + .into(); + + // call object without any arguments + fun.call0(py)?; + + // pass object with Rust tuple of positional arguments + let args = (arg1, arg2, arg3); + fun.call1(py, args)?; + + // call object with Python tuple of positional arguments + let args = PyTuple::new_bound(py, &[arg1, arg2, arg3]); + fun.call1(py, args)?; + Ok(()) + }) +} +``` + +## Creating keyword arguments + +For the `call` and `call_method` APIs, `kwargs` are `Option<&Bound<'py, PyDict>>`, so can either be `None` or `Some(&dict)`. You can use the [`IntoPyDict`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.IntoPyDict.html) trait to convert other dict-like containers, e.g. `HashMap` or `BTreeMap`, as well as tuples with up to 10 elements and `Vec`s where each element is a two-element tuple. + +```rust +use pyo3::prelude::*; +use pyo3::types::IntoPyDict; +use std::collections::HashMap; + +fn main() -> PyResult<()> { + let key1 = "key1"; + let val1 = 1; + let key2 = "key2"; + let val2 = 2; + + Python::with_gil(|py| { + let fun: Py = PyModule::from_code_bound( + py, + "def example(*args, **kwargs): + if args != (): + print('called with args', args) + if kwargs != {}: + print('called with kwargs', kwargs) + if args == () and kwargs == {}: + print('called with no arguments')", + "", + "", + )? + .getattr("example")? + .into(); + + // call object with PyDict + let kwargs = [(key1, val1)].into_py_dict_bound(py); + fun.call_bound(py, (), Some(&kwargs))?; + + // pass arguments as Vec + let kwargs = vec![(key1, val1), (key2, val2)]; + fun.call_bound(py, (), Some(&kwargs.into_py_dict_bound(py)))?; + + // pass arguments as HashMap + let mut kwargs = HashMap::<&str, i32>::new(); + kwargs.insert(key1, 1); + fun.call_bound(py, (), Some(&kwargs.into_py_dict_bound(py)))?; + + Ok(()) + }) +} +``` + +
+ +During PyO3's [migration from "GIL Refs" to the `Bound` smart pointer](../migration.md#migrating-from-the-gil-refs-api-to-boundt), [`Py::call`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html#method.call) is temporarily named `call_bound` (and `call_method` is temporarily `call_method_bound`). + +(This temporary naming is only the case for the `Py` smart pointer. The methods on the `&PyAny` GIL Ref such as `call` have not been given replacements, and the methods on the `Bound` smart pointer such as [`Bound::call`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call) already use follow the newest API conventions.) + +
diff --git a/guide/src/rust-from-python.md b/guide/src/rust-from-python.md new file mode 100644 index 00000000000..470d5719098 --- /dev/null +++ b/guide/src/rust-from-python.md @@ -0,0 +1,13 @@ +# Using Rust from Python + +This chapter of the guide is dedicated to explaining how to wrap Rust code into Python objects. + +PyO3 uses Rust's "procedural macros" to provide a powerful yet simple API to denote what Rust code should map into Python objects. + +The three types of Python objects which PyO3 can produce are: + +- Python modules, via the `#[pymodule]` macro +- Python functions, via the `#[pyfunction]` macro +- Python classes, via the `#[pyclass]` macro (plus `#[pymethods]` to define methods for those clases) + +The following subchapters go through each of these in turn. diff --git a/guide/src/trait-bounds.md b/guide/src/trait-bounds.md index b0eee80c80a..0644e679190 100644 --- a/guide/src/trait-bounds.md +++ b/guide/src/trait-bounds.md @@ -66,6 +66,7 @@ The following wrapper will call the Python model from Rust, using a struct to ho ```rust # #![allow(dead_code)] use pyo3::prelude::*; +use pyo3::types::PyList; # pub trait Model { # fn set_variables(&mut self, inputs: &Vec); @@ -81,12 +82,9 @@ impl Model for UserModel { fn set_variables(&mut self, var: &Vec) { println!("Rust calling Python to set the variables"); Python::with_gil(|py| { - let values: Vec = var.clone(); - let list: PyObject = values.into_py(py); - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. - let py_model = self.model.as_ref(py); - py_model - .call_method("set_variables", (list,), None) + self.model + .bind(py) + .call_method("set_variables", (PyList::new_bound(py, var),), None) .unwrap(); }) } @@ -94,9 +92,8 @@ impl Model for UserModel { fn get_results(&self) -> Vec { println!("Rust calling Python to get the results"); Python::with_gil(|py| { - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. self.model - .as_ref(py) + .bind(py) .call_method("get_results", (), None) .unwrap() .extract() @@ -107,9 +104,8 @@ impl Model for UserModel { fn compute(&mut self) { println!("Rust calling Python to perform the computation"); Python::with_gil(|py| { - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. self.model - .as_ref(py) + .bind(py) .call_method("compute", (), None) .unwrap(); }) @@ -168,6 +164,7 @@ This wrapper will also perform the type conversions between Python and Rust. ```rust # #![allow(dead_code)] # use pyo3::prelude::*; +# use pyo3::types::PyList; # # pub trait Model { # fn set_variables(&mut self, inputs: &Vec); @@ -184,12 +181,8 @@ This wrapper will also perform the type conversions between Python and Rust. # fn set_variables(&mut self, var: &Vec) { # println!("Rust calling Python to set the variables"); # Python::with_gil(|py| { -# let values: Vec = var.clone(); -# let list: PyObject = values.into_py(py); -# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. -# let py_model = self.model.as_ref(py); -# py_model -# .call_method("set_variables", (list,), None) +# self.model.bind(py) +# .call_method("set_variables", (PyList::new_bound(py, var),), None) # .unwrap(); # }) # } @@ -197,9 +190,8 @@ This wrapper will also perform the type conversions between Python and Rust. # fn get_results(&self) -> Vec { # println!("Rust calling Python to get the results"); # Python::with_gil(|py| { -# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # self.model -# .as_ref(py) +# .bind(py) # .call_method("get_results", (), None) # .unwrap() # .extract() @@ -210,9 +202,8 @@ This wrapper will also perform the type conversions between Python and Rust. # fn compute(&mut self) { # println!("Rust calling Python to perform the computation"); # Python::with_gil(|py| { -# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # self.model -# .as_ref(py) +# .bind(py) # .call_method("compute", (), None) # .unwrap(); # }) @@ -340,6 +331,7 @@ We used in our `get_results` method the following call that performs the type co ```rust # #![allow(dead_code)] # use pyo3::prelude::*; +# use pyo3::types::PyList; # # pub trait Model { # fn set_variables(&mut self, inputs: &Vec); @@ -355,10 +347,9 @@ We used in our `get_results` method the following call that performs the type co impl Model for UserModel { fn get_results(&self) -> Vec { println!("Rust calling Python to get the results"); - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. Python::with_gil(|py| { self.model - .as_ref(py) + .bind(py) .call_method("get_results", (), None) .unwrap() .extract() @@ -368,12 +359,8 @@ impl Model for UserModel { # fn set_variables(&mut self, var: &Vec) { # println!("Rust calling Python to set the variables"); # Python::with_gil(|py| { -# let values: Vec = var.clone(); -# let list: PyObject = values.into_py(py); -# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. -# let py_model = self.model.as_ref(py); -# py_model -# .call_method("set_variables", (list,), None) +# self.model.bind(py) +# .call_method("set_variables", (PyList::new_bound(py, var),), None) # .unwrap(); # }) # } @@ -381,9 +368,8 @@ impl Model for UserModel { # fn compute(&mut self) { # println!("Rust calling Python to perform the computation"); # Python::with_gil(|py| { -# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # self.model -# .as_ref(py) +# .bind(py) # .call_method("compute", (), None) # .unwrap(); # }) @@ -396,6 +382,7 @@ Let's break it down in order to perform better error handling: ```rust # #![allow(dead_code)] # use pyo3::prelude::*; +# use pyo3::types::PyList; # # pub trait Model { # fn set_variables(&mut self, inputs: &Vec); @@ -412,10 +399,9 @@ impl Model for UserModel { fn get_results(&self) -> Vec { println!("Get results from Rust calling Python"); Python::with_gil(|py| { - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. - let py_result: &PyAny = self + let py_result: Bound<'_, PyAny> = self .model - .as_ref(py) + .bind(py) .call_method("get_results", (), None) .unwrap(); @@ -432,12 +418,8 @@ impl Model for UserModel { # fn set_variables(&mut self, var: &Vec) { # println!("Rust calling Python to set the variables"); # Python::with_gil(|py| { -# let values: Vec = var.clone(); -# let list: PyObject = values.into_py(py); -# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. -# let py_model = self.model.as_ref(py); -# py_model -# .call_method("set_variables", (list,), None) +# let py_model = self.model.bind(py) +# .call_method("set_variables", (PyList::new_bound(py, var),), None) # .unwrap(); # }) # } @@ -445,9 +427,8 @@ impl Model for UserModel { # fn compute(&mut self) { # println!("Rust calling Python to perform the computation"); # Python::with_gil(|py| { -# #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. # self.model -# .as_ref(py) +# .bind(py) # .call_method("compute", (), None) # .unwrap(); # }) @@ -478,6 +459,7 @@ It is also required to make the struct public. ```rust # #![allow(dead_code)] use pyo3::prelude::*; +use pyo3::types::PyList; pub trait Model { fn set_variables(&mut self, var: &Vec); @@ -533,12 +515,9 @@ impl Model for UserModel { fn set_variables(&mut self, var: &Vec) { println!("Rust calling Python to set the variables"); Python::with_gil(|py| { - let values: Vec = var.clone(); - let list: PyObject = values.into_py(py); - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. - let py_model = self.model.as_ref(py); - py_model - .call_method("set_variables", (list,), None) + self.model + .bind(py) + .call_method("set_variables", (PyList::new_bound(py, var),), None) .unwrap(); }) } @@ -546,10 +525,9 @@ impl Model for UserModel { fn get_results(&self) -> Vec { println!("Get results from Rust calling Python"); Python::with_gil(|py| { - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. - let py_result: &PyAny = self + let py_result: Bound<'_, PyAny> = self .model - .as_ref(py) + .bind(py) .call_method("get_results", (), None) .unwrap(); @@ -567,9 +545,8 @@ impl Model for UserModel { fn compute(&mut self) { println!("Rust calling Python to perform the computation"); Python::with_gil(|py| { - #[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. self.model - .as_ref(py) + .bind(py) .call_method("compute", (), None) .unwrap(); }) diff --git a/guide/src/types.md b/guide/src/types.md index 372c6c8632f..0f1fa3d0af1 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -1,66 +1,289 @@ -# GIL lifetimes, mutability and Python object types +# Python object types -On first glance, PyO3 provides a huge number of different types that can be used -to wrap or refer to Python objects. This page delves into the details and gives -an overview of their intended meaning, with examples when each type is best -used. +PyO3 offers two main sets of types to interact with Python objects. This section of the guide expands into detail about these types and how to choose which to use. +The first set of types is are the [smart pointers][smart-pointers] which all Python objects are wrapped in. These are `Py`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`. The [first section below](#pyo3s-smart-pointers) expands on each of these in detail and why there are three of them. -## The Python GIL, mutability, and Rust types +The second set of types are types which fill in the generic parameter `T` of the smart pointers. The most common is `PyAny`, which represents any Python object (similar to Python's `typing.Any`). There are also concrete types for many Python built-in types, such as `PyList`, `PyDict`, and `PyTuple`. User defined `#[pyclass]` types also fit this category. The [second section below](#concrete-python-types) expands on how to use these types. -Since Python has no concept of ownership, and works solely with boxed objects, -any Python object can be referenced any number of times, and mutation is allowed -from any reference. +Before PyO3 0.21, PyO3's main API to interact with Python objects was a deprecated API known as the "GIL Refs" API, containing reference types such as `&PyAny`, `&PyList`, and `&PyCell` for user-defined `#[pyclass]` types. The [third section below](#the-gil-refs-api) details this deprecated API. -The situation is helped a little by the Global Interpreter Lock (GIL), which -ensures that only one thread can use the Python interpreter and its API at the -same time, while non-Python operations (system calls and extension code) can -unlock the GIL. (See [the section on parallelism](parallelism.md) for how to do -that in PyO3.) +## PyO3's smart pointers -In PyO3, holding the GIL is modeled by acquiring a token of the type -`Python<'py>`, which serves three purposes: +PyO3's API offers three generic smart pointers: `Py`, `Bound<'py, T>` and `Borrowed<'a, 'py, T>`. For each of these the type parameter `T` will be filled by a [concrete Python type](#concrete-python-types). For example, a Python list object can be represented by `Py`, `Bound<'py, PyList>`, and `Borrowed<'a, 'py, PyList>`. -* It provides some global API for the Python interpreter, such as - [`eval`][eval]. -* It can be passed to functions that require a proof of holding the GIL, - such as [`Py::clone_ref`][clone_ref]. -* Its lifetime can be used to create Rust references that implicitly guarantee - holding the GIL, such as [`&'py PyAny`][PyAny]. +These smart pointers behave differently due to their lifetime parameters. `Py` has no lifetime parameters, `Bound<'py, T>` has [the `'py` lifetime](./python-from-rust.md#the-py-lifetime) as a parameter, and `Borrowed<'a, 'py, T>` has the `'py` lifetime plus an additional lifetime `'a` to denote the lifetime it is borrowing data for. (You can read more about these lifetimes in the subsections below). -The latter two points are the reason why some APIs in PyO3 require the `py: -Python` argument, while others don't. +Python objects are reference counted, like [`std::sync::Arc`](https://doc.rust-lang.org/stable/std/sync/struct.Arc.html). A major reason for these smart pointers is to bring Python's reference counting to a Rust API. -The PyO3 API for Python objects is written such that instead of requiring a -mutable Rust reference for mutating operations such as -[`PyList::append`][PyList_append], a shared reference (which, in turn, can only -be created through `Python<'_>` with a GIL lifetime) is sufficient. +The recommendation of when to use each of these smart pointers is as follows: -However, Rust structs wrapped as Python objects (called `pyclass` types) usually -*do* need `&mut` access. Due to the GIL, PyO3 *can* guarantee thread-safe access -to them, but it cannot statically guarantee uniqueness of `&mut` references once -an object's ownership has been passed to the Python interpreter, ensuring -references is done at runtime using `PyCell`, a scheme very similar to -`std::cell::RefCell`. +- Use `Bound<'py, T>` for as much as possible, as it offers the most efficient and complete API. +- Use `Py` mostly just for storage inside Rust `struct`s which do not want to or can't add a lifetime parameter for `Bound<'py, T>`. +- `Borrowed<'a, 'py, T>` is almost never used. It is occasionally present at the boundary between Rust and the Python interpreter, for example when borrowing data from Python tuples (which is safe because they are immutable). -### Accessing the Python GIL +The sections below also explain these smart pointers in a little more detail. -To get hold of a `Python<'py>` token to prove the GIL is held, consult [PyO3's documentation][obtaining-py]. +### `Py` (and `PyObject`) -## Object types +[`Py`][Py] is the foundational smart pointer in PyO3's API. The type parameter `T` denotes the type of the Python object. Very frequently this is `PyAny`, meaning any Python object. This is so common that `Py` has a type alias `PyObject`. -### [`PyAny`][PyAny] +Because `Py` is not bound to [the `'py` lifetime](./python-from-rust.md#the-py-lifetime), it is the type to use when storing a Python object inside a Rust `struct` or `enum` which do not want to have a lifetime parameter. In particular, [`#[pyclass]`][pyclass] types are not permitted to have a lifetime, so `Py` is the correct type to store Python objects inside them. + +The lack of binding to the `'py` lifetime also carries drawbacks: + - Almost all methods on `Py` require a `Python<'py>` token as the first argument + - Other functionality, such as [`Drop`][Drop], needs to check at runtime for attachment to the Python GIL, at a small performance cost + +Because of the drawbacks `Bound<'py, T>` is preferred for many of PyO3's APIs. In particular, `Bound<'py, T>` is the better for function arguments. + +To convert a `Py` into a `Bound<'py, T>`, the `Py::bind` and `Py::into_bound` methods are available. `Bound<'py, T>` can be converted back into `Py` using [`Bound::unbind`]. + +### `Bound<'py, T>` + +[`Bound<'py, T>`][Bound] is the counterpart to `Py` which is also bound to the `'py` lifetime. It can be thought of as equivalent to the Rust tuple `(Python<'py>, Py)`. + +By having the binding to the `'py` lifetime, `Bound<'py, T>` can offer the complete PyO3 API at maximum efficiency. This means that in almost all cases where `Py` is not necessary for lifetime reasons, `Bound<'py, T>` should be used. + +`Bound<'py, T>` engages in Python reference counting. This means that `Bound<'py, T>` owns a Python object. Rust code which just wants to borrow a Python object should use a shared reference `&Bound<'py, T>`. Just like `std::sync::Arc`, using `.clone()` and `drop()` will cheaply increment and decrement the reference count of the object (just in this case, the reference counting is implemented by the Python interpreter itself). + +To give an example of how `Bound<'py, T>` is PyO3's primary API type, consider the following Python code: + +```python +def example(): + x = list() # create a Python list + x.append(1) # append the integer 1 to it + y = x # create a second reference to the list + del x # delete the original reference +``` + +Using PyO3's API, and in particular `Bound<'py, PyList>`, this code translates into the following Rust code: + +```rust +use pyo3::prelude::*; +use pyo3::types::PyList; + +fn example<'py>(py: Python<'py>) -> PyResult<()> { + let x: Bound<'py, PyList> = PyList::empty_bound(py); + x.append(1)?; + let y: Bound<'py, PyList> = x.clone(); // y is a new reference to the same list + drop(x); // release the original reference x + Ok(()) +} +# Python::with_gil(example).unwrap(); +``` + +Or, without the type annotations: + +```rust +use pyo3::prelude::*; +use pyo3::types::PyList; + +# fn example(py: Python<'_>) -> PyResult<()> { + let x = PyList::empty_bound(py); + x.append(1)?; + let y = x.clone(); + drop(x); + Ok(()) +} +# Python::with_gil(example).unwrap(); +``` + +#### Function argument lifetimes + +Because the `'py` lifetime often appears in many function arguments as part of the `Bound<'py, T>` smart pointer, the Rust compiler will often require annotations of input and output lifetimes. This occurs when the function output has at least one lifetime, and there is more than one lifetime present on the inputs. + +To demonstrate, consider this function which takes accepts Python objects and applies the [Python `+` operation][PyAnyMethods::add] to them: + +```rust,compile_fail +# use pyo3::prelude::*; +fn add(left: &'_ Bound<'_, PyAny>, right: &'_ Bound<'_, PyAny>) -> PyResult> { + left.add(right) +} +``` + +Because the Python `+` operation might raise an exception, this function returns `PyResult>`. It doesn't need ownership of the inputs, so it takes `&Bound<'_, PyAny>` shared references. To demonstrate the point, all lifetimes have used the wildcard `'_` to allow the Rust compiler to attempt to infer them. Because there are four input lifetimes (two lifetimes of the shared references, and two `'py` lifetimes unnamed inside the `Bound<'_, PyAny>` pointers), the compiler cannot reason about which must be connected to the output. + +The correct way to solve this is to add the `'py` lifetime as a parameter for the function, and name all the `'py` lifetimes inside the `Bound<'py, PyAny>` smart pointers. For the shared references, it's also fine to reduce `&'_` to just `&`. The working end result is below: + +```rust +# use pyo3::prelude::*; +fn add<'py>( + left: &Bound<'py, PyAny>, + right: &Bound<'py, PyAny>, +) -> PyResult> { + left.add(right) +} +# Python::with_gil(|py| { +# let s = pyo3::types::PyString::new_bound(py, "s"); +# assert!(add(&s, &s).unwrap().eq("ss").unwrap()); +# }) +``` + +If naming the `'py` lifetime adds unwanted complexity to the function signature, it is also acceptable to return `PyObject` (aka `Py`), which has no lifetime. The cost is instead paid by a slight increase in implementation complexity, as seen by the introduction of a call to [`Bound::unbind`]: + +```rust +# use pyo3::prelude::*; +fn add(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { + let output: Bound<'_, PyAny> = left.add(right)?; + Ok(output.unbind()) +} +# Python::with_gil(|py| { +# let s = pyo3::types::PyString::new_bound(py, "s"); +# assert!(add(&s, &s).unwrap().bind(py).eq("ss").unwrap()); +# }) +``` + +### `Borrowed<'a, 'py, T>` + +[`Borrowed<'a, 'py, T>`][Borrowed] is an advanced type used just occasionally at the edge of interaction with the Python interpreter. It can be thought of as analogous to the shared reference `&'a Bound<'py, T>`. The difference is that `Borrowed<'a, 'py, T>` is just a smart pointer rather than a reference-to-a-smart-pointer, which is a helpful reduction in indirection in specific interactions with the Python interpreter. + +`Borrowed<'a, 'py, T>` dereferences to `Bound<'py, T>`, so all methods on `Bound<'py, T>` are available on `Borrowed<'a, 'py, T>`. + +An example where `Borrowed<'a, 'py, T>` is used is in [`PyTupleMethods::get_borrowed_item`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyTupleMethods.html#tymethod.get_item): + +```rust +use pyo3::prelude::*; +use pyo3::types::PyTuple; + +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +// Create a new tuple with the elements (0, 1, 2) +let t = PyTuple::new_bound(py, [0, 1, 2]); +for i in 0..=2 { + let entry: Borrowed<'_, 'py, PyAny> = t.get_borrowed_item(i)?; + // `PyAnyMethods::extract` is available on `Borrowed` + // via the dereference to `Bound` + let value: usize = entry.extract()?; + assert_eq!(i, value); +} +# Ok(()) +# } +# Python::with_gil(example).unwrap(); +``` + +## Concrete Python types + +In all of `Py`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`, the type parameter `T` denotes the type of the Python object referred to by the smart pointer. + +This parameter `T` can be filled by: + - [`PyAny`][PyAny], which represents any Python object, + - Native Python types such as `PyList`, `PyTuple`, and `PyDict`, and + - [`#[pyclass]`][pyclass] types defined from Rust + +The following subsections covers some further detail about how to work with these types: +- the APIs that are available for these concrete types, +- how to cast `Bound<'py, T>` to a specific concrete type, and +- how to get Rust data out of a `Bound<'py, T>`. + +### Using APIs for concrete Python types + +Each concrete Python type such as `PyAny`, `PyTuple` and `PyDict` exposes its API on the corresponding bound smart pointer `Bound<'py, PyAny>`, `Bound<'py, PyTuple>` and `Bound<'py, PyDict>`. + +Each type's API is exposed as a trait: [`PyAnyMethods`], [`PyTupleMethods`], [`PyDictMethods`], and so on for all concrete types. Using traits rather than associated methods on the `Bound` smart pointer is done for a couple of reasons: +- Clarity of documentation: each trait gets its own documentation page in the PyO3 API docs. If all methods were on the `Bound` smart pointer directly, the vast majority of PyO3's API would be on a single, extremely long, documentation page. +- Consistency: downstream code implementing Rust APIs for existing Python types can also follow this pattern of using a trait. Downstream code would not be allowed to add new associated methods directly on the `Bound` type. +- Future design: it is hoped that a future Rust with [arbitrary self types](https://github.com/rust-lang/rust/issues/44874) will remove the need for these traits in favour of placing the methods directly on `PyAny`, `PyTuple`, `PyDict`, and so on. + +These traits are all included in the `pyo3::prelude` module, so with the glob import `use pyo3::prelude::*` the full PyO3 API is made available to downstream code. + +The following function accesses the first item in the input Python list, using the `.get_item()` method from the `PyListMethods` trait: + +```rust +use pyo3::prelude::*; +use pyo3::types::PyList; + +fn get_first_item<'py>(list: &Bound<'py, PyList>) -> PyResult> { + list.get_item(0) +} +# Python::with_gil(|py| { +# let l = PyList::new_bound(py, ["hello world"]); +# assert!(get_first_item(&l).unwrap().eq("hello world").unwrap()); +# }) +``` + +### Casting between Python object types + +To cast `Bound<'py, T>` smart pointers to some other type, use the [`.downcast()`][PyAnyMethods::downcast] family of functions. This converts `&Bound<'py, T>` to a different `&Bound<'py, U>`, without transferring ownership. There is also [`.downcast_into()`][PyAnyMethods::downcast_into] to convert `Bound<'py, T>` to `Bound<'py, U>` with transfer of ownership. These methods are available for all types `T` which implement the [`PyTypeCheck`] trait. + +Casting to `Bound<'py, PyAny>` can be done with `.as_any()` or `.into_any()`. + +For example, the following snippet shows how to cast `Bound<'py, PyAny>` to `Bound<'py, PyTuple>`: + +```rust +# use pyo3::prelude::*; +# use pyo3::types::PyTuple; +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +// create a new Python `tuple`, and use `.into_any()` to erase the type +let obj: Bound<'py, PyAny> = PyTuple::empty_bound(py).into_any(); + +// use `.downcast()` to cast to `PyTuple` without transferring ownership +let _: &Bound<'py, PyTuple> = obj.downcast()?; + +// use `.downcast_into()` to cast to `PyTuple` with transfer of ownership +let _: Bound<'py, PyTuple> = obj.downcast_into()?; +# Ok(()) +# } +# Python::with_gil(example).unwrap() +``` + +Custom [`#[pyclass]`][pyclass] types implement [`PyTypeCheck`], so `.downcast()` also works for these types. The snippet below is the same as the snippet above casting instead to a custom type `MyClass`: + +```rust +use pyo3::prelude::*; + +#[pyclass] +struct MyClass {} + +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +// create a new Python `tuple`, and use `.into_any()` to erase the type +let obj: Bound<'py, PyAny> = Bound::new(py, MyClass {})?.into_any(); + +// use `.downcast()` to cast to `MyClass` without transferring ownership +let _: &Bound<'py, MyClass> = obj.downcast()?; + +// use `.downcast_into()` to cast to `MyClass` with transfer of ownership +let _: Bound<'py, MyClass> = obj.downcast_into()?; +# Ok(()) +# } +# Python::with_gil(example).unwrap() +``` + +### Extracting Rust data from Python objects + +To extract Rust data from Python objects, use [`.extract()`][PyAnyMethods::extract] instead of `.downcast()`. This method is available for all types which implement the [`FromPyObject`] trait. -**Represents:** a Python object of unspecified type, restricted to a GIL -lifetime. Currently, `PyAny` can only ever occur as a reference, `&PyAny`. +For example, the following snippet extracts a Rust tuple of integers from a Python tuple: -**Used:** Whenever you want to refer to some Python object and will have the -GIL for the whole duration you need to access that object. For example, -intermediate values and arguments to `pyfunction`s or `pymethod`s implemented -in Rust where any type is allowed. +```rust +# use pyo3::prelude::*; +# use pyo3::types::PyTuple; +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +// create a new Python `tuple`, and use `.into_any()` to erase the type +let obj: Bound<'py, PyAny> = PyTuple::new_bound(py, [1, 2, 3]).into_any(); + +// extracting the Python `tuple` to a rust `(i32, i32, i32)` tuple +let (x, y, z) = obj.extract::<(i32, i32, i32)>()?; +assert_eq!((x, y, z), (1, 2, 3)); +# Ok(()) +# } +# Python::with_gil(example).unwrap() +``` + +To avoid copying data, [`#[pyclass]`][pyclass] types can directly reference Rust data stored within the Python objects without needing to `.extract()`. See the [corresponding documentation in the class section of the guide](./class. +md#bound-and-interior-mutability) for more detail. -Many general methods for interacting with Python objects are on the `PyAny` struct, -such as `getattr`, `setattr`, and `.call`. +## The GIL Refs API + +The GIL Refs API was PyO3's primary API prior to PyO3 0.21. The main difference was that instead of the `Bound<'py, PyAny>` smart pointer, the "GIL Reference" `&'py PyAny` was used. (This was similar for other Python types.) + +As of PyO3 0.21, the GIL Refs API is deprecated. See the [migration guide](./migration.md#from-020-to-021) for details on how to upgrade. + +The following sections note some historical detail about the GIL Refs API. + +### [`PyAny`][PyAny] + +**Represented:** a Python object of unspecified type. In the GIL Refs API, this was only accessed as the GIL Ref `&'py PyAny`. + +**Used:** `&'py PyAny` was used to refer to some Python object when the GIL lifetime was available for the whole duration access was needed. For example, intermediate values and arguments to `pyfunction`s or `pymethod`s implemented in Rust where any type is allowed. **Conversions:** @@ -71,7 +294,7 @@ a list: # use pyo3::prelude::*; # use pyo3::types::PyList; # Python::with_gil(|py| -> PyResult<()> { -#[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. +#[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. let obj: &PyAny = PyList::empty(py); // To &PyList with PyAny::downcast @@ -92,11 +315,11 @@ For a `&PyAny` object reference `any` where the underlying object is a `#[pyclas # use pyo3::prelude::*; # #[pyclass] #[derive(Clone)] struct MyClass { } # Python::with_gil(|py| -> PyResult<()> { -# #[allow(deprecated)] +#[allow(deprecated)] // into_ref is part of the deprecated GIL Refs API let obj: &PyAny = Py::new(py, MyClass {})?.into_ref(py); // To &PyCell with PyAny::downcast -# #[allow(deprecated)] +#[allow(deprecated)] // &PyCell is part of the deprecated GIL Refs API let _: &PyCell = obj.downcast()?; // To Py (aka PyObject) with .into() @@ -117,18 +340,13 @@ let _: PyRefMut<'_, MyClass> = obj.extract()?; ### `PyTuple`, `PyDict`, and many more -**Represents:** a native Python object of known type, restricted to a GIL -lifetime just like `PyAny`. +**Represented:** a native Python object of known type. In the GIL Refs API, they were only accessed as the GIL Refs `&'py PyTuple`, `&'py PyDict`. -**Used:** Whenever you want to operate with native Python types while holding -the GIL. Like `PyAny`, this is the most convenient form to use for function -arguments and intermediate values. +**Used:** `&'py PyTuple` and similar were used to operate with native Python types while holding the GIL. Like `PyAny`, this is the most convenient form to use for function arguments and intermediate values. -These types all implement `Deref`, so they all expose the same -methods which can be found on `PyAny`. +These GIL Refs implement `Deref`, so they all expose the same methods which can be found on `PyAny`. -To see all Python types exposed by `PyO3` you should consult the -[`pyo3::types`][pyo3::types] module. +To see all Python types exposed by `PyO3` consult the [`pyo3::types`][pyo3::types] module. **Conversions:** @@ -136,7 +354,7 @@ To see all Python types exposed by `PyO3` you should consult the # use pyo3::prelude::*; # use pyo3::types::PyList; # Python::with_gil(|py| -> PyResult<()> { -#[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. +#[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. let list = PyList::empty(py); // Use methods from PyAny on all Python types with Deref implementation @@ -146,7 +364,7 @@ let _ = list.repr()?; let _: &PyAny = list; // To &PyAny explicitly with .as_ref() -#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. +#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let _: &PyAny = list.as_ref(); // To Py with .into() or Py::from() @@ -160,7 +378,7 @@ let _: PyObject = list.into(); ### `Py` and `PyObject` -**Represents:** a GIL-independent reference to a Python object. This can be a Python native type +**Represented:** a GIL-independent reference to a Python object. This can be a Python native type (like `PyTuple`), or a `pyclass` type implemented in Rust. The most commonly-used variant, `Py`, is also known as `PyObject`. @@ -170,86 +388,23 @@ Python-Rust FFI boundary, or returning objects from functions implemented in Rus Can be cloned using Python reference counts with `.clone()`. -**Conversions:** - -For a `Py`, the conversions are as below: - -```rust -# use pyo3::prelude::*; -# use pyo3::types::PyList; -# Python::with_gil(|py| { -let list: Py = PyList::empty_bound(py).unbind(); - -// To &PyList with Py::as_ref() (borrows from the Py) -#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. -let _: &PyList = list.as_ref(py); - -# let list_clone = list.clone(); // Because `.into_ref()` will consume `list`. -// To &PyList with Py::into_ref() (moves the pointer into PyO3's object storage) -# #[allow(deprecated)] -let _: &PyList = list.into_ref(py); - -# let list = list_clone; -// To Py (aka PyObject) with .into() -let _: Py = list.into(); -# }) -``` - -For a `#[pyclass] struct MyClass`, the conversions for `Py` are below: - -```rust -# use pyo3::prelude::*; -# Python::with_gil(|py| { -# #[pyclass] struct MyClass { } -# Python::with_gil(|py| -> PyResult<()> { -let my_class: Py = Py::new(py, MyClass { })?; - -// To &PyCell with Py::as_ref() (borrows from the Py) -#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. -let _: &PyCell = my_class.as_ref(py); - -# let my_class_clone = my_class.clone(); // Because `.into_ref()` will consume `my_class`. -// To &PyCell with Py::into_ref() (moves the pointer into PyO3's object storage) -# #[allow(deprecated)] -let _: &PyCell = my_class.into_ref(py); - -# let my_class = my_class_clone.clone(); -// To Py (aka PyObject) with .into_py(py) -let _: Py = my_class.into_py(py); - -# let my_class = my_class_clone; -// To PyRef<'_, MyClass> with Py::borrow or Py::try_borrow -let _: PyRef<'_, MyClass> = my_class.try_borrow(py)?; - -// To PyRefMut<'_, MyClass> with Py::borrow_mut or Py::try_borrow_mut -let _: PyRefMut<'_, MyClass> = my_class.try_borrow_mut(py)?; -# Ok(()) -# }).unwrap(); -# }); -``` - ### `PyCell` -**Represents:** a reference to a Rust object (instance of `PyClass`) which is -wrapped in a Python object. The cell part is an analog to stdlib's -[`RefCell`][RefCell] to allow access to `&mut` references. +**Represented:** a reference to a Rust object (instance of `PyClass`) wrapped in a Python object. The cell part is an analog to stdlib's [`RefCell`][RefCell] to allow access to `&mut` references. -**Used:** for accessing pure-Rust API of the instance (members and functions -taking `&SomeType` or `&mut SomeType`) while maintaining the aliasing rules of -Rust references. +**Used:** for accessing pure-Rust API of the instance (members and functions taking `&SomeType` or `&mut SomeType`) while maintaining the aliasing rules of Rust references. -Like PyO3's Python native types, `PyCell` implements `Deref`, -so it also exposes all of the methods on `PyAny`. +Like PyO3's Python native types, the GIL Ref `&PyCell` implements `Deref`, so it also exposed all of the methods on `PyAny`. **Conversions:** -`PyCell` can be used to access `&T` and `&mut T` via `PyRef` and `PyRefMut` respectively. +`PyCell` was used to access `&T` and `&mut T` via `PyRef` and `PyRefMut` respectively. ```rust # use pyo3::prelude::*; # #[pyclass] struct MyClass { } # Python::with_gil(|py| -> PyResult<()> { -# #[allow(deprecated)] +#[allow(deprecated)] // &PyCell is part of the deprecated GIL Refs API let cell: &PyCell = PyCell::new(py, MyClass {})?; // To PyRef with .borrow() or .try_borrow() @@ -264,13 +419,13 @@ let _: &mut MyClass = &mut *py_ref_mut; # }).unwrap(); ``` -`PyCell` can also be accessed like a Python-native type. +`PyCell` was also accessed like a Python-native type. ```rust # use pyo3::prelude::*; # #[pyclass] struct MyClass { } # Python::with_gil(|py| -> PyResult<()> { -# #[allow(deprecated)] +#[allow(deprecated)] // &PyCell is part of the deprecate GIL Refs API let cell: &PyCell = PyCell::new(py, MyClass {})?; // Use methods from PyAny on PyCell with Deref implementation @@ -280,37 +435,30 @@ let _ = cell.repr()?; let _: &PyAny = cell; // To &PyAny explicitly with .as_ref() -#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. +#[allow(deprecated)] // as_ref is part of the deprecated "GIL Refs" API. let _: &PyAny = cell.as_ref(); # Ok(()) # }).unwrap(); ``` -### `PyRef` and `PyRefMut` - -**Represents:** reference wrapper types employed by `PyCell` to keep track of -borrows, analog to `Ref` and `RefMut` used by `RefCell`. - -**Used:** while borrowing a `PyCell`. They can also be used with `.extract()` -on types like `Py` and `PyAny` to get a reference quickly. - - -## Related traits and types - -### `PyClass` - -This trait marks structs defined in Rust that are also usable as Python classes, -usually defined using the `#[pyclass]` macro. - -### `PyNativeType` - -This trait marks structs that mirror native Python types, such as `PyList`. - - +[Bound]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html +[`Bound::unbind`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html#method.unbind +[Py]: {{#PYO3_DOCS_URL}}/pyo3/struct.Py.html +[PyAnyMethods::add]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.add +[PyAnyMethods::extract]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.extract +[PyAnyMethods::downcast]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.downcast +[PyAnyMethods::downcast_into]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.downcast_into +[`PyTypeCheck`]: {{#PYO3_DOCS_URL}}/pyo3/type_object/trait.PyTypeCheck.html +[`PyAnyMethods`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html +[`PyDictMethods`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyDictMethods.html +[`PyTupleMethods`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyTupleMethods.html +[pyclass]: class.md +[Borrowed]: {{#PYO3_DOCS_URL}}/pyo3/struct.Borrowed.html +[Drop]: https://doc.rust-lang.org/std/ops/trait.Drop.html [eval]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval [clone_ref]: {{#PYO3_DOCS_URL}}/pyo3/struct.Py.html#method.clone_ref [pyo3::types]: {{#PYO3_DOCS_URL}}/pyo3/types/index.html [PyAny]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html [PyList_append]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyList.html#method.append [RefCell]: https://doc.rust-lang.org/std/cell/struct.RefCell.html -[obtaining-py]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#obtaining-a-python-token +[smart-pointers]: https://doc.rust-lang.org/book/ch15-00-smart-pointers.html diff --git a/src/lib.rs b/src/lib.rs index bc88838d4b1..a7c03d1b6cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,29 +30,38 @@ //! //! PyO3 has several core types that you should familiarize yourself with: //! -//! ## The Python<'py> object -//! -//! Holding the [global interpreter lock] (GIL) is modeled with the [`Python<'py>`](crate::Python) -//! token. All APIs that require that the GIL is held require this token as proof that you really -//! are holding the GIL. It can be explicitly acquired and is also implicitly acquired by PyO3 as -//! it wraps Rust functions and structs into Python functions and objects. -//! -//! ## The GIL-dependent types -//! -//! For example `&`[`PyAny`]. These are only ever seen as references, with a lifetime that is only -//! valid for as long as the GIL is held, which is why using them doesn't require a -//! [`Python<'py>`](crate::Python) token. The underlying Python object, if mutable, can be mutated -//! through any reference. +//! ## The `Python<'py>` object, and the `'py` lifetime +//! +//! Holding the [global interpreter lock] (GIL) is modeled with the [`Python<'py>`](Python) token. Many +//! Python APIs require that the GIL is held, and PyO3 uses this token as proof that these APIs +//! can be called safely. It can be explicitly acquired and is also implicitly acquired by PyO3 +//! as it wraps Rust functions and structs into Python functions and objects. +//! +//! The [`Python<'py>`](Python) token's lifetime `'py` is common to many PyO3 APIs: +//! - Types that also have the `'py` lifetime, such as the [`Bound<'py, T>`](Bound) smart pointer, are +//! bound to the Python GIL and rely on this to offer their functionality. These types often +//! have a [`.py()`](Bound::py) method to get the associated [`Python<'py>`](Python) token. +//! - Functions which depend on the `'py` lifetime, such as [`PyList::new_bound`](types::PyList::new_bound), +//! require a [`Python<'py>`](Python) token as an input. Sometimes the token is passed implicitly by +//! taking a [`Bound<'py, T>`](Bound) or other type which is bound to the `'py` lifetime. +//! - Traits which depend on the `'py` lifetime, such as [`FromPyObject<'py>`](FromPyObject), usually have +//! inputs or outputs which depend on the lifetime. Adding the lifetime to the trait allows +//! these inputs and outputs to express their binding to the GIL in the Rust type system. +//! +//! ## Python object smart pointers +//! +//! PyO3 has two core smart pointers to refer to Python objects, [`Py`](Py) and its GIL-bound +//! form [`Bound<'py, T>`](Bound) which carries the `'py` lifetime. (There is also +//! [`Borrowed<'a, 'py, T>`](instance::Borrowed), but it is used much more rarely). +//! +//! The type parameter `T` in these smart pointers can be filled by: +//! - [`PyAny`], e.g. `Py` or `Bound<'py, PyAny>`, where the Python object type is not +//! known. `Py` is so common it has a type alias [`PyObject`]. +//! - Concrete Python types like [`PyList`](types::PyList) or [`PyTuple`](types::PyTuple). +//! - Rust types which are exposed to Python using the [`#[pyclass]`](macro@pyclass) macro. //! //! See the [guide][types] for an explanation of the different Python object types. //! -//! ## The GIL-independent types -//! -//! When wrapped in [`Py`]`<...>`, like with [`Py`]`<`[`PyAny`]`>` or [`Py`]``, Python -//! objects no longer have a limited lifetime which makes them easier to store in structs and pass -//! between functions. However, you cannot do much with them without a -//! [`Python<'py>`](crate::Python) token, for which you’d need to reacquire the GIL. -//! //! ## PyErr //! //! The vast majority of operations in this library will return [`PyResult<...>`](PyResult). @@ -512,7 +521,10 @@ pub mod doc_test { "guide/src/parallelism.md" => guide_parallelism_md, "guide/src/performance.md" => guide_performance_md, "guide/src/python-from-rust.md" => guide_python_from_rust_md, + "guide/src/python-from-rust/calling-existing-code.md" => guide_pfr_calling_existing_code_md, + "guide/src/python-from-rust/function-calls.md" => guide_pfr_function_calls_md, "guide/src/python-typing-hints.md" => guide_python_typing_hints_md, + "guide/src/rust-from-python.md" => guide_rust_from_python_md, "guide/src/trait-bounds.md" => guide_trait_bounds_md, "guide/src/types.md" => guide_types_md, } diff --git a/src/sync.rs b/src/sync.rs index f4ffe0409a7..e89a8edd853 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,4 +1,9 @@ //! Synchronization mechanisms based on the Python GIL. +//! +//! With the acceptance of [PEP 703] (aka a "freethreaded Python") for Python 3.13, these +//! are likely to undergo significant developments in the future. +//! +//! [PEP 703]: https://peps.python.org/pep-703/ use crate::{ types::{any::PyAnyMethods, PyString, PyType}, Bound, Py, PyResult, PyVisit, Python, From db0a98c0404cb9e7554779836d9f165078b86298 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 10 Mar 2024 16:17:15 +0000 Subject: [PATCH 211/349] ci: pin pytest < 8.1 (#3946) --- pytests/pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index 126eaf77b09..43403a1241c 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -24,6 +24,7 @@ dev = [ "hypothesis>=3.55", "pytest-asyncio>=0.21", "pytest-benchmark>=3.4", - "pytest>=6.0", + # pinned < 8.1 because https://github.com/CodSpeedHQ/pytest-codspeed/issues/27 + "pytest>=8,<8.1", "typing_extensions>=4.0.0" ] From 67b1b3501300bd8842c75586b7a3c37381da701d Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 10 Mar 2024 22:16:18 +0000 Subject: [PATCH 212/349] release: 0.21.0-beta.0 (#3944) --- CHANGELOG.md | 68 +++++++++++++++++++++++++++++++++- Cargo.toml | 8 ++-- newsfragments/3514.added.md | 1 - newsfragments/3532.changed.md | 1 - newsfragments/3540.added.md | 1 - newsfragments/3577.added.md | 1 - newsfragments/3577.changed.md | 1 - newsfragments/3582.added.md | 1 - newsfragments/3588.added.md | 1 - newsfragments/3599.added.md | 1 - newsfragments/3600.changed.md | 1 - newsfragments/3601.changed.md | 1 - newsfragments/3603.removed.md | 1 - newsfragments/3609.changed.md | 1 - newsfragments/3632.added.md | 1 - newsfragments/3638.changed.md | 1 - newsfragments/3653.changed.md | 1 - newsfragments/3657.changed.md | 1 - newsfragments/3660.changed.md | 1 - newsfragments/3661.changed.md | 1 - newsfragments/3663.changed.md | 1 - newsfragments/3664.changed.md | 1 - newsfragments/3670.added.md | 1 - newsfragments/3677.added.md | 1 - newsfragments/3679.changed.md | 1 - newsfragments/3686.added.md | 1 - newsfragments/3689.changed.md | 1 - newsfragments/3692.added.md | 1 - newsfragments/3692.changed.md | 1 - newsfragments/3706.added.md | 1 - newsfragments/3707.added.md | 1 - newsfragments/3712.added.md | 1 - newsfragments/3730.added.md | 1 - newsfragments/3734.added.md | 1 - newsfragments/3736.added.md | 1 - newsfragments/3742.changed.md | 1 - newsfragments/3757.fixed.md | 1 - newsfragments/3776.changed.md | 1 - newsfragments/3785.added.md | 1 - newsfragments/3801.added.md | 1 - newsfragments/3802.added.md | 1 - newsfragments/3815.added.md | 2 - newsfragments/3818.fixed.md | 1 - newsfragments/3849.added.md | 1 - newsfragments/3849.changed.md | 1 - newsfragments/3871.added.md | 1 - newsfragments/3901.fixed.md | 1 - newsfragments/3905.changed.md | 1 - newsfragments/3928.added.md | 1 - newsfragments/3928.changed.md | 1 - newsfragments/3931.added.md | 1 - newsfragments/3934.removed.md | 1 - pyo3-build-config/Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 4 +- pyo3-macros-backend/Cargo.toml | 4 +- pyo3-macros/Cargo.toml | 4 +- pyproject.toml | 2 +- 57 files changed, 79 insertions(+), 64 deletions(-) delete mode 100644 newsfragments/3514.added.md delete mode 100644 newsfragments/3532.changed.md delete mode 100644 newsfragments/3540.added.md delete mode 100644 newsfragments/3577.added.md delete mode 100644 newsfragments/3577.changed.md delete mode 100644 newsfragments/3582.added.md delete mode 100644 newsfragments/3588.added.md delete mode 100644 newsfragments/3599.added.md delete mode 100644 newsfragments/3600.changed.md delete mode 100644 newsfragments/3601.changed.md delete mode 100644 newsfragments/3603.removed.md delete mode 100644 newsfragments/3609.changed.md delete mode 100644 newsfragments/3632.added.md delete mode 100644 newsfragments/3638.changed.md delete mode 100644 newsfragments/3653.changed.md delete mode 100644 newsfragments/3657.changed.md delete mode 100644 newsfragments/3660.changed.md delete mode 100644 newsfragments/3661.changed.md delete mode 100644 newsfragments/3663.changed.md delete mode 100644 newsfragments/3664.changed.md delete mode 100644 newsfragments/3670.added.md delete mode 100644 newsfragments/3677.added.md delete mode 100644 newsfragments/3679.changed.md delete mode 100644 newsfragments/3686.added.md delete mode 100644 newsfragments/3689.changed.md delete mode 100644 newsfragments/3692.added.md delete mode 100644 newsfragments/3692.changed.md delete mode 100644 newsfragments/3706.added.md delete mode 100644 newsfragments/3707.added.md delete mode 100644 newsfragments/3712.added.md delete mode 100644 newsfragments/3730.added.md delete mode 100644 newsfragments/3734.added.md delete mode 100644 newsfragments/3736.added.md delete mode 100644 newsfragments/3742.changed.md delete mode 100644 newsfragments/3757.fixed.md delete mode 100644 newsfragments/3776.changed.md delete mode 100644 newsfragments/3785.added.md delete mode 100644 newsfragments/3801.added.md delete mode 100644 newsfragments/3802.added.md delete mode 100644 newsfragments/3815.added.md delete mode 100644 newsfragments/3818.fixed.md delete mode 100644 newsfragments/3849.added.md delete mode 100644 newsfragments/3849.changed.md delete mode 100644 newsfragments/3871.added.md delete mode 100644 newsfragments/3901.fixed.md delete mode 100644 newsfragments/3905.changed.md delete mode 100644 newsfragments/3928.added.md delete mode 100644 newsfragments/3928.changed.md delete mode 100644 newsfragments/3931.added.md delete mode 100644 newsfragments/3934.removed.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fcd4ca4f495..36b1b552acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,72 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.21.0-beta.0] - 2024-03-10 + +### Added + +- Add `PyMemoryView` type. [#3514](https://github.com/PyO3/pyo3/pull/3514) +- Support `async fn` in macros with coroutine implementation [#3540](https://github.com/PyO3/pyo3/pull/3540) +- Implement `PyTypeInfo` for `PyEllipsis`, `PyNone` and `PyNotImplemented`. [#3577](https://github.com/PyO3/pyo3/pull/3577) +- Support `#[pyclass]` on enums that have non-unit variants. [#3582](https://github.com/PyO3/pyo3/pull/3582) +- Add `__name__`/`__qualname__` attributes to `Coroutine` [#3588](https://github.com/PyO3/pyo3/pull/3588) +- Add `coroutine::CancelHandle` to catch coroutine cancellation [#3599](https://github.com/PyO3/pyo3/pull/3599) +- Add support for extracting Rust set types from `frozenset`. [#3632](https://github.com/PyO3/pyo3/pull/3632) +- `FromPyObject`, `IntoPy` and `ToPyObject` are implemented on `std::duration::Duration` [#3670](https://github.com/PyO3/pyo3/pull/3670) +- Add `PyString::to_cow`. Add `Py::to_str`, `Py::to_cow`, and `Py::to_string_lossy`, as ways to access Python string data safely beyond the GIL lifetime. [#3677](https://github.com/PyO3/pyo3/pull/3677) +- Add `Bound` and `Borrowed` smart pointers as a new API for accessing Python objects. [#3686](https://github.com/PyO3/pyo3/pull/3686) +- Add `PyNativeType::as_bound` to convert "GIL refs" to the new `Bound` smart pointer. [#3692](https://github.com/PyO3/pyo3/pull/3692) +- Add `FromPyObject::extract_bound` method, which can be implemented to avoid using the GIL Ref API in `FromPyObject` implementations. [#3706](https://github.com/PyO3/pyo3/pull/3706) +- Add `gil-refs` feature to allow continued use of the deprecated GIL Refs APIs. [#3707](https://github.com/PyO3/pyo3/pull/3707) +- Added methods to `PyAnyMethods` for binary operators (`add`, `sub`, etc.) [#3712](https://github.com/PyO3/pyo3/pull/3712) +- `chrono-tz` feature allowing conversion between `chrono_tz::Tz` and `zoneinfo.ZoneInfo` [#3730](https://github.com/PyO3/pyo3/pull/3730) +- Add definition for `PyType_GetModuleByDef` to `pyo3_ffi`. [#3734](https://github.com/PyO3/pyo3/pull/3734) +- Conversion between `std::time::SystemTime` and `datetime.datetime` [#3736](https://github.com/PyO3/pyo3/pull/3736) +- Add `Py::as_any` and `Py::into_any`. [#3785](https://github.com/PyO3/pyo3/pull/3785) +- Add `PyStringMethods::encode_utf8`. [#3801](https://github.com/PyO3/pyo3/pull/3801) +- Add `PyBackedStr` and `PyBackedBytes`, as alternatives to `&str` and `&bytes` where a Python object owns the data. [#3802](https://github.com/PyO3/pyo3/pull/3802) +- The ability to create Python modules with a Rust `mod` block + behind the `experimental-declarative-modules` feature. [#3815](https://github.com/PyO3/pyo3/pull/3815) +- Implement `ExactSizeIterator` for `set` and `frozenset` iterators on `abi3` feature. [#3849](https://github.com/PyO3/pyo3/pull/3849) +- Add `Py::drop_ref` to explicitly drop a `Py`` and immediately decrease the Python reference count if the GIL is already held. [#3871](https://github.com/PyO3/pyo3/pull/3871) +- Implement `FromPyObject` for `Cow`. [#3928](https://github.com/PyO3/pyo3/pull/3928) +- Add `experimental-async` feature. [#3931](https://github.com/PyO3/pyo3/pull/3931) + +### Changed + +- - `PyDict::from_sequence` now takes a single argument of type `&PyAny` (previously took two arguments `Python` and `PyObject`). [#3532](https://github.com/PyO3/pyo3/pull/3532) +- Deprecate `Py::is_ellipsis` and `PyAny::is_ellipsis` in favour of `any.is(py.Ellipsis())`. [#3577](https://github.com/PyO3/pyo3/pull/3577) +- Split some `PyTypeInfo` functionality into new traits `HasPyGilRef` and `PyTypeCheck`. [#3600](https://github.com/PyO3/pyo3/pull/3600) +- Deprecate `PyTryFrom` and `PyTryInto` traits in favor of `any.downcast()` via the `PyTypeCheck` and `PyTypeInfo` traits. [#3601](https://github.com/PyO3/pyo3/pull/3601) +- Allow async methods to accept `&self`/`&mut self` [#3609](https://github.com/PyO3/pyo3/pull/3609) +- Values of type `bool` can now be extracted from NumPy's `bool_`. [#3638](https://github.com/PyO3/pyo3/pull/3638) +- Add `AsRefSource` to `PyNativeType`. [#3653](https://github.com/PyO3/pyo3/pull/3653) +- Changed `.is_true` to `.is_truthy` on `PyAny` and `Py` to clarify that the test is not based on identity with or equality to the True singleton. [#3657](https://github.com/PyO3/pyo3/pull/3657) +- `PyType::name` is now `PyType::qualname` whereas `PyType::name` efficiently accesses the full name which includes the module name. [#3660](https://github.com/PyO3/pyo3/pull/3660) +- The `Iter(A)NextOutput` types are now deprecated and `__(a)next__` can directly return anything which can be converted into Python objects, i.e. awaitables do not need to be wrapped into `IterANextOutput` or `Option` any more. `Option` can still be used as well and returning `None` will trigger the fast path for `__next__`, stopping iteration without having to raise a `StopIteration` exception. [#3661](https://github.com/PyO3/pyo3/pull/3661) +- Implements `FromPyObject` on `chrono::DateTime` for all `Tz` and not only `FixedOffset` and `Utc` [#3663](https://github.com/PyO3/pyo3/pull/3663) +- `chrono` conversions are compatible with `abi3` [#3664](https://github.com/PyO3/pyo3/pull/3664) +- Add lifetime parameter to `PyTzInfoAccess` trait. For the deprecated gil-ref API, the trait is now implemented for `&'py PyTime` and `&'py PyDateTime` instead of `PyTime` and `PyDate`. [#3679](https://github.com/PyO3/pyo3/pull/3679) +- Calls to `__traverse__` become no-ops for unsendable pyclasses if on the wrong thread, thereby avoiding hard aborts at the cost of potential leakage. [#3689](https://github.com/PyO3/pyo3/pull/3689) +- Include `PyNativeType` in `pyo3::prelude`. [#3692](https://github.com/PyO3/pyo3/pull/3692) +- Improve performance of `extract::` (and other integer types) by avoiding call to `__index__()` converting the value to an integer for 3.10+. Gives performance improvement of around 30% for successful extraction. [#3742](https://github.com/PyO3/pyo3/pull/3742) +- Relax bound of `FromPyObject` for `Py` to just `T: PyTypeCheck`. [#3776](https://github.com/PyO3/pyo3/pull/3776) +- `PySet` and `PyFrozenSet` iterators now always iterate the equivalent of `iter(set)`. (A "fast path" with no noticeable performance benefit was removed.) [#3849](https://github.com/PyO3/pyo3/pull/3849) +- The `#[pymodule]` macro now supports module functions that take a single argument as a `&Bound<'_, PyModule>`. [#3905](https://github.com/PyO3/pyo3/pull/3905) +- Move implementations of `FromPyObject` for `&str`, `Cow`, `&[u8]` and `Cow<[u8]>` onto a temporary trait `FromPyObjectBound` when `gil-refs` feature is deactivated. [#3928](https://github.com/PyO3/pyo3/pull/3928) + +### Removed + +- Remove all functionality deprecated in PyO3 0.19. [#3603](https://github.com/PyO3/pyo3/pull/3603) +- Remove `PyCode` and `PyCode_Type` on PyPy: `PyCode_Type` is not exposed by PyPy. [#3934](https://github.com/PyO3/pyo3/pull/3934) + +### Fixed + +- Match PyPy 7.3.14 in removing PyPy-only symbol `Py_MAX_NDIMS` in favour of `PyBUF_MAX_NDIM`. [#3757](https://github.com/PyO3/pyo3/pull/3757) +- Fix segmentation fault using `datetime` types when an invalid `datetime` module is on sys.path. [#3818](https://github.com/PyO3/pyo3/pull/3818) +- Fix `non_local_definitions` lint warning triggered by many PyO3 macros. [#3901](https://github.com/PyO3/pyo3/pull/3901) + + ## [0.20.3] - 2024-02-23 ### Packaging @@ -1640,7 +1706,7 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.20.3...HEAD +[0.21.0-beta.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0-beta.0 [0.20.3]: https://github.com/pyo3/pyo3/compare/v0.20.2...v0.20.3 [0.20.2]: https://github.com/pyo3/pyo3/compare/v0.20.1...v0.20.2 [0.20.1]: https://github.com/pyo3/pyo3/compare/v0.20.0...v0.20.1 diff --git a/Cargo.toml b/Cargo.toml index 32dc0ba75ef..ba5c3461180 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.21.0-dev" +version = "0.21.0-beta.0" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -22,10 +22,10 @@ memoffset = "0.9" portable-atomic = "1.0" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.0-dev" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.0-beta.0" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.21.0-dev", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.21.0-beta.0", optional = true } indoc = { version = "2.0.1", optional = true } unindent = { version = "0.2.1", optional = true } @@ -60,7 +60,7 @@ rayon = "1.6.1" futures = "0.3.28" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.0-beta.0", features = ["resolve-config"] } [features] default = ["macros"] diff --git a/newsfragments/3514.added.md b/newsfragments/3514.added.md deleted file mode 100644 index 7fbf662b2ec..00000000000 --- a/newsfragments/3514.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyMemoryView` type. diff --git a/newsfragments/3532.changed.md b/newsfragments/3532.changed.md deleted file mode 100644 index b65f240931e..00000000000 --- a/newsfragments/3532.changed.md +++ /dev/null @@ -1 +0,0 @@ -- `PyDict::from_sequence` now takes a single argument of type `&PyAny` (previously took two arguments `Python` and `PyObject`). diff --git a/newsfragments/3540.added.md b/newsfragments/3540.added.md deleted file mode 100644 index 2b113193bef..00000000000 --- a/newsfragments/3540.added.md +++ /dev/null @@ -1 +0,0 @@ -Support `async fn` in macros with coroutine implementation \ No newline at end of file diff --git a/newsfragments/3577.added.md b/newsfragments/3577.added.md deleted file mode 100644 index 632274984ec..00000000000 --- a/newsfragments/3577.added.md +++ /dev/null @@ -1 +0,0 @@ -Implement `PyTypeInfo` for `PyEllipsis`, `PyNone` and `PyNotImplemented`. diff --git a/newsfragments/3577.changed.md b/newsfragments/3577.changed.md deleted file mode 100644 index a7e6629d6a5..00000000000 --- a/newsfragments/3577.changed.md +++ /dev/null @@ -1 +0,0 @@ -Deprecate `Py::is_ellipsis` and `PyAny::is_ellipsis` in favour of `any.is(py.Ellipsis())`. diff --git a/newsfragments/3582.added.md b/newsfragments/3582.added.md deleted file mode 100644 index 59659a8819d..00000000000 --- a/newsfragments/3582.added.md +++ /dev/null @@ -1 +0,0 @@ -Support `#[pyclass]` on enums that have non-unit variants. diff --git a/newsfragments/3588.added.md b/newsfragments/3588.added.md deleted file mode 100644 index acddf296a6f..00000000000 --- a/newsfragments/3588.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `__name__`/`__qualname__` attributes to `Coroutine` \ No newline at end of file diff --git a/newsfragments/3599.added.md b/newsfragments/3599.added.md deleted file mode 100644 index 36078fbcdb6..00000000000 --- a/newsfragments/3599.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `coroutine::CancelHandle` to catch coroutine cancellation \ No newline at end of file diff --git a/newsfragments/3600.changed.md b/newsfragments/3600.changed.md deleted file mode 100644 index c8701ef4b25..00000000000 --- a/newsfragments/3600.changed.md +++ /dev/null @@ -1 +0,0 @@ -Split some `PyTypeInfo` functionality into new traits `HasPyGilRef` and `PyTypeCheck`. diff --git a/newsfragments/3601.changed.md b/newsfragments/3601.changed.md deleted file mode 100644 index 413765ecad5..00000000000 --- a/newsfragments/3601.changed.md +++ /dev/null @@ -1 +0,0 @@ -Deprecate `PyTryFrom` and `PyTryInto` traits in favor of `any.downcast()` via the `PyTypeCheck` and `PyTypeInfo` traits. diff --git a/newsfragments/3603.removed.md b/newsfragments/3603.removed.md deleted file mode 100644 index e8f5004e3b9..00000000000 --- a/newsfragments/3603.removed.md +++ /dev/null @@ -1 +0,0 @@ -Remove all functionality deprecated in PyO3 0.19. diff --git a/newsfragments/3609.changed.md b/newsfragments/3609.changed.md deleted file mode 100644 index 7979ea71960..00000000000 --- a/newsfragments/3609.changed.md +++ /dev/null @@ -1 +0,0 @@ -Allow async methods to accept `&self`/`&mut self` \ No newline at end of file diff --git a/newsfragments/3632.added.md b/newsfragments/3632.added.md deleted file mode 100644 index d9c954fa0b4..00000000000 --- a/newsfragments/3632.added.md +++ /dev/null @@ -1 +0,0 @@ -Add support for extracting Rust set types from `frozenset`. diff --git a/newsfragments/3638.changed.md b/newsfragments/3638.changed.md deleted file mode 100644 index 6bdafde8422..00000000000 --- a/newsfragments/3638.changed.md +++ /dev/null @@ -1 +0,0 @@ -Values of type `bool` can now be extracted from NumPy's `bool_`. diff --git a/newsfragments/3653.changed.md b/newsfragments/3653.changed.md deleted file mode 100644 index 75fea03cb71..00000000000 --- a/newsfragments/3653.changed.md +++ /dev/null @@ -1 +0,0 @@ -Add `AsRefSource` to `PyNativeType`. diff --git a/newsfragments/3657.changed.md b/newsfragments/3657.changed.md deleted file mode 100644 index 0a519b09d62..00000000000 --- a/newsfragments/3657.changed.md +++ /dev/null @@ -1 +0,0 @@ -Changed `.is_true` to `.is_truthy` on `PyAny` and `Py` to clarify that the test is not based on identity with or equality to the True singleton. diff --git a/newsfragments/3660.changed.md b/newsfragments/3660.changed.md deleted file mode 100644 index 8b4a3f734e1..00000000000 --- a/newsfragments/3660.changed.md +++ /dev/null @@ -1 +0,0 @@ -`PyType::name` is now `PyType::qualname` whereas `PyType::name` efficiently accesses the full name which includes the module name. diff --git a/newsfragments/3661.changed.md b/newsfragments/3661.changed.md deleted file mode 100644 index 8245a6f1a80..00000000000 --- a/newsfragments/3661.changed.md +++ /dev/null @@ -1 +0,0 @@ -The `Iter(A)NextOutput` types are now deprecated and `__(a)next__` can directly return anything which can be converted into Python objects, i.e. awaitables do not need to be wrapped into `IterANextOutput` or `Option` any more. `Option` can still be used as well and returning `None` will trigger the fast path for `__next__`, stopping iteration without having to raise a `StopIteration` exception. diff --git a/newsfragments/3663.changed.md b/newsfragments/3663.changed.md deleted file mode 100644 index 13c07e01f2d..00000000000 --- a/newsfragments/3663.changed.md +++ /dev/null @@ -1 +0,0 @@ -Implements `FromPyObject` on `chrono::DateTime` for all `Tz` and not only `FixedOffset` and `Utc` \ No newline at end of file diff --git a/newsfragments/3664.changed.md b/newsfragments/3664.changed.md deleted file mode 100644 index 3a167d2f9d2..00000000000 --- a/newsfragments/3664.changed.md +++ /dev/null @@ -1 +0,0 @@ -`chrono` conversions are compatible with `abi3` \ No newline at end of file diff --git a/newsfragments/3670.added.md b/newsfragments/3670.added.md deleted file mode 100644 index a524261e9d9..00000000000 --- a/newsfragments/3670.added.md +++ /dev/null @@ -1 +0,0 @@ -`FromPyObject`, `IntoPy` and `ToPyObject` are implemented on `std::duration::Duration` \ No newline at end of file diff --git a/newsfragments/3677.added.md b/newsfragments/3677.added.md deleted file mode 100644 index 3e6bc56d582..00000000000 --- a/newsfragments/3677.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyString::to_cow`. Add `Py::to_str`, `Py::to_cow`, and `Py::to_string_lossy`, as ways to access Python string data safely beyond the GIL lifetime. diff --git a/newsfragments/3679.changed.md b/newsfragments/3679.changed.md deleted file mode 100644 index ab46598ad65..00000000000 --- a/newsfragments/3679.changed.md +++ /dev/null @@ -1 +0,0 @@ -Add lifetime parameter to `PyTzInfoAccess` trait. For the deprecated gil-ref API, the trait is now implemented for `&'py PyTime` and `&'py PyDateTime` instead of `PyTime` and `PyDate`. diff --git a/newsfragments/3686.added.md b/newsfragments/3686.added.md deleted file mode 100644 index f808df3685a..00000000000 --- a/newsfragments/3686.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `Bound` and `Borrowed` smart pointers as a new API for accessing Python objects. diff --git a/newsfragments/3689.changed.md b/newsfragments/3689.changed.md deleted file mode 100644 index 30928e82f64..00000000000 --- a/newsfragments/3689.changed.md +++ /dev/null @@ -1 +0,0 @@ -Calls to `__traverse__` become no-ops for unsendable pyclasses if on the wrong thread, thereby avoiding hard aborts at the cost of potential leakage. diff --git a/newsfragments/3692.added.md b/newsfragments/3692.added.md deleted file mode 100644 index 45cdd5aba28..00000000000 --- a/newsfragments/3692.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyNativeType::as_bound` to convert "GIL refs" to the new `Bound` smart pointer. diff --git a/newsfragments/3692.changed.md b/newsfragments/3692.changed.md deleted file mode 100644 index 9535cbb23db..00000000000 --- a/newsfragments/3692.changed.md +++ /dev/null @@ -1 +0,0 @@ -Include `PyNativeType` in `pyo3::prelude`. diff --git a/newsfragments/3706.added.md b/newsfragments/3706.added.md deleted file mode 100644 index 31db8b96cef..00000000000 --- a/newsfragments/3706.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `FromPyObject::extract_bound` method, which can be implemented to avoid using the GIL Ref API in `FromPyObject` implementations. diff --git a/newsfragments/3707.added.md b/newsfragments/3707.added.md deleted file mode 100644 index bc92e2c0f95..00000000000 --- a/newsfragments/3707.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `gil-refs` feature to allow continued use of the deprecated GIL Refs APIs. diff --git a/newsfragments/3712.added.md b/newsfragments/3712.added.md deleted file mode 100644 index d7390f77c14..00000000000 --- a/newsfragments/3712.added.md +++ /dev/null @@ -1 +0,0 @@ -Added methods to `PyAnyMethods` for binary operators (`add`, `sub`, etc.) diff --git a/newsfragments/3730.added.md b/newsfragments/3730.added.md deleted file mode 100644 index 7e287245eb1..00000000000 --- a/newsfragments/3730.added.md +++ /dev/null @@ -1 +0,0 @@ -`chrono-tz` feature allowing conversion between `chrono_tz::Tz` and `zoneinfo.ZoneInfo` \ No newline at end of file diff --git a/newsfragments/3734.added.md b/newsfragments/3734.added.md deleted file mode 100644 index e58c2038e70..00000000000 --- a/newsfragments/3734.added.md +++ /dev/null @@ -1 +0,0 @@ -Add definition for `PyType_GetModuleByDef` to `pyo3_ffi`. diff --git a/newsfragments/3736.added.md b/newsfragments/3736.added.md deleted file mode 100644 index 0d3a4a08c1a..00000000000 --- a/newsfragments/3736.added.md +++ /dev/null @@ -1 +0,0 @@ -Conversion between `std::time::SystemTime` and `datetime.datetime` \ No newline at end of file diff --git a/newsfragments/3742.changed.md b/newsfragments/3742.changed.md deleted file mode 100644 index b8805abafda..00000000000 --- a/newsfragments/3742.changed.md +++ /dev/null @@ -1 +0,0 @@ -Improve performance of `extract::` (and other integer types) by avoiding call to `__index__()` converting the value to an integer for 3.10+. Gives performance improvement of around 30% for successful extraction. diff --git a/newsfragments/3757.fixed.md b/newsfragments/3757.fixed.md deleted file mode 100644 index 103a634af9f..00000000000 --- a/newsfragments/3757.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Match PyPy 7.3.14 in removing PyPy-only symbol `Py_MAX_NDIMS` in favour of `PyBUF_MAX_NDIM`. diff --git a/newsfragments/3776.changed.md b/newsfragments/3776.changed.md deleted file mode 100644 index 71ffd893a18..00000000000 --- a/newsfragments/3776.changed.md +++ /dev/null @@ -1 +0,0 @@ -Relax bound of `FromPyObject` for `Py` to just `T: PyTypeCheck`. diff --git a/newsfragments/3785.added.md b/newsfragments/3785.added.md deleted file mode 100644 index 6af3bb999f8..00000000000 --- a/newsfragments/3785.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `Py::as_any` and `Py::into_any`. diff --git a/newsfragments/3801.added.md b/newsfragments/3801.added.md deleted file mode 100644 index 78f45032ba2..00000000000 --- a/newsfragments/3801.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyStringMethods::encode_utf8`. diff --git a/newsfragments/3802.added.md b/newsfragments/3802.added.md deleted file mode 100644 index 86b98e9df97..00000000000 --- a/newsfragments/3802.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `PyBackedStr` and `PyBackedBytes`, as alternatives to `&str` and `&bytes` where a Python object owns the data. diff --git a/newsfragments/3815.added.md b/newsfragments/3815.added.md deleted file mode 100644 index e4fd3e9315a..00000000000 --- a/newsfragments/3815.added.md +++ /dev/null @@ -1,2 +0,0 @@ -The ability to create Python modules with a Rust `mod` block -behind the `experimental-declarative-modules` feature. \ No newline at end of file diff --git a/newsfragments/3818.fixed.md b/newsfragments/3818.fixed.md deleted file mode 100644 index 76fe01a545c..00000000000 --- a/newsfragments/3818.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix segmentation fault using `datetime` types when an invalid `datetime` module is on sys.path. diff --git a/newsfragments/3849.added.md b/newsfragments/3849.added.md deleted file mode 100644 index 8aa9a55df03..00000000000 --- a/newsfragments/3849.added.md +++ /dev/null @@ -1 +0,0 @@ -Implement `ExactSizeIterator` for `set` and `frozenset` iterators on `abi3` feature. diff --git a/newsfragments/3849.changed.md b/newsfragments/3849.changed.md deleted file mode 100644 index ee8dfbf5b2c..00000000000 --- a/newsfragments/3849.changed.md +++ /dev/null @@ -1 +0,0 @@ -`PySet` and `PyFrozenSet` iterators now always iterate the equivalent of `iter(set)`. (A "fast path" with no noticeable performance benefit was removed.) diff --git a/newsfragments/3871.added.md b/newsfragments/3871.added.md deleted file mode 100644 index f90e92fdfff..00000000000 --- a/newsfragments/3871.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `Py::drop_ref` to explicitly drop a `Py`` and immediately decrease the Python reference count if the GIL is already held. diff --git a/newsfragments/3901.fixed.md b/newsfragments/3901.fixed.md deleted file mode 100644 index 0845c2bbcf5..00000000000 --- a/newsfragments/3901.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix `non_local_definitions` lint warning triggered by many PyO3 macros. diff --git a/newsfragments/3905.changed.md b/newsfragments/3905.changed.md deleted file mode 100644 index 917584eb72a..00000000000 --- a/newsfragments/3905.changed.md +++ /dev/null @@ -1 +0,0 @@ -The `#[pymodule]` macro now supports module functions that take a single argument as a `&Bound<'_, PyModule>`. diff --git a/newsfragments/3928.added.md b/newsfragments/3928.added.md deleted file mode 100644 index e768e6b0163..00000000000 --- a/newsfragments/3928.added.md +++ /dev/null @@ -1 +0,0 @@ -Implement `FromPyObject` for `Cow`. diff --git a/newsfragments/3928.changed.md b/newsfragments/3928.changed.md deleted file mode 100644 index a4734c41ed3..00000000000 --- a/newsfragments/3928.changed.md +++ /dev/null @@ -1 +0,0 @@ -Move implementations of `FromPyObject` for `&str`, `Cow`, `&[u8]` and `Cow<[u8]>` onto a temporary trait `FromPyObjectBound` when `gil-refs` feature is deactivated. diff --git a/newsfragments/3931.added.md b/newsfragments/3931.added.md deleted file mode 100644 index b532adeeae5..00000000000 --- a/newsfragments/3931.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `experimental-async` feature. diff --git a/newsfragments/3934.removed.md b/newsfragments/3934.removed.md deleted file mode 100644 index 66741d3f6b3..00000000000 --- a/newsfragments/3934.removed.md +++ /dev/null @@ -1 +0,0 @@ -Remove `PyCode` and `PyCode_Type` on PyPy: `PyCode_Type` is not exposed by PyPy. \ No newline at end of file diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 702e99a4aac..c00427eb18e 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.21.0-dev" +version = "0.21.0-beta.0" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 8021dc72b69..8dea584e304 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.21.0-dev" +version = "0.21.0-beta.0" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,7 +38,7 @@ abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312"] generate-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0-beta.0", features = ["resolve-config"] } [lints] workspace = true diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 665c8c3510d..abf71902e20 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.21.0-dev" +version = "0.21.0-beta.0" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -16,7 +16,7 @@ edition = "2021" [dependencies] heck = "0.4" proc-macro2 = { version = "1", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0-beta.0", features = ["resolve-config"] } quote = { version = "1", default-features = false } [dependencies.syn] diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 97d2de07cba..008224e78ef 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.21.0-dev" +version = "0.21.0-beta.0" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -22,7 +22,7 @@ experimental-declarative-modules = [] proc-macro2 = { version = "1", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.0-dev" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.0-beta.0" } [lints] workspace = true diff --git a/pyproject.toml b/pyproject.toml index 866645d2ffc..5d5ef42b1db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.towncrier] filename = "CHANGELOG.md" -version = "0.21.0-dev" +version = "0.21.0-beta.0" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" From 93323bc9221ef5fabfd8cef68c1995a4faab67f0 Mon Sep 17 00:00:00 2001 From: acceptacross <150119116+acceptacross@users.noreply.github.com> Date: Mon, 11 Mar 2024 18:15:03 +0800 Subject: [PATCH 213/349] chore: remove repetitive words (#3950) Signed-off-by: acceptacross --- CHANGELOG.md | 2 +- guide/src/class.md | 2 +- guide/src/class/protocols.md | 2 +- guide/src/faq.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36b1b552acc..3581d789ff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -678,7 +678,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h - Respect Rust privacy rules for items wrapped with `wrap_pyfunction` and `wrap_pymodule`. [#2081](https://github.com/PyO3/pyo3/pull/2081) - Add modulo argument to `__ipow__` magic method. [#2083](https://github.com/PyO3/pyo3/pull/2083) - Fix FFI definition for `_PyCFunctionFast`. [#2126](https://github.com/PyO3/pyo3/pull/2126) -- `PyDateTimeAPI` and `PyDateTime_TimeZone_UTC` are are now unsafe functions instead of statics. [#2126](https://github.com/PyO3/pyo3/pull/2126) +- `PyDateTimeAPI` and `PyDateTime_TimeZone_UTC` are now unsafe functions instead of statics. [#2126](https://github.com/PyO3/pyo3/pull/2126) - `PyDateTimeAPI` does not implicitly call `PyDateTime_IMPORT` anymore to reflect the original Python API more closely. Before the first call to `PyDateTime_IMPORT` a null pointer is returned. Therefore before calling any of the following FFI functions `PyDateTime_IMPORT` must be called to avoid undefined behavior: [#2126](https://github.com/PyO3/pyo3/pull/2126) - `PyDateTime_TimeZone_UTC` - `PyDate_Check` diff --git a/guide/src/class.md b/guide/src/class.md index 2c6d854ff08..f353cc4787e 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -191,7 +191,7 @@ fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { ## Bound and interior mutability -Often is is useful to turn a `#[pyclass]` type `T` into a Python object and access it from Rust code. The [`Py`] and [`Bound<'py, T>`] smart pointers are the ways to represent a Python object in PyO3's API. More detail can be found about them [in the Python objects](./types.md#pyo3s-smart-pointers) section of the guide. +Often is useful to turn a `#[pyclass]` type `T` into a Python object and access it from Rust code. The [`Py`] and [`Bound<'py, T>`] smart pointers are the ways to represent a Python object in PyO3's API. More detail can be found about them [in the Python objects](./types.md#pyo3s-smart-pointers) section of the guide. Most Python objects do not offer exclusive (`&mut`) access (see the [section on Python's memory model](./python-from-rust.md#pythons-memory-model)). However, Rust structs wrapped as Python objects (called `pyclass` types) often *do* need `&mut` access. Due to the GIL, PyO3 *can* guarantee exclusive access to them. diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 0a77cd7f2a9..3b12fd531c3 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -6,7 +6,7 @@ In the Python C-API which PyO3 is implemented upon, many of these magic methods If a function name in `#[pymethods]` is a recognised magic method, it will be automatically placed into the correct slot in the Python type object. The function name is taken from the usual rules for naming `#[pymethods]`: the `#[pyo3(name = "...")]` attribute is used if present, otherwise the Rust function name is used. -The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html). Some of the slots do not have a magic method in Python, which leads to a few additional magic methods defined only in PyO3: +The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html). Some of the slots do not have a magic method in Python, which leads to a few additional magic methods defined only in PyO3: - Magic methods for garbage collection - Magic methods for the buffer protocol diff --git a/guide/src/faq.md b/guide/src/faq.md index 1034e9ccc2a..19f9b5d50ab 100644 --- a/guide/src/faq.md +++ b/guide/src/faq.md @@ -163,7 +163,7 @@ b: ``` The downside to this approach is that any Rust code working on the `Outer` struct now has to acquire the GIL to do anything with its field. -## I want to use the `pyo3` crate re-exported from from dependency but the proc-macros fail! +## I want to use the `pyo3` crate re-exported from dependency but the proc-macros fail! All PyO3 proc-macros (`#[pyclass]`, `#[pyfunction]`, `#[derive(FromPyObject)]` and so on) expect the `pyo3` crate to be available under that name in your crate From a7fa1bdf22c67e93543c9c4aa1927c52a1124e24 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 11 Mar 2024 13:40:26 +0100 Subject: [PATCH 214/349] fix: remove "track_caller" cfg check (#3951) This "cfg" value does not seem to exist (any more?), and #[track_caller] is used a lot elsewhere without cfg_attr. Found using cargo check -Zcheck-cfg. --- src/err/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/err/mod.rs b/src/err/mod.rs index 5e054449bc9..cc4de79909b 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -485,7 +485,7 @@ impl PyErr { /// /// Use this function when the error is expected to have been set, for example from /// [PyErr::occurred] or by an error return value from a C FFI function. - #[cfg_attr(all(debug_assertions, track_caller), track_caller)] + #[cfg_attr(debug_assertions, track_caller)] #[inline] pub fn fetch(py: Python<'_>) -> PyErr { const FAILED_TO_FETCH: &str = "attempted to fetch exception but none was set"; From ee89b2e8e2015d8fdfa28dd69e459391809c601b Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:57:03 +0100 Subject: [PATCH 215/349] deprecate `wrap_pyfunction` with `py` argument (#3954) * deprecate `wrap_pyfunction` with `py` argument The Python token in `wrap_pyfunction` is not handled automatically by `WrapPyFunctionArg`, for backwards compatibility. This uses deref specialization to deprecate this variant. * merge `Extractor`s * add deprecation ui test, revert closure variant due to test failure * fix nightly --- guide/src/conversions/traits.md | 2 +- guide/src/migration.md | 7 +++--- pytests/src/enums.rs | 8 ++++--- src/conversion.rs | 1 + src/conversions/anyhow.rs | 3 +-- src/conversions/eyre.rs | 3 +-- src/coroutine/waker.rs | 7 +++--- src/err/mod.rs | 4 ++-- src/exceptions.rs | 4 ++-- src/impl_/pymethods.rs | 9 ++++++++ src/macros.rs | 4 +++- src/marker.rs | 2 +- src/pycell.rs | 6 ++--- src/sync.rs | 6 ++--- src/tests/hygiene/pyfunction.rs | 1 + src/types/bytearray.rs | 4 ++-- tests/test_anyhow.rs | 10 +++++---- tests/test_bytes.rs | 6 ++--- tests/test_coroutine.rs | 19 +++++++++------- tests/test_enum.rs | 6 ++--- tests/test_exceptions.rs | 4 ++-- tests/test_macros.rs | 2 +- tests/test_methods.rs | 2 +- tests/test_pyfunction.rs | 30 ++++++++++++------------- tests/test_string.rs | 2 +- tests/test_text_signature.rs | 18 +++++++-------- tests/test_various.rs | 4 ++-- tests/test_wrap_pyfunction_deduction.rs | 1 + tests/ui/deprecations.rs | 16 +++++++++++++ tests/ui/deprecations.stderr | 8 +++++++ tests/ui/invalid_result_conversion.rs | 8 ++++--- 31 files changed, 127 insertions(+), 80 deletions(-) diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index 3a00a160c65..65a5d150e79 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -499,7 +499,7 @@ _without_ having a unique python type. ```rust use pyo3::prelude::*; - +# #[allow(dead_code)] struct MyPyObjectWrapper(PyObject); impl IntoPy for MyPyObjectWrapper { diff --git a/guide/src/migration.md b/guide/src/migration.md index 5af6eb2336c..1c5ca59714a 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -436,7 +436,7 @@ impl SomeClass { When converting from `anyhow::Error` or `eyre::Report` to `PyErr`, if the inner error is a "simple" `PyErr` (with no source error), then the inner error will be used directly as the `PyErr` instead of wrapping it in a new `PyRuntimeError` with the original information converted into a string. -```rust +```rust,ignore # #[cfg(feature = "anyhow")] # #[allow(dead_code)] # mod anyhow_only { @@ -597,9 +597,9 @@ fn function_with_defaults(a: i32, b: i32, c: i32) {} # fn main() { # Python::with_gil(|py| { -# let simple = wrap_pyfunction!(simple_function, py).unwrap(); +# let simple = wrap_pyfunction_bound!(simple_function, py).unwrap(); # assert_eq!(simple.getattr("__text_signature__").unwrap().to_string(), "(a, b, c)"); -# let defaulted = wrap_pyfunction!(function_with_defaults, py).unwrap(); +# let defaulted = wrap_pyfunction_bound!(function_with_defaults, py).unwrap(); # assert_eq!(defaulted.getattr("__text_signature__").unwrap().to_string(), "(a, b=1, c=2)"); # }) # } @@ -1090,6 +1090,7 @@ impl FromPy for PyObject { After ```rust # use pyo3::prelude::*; +# #[allow(dead_code)] struct MyPyObjectWrapper(PyObject); impl IntoPy for MyPyObjectWrapper { diff --git a/pytests/src/enums.rs b/pytests/src/enums.rs index 32478cbefc2..4bb269fbbd2 100644 --- a/pytests/src/enums.rs +++ b/pytests/src/enums.rs @@ -1,11 +1,13 @@ -use pyo3::{pyclass, pyfunction, pymodule, types::PyModule, wrap_pyfunction, Bound, PyResult}; +use pyo3::{ + pyclass, pyfunction, pymodule, types::PyModule, wrap_pyfunction_bound, Bound, PyResult, +}; #[pymodule] pub fn enums(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; - m.add_wrapped(wrap_pyfunction!(do_simple_stuff))?; - m.add_wrapped(wrap_pyfunction!(do_complex_stuff))?; + m.add_wrapped(wrap_pyfunction_bound!(do_simple_stuff))?; + m.add_wrapped(wrap_pyfunction_bound!(do_complex_stuff))?; Ok(()) } diff --git a/src/conversion.rs b/src/conversion.rs index efb09b6b341..8d4ad776837 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -96,6 +96,7 @@ pub trait ToPyObject { /// ```rust /// use pyo3::prelude::*; /// +/// # #[allow(dead_code)] /// struct Number { /// value: i32, /// } diff --git a/src/conversions/anyhow.rs b/src/conversions/anyhow.rs index fba8816d23a..623ee7d548c 100644 --- a/src/conversions/anyhow.rs +++ b/src/conversions/anyhow.rs @@ -35,7 +35,6 @@ //! //! ```rust //! use pyo3::prelude::*; -//! use pyo3::wrap_pyfunction; //! use std::path::PathBuf; //! //! // A wrapper around a Rust function. @@ -48,7 +47,7 @@ //! //! fn main() { //! let error = Python::with_gil(|py| -> PyResult> { -//! let fun = wrap_pyfunction!(py_open, py)?; +//! let fun = wrap_pyfunction_bound!(py_open, py)?; //! let text = fun.call1(("foo.txt",))?.extract::>()?; //! Ok(text) //! }).unwrap_err(); diff --git a/src/conversions/eyre.rs b/src/conversions/eyre.rs index 236e2c8bc92..d4704e411c5 100644 --- a/src/conversions/eyre.rs +++ b/src/conversions/eyre.rs @@ -34,7 +34,6 @@ //! //! ```rust //! use pyo3::prelude::*; -//! use pyo3::wrap_pyfunction; //! use std::path::PathBuf; //! //! // A wrapper around a Rust function. @@ -47,7 +46,7 @@ //! //! fn main() { //! let error = Python::with_gil(|py| -> PyResult> { -//! let fun = wrap_pyfunction!(py_open, py)?; +//! let fun = wrap_pyfunction_bound!(py_open, py)?; //! let text = fun.call1(("foo.txt",))?.extract::>()?; //! Ok(text) //! }).unwrap_err(); diff --git a/src/coroutine/waker.rs b/src/coroutine/waker.rs index 096146f8292..b524b6d7298 100644 --- a/src/coroutine/waker.rs +++ b/src/coroutine/waker.rs @@ -1,7 +1,7 @@ use crate::sync::GILOnceCell; use crate::types::any::PyAnyMethods; use crate::types::PyCFunction; -use crate::{intern, wrap_pyfunction, Bound, Py, PyAny, PyObject, PyResult, Python}; +use crate::{intern, wrap_pyfunction_bound, Bound, Py, PyAny, PyObject, PyResult, Python}; use pyo3_macros::pyfunction; use std::sync::Arc; use std::task::Wake; @@ -70,8 +70,9 @@ impl LoopAndFuture { fn set_result(&self, py: Python<'_>) -> PyResult<()> { static RELEASE_WAITER: GILOnceCell> = GILOnceCell::new(); - let release_waiter = RELEASE_WAITER - .get_or_try_init(py, || wrap_pyfunction!(release_waiter, py).map(Into::into))?; + let release_waiter = RELEASE_WAITER.get_or_try_init(py, || { + wrap_pyfunction_bound!(release_waiter, py).map(Bound::unbind) + })?; // `Future.set_result` must be called in event loop thread, // so it requires `call_soon_threadsafe` let call_soon_threadsafe = self.event_loop.call_method1( diff --git a/src/err/mod.rs b/src/err/mod.rs index cc4de79909b..7610f49951c 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -145,7 +145,7 @@ impl PyErr { /// } /// # /// # Python::with_gil(|py| { - /// # let fun = pyo3::wrap_pyfunction!(always_throws, py).unwrap(); + /// # let fun = pyo3::wrap_pyfunction_bound!(always_throws, py).unwrap(); /// # let err = fun.call0().expect_err("called a function that should always return an error but the return value was Ok"); /// # assert!(err.is_instance_of::(py)) /// # }); @@ -163,7 +163,7 @@ impl PyErr { /// } /// # /// # Python::with_gil(|py| { - /// # let fun = pyo3::wrap_pyfunction!(always_throws, py).unwrap(); + /// # let fun = pyo3::wrap_pyfunction_bound!(always_throws, py).unwrap(); /// # let err = fun.call0().expect_err("called a function that should always return an error but the return value was Ok"); /// # assert!(err.is_instance_of::(py)) /// # }); diff --git a/src/exceptions.rs b/src/exceptions.rs index 3401679b25e..bd9c89c425f 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -171,7 +171,7 @@ macro_rules! import_exception { /// } /// # fn main() -> PyResult<()> { /// # Python::with_gil(|py| -> PyResult<()> { -/// # let fun = wrap_pyfunction!(raise_myerror, py)?; +/// # let fun = wrap_pyfunction_bound!(raise_myerror, py)?; /// # let locals = pyo3::types::PyDict::new_bound(py); /// # locals.set_item("MyError", py.get_type_bound::())?; /// # locals.set_item("raise_myerror", fun)?; @@ -322,7 +322,7 @@ fn always_throws() -> PyResult<()> { } # # Python::with_gil(|py| { -# let fun = pyo3::wrap_pyfunction!(always_throws, py).unwrap(); +# let fun = pyo3::wrap_pyfunction_bound!(always_throws, py).unwrap(); # let err = fun.call0().expect_err(\"called a function that should always return an error but the return value was Ok\"); # assert!(err.is_instance_of::(py)) # }); diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index a7df90b572d..bc1125f97fd 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -599,6 +599,14 @@ impl Extractor { } } +impl Extractor> { + #[cfg_attr( + not(feature = "gil-refs"), + deprecated(since = "0.21.0", note = "use `wrap_pyfunction_bound!` instead") + )] + pub fn is_python(&self) {} +} + impl Extractor { #[cfg_attr( not(feature = "gil-refs"), @@ -612,6 +620,7 @@ impl Extractor { impl NotAGilRef { pub fn extract_gil_ref(&self) {} + pub fn is_python(&self) {} } impl std::ops::Deref for Extractor { diff --git a/src/macros.rs b/src/macros.rs index 33f378e7b83..bd0bd81421c 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -144,8 +144,10 @@ macro_rules! wrap_pyfunction { }; ($function:path, $py_or_module:expr) => {{ use $function as wrapped_pyfunction; + let (py_or_module, e) = $crate::impl_::pymethods::inspect_type($py_or_module); + e.is_python(); $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( - $py_or_module, + py_or_module, &wrapped_pyfunction::DEF, ) }}; diff --git a/src/marker.rs b/src/marker.rs index 9b3d7329316..2a38b83cba5 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -509,7 +509,7 @@ impl<'py> Python<'py> { /// # /// # fn main() -> PyResult<()> { /// # Python::with_gil(|py| -> PyResult<()> { - /// # let fun = pyo3::wrap_pyfunction!(sum_numbers, py)?; + /// # let fun = pyo3::wrap_pyfunction_bound!(sum_numbers, py)?; /// # let res = fun.call1((vec![1_u32, 2, 3],))?; /// # assert_eq!(res.extract::()?, 6_u32); /// # Ok(()) diff --git a/src/pycell.rs b/src/pycell.rs index 397e7b6a22a..636fb90cef6 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -132,7 +132,7 @@ //! # let n = Py::new(py, Number{inner: 35}).unwrap(); //! # let n2 = n.clone_ref(py); //! # assert!(n.is(&n2)); -//! # let fun = pyo3::wrap_pyfunction!(swap_numbers, py).unwrap(); +//! # let fun = pyo3::wrap_pyfunction_bound!(swap_numbers, py).unwrap(); //! # fun.call1((n, n2)).expect_err("Managed to create overlapping mutable references. Note: this is undefined behaviour."); //! # }); //! # } @@ -170,7 +170,7 @@ //! # let n = Py::new(py, Number{inner: 35}).unwrap(); //! # let n2 = n.clone_ref(py); //! # assert!(n.is(&n2)); -//! # let fun = pyo3::wrap_pyfunction!(swap_numbers, py).unwrap(); +//! # let fun = pyo3::wrap_pyfunction_bound!(swap_numbers, py).unwrap(); //! # fun.call1((n, n2)).unwrap(); //! # }); //! # @@ -179,7 +179,7 @@ //! # let n = Py::new(py, Number{inner: 35}).unwrap(); //! # let n2 = Py::new(py, Number{inner: 42}).unwrap(); //! # assert!(!n.is(&n2)); -//! # let fun = pyo3::wrap_pyfunction!(swap_numbers, py).unwrap(); +//! # let fun = pyo3::wrap_pyfunction_bound!(swap_numbers, py).unwrap(); //! # fun.call1((&n, &n2)).unwrap(); //! # let n: u32 = n.borrow(py).inner; //! # let n2: u32 = n2.borrow(py).inner; diff --git a/src/sync.rs b/src/sync.rs index e89a8edd853..38471fb7ca3 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -220,7 +220,7 @@ impl GILOnceCell> { /// /// ``` /// use pyo3::intern; -/// # use pyo3::{pyfunction, types::PyDict, wrap_pyfunction, PyResult, Python, prelude::PyDictMethods, Bound}; +/// # use pyo3::{prelude::*, types::PyDict}; /// /// #[pyfunction] /// fn create_dict(py: Python<'_>) -> PyResult> { @@ -241,10 +241,10 @@ impl GILOnceCell> { /// } /// # /// # Python::with_gil(|py| { -/// # let fun_slow = wrap_pyfunction!(create_dict, py).unwrap(); +/// # let fun_slow = wrap_pyfunction_bound!(create_dict, py).unwrap(); /// # let dict = fun_slow.call0().unwrap(); /// # assert!(dict.contains("foo").unwrap()); -/// # let fun = wrap_pyfunction!(create_dict_faster, py).unwrap(); +/// # let fun = wrap_pyfunction_bound!(create_dict_faster, py).unwrap(); /// # let dict = fun.call0().unwrap(); /// # assert!(dict.contains("foo").unwrap()); /// # }); diff --git a/src/tests/hygiene/pyfunction.rs b/src/tests/hygiene/pyfunction.rs index 9cfad0db6c6..edc8b6e35d3 100644 --- a/src/tests/hygiene/pyfunction.rs +++ b/src/tests/hygiene/pyfunction.rs @@ -10,6 +10,7 @@ fn do_something(x: i32) -> crate::PyResult { #[test] fn invoke_wrap_pyfunction() { crate::Python::with_gil(|py| { + #[allow(deprecated)] let func = crate::wrap_pyfunction!(do_something)(py).unwrap(); crate::py_run!(py, func, r#"func(5)"#); }); diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index a860b4d4cca..55cda1a355a 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -193,7 +193,7 @@ impl PyByteArray { /// } /// # fn main() -> PyResult<()> { /// # Python::with_gil(|py| -> PyResult<()> { - /// # let fun = wrap_pyfunction!(a_valid_function, py)?; + /// # let fun = wrap_pyfunction_bound!(a_valid_function, py)?; /// # let locals = pyo3::types::PyDict::new_bound(py); /// # locals.set_item("a_valid_function", fun)?; /// # @@ -355,7 +355,7 @@ pub trait PyByteArrayMethods<'py>: crate::sealed::Sealed { /// } /// # fn main() -> PyResult<()> { /// # Python::with_gil(|py| -> PyResult<()> { - /// # let fun = wrap_pyfunction!(a_valid_function, py)?; + /// # let fun = wrap_pyfunction_bound!(a_valid_function, py)?; /// # let locals = pyo3::types::PyDict::new_bound(py); /// # locals.set_item("a_valid_function", fun)?; /// # diff --git a/tests/test_anyhow.rs b/tests/test_anyhow.rs index 1807cfe9708..d6f5c036b74 100644 --- a/tests/test_anyhow.rs +++ b/tests/test_anyhow.rs @@ -1,8 +1,10 @@ #![cfg(feature = "anyhow")] +use pyo3::wrap_pyfunction_bound; + #[test] fn test_anyhow_py_function_ok_result() { - use pyo3::{py_run, pyfunction, wrap_pyfunction, Python}; + use pyo3::{py_run, pyfunction, Python}; #[pyfunction] #[allow(clippy::unnecessary_wraps)] @@ -11,7 +13,7 @@ fn test_anyhow_py_function_ok_result() { } Python::with_gil(|py| { - let func = wrap_pyfunction!(produce_ok_result)(py).unwrap(); + let func = wrap_pyfunction_bound!(produce_ok_result)(py).unwrap(); py_run!( py, @@ -26,7 +28,7 @@ fn test_anyhow_py_function_ok_result() { #[test] fn test_anyhow_py_function_err_result() { use pyo3::prelude::PyDictMethods; - use pyo3::{pyfunction, types::PyDict, wrap_pyfunction, Python}; + use pyo3::{pyfunction, types::PyDict, Python}; #[pyfunction] fn produce_err_result() -> anyhow::Result { @@ -34,7 +36,7 @@ fn test_anyhow_py_function_err_result() { } Python::with_gil(|py| { - let func = wrap_pyfunction!(produce_err_result)(py).unwrap(); + let func = wrap_pyfunction_bound!(produce_err_result)(py).unwrap(); let locals = PyDict::new_bound(py); locals.set_item("func", func).unwrap(); diff --git a/tests/test_bytes.rs b/tests/test_bytes.rs index cdb4ec15750..a3f1e2fcafe 100644 --- a/tests/test_bytes.rs +++ b/tests/test_bytes.rs @@ -14,7 +14,7 @@ fn bytes_pybytes_conversion(bytes: &[u8]) -> &[u8] { #[test] fn test_pybytes_bytes_conversion() { Python::with_gil(|py| { - let f = wrap_pyfunction!(bytes_pybytes_conversion)(py).unwrap(); + let f = wrap_pyfunction_bound!(bytes_pybytes_conversion)(py).unwrap(); py_assert!(py, f, "f(b'Hello World') == b'Hello World'"); }); } @@ -27,7 +27,7 @@ fn bytes_vec_conversion(py: Python<'_>, bytes: Vec) -> Bound<'_, PyBytes> { #[test] fn test_pybytes_vec_conversion() { Python::with_gil(|py| { - let f = wrap_pyfunction!(bytes_vec_conversion)(py).unwrap(); + let f = wrap_pyfunction_bound!(bytes_vec_conversion)(py).unwrap(); py_assert!(py, f, "f(b'Hello World') == b'Hello World'"); }); } @@ -35,7 +35,7 @@ fn test_pybytes_vec_conversion() { #[test] fn test_bytearray_vec_conversion() { Python::with_gil(|py| { - let f = wrap_pyfunction!(bytes_vec_conversion)(py).unwrap(); + let f = wrap_pyfunction_bound!(bytes_vec_conversion)(py).unwrap(); py_assert!(py, f, "f(bytearray(b'Hello World')) == b'Hello World'"); }); } diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index 17539fa113e..23f6a6722d4 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -30,7 +30,7 @@ fn noop_coroutine() { 42 } Python::with_gil(|gil| { - let noop = wrap_pyfunction!(noop, gil).unwrap(); + let noop = wrap_pyfunction_bound!(noop, gil).unwrap(); let test = "import asyncio; assert asyncio.run(noop()) == 42"; py_run!(gil, noop, &handle_windows(test)); }) @@ -68,7 +68,10 @@ fn test_coroutine_qualname() { let locals = [ ( "my_fn", - wrap_pyfunction!(my_fn, gil).unwrap().as_borrowed().as_any(), + wrap_pyfunction_bound!(my_fn, gil) + .unwrap() + .as_borrowed() + .as_any(), ), ("MyClass", gil.get_type_bound::().as_any()), ] @@ -93,7 +96,7 @@ fn sleep_0_like_coroutine() { .await } Python::with_gil(|gil| { - let sleep_0 = wrap_pyfunction!(sleep_0, gil).unwrap(); + let sleep_0 = wrap_pyfunction_bound!(sleep_0, gil).unwrap(); let test = "import asyncio; assert asyncio.run(sleep_0()) == 42"; py_run!(gil, sleep_0, &handle_windows(test)); }) @@ -112,7 +115,7 @@ async fn sleep(seconds: f64) -> usize { #[test] fn sleep_coroutine() { Python::with_gil(|gil| { - let sleep = wrap_pyfunction!(sleep, gil).unwrap(); + let sleep = wrap_pyfunction_bound!(sleep, gil).unwrap(); let test = r#"import asyncio; assert asyncio.run(sleep(0.1)) == 42"#; py_run!(gil, sleep, &handle_windows(test)); }) @@ -121,7 +124,7 @@ fn sleep_coroutine() { #[test] fn cancelled_coroutine() { Python::with_gil(|gil| { - let sleep = wrap_pyfunction!(sleep, gil).unwrap(); + let sleep = wrap_pyfunction_bound!(sleep, gil).unwrap(); let test = r#" import asyncio async def main(): @@ -160,7 +163,7 @@ fn coroutine_cancel_handle() { } } Python::with_gil(|gil| { - let cancellable_sleep = wrap_pyfunction!(cancellable_sleep, gil).unwrap(); + let cancellable_sleep = wrap_pyfunction_bound!(cancellable_sleep, gil).unwrap(); let test = r#" import asyncio; async def main(): @@ -192,7 +195,7 @@ fn coroutine_is_cancelled() { } } Python::with_gil(|gil| { - let sleep_loop = wrap_pyfunction!(sleep_loop, gil).unwrap(); + let sleep_loop = wrap_pyfunction_bound!(sleep_loop, gil).unwrap(); let test = r#" import asyncio; async def main(): @@ -220,7 +223,7 @@ fn coroutine_panic() { panic!("test panic"); } Python::with_gil(|gil| { - let panic = wrap_pyfunction!(panic, gil).unwrap(); + let panic = wrap_pyfunction_bound!(panic, gil).unwrap(); let test = r#" import asyncio coro = panic() diff --git a/tests/test_enum.rs b/tests/test_enum.rs index d73316e5512..63492b8d3cd 100644 --- a/tests/test_enum.rs +++ b/tests/test_enum.rs @@ -1,7 +1,7 @@ #![cfg(feature = "macros")] use pyo3::prelude::*; -use pyo3::{py_run, wrap_pyfunction}; +use pyo3::py_run; #[path = "../src/tests/common.rs"] mod common; @@ -30,7 +30,7 @@ fn return_enum() -> MyEnum { #[test] fn test_return_enum() { Python::with_gil(|py| { - let f = wrap_pyfunction!(return_enum)(py).unwrap(); + let f = wrap_pyfunction_bound!(return_enum)(py).unwrap(); let mynum = py.get_type_bound::(); py_run!(py, f mynum, "assert f() == mynum.Variant") @@ -45,7 +45,7 @@ fn enum_arg(e: MyEnum) { #[test] fn test_enum_arg() { Python::with_gil(|py| { - let f = wrap_pyfunction!(enum_arg)(py).unwrap(); + let f = wrap_pyfunction_bound!(enum_arg)(py).unwrap(); let mynum = py.get_type_bound::(); py_run!(py, f mynum, "f(mynum.OtherVariant)") diff --git a/tests/test_exceptions.rs b/tests/test_exceptions.rs index 3603586033e..ec2fe156b29 100644 --- a/tests/test_exceptions.rs +++ b/tests/test_exceptions.rs @@ -22,7 +22,7 @@ fn fail_to_open_file() -> PyResult<()> { #[cfg(not(target_os = "windows"))] fn test_filenotfounderror() { Python::with_gil(|py| { - let fail_to_open_file = wrap_pyfunction!(fail_to_open_file)(py).unwrap(); + let fail_to_open_file = wrap_pyfunction_bound!(fail_to_open_file)(py).unwrap(); py_run!( py, @@ -68,7 +68,7 @@ fn call_fail_with_custom_error() -> PyResult<()> { fn test_custom_error() { Python::with_gil(|py| { let call_fail_with_custom_error = - wrap_pyfunction!(call_fail_with_custom_error)(py).unwrap(); + wrap_pyfunction_bound!(call_fail_with_custom_error)(py).unwrap(); py_run!( py, diff --git a/tests/test_macros.rs b/tests/test_macros.rs index 0d2b125b870..6a50e5b36e4 100644 --- a/tests/test_macros.rs +++ b/tests/test_macros.rs @@ -76,7 +76,7 @@ fn test_macro_rules_interactions() { let my_base = py.get_type_bound::(); py_assert!(py, my_base, "my_base.__name__ == 'MyClass'"); - let my_func = wrap_pyfunction!(my_function_in_macro, py).unwrap(); + let my_func = wrap_pyfunction_bound!(my_function_in_macro, py).unwrap(); py_assert!( py, my_func, diff --git a/tests/test_methods.rs b/tests/test_methods.rs index a9e7d37d64a..e7eacf26b40 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -1124,7 +1124,7 @@ fn test_option_pyclass_arg() { } Python::with_gil(|py| { - let f = wrap_pyfunction!(option_class_arg, py).unwrap(); + let f = wrap_pyfunction_bound!(option_class_arg, py).unwrap(); assert!(f.call0().unwrap().is_none()); let obj = Py::new(py, SomePyClass {}).unwrap(); assert!(f diff --git a/tests/test_pyfunction.rs b/tests/test_pyfunction.rs index 32c3ede6309..5c4350467c4 100644 --- a/tests/test_pyfunction.rs +++ b/tests/test_pyfunction.rs @@ -23,7 +23,7 @@ fn optional_bool(arg: Option) -> String { fn test_optional_bool() { // Regression test for issue #932 Python::with_gil(|py| { - let f = wrap_pyfunction!(optional_bool)(py).unwrap(); + let f = wrap_pyfunction_bound!(optional_bool)(py).unwrap(); py_assert!(py, f, "f() == 'Some(true)'"); py_assert!(py, f, "f(True) == 'Some(true)'"); @@ -47,7 +47,7 @@ fn buffer_inplace_add(py: Python<'_>, x: PyBuffer, y: PyBuffer) { #[test] fn test_buffer_add() { Python::with_gil(|py| { - let f = wrap_pyfunction!(buffer_inplace_add)(py).unwrap(); + let f = wrap_pyfunction_bound!(buffer_inplace_add)(py).unwrap(); py_expect_exception!( py, @@ -89,8 +89,8 @@ fn function_with_pycfunction_arg(fun: &PyCFunction) -> PyResult<&PyAny> { #[test] fn test_functions_with_function_args() { Python::with_gil(|py| { - let py_cfunc_arg = wrap_pyfunction!(function_with_pycfunction_arg)(py).unwrap(); - let bool_to_string = wrap_pyfunction!(optional_bool)(py).unwrap(); + let py_cfunc_arg = wrap_pyfunction_bound!(function_with_pycfunction_arg)(py).unwrap(); + let bool_to_string = wrap_pyfunction_bound!(optional_bool)(py).unwrap(); pyo3::py_run!( py, @@ -103,7 +103,7 @@ fn test_functions_with_function_args() { #[cfg(not(any(Py_LIMITED_API, PyPy)))] { - let py_func_arg = wrap_pyfunction!(function_with_pyfunction_arg)(py).unwrap(); + let py_func_arg = wrap_pyfunction_bound!(function_with_pyfunction_arg)(py).unwrap(); pyo3::py_run!( py, @@ -137,7 +137,7 @@ fn function_with_custom_conversion( #[test] fn test_function_with_custom_conversion() { Python::with_gil(|py| { - let custom_conv_func = wrap_pyfunction!(function_with_custom_conversion)(py).unwrap(); + let custom_conv_func = wrap_pyfunction_bound!(function_with_custom_conversion)(py).unwrap(); pyo3::py_run!( py, @@ -156,7 +156,7 @@ fn test_function_with_custom_conversion() { #[test] fn test_function_with_custom_conversion_error() { Python::with_gil(|py| { - let custom_conv_func = wrap_pyfunction!(function_with_custom_conversion)(py).unwrap(); + let custom_conv_func = wrap_pyfunction_bound!(function_with_custom_conversion)(py).unwrap(); py_expect_exception!( py, @@ -190,13 +190,13 @@ fn test_from_py_with_defaults() { } Python::with_gil(|py| { - let f = wrap_pyfunction!(from_py_with_option)(py).unwrap(); + let f = wrap_pyfunction_bound!(from_py_with_option)(py).unwrap(); assert_eq!(f.call0().unwrap().extract::().unwrap(), 0); assert_eq!(f.call1((123,)).unwrap().extract::().unwrap(), 123); assert_eq!(f.call1((999,)).unwrap().extract::().unwrap(), 999); - let f2 = wrap_pyfunction!(from_py_with_default)(py).unwrap(); + let f2 = wrap_pyfunction_bound!(from_py_with_default)(py).unwrap(); assert_eq!(f2.call0().unwrap().extract::().unwrap(), 0); assert_eq!(f2.call1(("123",)).unwrap().extract::().unwrap(), 3); @@ -228,7 +228,7 @@ fn conversion_error( #[test] fn test_conversion_error() { Python::with_gil(|py| { - let conversion_error = wrap_pyfunction!(conversion_error)(py).unwrap(); + let conversion_error = wrap_pyfunction_bound!(conversion_error)(py).unwrap(); py_expect_exception!( py, conversion_error, @@ -473,12 +473,12 @@ fn use_pyfunction() { use function_in_module::foo; // check imported name can be wrapped - let f = wrap_pyfunction!(foo, py).unwrap(); + let f = wrap_pyfunction_bound!(foo, py).unwrap(); assert_eq!(f.call1((5,)).unwrap().extract::().unwrap(), 5); assert_eq!(f.call1((42,)).unwrap().extract::().unwrap(), 42); // check path import can be wrapped - let f2 = wrap_pyfunction!(function_in_module::foo, py).unwrap(); + let f2 = wrap_pyfunction_bound!(function_in_module::foo, py).unwrap(); assert_eq!(f2.call1((5,)).unwrap().extract::().unwrap(), 5); assert_eq!(f2.call1((42,)).unwrap().extract::().unwrap(), 42); }) @@ -506,7 +506,7 @@ fn return_value_borrows_from_arguments<'py>( #[test] fn test_return_value_borrows_from_arguments() { Python::with_gil(|py| { - let function = wrap_pyfunction!(return_value_borrows_from_arguments, py).unwrap(); + let function = wrap_pyfunction_bound!(return_value_borrows_from_arguments, py).unwrap(); let key = Py::new(py, Key("key".to_owned())).unwrap(); let value = Py::new(py, Value(42)).unwrap(); @@ -530,7 +530,7 @@ fn test_some_wrap_arguments() { } Python::with_gil(|py| { - let function = wrap_pyfunction!(some_wrap_arguments, py).unwrap(); + let function = wrap_pyfunction_bound!(some_wrap_arguments, py).unwrap(); py_assert!(py, function, "function() == [1, 2, None, None]"); }) } @@ -546,7 +546,7 @@ fn test_reference_to_bound_arguments() { } Python::with_gil(|py| { - let function = wrap_pyfunction!(reference_args, py).unwrap(); + let function = wrap_pyfunction_bound!(reference_args, py).unwrap(); py_assert!(py, function, "function(1) == 1"); py_assert!(py, function, "function(1, 2) == 3"); }) diff --git a/tests/test_string.rs b/tests/test_string.rs index 02bf2ecd4df..d90c5a81b83 100644 --- a/tests/test_string.rs +++ b/tests/test_string.rs @@ -11,7 +11,7 @@ fn take_str(_s: &str) {} #[test] fn test_unicode_encode_error() { Python::with_gil(|py| { - let take_str = wrap_pyfunction!(take_str)(py).unwrap(); + let take_str = wrap_pyfunction_bound!(take_str)(py).unwrap(); py_expect_exception!( py, take_str, diff --git a/tests/test_text_signature.rs b/tests/test_text_signature.rs index 0b93500db7e..32a78346e9a 100644 --- a/tests/test_text_signature.rs +++ b/tests/test_text_signature.rs @@ -101,7 +101,7 @@ fn test_function() { } Python::with_gil(|py| { - let f = wrap_pyfunction!(my_function)(py).unwrap(); + let f = wrap_pyfunction_bound!(my_function)(py).unwrap(); py_assert!(py, f, "f.__text_signature__ == '(a, b=None, *, c=42)'"); }); @@ -147,42 +147,42 @@ fn test_auto_test_signature_function() { } Python::with_gil(|py| { - let f = wrap_pyfunction!(my_function)(py).unwrap(); + let f = wrap_pyfunction_bound!(my_function)(py).unwrap(); py_assert!( py, f, "f.__text_signature__ == '(a, b, c)', f.__text_signature__" ); - let f = wrap_pyfunction!(my_function_2)(py).unwrap(); + let f = wrap_pyfunction_bound!(my_function_2)(py).unwrap(); py_assert!( py, f, "f.__text_signature__ == '($module, a, b, c)', f.__text_signature__" ); - let f = wrap_pyfunction!(my_function_3)(py).unwrap(); + let f = wrap_pyfunction_bound!(my_function_3)(py).unwrap(); py_assert!( py, f, "f.__text_signature__ == '(a, /, b=None, *, c=5)', f.__text_signature__" ); - let f = wrap_pyfunction!(my_function_4)(py).unwrap(); + let f = wrap_pyfunction_bound!(my_function_4)(py).unwrap(); py_assert!( py, f, "f.__text_signature__ == '(a, /, b=None, *args, c, d=5, **kwargs)', f.__text_signature__" ); - let f = wrap_pyfunction!(my_function_5)(py).unwrap(); + let f = wrap_pyfunction_bound!(my_function_5)(py).unwrap(); py_assert!( py, f, "f.__text_signature__ == '(a=1, /, b=None, c=1.5, d=5, e=\"pyo3\", f=\\'f\\', h=True)', f.__text_signature__" ); - let f = wrap_pyfunction!(my_function_6)(py).unwrap(); + let f = wrap_pyfunction_bound!(my_function_6)(py).unwrap(); py_assert!( py, f, @@ -317,10 +317,10 @@ fn test_auto_test_signature_opt_out() { } Python::with_gil(|py| { - let f = wrap_pyfunction!(my_function)(py).unwrap(); + let f = wrap_pyfunction_bound!(my_function)(py).unwrap(); py_assert!(py, f, "f.__text_signature__ == None"); - let f = wrap_pyfunction!(my_function_2)(py).unwrap(); + let f = wrap_pyfunction_bound!(my_function_2)(py).unwrap(); py_assert!(py, f, "f.__text_signature__ == None"); let cls = py.get_type_bound::(); diff --git a/tests/test_various.rs b/tests/test_various.rs index 250f39834d1..0e619f49a28 100644 --- a/tests/test_various.rs +++ b/tests/test_various.rs @@ -56,7 +56,7 @@ fn return_custom_class() { assert_eq!(get_zero().value, 0); // Using from python - let get_zero = wrap_pyfunction!(get_zero)(py).unwrap(); + let get_zero = wrap_pyfunction_bound!(get_zero)(py).unwrap(); py_assert!(py, get_zero, "get_zero().value == 0"); }); } @@ -201,6 +201,6 @@ fn result_conversion_function() -> Result<(), MyError> { #[test] fn test_result_conversion() { Python::with_gil(|py| { - wrap_pyfunction!(result_conversion_function)(py).unwrap(); + wrap_pyfunction_bound!(result_conversion_function)(py).unwrap(); }); } diff --git a/tests/test_wrap_pyfunction_deduction.rs b/tests/test_wrap_pyfunction_deduction.rs index 52c9adcb6d7..845cf2a39d7 100644 --- a/tests/test_wrap_pyfunction_deduction.rs +++ b/tests/test_wrap_pyfunction_deduction.rs @@ -11,6 +11,7 @@ pub fn add_wrapped(wrapper: &impl Fn(Python<'_>) -> PyResult<&PyCFunction>) { #[test] fn wrap_pyfunction_deduction() { + #[allow(deprecated)] add_wrapped(wrap_pyfunction!(f)); } diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index c062cead3ad..1f3cc302c61 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -72,3 +72,19 @@ fn module_bound_by_value(m: Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(double, &m)?)?; Ok(()) } + +fn test_wrap_pyfunction(py: Python<'_>, m: &Bound<'_, PyModule>) { + // should lint + let _ = wrap_pyfunction!(double, py); + + // should lint but currently does not + let _ = wrap_pyfunction!(double)(py); + + // should not lint + let _ = wrap_pyfunction!(double, m); + let _ = wrap_pyfunction!(double)(m); + let _ = wrap_pyfunction!(double, m.as_gil_ref()); + let _ = wrap_pyfunction!(double)(m.as_gil_ref()); + let _ = wrap_pyfunction_bound!(double, py); + let _ = wrap_pyfunction_bound!(double)(py); +} diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index edb74f97b74..d4a3e01f99f 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -45,3 +45,11 @@ error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref` | 53 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { | ^ + +error: use of deprecated method `pyo3::methods::Extractor::>::is_python`: use `wrap_pyfunction_bound!` instead + --> tests/ui/deprecations.rs:78:13 + | +78 | let _ = wrap_pyfunction!(double, py); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/invalid_result_conversion.rs b/tests/ui/invalid_result_conversion.rs index 4cf3e0bd8fc..373d3cacd9d 100644 --- a/tests/ui/invalid_result_conversion.rs +++ b/tests/ui/invalid_result_conversion.rs @@ -20,11 +20,13 @@ impl fmt::Display for MyError { #[pyfunction] fn should_not_work() -> Result<(), MyError> { - Err(MyError { descr: "something went wrong" }) + Err(MyError { + descr: "something went wrong", + }) } fn main() { - Python::with_gil(|py|{ - wrap_pyfunction!(should_not_work)(py); + Python::with_gil(|py| { + wrap_pyfunction_bound!(should_not_work)(py); }); } From 7cde95bba4f672b4840dcd8543e3da0076bd49ec Mon Sep 17 00:00:00 2001 From: Thomas Tanon Date: Tue, 12 Mar 2024 23:57:31 +0100 Subject: [PATCH 216/349] Documents experimental-declarative-modules feature (#3953) * Documents experimental-declarative-modules feature * More details on experimental-declarative-modules progress --- guide/src/features.md | 6 +++++ guide/src/module.md | 52 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/guide/src/features.md b/guide/src/features.md index 6e4e5ab70b1..0816770a781 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -57,6 +57,12 @@ This feature adds support for `async fn` in `#[pyfunction]` and `#[pymethods]`. The feature has some unfinished refinements and performance improvements. To help finish this off, see [issue #1632](https://github.com/PyO3/pyo3/issues/1632) and its associated draft PRs. +### `experimental-declarative-modules` + +This feature allows to declare Python modules using `#[pymodule] mod my_module { ... }` syntax. + +The feature has some unfinished refinements and edge cases. To help finish this off, see [issue #3900](https://github.com/PyO3/pyo3/issues/3900). + ### `experimental-inspect` This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types. diff --git a/guide/src/module.md b/guide/src/module.md index 8cac9a5be4c..c9c7f78aaf5 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -105,3 +105,55 @@ submodules by using `from parent_module import child_module`. For more informati [#1517](https://github.com/PyO3/pyo3/issues/1517#issuecomment-808664021). It is not necessary to add `#[pymodule]` on nested modules, which is only required on the top-level module. + +## Declarative modules (experimental) + +Another syntax based on Rust inline modules is also available to declare modules. +The `experimental-declarative-modules` feature must be enabled to use it. + +For example: +```rust +# #[cfg(feature = "experimental-declarative-modules")] +# mod declarative_module_test { +use pyo3::prelude::*; + +#[pyfunction] +fn double(x: usize) -> usize { + x * 2 +} + +#[pymodule] +mod my_extension { + use super::*; + + #[pymodule_export] + use super::double; // Exports the double function as part of the module + + #[pyfunction] // This will be part of the module + fn triple(x: usize) -> usize { + x * 3 + } + + #[pyclass] // This will be part of the module + struct Unit; + + #[pymodule] + mod submodule { + // This is a submodule + } + + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + // Arbitrary code to run at the module initialization + m.add("double2", m.getattr("double")?)?; + Ok(()) + } +} +# } +``` + +Some changes are planned to this feature before stabilization, like automatically +filling submodules into `sys.modules` to allow easier imports (see [issue #759](https://github.com/PyO3/pyo3/issues/759)) +and filling the `module` argument of inlined `#[pyclass]` automatically with the proper module name. +Macro names might also change. +See [issue #3900](https://github.com/PyO3/pyo3/issues/3900) to track this feature progress. From 989d2c53ab2e2c0cfb36c1ab86e943e7486ef470 Mon Sep 17 00:00:00 2001 From: Bruno Kolenbrander <59372212+mejrs@users.noreply.github.com> Date: Tue, 12 Mar 2024 23:59:33 +0100 Subject: [PATCH 217/349] Fix non_local_definitions lint triggers (#3955) --- pyo3-macros-backend/src/frompyobject.rs | 34 +++++++--------- pyo3-macros-backend/src/module.rs | 35 +++++++--------- pyo3-macros-backend/src/pyclass.rs | 54 ++++++++++--------------- pyo3-macros-backend/src/pyfunction.rs | 14 +++---- pyo3-macros-backend/src/pyimpl.rs | 18 ++++----- src/tests/hygiene/misc.rs | 1 + 6 files changed, 63 insertions(+), 93 deletions(-) diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 24471c1aae8..68ef72ea157 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -310,7 +310,7 @@ impl<'a> Container<'a> { } }); quote!( - match obj.extract() { + match #pyo3_path::types::PyAnyMethods::extract(obj) { ::std::result::Result::Ok((#(#field_idents),*)) => ::std::result::Result::Ok(#self_ty(#(#fields),*)), ::std::result::Result::Err(err) => ::std::result::Result::Err(err), } @@ -327,27 +327,29 @@ impl<'a> Container<'a> { let field_name = ident.to_string(); let getter = match field.getter.as_ref().unwrap_or(&FieldGetter::GetAttr(None)) { FieldGetter::GetAttr(Some(name)) => { - quote!(getattr(#pyo3_path::intern!(obj.py(), #name))) + quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) } FieldGetter::GetAttr(None) => { - quote!(getattr(#pyo3_path::intern!(obj.py(), #field_name))) + quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #field_name))) } FieldGetter::GetItem(Some(syn::Lit::Str(key))) => { - quote!(get_item(#pyo3_path::intern!(obj.py(), #key))) + quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #pyo3_path::intern!(obj.py(), #key))) + } + FieldGetter::GetItem(Some(key)) => { + quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #key)) } - FieldGetter::GetItem(Some(key)) => quote!(get_item(#key)), FieldGetter::GetItem(None) => { - quote!(get_item(#pyo3_path::intern!(obj.py(), #field_name))) + quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #pyo3_path::intern!(obj.py(), #field_name))) } }; let extractor = match &field.from_py_with { None => { - quote!(#pyo3_path::impl_::frompyobject::extract_struct_field(&obj.#getter?, #struct_name, #field_name)?) + quote!(#pyo3_path::impl_::frompyobject::extract_struct_field(&#getter?, #struct_name, #field_name)?) } Some(FromPyWithAttribute { value: expr_path, .. }) => { - quote! (#pyo3_path::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, &obj.#getter?, #struct_name, #field_name)?) + quote! (#pyo3_path::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, &#getter?, #struct_name, #field_name)?) } }; @@ -609,17 +611,11 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { let ident = &tokens.ident; Ok(quote!( - // FIXME https://github.com/PyO3/pyo3/issues/3903 - #[allow(unknown_lints, non_local_definitions)] - const _: () = { - use #pyo3_path::prelude::PyAnyMethods; - - #[automatically_derived] - impl #trait_generics #pyo3_path::FromPyObject<#lt_param> for #ident #generics #where_clause { - fn extract_bound(obj: &#pyo3_path::Bound<#lt_param, #pyo3_path::PyAny>) -> #pyo3_path::PyResult { - #derives - } + #[automatically_derived] + impl #trait_generics #pyo3_path::FromPyObject<#lt_param> for #ident #generics #where_clause { + fn extract_bound(obj: &#pyo3_path::Bound<#lt_param, #pyo3_path::PyAny>) -> #pyo3_path::PyResult { + #derives } - }; + } )) } diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index fb02c074996..1cc3e836404 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -295,7 +295,7 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result if function.sig.inputs.len() == 2 { module_args.push(quote!(module.py())); } - module_args.push(quote!(::std::convert::Into::into(BoundRef(module)))); + module_args.push(quote!(::std::convert::Into::into(#pyo3_path::methods::BoundRef(module)))); let extractors = function .sig @@ -330,29 +330,22 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result // this avoids complications around the fact that the generated module has a different scope // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pymodule] is // inside a function body) - // FIXME https://github.com/PyO3/pyo3/issues/3903 - #[allow(unknown_lints, non_local_definitions)] - const _: () = { - use #pyo3_path::impl_::pymodule as impl_; - use #pyo3_path::impl_::pymethods::BoundRef; - - fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { - #ident(#(#module_args),*) - } + impl #ident::MakeDef { + const fn make_def() -> #pyo3_path::impl_::pymodule::ModuleDef { + fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { + #ident(#(#module_args),*) + } - impl #ident::MakeDef { - const fn make_def() -> impl_::ModuleDef { - const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule); - unsafe { - impl_::ModuleDef::new( - #ident::__PYO3_NAME, - #doc, - INITIALIZER - ) - } + const INITIALIZER: #pyo3_path::impl_::pymodule::ModuleInitializer = #pyo3_path::impl_::pymodule::ModuleInitializer(__pyo3_pymodule); + unsafe { + #pyo3_path::impl_::pymodule::ModuleDef::new( + #ident::__PYO3_NAME, + #doc, + INITIALIZER + ) } } - }; + } }) } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 3eca80861b7..27a7ee969d5 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -366,15 +366,11 @@ fn impl_class( .impl_all(ctx)?; Ok(quote! { - // FIXME https://github.com/PyO3/pyo3/issues/3903 - #[allow(unknown_lints, non_local_definitions)] - const _: () = { - impl #pyo3_path::types::DerefToPyAny for #cls {} + impl #pyo3_path::types::DerefToPyAny for #cls {} - #pytypeinfo_impl + #pytypeinfo_impl - #py_class_impl - }; + #py_class_impl }) } @@ -794,21 +790,17 @@ fn impl_simple_enum( .impl_all(ctx)?; Ok(quote! { - // FIXME https://github.com/PyO3/pyo3/issues/3903 - #[allow(unknown_lints, non_local_definitions)] - const _: () = { - #pytypeinfo + #pytypeinfo - #pyclass_impls + #pyclass_impls - #[doc(hidden)] - #[allow(non_snake_case)] - impl #cls { - #default_repr - #default_int - #default_richcmp - } - }; + #[doc(hidden)] + #[allow(non_snake_case)] + impl #cls { + #default_repr + #default_int + #default_richcmp + } }) } @@ -933,25 +925,21 @@ fn impl_complex_enum( } Ok(quote! { - // FIXME https://github.com/PyO3/pyo3/issues/3903 - #[allow(unknown_lints, non_local_definitions)] - const _: () = { - #pytypeinfo + #pytypeinfo - #pyclass_impls + #pyclass_impls - #[doc(hidden)] - #[allow(non_snake_case)] - impl #cls {} + #[doc(hidden)] + #[allow(non_snake_case)] + impl #cls {} - #(#variant_cls_zsts)* + #(#variant_cls_zsts)* - #(#variant_cls_pytypeinfos)* + #(#variant_cls_pytypeinfos)* - #(#variant_cls_pyclass_impls)* + #(#variant_cls_pyclass_impls)* - #(#variant_cls_impls)* - }; + #(#variant_cls_impls)* }) } diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 4b1e0eadeba..cce9f74824c 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -282,16 +282,12 @@ pub fn impl_wrap_pyfunction( // this avoids complications around the fact that the generated module has a different scope // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pyfunction] is // inside a function body) - // FIXME https://github.com/PyO3/pyo3/issues/3903 - #[allow(unknown_lints, non_local_definitions)] - const _: () = { - impl #name::MakeDef { - const DEF: #pyo3_path::impl_::pymethods::PyMethodDef = #methoddef; - } + impl #name::MakeDef { + const DEF: #pyo3_path::impl_::pymethods::PyMethodDef = #methoddef; + } - #[allow(non_snake_case)] - #wrapper - }; + #[allow(non_snake_case)] + #wrapper }; Ok(wrapped_pyfunction) } diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index 30a6d6dd17e..cf27cf37066 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -170,19 +170,15 @@ pub fn impl_methods( }; Ok(quote! { - // FIXME https://github.com/PyO3/pyo3/issues/3903 - #[allow(unknown_lints, non_local_definitions)] - const _: () = { - #(#trait_impls)* + #(#trait_impls)* - #items + #items - #[doc(hidden)] - #[allow(non_snake_case)] - impl #ty { - #(#associated_methods)* - } - }; + #[doc(hidden)] + #[allow(non_snake_case)] + impl #ty { + #(#associated_methods)* + } }) } diff --git a/src/tests/hygiene/misc.rs b/src/tests/hygiene/misc.rs index 66db7f3a28a..24dad7ec196 100644 --- a/src/tests/hygiene/misc.rs +++ b/src/tests/hygiene/misc.rs @@ -14,6 +14,7 @@ struct Derive2(i32, i32); // tuple case #[allow(dead_code)] struct Derive3 { f: i32, + #[pyo3(item(42))] g: i32, } // struct case From 5c86dc35c1108591e553ff3829bca88678d3ccf8 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 15 Mar 2024 08:53:52 +0100 Subject: [PATCH 218/349] allow borrowed extracts with `gil-refs` disabled (#3959) --- src/conversion.rs | 10 ++++++---- src/conversions/std/slice.rs | 13 ++++++------- src/conversions/std/string.rs | 4 ++-- src/err/mod.rs | 23 ++++++++++++++++------- src/instance.rs | 14 ++++++++++++++ src/types/any.rs | 14 ++++++++------ src/types/bytes.rs | 2 +- src/types/string.rs | 2 +- 8 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index 8d4ad776837..dfa53eac83e 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -6,7 +6,9 @@ use crate::pyclass::boolean_struct::False; use crate::type_object::PyTypeInfo; use crate::types::any::PyAnyMethods; use crate::types::PyTuple; -use crate::{ffi, gil, Bound, Py, PyAny, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python}; +use crate::{ + ffi, gil, Borrowed, Bound, Py, PyAny, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, +}; use std::ptr::NonNull; /// Returns a borrowed pointer to a Python object. @@ -288,7 +290,7 @@ pub trait FromPyObjectBound<'a, 'py>: Sized + from_py_object_bound_sealed::Seale /// /// Users are advised against calling this method directly: instead, use this via /// [`Bound<'_, PyAny>::extract`] or [`Py::extract`]. - fn from_py_object_bound(ob: &'a Bound<'py, PyAny>) -> PyResult; + fn from_py_object_bound(ob: Borrowed<'a, 'py, PyAny>) -> PyResult; /// Extracts the type hint information for this type when it appears as an argument. /// @@ -307,8 +309,8 @@ impl<'py, T> FromPyObjectBound<'_, 'py> for T where T: FromPyObject<'py>, { - fn from_py_object_bound(ob: &Bound<'py, PyAny>) -> PyResult { - Self::extract_bound(ob) + fn from_py_object_bound(ob: Borrowed<'_, 'py, PyAny>) -> PyResult { + Self::extract_bound(&ob) } #[cfg(feature = "experimental-inspect")] diff --git a/src/conversions/std/slice.rs b/src/conversions/std/slice.rs index 18bb52cdd1a..b3932302ef3 100644 --- a/src/conversions/std/slice.rs +++ b/src/conversions/std/slice.rs @@ -2,11 +2,9 @@ use std::borrow::Cow; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; -#[cfg(not(feature = "gil-refs"))] -use crate::types::PyBytesMethods; use crate::{ - types::{PyAnyMethods, PyByteArray, PyByteArrayMethods, PyBytes}, - Bound, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, + types::{PyByteArray, PyByteArrayMethods, PyBytes}, + IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject, }; impl<'a> IntoPy for &'a [u8] { @@ -34,7 +32,7 @@ impl<'py> crate::FromPyObject<'py> for &'py [u8] { #[cfg(not(feature = "gil-refs"))] impl<'a> crate::conversion::FromPyObjectBound<'a, '_> for &'a [u8] { - fn from_py_object_bound(obj: &'a Bound<'_, PyAny>) -> PyResult { + fn from_py_object_bound(obj: crate::Borrowed<'a, '_, PyAny>) -> PyResult { Ok(obj.downcast::()?.as_bytes()) } @@ -51,7 +49,8 @@ impl<'a> crate::conversion::FromPyObjectBound<'a, '_> for &'a [u8] { /// If it is a `bytearray`, its contents will be copied to an owned `Cow`. #[cfg(feature = "gil-refs")] impl<'py> crate::FromPyObject<'py> for Cow<'py, [u8]> { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { + fn extract_bound(ob: &crate::Bound<'py, PyAny>) -> PyResult { + use crate::types::PyAnyMethods; if let Ok(bytes) = ob.downcast::() { return Ok(Cow::Borrowed(bytes.clone().into_gil_ref().as_bytes())); } @@ -63,7 +62,7 @@ impl<'py> crate::FromPyObject<'py> for Cow<'py, [u8]> { #[cfg(not(feature = "gil-refs"))] impl<'a> crate::conversion::FromPyObjectBound<'a, '_> for Cow<'a, [u8]> { - fn from_py_object_bound(ob: &'a Bound<'_, PyAny>) -> PyResult { + fn from_py_object_bound(ob: crate::Borrowed<'a, '_, PyAny>) -> PyResult { if let Ok(bytes) = ob.downcast::() { return Ok(Cow::Borrowed(bytes.as_bytes())); } diff --git a/src/conversions/std/string.rs b/src/conversions/std/string.rs index d013890a77e..9c276d1d3d9 100644 --- a/src/conversions/std/string.rs +++ b/src/conversions/std/string.rs @@ -128,7 +128,7 @@ impl<'py> FromPyObject<'py> for &'py str { #[cfg(all(not(feature = "gil-refs"), any(Py_3_10, not(Py_LIMITED_API))))] impl<'a> crate::conversion::FromPyObjectBound<'a, '_> for &'a str { - fn from_py_object_bound(ob: &'a Bound<'_, PyAny>) -> PyResult { + fn from_py_object_bound(ob: crate::Borrowed<'a, '_, PyAny>) -> PyResult { ob.downcast::()?.to_str() } @@ -152,7 +152,7 @@ impl<'py> FromPyObject<'py> for Cow<'py, str> { #[cfg(not(feature = "gil-refs"))] impl<'a> crate::conversion::FromPyObjectBound<'a, '_> for Cow<'a, str> { - fn from_py_object_bound(ob: &'a Bound<'_, PyAny>) -> PyResult { + fn from_py_object_bound(ob: crate::Borrowed<'a, '_, PyAny>) -> PyResult { ob.downcast::()?.to_cow() } diff --git a/src/err/mod.rs b/src/err/mod.rs index 7610f49951c..de1e621fc13 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -7,7 +7,7 @@ use crate::{ exceptions::{self, PyBaseException}, ffi, }; -use crate::{IntoPy, Py, PyAny, PyNativeType, PyObject, Python, ToPyObject}; +use crate::{Borrowed, IntoPy, Py, PyAny, PyNativeType, PyObject, Python, ToPyObject}; use std::borrow::Cow; use std::cell::UnsafeCell; use std::ffi::CString; @@ -65,17 +65,16 @@ impl<'a> PyDowncastError<'a> { /// Compatibility API to convert the Bound variant `DowncastError` into the /// gil-ref variant pub(crate) fn from_downcast_err(DowncastError { from, to }: DowncastError<'a, 'a>) -> Self { - Self { - from: from.as_gil_ref(), - to, - } + #[allow(deprecated)] + let from = unsafe { from.py().from_borrowed_ptr(from.as_ptr()) }; + Self { from, to } } } /// Error that indicates a failure to convert a PyAny to a more specific Python type. #[derive(Debug)] pub struct DowncastError<'a, 'py> { - from: &'a Bound<'py, PyAny>, + from: Borrowed<'a, 'py, PyAny>, to: Cow<'static, str>, } @@ -83,6 +82,16 @@ impl<'a, 'py> DowncastError<'a, 'py> { /// Create a new `PyDowncastError` representing a failure to convert the object /// `from` into the type named in `to`. pub fn new(from: &'a Bound<'py, PyAny>, to: impl Into>) -> Self { + DowncastError { + from: from.as_borrowed(), + to: to.into(), + } + } + #[cfg(not(feature = "gil-refs"))] + pub(crate) fn new_from_borrowed( + from: Borrowed<'a, 'py, PyAny>, + to: impl Into>, + ) -> Self { DowncastError { from, to: to.into(), @@ -1036,7 +1045,7 @@ impl std::error::Error for DowncastError<'_, '_> {} impl std::fmt::Display for DowncastError<'_, '_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - display_downcast_error(f, self.from, &self.to) + display_downcast_error(f, &self.from, &self.to) } } diff --git a/src/instance.rs b/src/instance.rs index 069bea69f8c..83775708e1a 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -579,6 +579,20 @@ impl<'a, 'py> Borrowed<'a, 'py, PyAny> { Self(NonNull::new_unchecked(ptr), PhantomData, py) } + #[inline] + #[cfg(not(feature = "gil-refs"))] + pub(crate) fn downcast(self) -> Result, DowncastError<'a, 'py>> + where + T: PyTypeCheck, + { + if T::type_check(&self) { + // Safety: type_check is responsible for ensuring that the type is correct + Ok(unsafe { self.downcast_unchecked() }) + } else { + Err(DowncastError::new_from_borrowed(self, T::NAME)) + } + } + /// Converts this `PyAny` to a concrete Python type without checking validity. /// /// # Safety diff --git a/src/types/any.rs b/src/types/any.rs index e4dbaf80200..19855abbb9a 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1,5 +1,5 @@ use crate::class::basic::CompareOp; -use crate::conversion::{AsPyPointer, FromPyObject, FromPyObjectBound, IntoPy, ToPyObject}; +use crate::conversion::{AsPyPointer, FromPyObjectBound, IntoPy, ToPyObject}; use crate::err::{DowncastError, DowncastIntoError, PyDowncastError, PyErr, PyResult}; use crate::exceptions::{PyAttributeError, PyTypeError}; use crate::ffi_ptr_ext::FfiPtrExt; @@ -799,13 +799,14 @@ impl PyAny { /// Extracts some type from the Python object. /// - /// This is a wrapper function around [`FromPyObject::extract()`]. + /// This is a wrapper function around + /// [`FromPyObject::extract()`](crate::FromPyObject::extract). #[inline] pub fn extract<'py, D>(&'py self) -> PyResult where - D: FromPyObject<'py>, + D: FromPyObjectBound<'py, 'py>, { - FromPyObject::extract(self) + FromPyObjectBound::from_py_object_bound(self.as_borrowed()) } /// Returns the reference count for the Python object. @@ -1641,7 +1642,8 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// Extracts some type from the Python object. /// - /// This is a wrapper function around [`FromPyObject::extract()`]. + /// This is a wrapper function around + /// [`FromPyObject::extract()`](crate::FromPyObject::extract). fn extract<'a, T>(&'a self) -> PyResult where T: FromPyObjectBound<'a, 'py>; @@ -2182,7 +2184,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { where T: FromPyObjectBound<'a, 'py>, { - FromPyObjectBound::from_py_object_bound(self) + FromPyObjectBound::from_py_object_bound(self.as_borrowed()) } fn get_refcnt(&self) -> isize { diff --git a/src/types/bytes.rs b/src/types/bytes.rs index 55c295c416f..44c87560514 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -158,7 +158,7 @@ impl<'py> PyBytesMethods<'py> for Bound<'py, PyBytes> { impl<'a> Borrowed<'a, '_, PyBytes> { /// Gets the Python string as a byte slice. #[allow(clippy::wrong_self_convention)] - fn as_bytes(self) -> &'a [u8] { + pub(crate) fn as_bytes(self) -> &'a [u8] { unsafe { let buffer = ffi::PyBytes_AsString(self.as_ptr()) as *const u8; let length = ffi::PyBytes_Size(self.as_ptr()) as usize; diff --git a/src/types/string.rs b/src/types/string.rs index 93f3682c3a4..81c41adb545 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -369,7 +369,7 @@ impl<'a> Borrowed<'a, '_, PyString> { } #[allow(clippy::wrong_self_convention)] - fn to_cow(self) -> PyResult> { + pub(crate) fn to_cow(self) -> PyResult> { // TODO: this method can probably be deprecated once Python 3.9 support is dropped, // because all versions then support the more efficient `to_str`. #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] From dcba984b51be253cb5d385e72008ee9578807504 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 15 Mar 2024 10:25:27 +0000 Subject: [PATCH 219/349] deprecate `GILPool` (#3947) * deprecate `GILPool` * review: adamreichold * fix deprecation warnings in tests --- guide/src/memory.md | 14 +++++++++++++- guide/src/migration.md | 2 ++ newsfragments/3947.changed.md | 1 + src/gil.rs | 30 +++++++++++++++++++++--------- src/impl_/trampoline.rs | 8 +++++++- src/lib.rs | 1 + src/marker.rs | 32 ++++++++++++++++++++++++++------ tests/ui/deprecations.rs | 1 + tests/ui/deprecations.stderr | 32 ++++++++++++++++---------------- 9 files changed, 88 insertions(+), 33 deletions(-) create mode 100644 newsfragments/3947.changed.md diff --git a/guide/src/memory.md b/guide/src/memory.md index fe98184e3e1..78f9f348f40 100644 --- a/guide/src/memory.md +++ b/guide/src/memory.md @@ -83,6 +83,13 @@ bound to the `GILPool`, not the for loop. The `GILPool` isn't dropped until the end of the `with_gil()` closure, at which point the 10 copies of `hello` are finally released to the Python garbage collector. +
+⚠️ Warning: `GILPool` is no longer the preferred way to manage memory with PyO3 🛠️ + +PyO3 0.21 has introduced a new API known as the Bound API, which doesn't have the same surprising results. Instead, each `Bound` smart pointer releases the Python reference immediately on drop. See [the smart pointer types](./types.md#pyo3s-smart-pointers) for more details. +
+ + In general we don't want unbounded memory growth during loops! One workaround is to acquire and release the GIL with each iteration of the loop. @@ -114,6 +121,7 @@ this is unsafe. # fn main() -> PyResult<()> { Python::with_gil(|py| -> PyResult<()> { for _ in 0..10 { + #[allow(deprecated)] // `new_pool` is not needed in code not using the GIL Refs API let pool = unsafe { py.new_pool() }; let py = pool.python(); #[allow(deprecated)] // py.eval() is part of the GIL Refs API @@ -146,7 +154,11 @@ function call, releasing objects when the function returns. Most functions only a few objects, meaning this doesn't have a significant impact. Occasionally functions with long complex loops may need to use `Python::new_pool` as shown above. -This behavior may change in future, see [issue #1056](https://github.com/PyO3/pyo3/issues/1056). +
+⚠️ Warning: `GILPool` is no longer the preferred way to manage memory with PyO3 🛠️ + +PyO3 0.21 has introduced a new API known as the Bound API, which doesn't have the same surprising results. Instead, each `Bound` smart pointer releases the Python reference immediately on drop. See [the smart pointer types](./types.md#pyo3s-smart-pointers) for more details. +
## GIL-independent memory diff --git a/guide/src/migration.md b/guide/src/migration.md index 1c5ca59714a..8ea3f0730c0 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -280,6 +280,8 @@ The expectation is that in 0.22 `extract_bound` will have the default implementa As a final step of migration, deactivating the `gil-refs` feature will set up code for best performance and is intended to set up a forward-compatible API for PyO3 0.22. +At this point code which needed to manage GIL Ref memory can safely remove uses of `GILPool` (which are constructed by calls to `Python::new_pool` and `Python::with_pool`). Deprecation warnings will highlight these cases. + There is one notable API removed when this feature is disabled. `FromPyObject` trait implementations for types which borrow directly from the input data cannot be implemented by PyO3 without GIL Refs (while the migration is ongoing). These types are `&str`, `Cow<'_, str>`, `&[u8]`, `Cow<'_, u8>`. To ease pain during migration, these types instead implement a new temporary trait `FromPyObjectBound` which is the expected future form of `FromPyObject`. The new temporary trait ensures is that `obj.extract::<&str>()` continues to work (with the new constraint that the extracted value now depends on the input `obj` lifetime), as well for these types in `#[pyfunction]` arguments. diff --git a/newsfragments/3947.changed.md b/newsfragments/3947.changed.md new file mode 100644 index 00000000000..4618e9378fb --- /dev/null +++ b/newsfragments/3947.changed.md @@ -0,0 +1 @@ +Deprecate `GILPool`, `Python::with_pool`, and `Python::new_pool`. diff --git a/src/gil.rs b/src/gil.rs index 91c3d1cdd27..5ca3167e66c 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -139,6 +139,7 @@ where ffi::Py_InitializeEx(0); // Safety: the GIL is already held because of the Py_IntializeEx call. + #[allow(deprecated)] // TODO: remove this with the GIL Refs feature in 0.22 let pool = GILPool::new(); // Import the threading module - this ensures that it will associate this thread as the "main" @@ -160,6 +161,7 @@ where /// RAII type that represents the Global Interpreter Lock acquisition. pub(crate) struct GILGuard { gstate: ffi::PyGILState_STATE, + #[allow(deprecated)] // TODO: remove this with the gil-refs feature in 0.22 pool: mem::ManuallyDrop, } @@ -222,6 +224,7 @@ impl GILGuard { } let gstate = unsafe { ffi::PyGILState_Ensure() }; // acquire GIL + #[allow(deprecated)] let pool = unsafe { mem::ManuallyDrop::new(GILPool::new()) }; Some(GILGuard { gstate, pool }) @@ -358,6 +361,13 @@ impl Drop for LockGIL { /// /// [Memory Management]: https://pyo3.rs/main/memory.html#gil-bound-memory +#[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "`GILPool` has no function if PyO3's deprecated GIL Refs API is not used" + ) +)] pub struct GILPool { /// Initial length of owned objects and anys. /// `Option` is used since TSL can be broken when `new` is called from `atexit`. @@ -365,6 +375,7 @@ pub struct GILPool { _not_send: NotSend, } +#[allow(deprecated)] impl GILPool { /// Creates a new [`GILPool`]. This function should only ever be called with the GIL held. /// @@ -401,6 +412,7 @@ impl GILPool { } } +#[allow(deprecated)] impl Drop for GILPool { fn drop(&mut self) { if let Some(start) = self.start { @@ -506,21 +518,17 @@ fn decrement_gil_count() { #[cfg(test)] mod tests { - use super::{gil_is_acquired, GILPool, GIL_COUNT, OWNED_OBJECTS, POOL}; + #[allow(deprecated)] + use super::GILPool; + use super::{gil_is_acquired, GIL_COUNT, OWNED_OBJECTS, POOL}; use crate::types::any::PyAnyMethods; - use crate::{ffi, gil, PyObject, Python, ToPyObject}; + use crate::{ffi, gil, PyObject, Python}; #[cfg(not(target_arch = "wasm32"))] use parking_lot::{const_mutex, Condvar, Mutex}; use std::ptr::NonNull; fn get_object(py: Python<'_>) -> PyObject { - // Convenience function for getting a single unique object, using `new_pool` so as to leave - // the original pool state unchanged. - let pool = unsafe { py.new_pool() }; - let py = pool.python(); - - let obj = py.eval_bound("object()", None, None).unwrap(); - obj.to_object(py) + py.eval_bound("object()", None, None).unwrap().unbind() } fn owned_object_count() -> usize { @@ -556,6 +564,7 @@ mod tests { } #[test] + #[allow(deprecated)] fn test_owned() { Python::with_gil(|py| { let obj = get_object(py); @@ -581,6 +590,7 @@ mod tests { } #[test] + #[allow(deprecated)] fn test_owned_nested() { Python::with_gil(|py| { let obj = get_object(py); @@ -666,6 +676,7 @@ mod tests { } #[test] + #[allow(deprecated)] fn test_gil_counts() { // Check with_gil and GILPool both increase counts correctly let get_gil_count = || GIL_COUNT.with(|c| c.get()); @@ -906,6 +917,7 @@ mod tests { unsafe extern "C" fn capsule_drop(capsule: *mut ffi::PyObject) { // This line will implicitly call update_counts // -> and so cause deadlock if update_counts is not handling recursion correctly. + #[allow(deprecated)] let pool = GILPool::new(); // Rebuild obj so that it can be dropped diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index 4b4eac17a15..4d77f329125 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -9,9 +9,11 @@ use std::{ panic::{self, UnwindSafe}, }; +#[allow(deprecated)] +use crate::gil::GILPool; use crate::{ callback::PyCallbackOutput, ffi, ffi_ptr_ext::FfiPtrExt, impl_::panic::PanicTrap, - methods::IPowModulo, panic::PanicException, types::PyModule, GILPool, Py, PyResult, Python, + methods::IPowModulo, panic::PanicException, types::PyModule, Py, PyResult, Python, }; #[inline] @@ -174,6 +176,8 @@ where R: PyCallbackOutput, { let trap = PanicTrap::new("uncaught panic at ffi boundary"); + // Necessary to construct a pool until PyO3 0.22 when the GIL Refs API is fully disabled + #[allow(deprecated)] let pool = unsafe { GILPool::new() }; let py = pool.python(); let out = panic_result_into_callback_output( @@ -219,6 +223,8 @@ where F: for<'py> FnOnce(Python<'py>) -> PyResult<()> + UnwindSafe, { let trap = PanicTrap::new("uncaught panic at ffi boundary"); + // Necessary to construct a pool until PyO3 0.22 when the GIL Refs API is fully disabled + #[allow(deprecated)] let pool = GILPool::new(); let py = pool.python(); if let Err(py_err) = panic::catch_unwind(move || body(py)) diff --git a/src/lib.rs b/src/lib.rs index a7c03d1b6cb..dab48fbe01b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -319,6 +319,7 @@ pub use crate::conversion::{FromPyPointer, PyTryFrom, PyTryInto}; pub use crate::err::{ DowncastError, DowncastIntoError, PyDowncastError, PyErr, PyErrArguments, PyResult, ToPyErr, }; +#[allow(deprecated)] pub use crate::gil::GILPool; #[cfg(not(PyPy))] pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter}; diff --git a/src/marker.rs b/src/marker.rs index 2a38b83cba5..08b9042491b 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -118,7 +118,7 @@ //! [`Py`]: crate::Py use crate::err::{self, PyDowncastError, PyErr, PyResult}; use crate::ffi_ptr_ext::FfiPtrExt; -use crate::gil::{GILGuard, GILPool, SuspendGIL}; +use crate::gil::{GILGuard, SuspendGIL}; use crate::impl_::not_send::NotSend; use crate::py_result_ext::PyResultExt; use crate::type_object::HasPyGilRef; @@ -127,9 +127,9 @@ use crate::types::{ PyAny, PyDict, PyEllipsis, PyModule, PyNone, PyNotImplemented, PyString, PyType, }; use crate::version::PythonVersionInfo; -#[allow(deprecated)] -use crate::FromPyPointer; use crate::{ffi, Bound, IntoPy, Py, PyNativeType, PyObject, PyTypeCheck, PyTypeInfo}; +#[allow(deprecated)] +use crate::{gil::GILPool, FromPyPointer}; use std::ffi::{CStr, CString}; use std::marker::PhantomData; use std::os::raw::c_int; @@ -1053,9 +1053,10 @@ impl<'py> Python<'py> { err::error_on_minusone(self, unsafe { ffi::PyErr_CheckSignals() }) } - /// Create a new pool for managing PyO3's owned references. + /// Create a new pool for managing PyO3's GIL Refs. This has no functional + /// use for code which does not use the deprecated GIL Refs API. /// - /// When this `GILPool` is dropped, all PyO3 owned references created after this `GILPool` will + /// When this `GILPool` is dropped, all GIL Refs created after this `GILPool` will /// all have their Python reference counts decremented, potentially allowing Python to drop /// the corresponding Python objects. /// @@ -1074,6 +1075,7 @@ impl<'py> Python<'py> { /// // Some long-running process like a webserver, which never releases the GIL. /// loop { /// // Create a new pool, so that PyO3 can clear memory at the end of the loop. + /// #[allow(deprecated)] // `new_pool` is not needed in code not using the GIL Refs API /// let pool = unsafe { py.new_pool() }; /// /// // It is recommended to *always* immediately set py to the pool's Python, to help @@ -1108,13 +1110,22 @@ impl<'py> Python<'py> { /// /// [`.python()`]: crate::GILPool::python #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "code not using the GIL Refs API can safely remove use of `Python::new_pool`" + ) + )] + #[allow(deprecated)] pub unsafe fn new_pool(self) -> GILPool { GILPool::new() } } impl Python<'_> { - /// Creates a scope using a new pool for managing PyO3's owned references. + /// Creates a scope using a new pool for managing PyO3's GIL Refs. This has no functional + /// use for code which does not use the deprecated GIL Refs API. /// /// This is a safe alterantive to [`new_pool`][Self::new_pool] as /// it limits the closure to using the new GIL token at the cost of @@ -1131,6 +1142,7 @@ impl Python<'_> { /// // Some long-running process like a webserver, which never releases the GIL. /// loop { /// // Create a new scope, so that PyO3 can clear memory at the end of the loop. + /// #[allow(deprecated)] // `with_pool` is not needed in code not using the GIL Refs API /// py.with_pool(|py| { /// // do stuff... /// }); @@ -1167,6 +1179,14 @@ impl Python<'_> { /// }); /// ``` #[inline] + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "code not using the GIL Refs API can safely remove use of `Python::with_pool`" + ) + )] + #[allow(deprecated)] pub fn with_pool(&self, f: F) -> R where F: for<'py> FnOnce(Python<'py>) -> R + Ungil, diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index 1f3cc302c61..d82c406f9c6 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -1,4 +1,5 @@ #![deny(deprecated)] +#![allow(dead_code)] use pyo3::prelude::*; use pyo3::types::{PyString, PyType}; diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index d4a3e01f99f..e54e1a4e266 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -1,7 +1,7 @@ error: use of deprecated constant `pyo3::impl_::deprecations::PYMETHODS_NEW_DEPRECATED_FORM`: use `#[new]` instead of `#[__new__]` - --> tests/ui/deprecations.rs:11:7 + --> tests/ui/deprecations.rs:12:7 | -11 | #[__new__] +12 | #[__new__] | ^^^^^^^ | note: the lint level is defined here @@ -11,45 +11,45 @@ note: the lint level is defined here | ^^^^^^^^^^ error: use of deprecated struct `pyo3::PyCell`: `PyCell` was merged into `Bound`, use that instead; see the migration guide for more info - --> tests/ui/deprecations.rs:22:30 + --> tests/ui/deprecations.rs:23:30 | -22 | fn method_gil_ref(_slf: &PyCell) {} +23 | fn method_gil_ref(_slf: &PyCell) {} | ^^^^^^ error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:17:33 + --> tests/ui/deprecations.rs:18:33 | -17 | fn cls_method_gil_ref(_cls: &PyType) {} +18 | fn cls_method_gil_ref(_cls: &PyType) {} | ^ error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:22:29 + --> tests/ui/deprecations.rs:23:29 | -22 | fn method_gil_ref(_slf: &PyCell) {} +23 | fn method_gil_ref(_slf: &PyCell) {} | ^ error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:37:43 + --> tests/ui/deprecations.rs:38:43 | -37 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { +38 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { | ^ error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:47:19 + --> tests/ui/deprecations.rs:48:19 | -47 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { +48 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:53:57 + --> tests/ui/deprecations.rs:54:57 | -53 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +54 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::methods::Extractor::>::is_python`: use `wrap_pyfunction_bound!` instead - --> tests/ui/deprecations.rs:78:13 + --> tests/ui/deprecations.rs:79:13 | -78 | let _ = wrap_pyfunction!(double, py); +79 | let _ = wrap_pyfunction!(double, py); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From da24f0cf937c833763dd94c341a0d4188def40dd Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sun, 17 Mar 2024 10:17:09 +0100 Subject: [PATCH 220/349] exposes `Borrowed::to_owned` as public API (#3963) * exposes `Borrowed::to_owned` as public API * add newsfragment --- newsfragments/3963.added.md | 1 + src/instance.rs | 27 +++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 newsfragments/3963.added.md diff --git a/newsfragments/3963.added.md b/newsfragments/3963.added.md new file mode 100644 index 00000000000..86da17acc89 --- /dev/null +++ b/newsfragments/3963.added.md @@ -0,0 +1 @@ +added `Borrowed::to_owned` \ No newline at end of file diff --git a/src/instance.rs b/src/instance.rs index 83775708e1a..024a9fe5b51 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -511,8 +511,31 @@ unsafe impl AsPyPointer for Bound<'_, T> { pub struct Borrowed<'a, 'py, T>(NonNull, PhantomData<&'a Py>, Python<'py>); impl<'py, T> Borrowed<'_, 'py, T> { - /// Creates a new owned `Bound` from this borrowed reference by increasing the reference count. - pub(crate) fn to_owned(self) -> Bound<'py, T> { + /// Creates a new owned [`Bound`] from this borrowed reference by + /// increasing the reference count. + /// + /// # Example + /// ``` + /// use pyo3::{prelude::*, types::PyTuple}; + /// + /// # fn main() -> PyResult<()> { + /// Python::with_gil(|py| -> PyResult<()> { + /// let tuple = PyTuple::new_bound(py, [1, 2, 3]); + /// + /// // borrows from `tuple`, so can only be + /// // used while `tuple` stays alive + /// let borrowed = tuple.get_borrowed_item(0)?; + /// + /// // creates a new owned reference, which + /// // can be used indendently of `tuple` + /// let bound = borrowed.to_owned(); + /// drop(tuple); + /// + /// assert_eq!(bound.extract::().unwrap(), 1); + /// Ok(()) + /// }) + /// # } + pub fn to_owned(self) -> Bound<'py, T> { (*self).clone() } } From ebeea943fea21b93e9ab89e0ea8d009bb5f704dc Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 19 Mar 2024 08:14:42 +0000 Subject: [PATCH 221/349] ci: fixes to actions caches (#3970) --- .github/workflows/cache-cleanup.yml | 4 ++-- .github/workflows/ci.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml index 2b81a69ce88..02e8a6ab3cb 100644 --- a/.github/workflows/cache-cleanup.yml +++ b/.github/workflows/cache-cleanup.yml @@ -1,6 +1,6 @@ name: CI Cache Cleanup on: - pull_request: + pull_request_target: types: - closed @@ -22,7 +22,7 @@ jobs: echo "Deleting caches..." for cacheKey in $cacheKeysForPR do - gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + gh actions-cache delete -R $REPO -B $BRANCH --confirm -- $cacheKey done echo "Done" env: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f4347aa6f1..148938fe016 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -526,6 +526,7 @@ jobs: workspaces: examples/maturin-starter save-if: ${{ github.event_name != 'merge_group' }} + key: ${{ matrix.target }} - name: Setup cross-compiler if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }} run: sudo apt-get install -y mingw-w64 llvm From b06e95727b1bda03a5352f162bb90fa483cc1a8b Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 19 Mar 2024 09:58:41 +0100 Subject: [PATCH 222/349] deprecate gil-refs in `from_py_with` (#3967) * deprecate gil-refs in `from_py_with` * review feedback davidhewitt --- pyo3-macros-backend/src/params.rs | 39 +++++++++++++++++++++++++++---- src/impl_/pymethods.rs | 13 +++++++++++ tests/test_methods.rs | 2 +- tests/test_pyfunction.rs | 10 ++++---- tests/ui/deprecations.rs | 15 ++++++++++++ tests/ui/deprecations.stderr | 10 ++++++-- 6 files changed, 77 insertions(+), 12 deletions(-) diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 7260362fa43..74b3eba411f 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -36,6 +36,23 @@ pub fn impl_arg_params( let args_array = syn::Ident::new("output", Span::call_site()); let Ctx { pyo3_path } = ctx; + let from_py_with = spec + .signature + .arguments + .iter() + .enumerate() + .filter_map(|(i, arg)| { + let from_py_with = &arg.attrs.from_py_with.as_ref()?.value; + let from_py_with_holder = + syn::Ident::new(&format!("from_py_with_{}", i), Span::call_site()); + Some(quote_spanned! { from_py_with.span() => + let e = #pyo3_path::impl_::pymethods::Extractor::new(); + let #from_py_with_holder = #pyo3_path::impl_::pymethods::inspect_fn(#from_py_with, &e); + e.extract_from_py_with(); + }) + }) + .collect::(); + if !fastcall && is_forwarded_args(&spec.signature) { // In the varargs convention, we can just pass though if the signature // is (*args, **kwds). @@ -43,12 +60,14 @@ pub fn impl_arg_params( .signature .arguments .iter() - .map(|arg| impl_arg_param(arg, &mut 0, &args_array, holders, ctx)) + .enumerate() + .map(|(i, arg)| impl_arg_param(arg, i, &mut 0, &args_array, holders, ctx)) .collect::>()?; return Ok(( quote! { let _args = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_args); let _kwargs = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_kwargs); + #from_py_with }, arg_convert, )); @@ -81,7 +100,8 @@ pub fn impl_arg_params( .signature .arguments .iter() - .map(|arg| impl_arg_param(arg, &mut option_pos, &args_array, holders, ctx)) + .enumerate() + .map(|(i, arg)| impl_arg_param(arg, i, &mut option_pos, &args_array, holders, ctx)) .collect::>()?; let args_handler = if spec.signature.python_signature.varargs.is_some() { @@ -136,6 +156,7 @@ pub fn impl_arg_params( }; let mut #args_array = [::std::option::Option::None; #num_params]; let (_args, _kwargs) = #extract_expression; + #from_py_with }, param_conversion, )) @@ -145,6 +166,7 @@ pub fn impl_arg_params( /// index and the index in option diverge when using py: Python fn impl_arg_param( arg: &FnArg<'_>, + pos: usize, option_pos: &mut usize, args_array: &syn::Ident, holders: &mut Vec, @@ -222,14 +244,21 @@ fn impl_arg_param( )); } - let tokens = if let Some(expr_path) = arg.attrs.from_py_with.as_ref().map(|attr| &attr.value) { + let tokens = if arg + .attrs + .from_py_with + .as_ref() + .map(|attr| &attr.value) + .is_some() + { + let from_py_with = syn::Ident::new(&format!("from_py_with_{}", pos), Span::call_site()); if let Some(default) = default { quote_arg_span! { #[allow(clippy::redundant_closure)] #pyo3_path::impl_::extract_argument::from_py_with_with_default( #arg_value.as_deref(), #name_str, - #expr_path as fn(_) -> _, + #from_py_with as fn(_) -> _, || #default )? } @@ -238,7 +267,7 @@ fn impl_arg_param( #pyo3_path::impl_::extract_argument::from_py_with( &#pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value), #name_str, - #expr_path as fn(_) -> _, + #from_py_with as fn(_) -> _, )? } } diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index bc1125f97fd..ca3f0a06bf3 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -585,6 +585,10 @@ pub fn inspect_type(t: T) -> (T, Extractor) { (t, Extractor::new()) } +pub fn inspect_fn(f: fn(A) -> PyResult, _: &Extractor) -> fn(A) -> PyResult { + f +} + pub struct Extractor(NotAGilRef); pub struct NotAGilRef(std::marker::PhantomData); @@ -616,10 +620,19 @@ impl Extractor { ) )] pub fn extract_gil_ref(&self) {} + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor" + ) + )] + pub fn extract_from_py_with(&self) {} } impl NotAGilRef { pub fn extract_gil_ref(&self) {} + pub fn extract_from_py_with(&self) {} pub fn is_python(&self) {} } diff --git a/tests/test_methods.rs b/tests/test_methods.rs index e7eacf26b40..7ca06b27c05 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -1146,7 +1146,7 @@ fn test_issue_2988() { _data: Vec, // The from_py_with here looks a little odd, we just need some way // to encourage the macro to expand the from_py_with default path too - #[pyo3(from_py_with = "PyAny::extract")] _data2: Vec, + #[pyo3(from_py_with = " as PyAnyMethods>::extract")] _data2: Vec, ) { } } diff --git a/tests/test_pyfunction.rs b/tests/test_pyfunction.rs index 5c4350467c4..5221b0f85a5 100644 --- a/tests/test_pyfunction.rs +++ b/tests/test_pyfunction.rs @@ -118,8 +118,8 @@ fn test_functions_with_function_args() { } #[cfg(not(Py_LIMITED_API))] -fn datetime_to_timestamp(dt: &PyAny) -> PyResult { - let dt: &PyDateTime = dt.extract()?; +fn datetime_to_timestamp(dt: &Bound<'_, PyAny>) -> PyResult { + let dt = dt.downcast::()?; let ts: f64 = dt.call_method0("timestamp")?.extract()?; Ok(ts as i64) @@ -170,7 +170,7 @@ fn test_function_with_custom_conversion_error() { #[test] fn test_from_py_with_defaults() { - fn optional_int(x: &PyAny) -> PyResult> { + fn optional_int(x: &Bound<'_, PyAny>) -> PyResult> { if x.is_none() { Ok(None) } else { @@ -185,7 +185,9 @@ fn test_from_py_with_defaults() { } #[pyfunction(signature = (len=0))] - fn from_py_with_default(#[pyo3(from_py_with = "PyAny::len")] len: usize) -> usize { + fn from_py_with_default( + #[pyo3(from_py_with = " as PyAnyMethods>::len")] len: usize, + ) -> usize { len } diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index d82c406f9c6..1dcc673a33a 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -74,6 +74,21 @@ fn module_bound_by_value(m: Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } +fn extract_gil_ref(obj: &PyAny) -> PyResult { + obj.extract() +} + +fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { + obj.extract() +} + +#[pyfunction] +fn pyfunction_from_py_with( + #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, + #[pyo3(from_py_with = "extract_bound")] _bound: i32, +) { +} + fn test_wrap_pyfunction(py: Python<'_>, m: &Bound<'_, PyModule>) { // should lint let _ = wrap_pyfunction!(double, py); diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index e54e1a4e266..10a21d3a5e8 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -46,10 +46,16 @@ error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref` 54 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { | ^ +error: use of deprecated method `pyo3::methods::Extractor::::extract_from_py_with`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor + --> tests/ui/deprecations.rs:87:27 + | +87 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, + | ^^^^^^^^^^^^^^^^^ + error: use of deprecated method `pyo3::methods::Extractor::>::is_python`: use `wrap_pyfunction_bound!` instead - --> tests/ui/deprecations.rs:79:13 + --> tests/ui/deprecations.rs:94:13 | -79 | let _ = wrap_pyfunction!(double, py); +94 | let _ = wrap_pyfunction!(double, py); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From e29fac9c463eb3c1ae9a72d918e886e5f0e21515 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 19 Mar 2024 08:59:05 +0000 Subject: [PATCH 223/349] docs: use details to condense migration guide (#3961) --- guide/src/migration.md | 172 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/guide/src/migration.md b/guide/src/migration.md index 8ea3f0730c0..16f9616e683 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -291,15 +291,21 @@ An unfortunate final point here is that PyO3 cannot offer this new implementatio ## from 0.19.* to 0.20 ### Drop support for older technologies +
+Click to expand PyO3 0.20 has increased minimum Rust version to 1.56. This enables use of newer language features and simplifies maintenance of the project. +
### `PyDict::get_item` now returns a `Result` +
+Click to expand `PyDict::get_item` in PyO3 0.19 and older was implemented using a Python API which would suppress all exceptions and return `None` in those cases. This included errors in `__hash__` and `__eq__` implementations of the key being looked up. Newer recommendations by the Python core developers advise against using these APIs which suppress exceptions, instead allowing exceptions to bubble upwards. `PyDict::get_item_with_error` already implemented this recommended behavior, so that API has been renamed to `PyDict::get_item`. + Before: ```rust,ignore @@ -349,8 +355,11 @@ Python::with_gil(|py| -> PyResult<()> { }); # } ``` +
### Required arguments are no longer accepted after optional arguments +
+Click to expand [Trailing `Option` arguments](./function/signature.md#trailing-optional-arguments) have an automatic default of `None`. To avoid unwanted changes when modifying function signatures, in PyO3 0.18 it was deprecated to have a required argument after an `Option` argument without using `#[pyo3(signature = (...))]` to specify the intended defaults. In PyO3 0.20, this becomes a hard error. @@ -375,8 +384,11 @@ fn x_or_y(x: Option, y: u64) -> u64 { x.unwrap_or(y) } ``` +
### Remove deprecated function forms +
+Click to expand In PyO3 0.18 the `#[args]` attribute for `#[pymethods]`, and directly specifying the function signature in `#[pyfunction]`, was deprecated. This functionality has been removed in PyO3 0.20. @@ -403,17 +415,27 @@ fn add(a: u64, b: u64) -> u64 { } ``` +
+ ### `IntoPyPointer` trait removed +
+Click to expand The trait `IntoPyPointer`, which provided the `into_ptr` method on many types, has been removed. `into_ptr` is now available as an inherent method on all types that previously implemented this trait. +
### `AsPyPointer` now `unsafe` trait +
+Click to expand The trait `AsPyPointer` is now `unsafe trait`, meaning any external implementation of it must be marked as `unsafe impl`, and ensure that they uphold the invariant of returning valid pointers. +
## from 0.18.* to 0.19 ### Access to `Python` inside `__traverse__` implementations are now forbidden +
+Click to expand During `__traverse__` implementations for Python's Garbage Collection it is forbidden to do anything other than visit the members of the `#[pyclass]` being traversed. This means making Python function calls or other API calls are forbidden. @@ -433,8 +455,11 @@ impl SomeClass { } } ``` +
### Smarter `anyhow::Error` / `eyre::Report` conversion when inner error is "simple" `PyErr` +
+Click to expand When converting from `anyhow::Error` or `eyre::Report` to `PyErr`, if the inner error is a "simple" `PyErr` (with no source error), then the inner error will be used directly as the `PyErr` instead of wrapping it in a new `PyRuntimeError` with the original information converted into a string. @@ -472,8 +497,11 @@ Before, the above code would have printed `RuntimeError('ValueError: original er After, the same code will print `ValueError: original error message`, which is more straightforward. However, if the `anyhow::Error` or `eyre::Report` has a source, then the original exception will still be wrapped in a `PyRuntimeError`. +
### The deprecated `Python::acquire_gil` was removed and `Python::with_gil` must be used instead +
+Click to expand While the API provided by [`Python::acquire_gil`](https://docs.rs/pyo3/0.18.3/pyo3/marker/struct.Python.html#method.acquire_gil) seems convenient, it is somewhat brittle as the design of the GIL token [`Python`](https://docs.rs/pyo3/0.18.3/pyo3/marker/struct.Python.html) relies on proper nesting and panics if not used correctly, e.g. @@ -542,10 +570,13 @@ Python::with_gil(|py| { ``` Furthermore, `Python::acquire_gil` provides ownership of a `GILGuard` which can be freely stored and passed around. This is usually not helpful as it may keep the lock held for a long time thereby blocking progress in other parts of the program. Due to the generative lifetime attached to the GIL token supplied by `Python::with_gil`, the problem is avoided as the GIL token can only be passed down the call chain. Often, this issue can also be avoided entirely as any GIL-bound reference `&'py PyAny` implies access to a GIL token `Python<'py>` via the [`PyAny::py`](https://docs.rs/pyo3/latest/pyo3/types/struct.PyAny.html#method.py) method. +
## from 0.17.* to 0.18 ### Required arguments after `Option<_>` arguments will no longer be automatically inferred +
+Click to expand In `#[pyfunction]` and `#[pymethods]`, if a "required" function input such as `i32` came after an `Option<_>` input, then the `Option<_>` would be implicitly treated as required. (All trailing `Option<_>` arguments were treated as optional with a default value of `None`). @@ -575,8 +606,11 @@ fn required_argument_after_option_a(x: Option, y: i32) {} #[pyfunction(signature = (x=None, y=0))] fn required_argument_after_option_b(x: Option, y: i32) {} ``` +
### `__text_signature__` is now automatically generated for `#[pyfunction]` and `#[pymethods]` +
+Click to expand The [`#[pyo3(text_signature = "...")]` option](./function/signature.md#making-the-function-signature-available-to-python) was previously the only supported way to set the `__text_signature__` attribute on generated Python functions. @@ -606,10 +640,13 @@ fn function_with_defaults(a: i32, b: i32, c: i32) {} # }) # } ``` +
## from 0.16.* to 0.17 ### Type checks have been changed for `PyMapping` and `PySequence` types +
+Click to expand Previously the type checks for `PyMapping` and `PySequence` (implemented in `PyTryFrom`) used the Python C-API functions `PyMapping_Check` and `PySequence_Check`. @@ -659,13 +696,19 @@ assert!(m.as_ref(py).downcast::().is_ok()); ``` Note that this requirement may go away in the future when a pyclass is able to inherit from the abstract base class directly (see [pyo3/pyo3#991](https://github.com/PyO3/pyo3/issues/991)). +
### The `multiple-pymethods` feature now requires Rust 1.62 +
+Click to expand Due to limitations in the `inventory` crate which the `multiple-pymethods` feature depends on, this feature now requires Rust 1.62. For more information see [dtolnay/inventory#32](https://github.com/dtolnay/inventory/issues/32). +
### Added `impl IntoPy> for &str` +
+Click to expand This may cause inference errors. @@ -692,12 +735,18 @@ Python::with_gil(|py| { }); # } ``` +
### The `pyproto` feature is now disabled by default +
+Click to expand In preparation for removing the deprecated `#[pyproto]` attribute macro in a future PyO3 version, it is now gated behind an opt-in feature flag. This also gives a slight saving to compile times for code which does not use the deprecated macro. +
### `PyTypeObject` trait has been deprecated +
+Click to expand The `PyTypeObject` trait already was near-useless; almost all functionality was already on the `PyTypeInfo` trait, which `PyTypeObject` had a blanket implementation based upon. In PyO3 0.17 the final method, `PyTypeObject::type_object` was moved to `PyTypeInfo::type_object`. @@ -727,22 +776,34 @@ fn get_type_object(py: Python<'_>) -> &PyType { # Python::with_gil(|py| { get_type_object::(py); }); ``` +
### `impl IntoPy for [T; N]` now requires `T: IntoPy` rather than `T: ToPyObject` +
+Click to expand If this leads to errors, simply implement `IntoPy`. Because pyclasses already implement `IntoPy`, you probably don't need to worry about this. +
### Each `#[pymodule]` can now only be initialized once per process +
+Click to expand To make PyO3 modules sound in the presence of Python sub-interpreters, for now it has been necessary to explicitly disable the ability to initialize a `#[pymodule]` more than once in the same process. Attempting to do this will now raise an `ImportError`. +
## from 0.15.* to 0.16 ### Drop support for older technologies +
+Click to expand PyO3 0.16 has increased minimum Rust version to 1.48 and minimum Python version to 3.7. This enables use of newer language features (enabling some of the other additions in 0.16) and simplifies maintenance of the project. +
### `#[pyproto]` has been deprecated +
+Click to expand In PyO3 0.15, the `#[pymethods]` attribute macro gained support for implementing "magic methods" such as `__str__` (aka "dunder" methods). This implementation was not quite finalized at the time, with a few edge cases to be decided upon. The existing `#[pyproto]` attribute macro was left untouched, because it covered these edge cases. @@ -795,8 +856,11 @@ impl MyClass { } } ``` +
### Removed `PartialEq` for object wrappers +
+Click to expand The Python object wrappers `Py` and `PyAny` had implementations of `PartialEq` so that `object_a == object_b` would compare the Python objects for pointer @@ -808,8 +872,11 @@ wrapper type for `object_a` and `object_b`; you can now directly compare a To check for Python object equality (the Python `==` operator), use the new method `eq()`. +
### Container magic methods now match Python behavior +
+Click to expand In PyO3 0.15, `__getitem__`, `__setitem__` and `__delitem__` in `#[pymethods]` would generate only the _mapping_ implementation for a `#[pyclass]`. To match the Python behavior, these methods now generate both the _mapping_ **and** _sequence_ implementations. @@ -838,8 +905,11 @@ The `__len__` and `__getitem__` methods are also used to implement a Python [map Because there is no such distinction from Python, implementing these methods will fill the mapping and sequence slots simultaneously. A Python class with `__len__` implemented, for example, will have both the `sq_length` and `mp_length` slots filled. The PyO3 behavior in 0.16 has been changed to be closer to this Python behavior by default. +
### `wrap_pymodule!` and `wrap_pyfunction!` now respect privacy correctly +
+Click to expand Prior to PyO3 0.16 the `wrap_pymodule!` and `wrap_pyfunction!` macros could use modules and functions whose defining `fn` was not reachable according Rust privacy rules. @@ -887,10 +957,13 @@ fn my_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> { Ok(()) } ``` +
## from 0.14.* to 0.15 ### Changes in sequence indexing +
+Click to expand For all types that take sequence indices (`PyList`, `PyTuple` and `PySequence`), the API has been made consistent to only take `usize` indices, for consistency @@ -919,20 +992,29 @@ Python::with_gil(|py| { assert_eq!(list[0..2].to_string(), "[1, 2]"); }); ``` +
## from 0.13.* to 0.14 ### `auto-initialize` feature is now opt-in +
+Click to expand For projects embedding Python in Rust, PyO3 no longer automatically initializes a Python interpreter on the first call to `Python::with_gil` (or `Python::acquire_gil`) unless the [`auto-initialize` feature](features.md#auto-initialize) is enabled. +
### New `multiple-pymethods` feature +
+Click to expand `#[pymethods]` have been reworked with a simpler default implementation which removes the dependency on the `inventory` crate. This reduces dependencies and compile times for the majority of users. The limitation of the new default implementation is that it cannot support multiple `#[pymethods]` blocks for the same `#[pyclass]`. If you need this functionality, you must enable the `multiple-pymethods` feature which will switch `#[pymethods]` to the inventory-based implementation. +
### Deprecated `#[pyproto]` methods +
+Click to expand Some protocol (aka `__dunder__`) methods such as `__bytes__` and `__format__` have been possible to implement two ways in PyO3 for some time: via a `#[pyproto]` (e.g. `PyObjectProtocol` for the methods listed here), or by writing them directly in `#[pymethods]`. This is only true for a handful of the `#[pyproto]` methods (for technical reasons to do with the way PyO3 currently interacts with the Python C-API). @@ -972,14 +1054,20 @@ impl MyClass { } } ``` +
## from 0.12.* to 0.13 ### Minimum Rust version increased to Rust 1.45 +
+Click to expand PyO3 `0.13` makes use of new Rust language features stabilized between Rust 1.40 and Rust 1.45. If you are using a Rust compiler older than Rust 1.45, you will need to update your toolchain to be able to continue using PyO3. +
### Runtime changes to support the CPython limited API +
+Click to expand In PyO3 `0.13` support was added for compiling against the CPython limited API. This had a number of implications for _all_ PyO3 users, described here. @@ -988,10 +1076,13 @@ The largest of these is that all types created from PyO3 are what CPython calls - If you wish to subclass one of these types _from Rust_ you must mark it `#[pyclass(subclass)]`, as you would if you wished to allow subclassing it from Python code. - Type objects are now mutable - Python code can set attributes on them. - `__module__` on types without `#[pyclass(module="mymodule")]` no longer returns `builtins`, it now raises `AttributeError`. +
## from 0.11.* to 0.12 ### `PyErr` has been reworked +
+Click to expand In PyO3 `0.12` the `PyErr` type has been re-implemented to be significantly more compatible with the standard Rust error handling ecosystem. Specifically `PyErr` now implements @@ -1000,28 +1091,40 @@ the standard Rust error handling ecosystem. Specifically `PyErr` now implements While this has necessitated the removal of a number of APIs, the resulting `PyErr` type should now be much more easier to work with. The following sections list the changes in detail and how to migrate to the new APIs. +
#### `PyErr::new` and `PyErr::from_type` now require `Send + Sync` for their argument +
+Click to expand For most uses no change will be needed. If you are trying to construct `PyErr` from a value that is not `Send + Sync`, you will need to first create the Python object and then use `PyErr::from_instance`. Similarly, any types which implemented `PyErrArguments` will now need to be `Send + Sync`. +
#### `PyErr`'s contents are now private +
+Click to expand It is no longer possible to access the fields `.ptype`, `.pvalue` and `.ptraceback` of a `PyErr`. You should instead now use the new methods `PyErr::ptype`, `PyErr::pvalue` and `PyErr::ptraceback`. +
#### `PyErrValue` and `PyErr::from_value` have been removed +
+Click to expand As these were part the internals of `PyErr` which have been reworked, these APIs no longer exist. If you used this API, it is recommended to use `PyException::new_err` (see [the section on Exception types](#exception-types-have-been-reworked)). +
#### `Into>` for `PyErr` has been removed +
+Click to expand This implementation was redundant. Just construct the `Result::Err` variant directly. @@ -1035,8 +1138,11 @@ After (also using the new reworked exception types; see the following section): # use pyo3::{PyResult, exceptions::PyTypeError}; let result: PyResult<()> = Err(PyTypeError::new_err("error message")); ``` +
### Exception types have been reworked +
+Click to expand Previously exception types were zero-sized marker types purely used to construct `PyErr`. In PyO3 0.12, these types have been replaced with full definitions and are usable in the same way as `PyAny`, `PyDict` etc. This @@ -1068,8 +1174,12 @@ assert_eq!( # Ok(()) # }).unwrap(); ``` +
### `FromPy` has been removed +
+Click to expand + To simplify the PyO3 conversion traits, the `FromPy` trait has been removed. Previously there were two ways to define the to-Python conversion for a type: `FromPy for PyObject` and `IntoPy for T`. @@ -1119,12 +1229,20 @@ After: let obj: PyObject = 1.234.into_py(py); # }) ``` +
### `PyObject` is now a type alias of `Py` +
+Click to expand + This should change very little from a usage perspective. If you implemented traits for both `PyObject` and `Py`, you may find you can just remove the `PyObject` implementation. +
### `AsPyRef` has been removed +
+Click to expand + As `PyObject` has been changed to be just a type alias, the only remaining implementor of `AsPyRef` was `Py`. This removed the need for a trait, so the `AsPyRef::as_ref` method has been moved to `Py::as_ref`. @@ -1149,13 +1267,21 @@ let list_py: Py = PyList::empty(py).into(); let list_ref: &PyList = list_py.as_ref(py); # }) ``` +
## from 0.10.* to 0.11 ### Stable Rust +
+Click to expand + PyO3 now supports the stable Rust toolchain. The minimum required version is 1.39.0. +
### `#[pyclass]` structs must now be `Send` or `unsendable` +
+Click to expand + Because `#[pyclass]` structs can be sent between threads by the Python interpreter, they must implement `Send` or declared as `unsendable` (by `#[pyclass(unsendable)]`). Note that `unsendable` is added in PyO3 `0.11.1` and `Send` is always required in PyO3 `0.11.0`. @@ -1222,8 +1348,12 @@ There can be two fixes: pointers: Vec<*mut std::os::raw::c_char>, } ``` +
### All `PyObject` and `Py` methods now take `Python` as an argument +
+Click to expand + Previously, a few methods such as `Object::get_refcnt` did not take `Python` as an argument (to ensure that the Python GIL was held by the current thread). Technically, this was not sound. To migrate, just pass a `py` argument to any calls to these methods. @@ -1241,10 +1371,14 @@ After: py.None().get_refcnt(py); # }) ``` +
## from 0.9.* to 0.10 ### `ObjectProtocol` is removed +
+Click to expand + All methods are moved to [`PyAny`]. And since now all native types (e.g., `PyList`) implements `Deref`, all you need to do is remove `ObjectProtocol` from your code. @@ -1269,14 +1403,22 @@ let hi: &pyo3::types::PyString = obj.call0().unwrap().downcast().unwrap(); assert_eq!(hi.len().unwrap(), 5); # }) ``` +
### No `#![feature(specialization)]` in user code +
+Click to expand + While PyO3 itself still requires specialization and nightly Rust, now you don't have to use `#![feature(specialization)]` in your crate. +
## from 0.8.* to 0.9 ### `#[new]` interface +
+Click to expand + [`PyRawObject`](https://docs.rs/pyo3/0.8.5/pyo3/type_object/struct.PyRawObject.html) is now removed and our syntax for constructors has changed. @@ -1311,8 +1453,12 @@ impl MyClass { Basically you can return `Self` or `Result` directly. For more, see [the constructor section](class.md#constructor) of this guide. +
### PyCell +
+Click to expand + PyO3 0.9 introduces [`PyCell`], which is a [`RefCell`]-like object wrapper for ensuring Rust's rules regarding aliasing of references are upheld. For more detail, see the @@ -1463,6 +1609,32 @@ impl PySequenceProtocol for ByteSequence { } } ``` +
+ + [`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html [`PyAny`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html From cbed7c11b639107462b938c1025508a60d82efae Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 19 Mar 2024 19:36:22 +0000 Subject: [PATCH 224/349] ci: tidy ups for 3.7 (#3969) --- pytests/pyproject.toml | 2 +- src/tests/hygiene/pymethods.rs | 396 --------------------------------- 2 files changed, 1 insertion(+), 397 deletions(-) diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index 43403a1241c..aace57dd4d4 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -25,6 +25,6 @@ dev = [ "pytest-asyncio>=0.21", "pytest-benchmark>=3.4", # pinned < 8.1 because https://github.com/CodSpeedHQ/pytest-codspeed/issues/27 - "pytest>=8,<8.1", + "pytest>=7,<8.1", "typing_extensions>=4.0.0" ] diff --git a/src/tests/hygiene/pymethods.rs b/src/tests/hygiene/pymethods.rs index a00e67d9a85..2ffaf68690f 100644 --- a/src/tests/hygiene/pymethods.rs +++ b/src/tests/hygiene/pymethods.rs @@ -9,7 +9,6 @@ pub struct Dummy; #[pyo3(crate = "crate")] pub struct DummyIter; -#[cfg(Py_3_8)] #[crate::pymethods] #[pyo3(crate = "crate")] impl Dummy { @@ -405,401 +404,6 @@ impl Dummy { // Buffer protocol? } -#[cfg(not(Py_3_8))] -#[crate::pymethods] -#[pyo3(crate = "crate")] -impl Dummy { - ////////////////////// - // Basic customization - ////////////////////// - fn __repr__(&self) -> &'static str { - "Dummy" - } - - fn __str__(&self) -> &'static str { - "Dummy" - } - - fn __bytes__<'py>(&self, py: crate::Python<'py>) -> crate::Bound<'py, crate::types::PyBytes> { - crate::types::PyBytes::new_bound(py, &[0]) - } - - fn __format__(&self, format_spec: ::std::string::String) -> ::std::string::String { - ::std::panic!("unimplemented isn't hygienic before 1.50") - } - - fn __lt__(&self, other: &Self) -> bool { - false - } - - fn __le__(&self, other: &Self) -> bool { - false - } - fn __eq__(&self, other: &Self) -> bool { - false - } - fn __ne__(&self, other: &Self) -> bool { - false - } - fn __gt__(&self, other: &Self) -> bool { - false - } - fn __ge__(&self, other: &Self) -> bool { - false - } - - fn __hash__(&self) -> u64 { - 42 - } - - fn __bool__(&self) -> bool { - true - } - - ////////////////////// - // Customizing attribute access - ////////////////////// - - fn __getattr__(&self, name: ::std::string::String) -> &crate::PyAny { - ::std::panic!("unimplemented isn't hygienic before 1.50") - } - - fn __getattribute__(&self, name: ::std::string::String) -> &crate::PyAny { - ::std::panic!("unimplemented isn't hygienic before 1.50") - } - - fn __setattr__(&mut self, name: ::std::string::String, value: ::std::string::String) {} - - fn __delattr__(&mut self, name: ::std::string::String) {} - - fn __dir__<'py>(&self, py: crate::Python<'py>) -> crate::Bound<'py, crate::types::PyList> { - crate::types::PyList::new_bound(py, ::std::vec![0_u8]) - } - - ////////////////////// - // Implementing Descriptors - ////////////////////// - - fn __get__( - &self, - instance: &crate::PyAny, - owner: &crate::PyAny, - ) -> crate::PyResult<&crate::PyAny> { - ::std::panic!("unimplemented isn't hygienic before 1.50") - } - - fn __set__(&self, instance: &crate::PyAny, owner: &crate::PyAny) {} - - fn __delete__(&self, instance: &crate::PyAny) {} - - fn __set_name__(&self, owner: &crate::PyAny, name: &crate::PyAny) {} - - ////////////////////// - // Implementing Descriptors - ////////////////////// - - fn __len__(&self) -> usize { - 0 - } - - fn __getitem__(&self, key: u32) -> crate::PyResult { - ::std::result::Result::Err(crate::exceptions::PyKeyError::new_err("boo")) - } - - fn __setitem__(&self, key: u32, value: u32) {} - - fn __delitem__(&self, key: u32) {} - - fn __iter__(_: crate::pycell::PyRef<'_, Self>, py: crate::Python<'_>) -> crate::Py { - crate::Py::new(py, DummyIter {}).unwrap() - } - - fn __next__(&mut self) -> ::std::option::Option<()> { - ::std::option::Option::None - } - - fn __reversed__( - slf: crate::pycell::PyRef<'_, Self>, - py: crate::Python<'_>, - ) -> crate::Py { - crate::Py::new(py, DummyIter {}).unwrap() - } - - fn __contains__(&self, item: u32) -> bool { - false - } - - ////////////////////// - // Emulating numeric types - ////////////////////// - - fn __add__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __sub__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __mul__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __truediv__(&self, _other: &Self) -> crate::PyResult<()> { - ::std::result::Result::Err(crate::exceptions::PyZeroDivisionError::new_err("boo")) - } - - fn __floordiv__(&self, _other: &Self) -> crate::PyResult<()> { - ::std::result::Result::Err(crate::exceptions::PyZeroDivisionError::new_err("boo")) - } - - fn __mod__(&self, _other: &Self) -> u32 { - 0 - } - - fn __divmod__(&self, _other: &Self) -> (u32, u32) { - (0, 0) - } - - fn __pow__(&self, _other: &Self, modulo: ::std::option::Option) -> Dummy { - Dummy {} - } - - fn __lshift__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __rshift__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __and__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __xor__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __or__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __radd__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __rrsub__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __rmul__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __rtruediv__(&self, _other: &Self) -> crate::PyResult<()> { - ::std::result::Result::Err(crate::exceptions::PyZeroDivisionError::new_err("boo")) - } - - fn __rfloordiv__(&self, _other: &Self) -> crate::PyResult<()> { - ::std::result::Result::Err(crate::exceptions::PyZeroDivisionError::new_err("boo")) - } - - fn __rmod__(&self, _other: &Self) -> u32 { - 0 - } - - fn __rdivmod__(&self, _other: &Self) -> (u32, u32) { - (0, 0) - } - - fn __rpow__(&self, _other: &Self, modulo: ::std::option::Option) -> Dummy { - Dummy {} - } - - fn __rlshift__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __rrshift__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __rand__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __rxor__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __ror__(&self, other: &Self) -> Dummy { - Dummy {} - } - - fn __iadd__(&mut self, other: &Self) {} - - fn __irsub__(&mut self, other: &Self) {} - - fn __imul__(&mut self, other: &Self) {} - - fn __itruediv__(&mut self, _other: &Self) {} - - fn __ifloordiv__(&mut self, _other: &Self) {} - - fn __imod__(&mut self, _other: &Self) {} - - fn __ipow__(&mut self, _other: &Self, _modulo: ::std::option::Option) {} - fn __ilshift__(&mut self, other: &Self) {} - - fn __irshift__(&mut self, other: &Self) {} - - fn __iand__(&mut self, other: &Self) {} - - fn __ixor__(&mut self, other: &Self) {} - - fn __ior__(&mut self, other: &Self) {} - - fn __neg__(slf: crate::pycell::PyRef<'_, Self>) -> crate::pycell::PyRef<'_, Self> { - slf - } - - fn __pos__(slf: crate::pycell::PyRef<'_, Self>) -> crate::pycell::PyRef<'_, Self> { - slf - } - - fn __abs__(slf: crate::pycell::PyRef<'_, Self>) -> crate::pycell::PyRef<'_, Self> { - slf - } - - fn __invert__(slf: crate::pycell::PyRef<'_, Self>) -> crate::pycell::PyRef<'_, Self> { - slf - } - - fn __complex__<'py>( - &self, - py: crate::Python<'py>, - ) -> crate::Bound<'py, crate::types::PyComplex> { - crate::types::PyComplex::from_doubles_bound(py, 0.0, 0.0) - } - - fn __int__(&self) -> u32 { - 0 - } - - fn __float__(&self) -> f64 { - 0.0 - } - - fn __index__(&self) -> u32 { - 0 - } - - fn __round__(&self, ndigits: ::std::option::Option) -> u32 { - 0 - } - - fn __trunc__(&self) -> u32 { - 0 - } - - fn __floor__(&self) -> u32 { - 0 - } - - fn __ceil__(&self) -> u32 { - 0 - } - - ////////////////////// - // With Statement Context Managers - ////////////////////// - - fn __enter__(&mut self) {} - - fn __exit__( - &mut self, - exc_type: &crate::PyAny, - exc_value: &crate::PyAny, - traceback: &crate::PyAny, - ) { - } - - ////////////////////// - // Awaitable Objects - ////////////////////// - - fn __await__(slf: crate::pycell::PyRef<'_, Self>) -> crate::pycell::PyRef<'_, Self> { - slf - } - - ////////////////////// - - // Asynchronous Iterators - ////////////////////// - - fn __aiter__( - slf: crate::pycell::PyRef<'_, Self>, - py: crate::Python<'_>, - ) -> crate::Py { - crate::Py::new(py, DummyIter {}).unwrap() - } - - fn __anext__(&mut self) -> ::std::option::Option<()> { - ::std::option::Option::None - } - - ////////////////////// - // Asynchronous Context Managers - ////////////////////// - - fn __aenter__(&mut self) {} - - fn __aexit__( - &mut self, - exc_type: &crate::PyAny, - exc_value: &crate::PyAny, - traceback: &crate::PyAny, - ) { - } - - // Things with attributes - - #[pyo3(signature = (_y, *, _z=2))] - fn test(&self, _y: &Dummy, _z: i32) {} - #[staticmethod] - fn staticmethod() {} - #[classmethod] - fn clsmethod(_: &crate::Bound<'_, crate::types::PyType>) {} - #[pyo3(signature = (*_args, **_kwds))] - fn __call__( - &self, - _args: &crate::types::PyTuple, - _kwds: ::std::option::Option<&crate::types::PyDict>, - ) -> crate::PyResult { - ::std::panic!("unimplemented isn't hygienic before 1.50") - } - #[new] - fn new(a: u8) -> Self { - Dummy {} - } - #[getter] - fn get(&self) -> i32 { - 0 - } - #[setter] - fn set(&mut self, _v: i32) {} - #[classattr] - fn class_attr() -> i32 { - 0 - } - - // Dunder methods invented for protocols - - // PyGcProtocol - // Buffer protocol? -} - // Ensure that crate argument is also accepted inline #[crate::pyclass(crate = "crate")] From 02e188e4b4151afb42c52866bd7e3a5f77dfb87d Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 19 Mar 2024 21:08:20 +0000 Subject: [PATCH 225/349] adjust path for GIL Refs deprecation warnings (#3968) --- pyo3-macros-backend/src/method.rs | 27 ++++++------- pyo3-macros-backend/src/module.rs | 7 ++-- pyo3-macros-backend/src/params.rs | 6 +-- src/impl_/deprecations.rs | 64 +++++++++++++++++++++++++++++++ src/impl_/pymethods.rs | 62 ------------------------------ src/lib.rs | 23 +++++++++-- src/macros.rs | 2 +- src/types/function.rs | 3 +- tests/ui/deprecations.stderr | 16 ++++---- 9 files changed, 113 insertions(+), 97 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 3cc2a96e899..93a54395c94 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -540,7 +540,7 @@ impl<'a> FnSpec<'a> { holders.pop().unwrap(); // does not actually use holder created by `self_arg` quote! {{ - #self_e = #pyo3_path::impl_::pymethods::Extractor::<()>::new(); + #self_e = #pyo3_path::impl_::deprecations::GilRefs::<()>::new(); let __guard = #pyo3_path::impl_::coroutine::RefGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; async move { function(&__guard, #(#args),*).await } }} @@ -549,7 +549,7 @@ impl<'a> FnSpec<'a> { holders.pop().unwrap(); // does not actually use holder created by `self_arg` quote! {{ - #self_e = #pyo3_path::impl_::pymethods::Extractor::<()>::new(); + #self_e = #pyo3_path::impl_::deprecations::GilRefs::<()>::new(); let mut __guard = #pyo3_path::impl_::coroutine::RefMutGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; async move { function(&mut __guard, #(#args),*).await } }} @@ -557,12 +557,12 @@ impl<'a> FnSpec<'a> { _ => { if self_arg.is_empty() { quote! {{ - #self_e = #pyo3_path::impl_::pymethods::Extractor::<()>::new(); + #self_e = #pyo3_path::impl_::deprecations::GilRefs::<()>::new(); function(#(#args),*) }} } else { quote! { function({ - let (self_arg, e) = #pyo3_path::impl_::pymethods::inspect_type(#self_arg); + let (self_arg, e) = #pyo3_path::impl_::deprecations::inspect_type(#self_arg); #self_e = e; self_arg }, #(#args),*) } @@ -588,13 +588,13 @@ impl<'a> FnSpec<'a> { call } else if self_arg.is_empty() { quote! {{ - #self_e = #pyo3_path::impl_::pymethods::Extractor::<()>::new(); + #self_e = #pyo3_path::impl_::deprecations::GilRefs::<()>::new(); function(#(#args),*) }} } else { quote! { function({ - let (self_arg, e) = #pyo3_path::impl_::pymethods::inspect_type(#self_arg); + let (self_arg, e) = #pyo3_path::impl_::deprecations::inspect_type(#self_arg); #self_e = e; self_arg }, #(#args),*) @@ -632,8 +632,7 @@ impl<'a> FnSpec<'a> { }) .collect(); let (call, self_arg_span) = rust_call(args, &self_e, &mut holders); - let extract_gil_ref = - quote_spanned! { self_arg_span => #self_e.extract_gil_ref(); }; + let function_arg = quote_spanned! { self_arg_span => #self_e.function_arg(); }; quote! { unsafe fn #ident<'py>( @@ -645,7 +644,7 @@ impl<'a> FnSpec<'a> { let #self_e; #( #holders )* let result = #call; - #extract_gil_ref + #function_arg result } } @@ -654,8 +653,7 @@ impl<'a> FnSpec<'a> { let mut holders = Vec::new(); let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders, ctx)?; let (call, self_arg_span) = rust_call(args, &self_e, &mut holders); - let extract_gil_ref = - quote_spanned! { self_arg_span => #self_e.extract_gil_ref(); }; + let function_arg = quote_spanned! { self_arg_span => #self_e.function_arg(); }; quote! { unsafe fn #ident<'py>( @@ -671,7 +669,7 @@ impl<'a> FnSpec<'a> { #arg_convert #( #holders )* let result = #call; - #extract_gil_ref + #function_arg result } } @@ -680,8 +678,7 @@ impl<'a> FnSpec<'a> { let mut holders = Vec::new(); let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx)?; let (call, self_arg_span) = rust_call(args, &self_e, &mut holders); - let extract_gil_ref = - quote_spanned! { self_arg_span => #self_e.extract_gil_ref(); }; + let function_arg = quote_spanned! { self_arg_span => #self_e.function_arg(); }; quote! { unsafe fn #ident<'py>( @@ -696,7 +693,7 @@ impl<'a> FnSpec<'a> { #arg_convert #( #holders )* let result = #call; - #extract_gil_ref + #function_arg result } } diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 1cc3e836404..2515d519287 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -295,7 +295,8 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result if function.sig.inputs.len() == 2 { module_args.push(quote!(module.py())); } - module_args.push(quote!(::std::convert::Into::into(#pyo3_path::methods::BoundRef(module)))); + module_args + .push(quote!(::std::convert::Into::into(#pyo3_path::impl_::pymethods::BoundRef(module)))); let extractors = function .sig @@ -306,8 +307,8 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result if let syn::Pat::Ident(pat_ident) = &*pat_type.pat { let ident = &pat_ident.ident; return Some([ - parse_quote! { let (#ident, e) = #pyo3_path::impl_::pymethods::inspect_type(#ident); }, - parse_quote_spanned! { pat_type.span() => e.extract_gil_ref(); }, + parse_quote! { let (#ident, e) = #pyo3_path::impl_::deprecations::inspect_type(#ident); }, + parse_quote_spanned! { pat_type.span() => e.function_arg(); }, ]); } } diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 74b3eba411f..ae1661fe2c1 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -46,9 +46,9 @@ pub fn impl_arg_params( let from_py_with_holder = syn::Ident::new(&format!("from_py_with_{}", i), Span::call_site()); Some(quote_spanned! { from_py_with.span() => - let e = #pyo3_path::impl_::pymethods::Extractor::new(); - let #from_py_with_holder = #pyo3_path::impl_::pymethods::inspect_fn(#from_py_with, &e); - e.extract_from_py_with(); + let e = #pyo3_path::impl_::deprecations::GilRefs::new(); + let #from_py_with_holder = #pyo3_path::impl_::deprecations::inspect_fn(#from_py_with, &e); + e.from_py_with_arg(); }) }) .collect::(); diff --git a/src/impl_/deprecations.rs b/src/impl_/deprecations.rs index 6b9930ac69b..0c749ff985b 100644 --- a/src/impl_/deprecations.rs +++ b/src/impl_/deprecations.rs @@ -1,4 +1,68 @@ //! Symbols used to denote deprecated usages of PyO3's proc macros. +use crate::{PyResult, Python}; + #[deprecated(since = "0.20.0", note = "use `#[new]` instead of `#[__new__]`")] pub const PYMETHODS_NEW_DEPRECATED_FORM: () = (); + +pub fn inspect_type(t: T) -> (T, GilRefs) { + (t, GilRefs::new()) +} + +pub fn inspect_fn(f: fn(A) -> PyResult, _: &GilRefs) -> fn(A) -> PyResult { + f +} + +pub struct GilRefs(NotAGilRef); +pub struct NotAGilRef(std::marker::PhantomData); + +pub trait IsGilRef {} + +impl IsGilRef for &'_ T {} + +impl GilRefs { + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + GilRefs(NotAGilRef(std::marker::PhantomData)) + } +} + +impl GilRefs> { + #[cfg_attr( + not(feature = "gil-refs"), + deprecated(since = "0.21.0", note = "use `wrap_pyfunction_bound!` instead") + )] + pub fn is_python(&self) {} +} + +impl GilRefs { + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `&Bound<'_, T>` instead for this function argument" + ) + )] + pub fn function_arg(&self) {} + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor" + ) + )] + pub fn from_py_with_arg(&self) {} +} + +impl NotAGilRef { + pub fn function_arg(&self) {} + pub fn from_py_with_arg(&self) {} + pub fn is_python(&self) {} +} + +impl std::ops::Deref for GilRefs { + type Target = NotAGilRef; + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index ca3f0a06bf3..df89dba7dbd 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -580,65 +580,3 @@ pub unsafe fn tp_new_impl( .create_class_object_of_type(py, target_type) .map(Bound::into_ptr) } - -pub fn inspect_type(t: T) -> (T, Extractor) { - (t, Extractor::new()) -} - -pub fn inspect_fn(f: fn(A) -> PyResult, _: &Extractor) -> fn(A) -> PyResult { - f -} - -pub struct Extractor(NotAGilRef); -pub struct NotAGilRef(std::marker::PhantomData); - -pub trait IsGilRef {} - -impl IsGilRef for &'_ T {} - -impl Extractor { - #[allow(clippy::new_without_default)] - pub fn new() -> Self { - Extractor(NotAGilRef(std::marker::PhantomData)) - } -} - -impl Extractor> { - #[cfg_attr( - not(feature = "gil-refs"), - deprecated(since = "0.21.0", note = "use `wrap_pyfunction_bound!` instead") - )] - pub fn is_python(&self) {} -} - -impl Extractor { - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `&Bound<'_, T>` instead for this function argument" - ) - )] - pub fn extract_gil_ref(&self) {} - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor" - ) - )] - pub fn extract_from_py_with(&self) {} -} - -impl NotAGilRef { - pub fn extract_gil_ref(&self) {} - pub fn extract_from_py_with(&self) {} - pub fn is_python(&self) {} -} - -impl std::ops::Deref for Extractor { - type Target = NotAGilRef; - fn deref(&self) -> &Self::Target { - &self.0 - } -} diff --git a/src/lib.rs b/src/lib.rs index dab48fbe01b..fd5a520fdb5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -346,9 +346,6 @@ pub(crate) mod sealed; /// For compatibility reasons this has not yet been removed, however will be done so /// once is resolved. pub mod class { - #[doc(hidden)] - pub use crate::impl_::pymethods as methods; - pub use self::gc::{PyTraverseError, PyVisit}; #[doc(hidden)] @@ -356,6 +353,16 @@ pub mod class { PyClassAttributeDef, PyGetterDef, PyMethodDef, PyMethodDefType, PyMethodType, PySetterDef, }; + #[doc(hidden)] + pub mod methods { + // frozen with the contents of the `impl_::pymethods` module in 0.20, + // this should probably all be replaced with deprecated type aliases and removed. + pub use crate::impl_::pymethods::{ + IPowModulo, PyClassAttributeDef, PyGetterDef, PyMethodDef, PyMethodDefType, + PyMethodType, PySetterDef, + }; + } + /// Old module which contained some implementation details of the `#[pyproto]` module. /// /// Prefer using the same content from `pyo3::pyclass`, e.g. `use pyo3::pyclass::CompareOp` instead @@ -479,6 +486,16 @@ mod macros; #[cfg(feature = "experimental-inspect")] pub mod inspect; +/// Ths module only contains re-exports of pyo3 deprecation warnings and exists +/// purely to make compiler error messages nicer. +/// +/// (The compiler uses this module in error messages, probably because it's a public +/// re-export at a shorter path than `pyo3::impl_::deprecations`.) +#[doc(hidden)] +pub mod deprecations { + pub use crate::impl_::deprecations::*; +} + /// Test readme and user guide #[cfg(doctest)] pub mod doc_test { diff --git a/src/macros.rs b/src/macros.rs index bd0bd81421c..db409515085 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -144,7 +144,7 @@ macro_rules! wrap_pyfunction { }; ($function:path, $py_or_module:expr) => {{ use $function as wrapped_pyfunction; - let (py_or_module, e) = $crate::impl_::pymethods::inspect_type($py_or_module); + let (py_or_module, e) = $crate::impl_::deprecations::inspect_type($py_or_module); e.is_python(); $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( py_or_module, diff --git a/src/types/function.rs b/src/types/function.rs index ea8201fb131..f5305e31886 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -1,12 +1,11 @@ use crate::derive_utils::PyFunctionArguments; use crate::ffi_ptr_ext::FfiPtrExt; -use crate::methods::PyMethodDefDestructor; use crate::py_result_ext::PyResultExt; use crate::types::capsule::PyCapsuleMethods; use crate::types::module::PyModuleMethods; use crate::{ ffi, - impl_::pymethods::{self, PyMethodDef}, + impl_::pymethods::{self, PyMethodDef, PyMethodDefDestructor}, types::{PyCapsule, PyDict, PyModule, PyString, PyTuple}, }; use crate::{Bound, IntoPy, Py, PyAny, PyNativeType, PyResult, Python}; diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index 10a21d3a5e8..8bd1450874f 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -1,4 +1,4 @@ -error: use of deprecated constant `pyo3::impl_::deprecations::PYMETHODS_NEW_DEPRECATED_FORM`: use `#[new]` instead of `#[__new__]` +error: use of deprecated constant `pyo3::deprecations::PYMETHODS_NEW_DEPRECATED_FORM`: use `#[new]` instead of `#[__new__]` --> tests/ui/deprecations.rs:12:7 | 12 | #[__new__] @@ -16,43 +16,43 @@ error: use of deprecated struct `pyo3::PyCell`: `PyCell` was merged into `Bound` 23 | fn method_gil_ref(_slf: &PyCell) {} | ^^^^^^ -error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument +error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument --> tests/ui/deprecations.rs:18:33 | 18 | fn cls_method_gil_ref(_cls: &PyType) {} | ^ -error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument +error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument --> tests/ui/deprecations.rs:23:29 | 23 | fn method_gil_ref(_slf: &PyCell) {} | ^ -error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument +error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument --> tests/ui/deprecations.rs:38:43 | 38 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { | ^ -error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument +error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument --> tests/ui/deprecations.rs:48:19 | 48 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { | ^ -error: use of deprecated method `pyo3::methods::Extractor::::extract_gil_ref`: use `&Bound<'_, T>` instead for this function argument +error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument --> tests/ui/deprecations.rs:54:57 | 54 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { | ^ -error: use of deprecated method `pyo3::methods::Extractor::::extract_from_py_with`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor +error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor --> tests/ui/deprecations.rs:87:27 | 87 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, | ^^^^^^^^^^^^^^^^^ -error: use of deprecated method `pyo3::methods::Extractor::>::is_python`: use `wrap_pyfunction_bound!` instead +error: use of deprecated method `pyo3::deprecations::GilRefs::>::is_python`: use `wrap_pyfunction_bound!` instead --> tests/ui/deprecations.rs:94:13 | 94 | let _ = wrap_pyfunction!(double, py); From caf80eca6607db6dfffa1f305c3118423320e9de Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 19 Mar 2024 21:41:27 +0000 Subject: [PATCH 226/349] handle clippy `new_without_default` warnings (#3971) * handle clippy `new_without_default` warnings * add newsfragment --- newsfragments/3971.added.md | 1 + pyo3-ffi/src/cpython/object.rs | 1 + pyo3-ffi/src/pybuffer.rs | 1 + src/impl_/pyclass/lazy_type_object.rs | 1 + src/sync.rs | 1 + 5 files changed, 5 insertions(+) create mode 100644 newsfragments/3971.added.md diff --git a/newsfragments/3971.added.md b/newsfragments/3971.added.md new file mode 100644 index 00000000000..12c0d2265bc --- /dev/null +++ b/newsfragments/3971.added.md @@ -0,0 +1 @@ +Implement `Default` for `GILOnceCell`. diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 161fb50cf24..d0c1634082b 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -46,6 +46,7 @@ mod bufferinfo { } impl Py_buffer { + #[allow(clippy::new_without_default)] pub const fn new() -> Self { Py_buffer { buf: ptr::null_mut(), diff --git a/pyo3-ffi/src/pybuffer.rs b/pyo3-ffi/src/pybuffer.rs index a414f333ce6..50bf4e6109c 100644 --- a/pyo3-ffi/src/pybuffer.rs +++ b/pyo3-ffi/src/pybuffer.rs @@ -27,6 +27,7 @@ pub struct Py_buffer { } impl Py_buffer { + #[allow(clippy::new_without_default)] pub const fn new() -> Self { Py_buffer { buf: ptr::null_mut(), diff --git a/src/impl_/pyclass/lazy_type_object.rs b/src/impl_/pyclass/lazy_type_object.rs index 1318e1abbbc..efb6ecf37e6 100644 --- a/src/impl_/pyclass/lazy_type_object.rs +++ b/src/impl_/pyclass/lazy_type_object.rs @@ -32,6 +32,7 @@ struct LazyTypeObjectInner { impl LazyTypeObject { /// Creates an uninitialized `LazyTypeObject`. + #[allow(clippy::new_without_default)] pub const fn new() -> Self { LazyTypeObject( LazyTypeObjectInner { diff --git a/src/sync.rs b/src/sync.rs index 38471fb7ca3..5af4940461d 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -90,6 +90,7 @@ unsafe impl Sync for GILProtected where T: Send {} /// } /// # Python::with_gil(|py| assert_eq!(get_shared_list(py).len(), 0)); /// ``` +#[derive(Default)] pub struct GILOnceCell(UnsafeCell>); // T: Send is needed for Sync because the thread which drops the GILOnceCell can be different From 2736cf670c7d7a0bc38a96f6c8833fd16dc8548d Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:27:38 +0100 Subject: [PATCH 227/349] deprecate gil-refs in `from_py_with` (Part 2) (#3972) * deprecate `from_py_with` in `#[derive(FromPyObject)]` (NewType) * deprecate `from_py_with` in `#[derive(FromPyObject)]` (Enum, Struct) --- pyo3-macros-backend/src/frompyobject.rs | 168 ++++++++++++++++++------ tests/ui/deprecations.rs | 39 ++++++ tests/ui/deprecations.stderr | 36 ++++- 3 files changed, 197 insertions(+), 46 deletions(-) diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 68ef72ea157..7f26e5b14fc 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -1,7 +1,7 @@ use crate::attributes::{self, get_pyo3_options, CrateAttribute, FromPyWithAttribute}; use crate::utils::Ctx; use proc_macro2::TokenStream; -use quote::{format_ident, quote}; +use quote::{format_ident, quote, quote_spanned}; use syn::{ parenthesized, parse::{Parse, ParseStream}, @@ -44,13 +44,16 @@ impl<'a> Enum<'a> { } /// Build derivation body for enums. - fn build(&self, ctx: &Ctx) -> TokenStream { + fn build(&self, ctx: &Ctx) -> (TokenStream, TokenStream) { let Ctx { pyo3_path } = ctx; let mut var_extracts = Vec::new(); let mut variant_names = Vec::new(); let mut error_names = Vec::new(); + + let mut deprecations = TokenStream::new(); for var in &self.variants { - let struct_derive = var.build(ctx); + let (struct_derive, dep) = var.build(ctx); + deprecations.extend(dep); let ext = quote!({ let maybe_ret = || -> #pyo3_path::PyResult { #struct_derive @@ -67,19 +70,22 @@ impl<'a> Enum<'a> { error_names.push(&var.err_name); } let ty_name = self.enum_ident.to_string(); - quote!( - let errors = [ - #(#var_extracts),* - ]; - ::std::result::Result::Err( - #pyo3_path::impl_::frompyobject::failed_to_extract_enum( - obj.py(), - #ty_name, - &[#(#variant_names),*], - &[#(#error_names),*], - &errors + ( + quote!( + let errors = [ + #(#var_extracts),* + ]; + ::std::result::Result::Err( + #pyo3_path::impl_::frompyobject::failed_to_extract_enum( + obj.py(), + #ty_name, + &[#(#variant_names),*], + &[#(#error_names),*], + &errors + ) ) - ) + ), + deprecations, ) } } @@ -238,7 +244,7 @@ impl<'a> Container<'a> { } /// Build derivation body for a struct. - fn build(&self, ctx: &Ctx) -> TokenStream { + fn build(&self, ctx: &Ctx) -> (TokenStream, TokenStream) { match &self.ty { ContainerType::StructNewtype(ident, from_py_with) => { self.build_newtype_struct(Some(ident), from_py_with, ctx) @@ -256,41 +262,73 @@ impl<'a> Container<'a> { field_ident: Option<&Ident>, from_py_with: &Option, ctx: &Ctx, - ) -> TokenStream { + ) -> (TokenStream, TokenStream) { let Ctx { pyo3_path } = ctx; let self_ty = &self.path; let struct_name = self.name(); if let Some(ident) = field_ident { let field_name = ident.to_string(); match from_py_with { - None => quote! { - Ok(#self_ty { - #ident: #pyo3_path::impl_::frompyobject::extract_struct_field(obj, #struct_name, #field_name)? - }) - }, + None => ( + quote! { + Ok(#self_ty { + #ident: #pyo3_path::impl_::frompyobject::extract_struct_field(obj, #struct_name, #field_name)? + }) + }, + TokenStream::new(), + ), Some(FromPyWithAttribute { value: expr_path, .. - }) => quote! { - Ok(#self_ty { - #ident: #pyo3_path::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, #field_name)? - }) - }, + }) => ( + quote! { + Ok(#self_ty { + #ident: #pyo3_path::impl_::frompyobject::extract_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, #field_name)? + }) + }, + quote_spanned! { expr_path.span() => + const _: () = { + fn check_from_py_with() { + let e = #pyo3_path::impl_::deprecations::GilRefs::new(); + #pyo3_path::impl_::deprecations::inspect_fn(#expr_path, &e); + e.from_py_with_arg(); + } + }; + }, + ), } } else { match from_py_with { - None => quote!( - #pyo3_path::impl_::frompyobject::extract_tuple_struct_field(obj, #struct_name, 0).map(#self_ty) + None => ( + quote!( + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field(obj, #struct_name, 0).map(#self_ty) + ), + TokenStream::new(), ), Some(FromPyWithAttribute { value: expr_path, .. - }) => quote! ( - #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, 0).map(#self_ty) + }) => ( + quote! ( + #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#expr_path as fn(_) -> _, obj, #struct_name, 0).map(#self_ty) + ), + quote_spanned! { expr_path.span() => + const _: () = { + fn check_from_py_with() { + let e = #pyo3_path::impl_::deprecations::GilRefs::new(); + #pyo3_path::impl_::deprecations::inspect_fn(#expr_path, &e); + e.from_py_with_arg(); + } + }; + }, ), } } } - fn build_tuple_struct(&self, struct_fields: &[TupleStructField], ctx: &Ctx) -> TokenStream { + fn build_tuple_struct( + &self, + struct_fields: &[TupleStructField], + ctx: &Ctx, + ) -> (TokenStream, TokenStream) { let Ctx { pyo3_path } = ctx; let self_ty = &self.path; let struct_name = &self.name(); @@ -309,15 +347,41 @@ impl<'a> Container<'a> { ), } }); - quote!( - match #pyo3_path::types::PyAnyMethods::extract(obj) { - ::std::result::Result::Ok((#(#field_idents),*)) => ::std::result::Result::Ok(#self_ty(#(#fields),*)), - ::std::result::Result::Err(err) => ::std::result::Result::Err(err), - } + + let deprecations = struct_fields + .iter() + .filter_map(|field| { + let FromPyWithAttribute { + value: expr_path, .. + } = field.from_py_with.as_ref()?; + Some(quote_spanned! { expr_path.span() => + const _: () = { + fn check_from_py_with() { + let e = #pyo3_path::impl_::deprecations::GilRefs::new(); + #pyo3_path::impl_::deprecations::inspect_fn(#expr_path, &e); + e.from_py_with_arg(); + } + }; + }) + }) + .collect::(); + + ( + quote!( + match #pyo3_path::types::PyAnyMethods::extract(obj) { + ::std::result::Result::Ok((#(#field_idents),*)) => ::std::result::Result::Ok(#self_ty(#(#fields),*)), + ::std::result::Result::Err(err) => ::std::result::Result::Err(err), + } + ), + deprecations, ) } - fn build_struct(&self, struct_fields: &[NamedStructField<'_>], ctx: &Ctx) -> TokenStream { + fn build_struct( + &self, + struct_fields: &[NamedStructField<'_>], + ctx: &Ctx, + ) -> (TokenStream, TokenStream) { let Ctx { pyo3_path } = ctx; let self_ty = &self.path; let struct_name = &self.name(); @@ -355,7 +419,29 @@ impl<'a> Container<'a> { fields.push(quote!(#ident: #extractor)); } - quote!(::std::result::Result::Ok(#self_ty{#fields})) + + let deprecations = struct_fields + .iter() + .filter_map(|field| { + let FromPyWithAttribute { + value: expr_path, .. + } = field.from_py_with.as_ref()?; + Some(quote_spanned! { expr_path.span() => + const _: () = { + fn check_from_py_with() { + let e = #pyo3_path::impl_::deprecations::GilRefs::new(); + #pyo3_path::impl_::deprecations::inspect_fn(#expr_path, &e); + e.from_py_with_arg(); + } + }; + }) + }) + .collect::(); + + ( + quote!(::std::result::Result::Ok(#self_ty{#fields})), + deprecations, + ) } } @@ -587,7 +673,7 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { let ctx = &Ctx::new(&options.krate); let Ctx { pyo3_path } = &ctx; - let derives = match &tokens.data { + let (derives, from_py_with_deprecations) = match &tokens.data { syn::Data::Enum(en) => { if options.transparent || options.annotation.is_some() { bail_spanned!(tokens.span() => "`transparent` or `annotation` is not supported \ @@ -617,5 +703,7 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { #derives } } + + #from_py_with_deprecations )) } diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index 1dcc673a33a..cbaba2fa4ff 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -89,6 +89,45 @@ fn pyfunction_from_py_with( ) { } +#[derive(Debug, FromPyObject)] +pub struct Zap { + #[pyo3(item)] + name: String, + + #[pyo3(from_py_with = "PyAny::len", item("my_object"))] + some_object_length: usize, + + #[pyo3(from_py_with = "extract_bound")] + some_number: i32, +} + +#[derive(Debug, FromPyObject)] +pub struct ZapTuple( + String, + #[pyo3(from_py_with = "PyAny::len")] usize, + #[pyo3(from_py_with = "extract_bound")] i32, +); + +#[derive(Debug, FromPyObject, PartialEq, Eq)] +pub enum ZapEnum { + Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), + Zap(String, #[pyo3(from_py_with = "extract_bound")] i32), +} + +#[derive(Debug, FromPyObject, PartialEq, Eq)] +#[pyo3(transparent)] +pub struct TransparentFromPyWithGilRef { + #[pyo3(from_py_with = "extract_gil_ref")] + len: i32, +} + +#[derive(Debug, FromPyObject, PartialEq, Eq)] +#[pyo3(transparent)] +pub struct TransparentFromPyWithBound { + #[pyo3(from_py_with = "extract_bound")] + len: i32, +} + fn test_wrap_pyfunction(py: Python<'_>, m: &Bound<'_, PyModule>) { // should lint let _ = wrap_pyfunction!(double, py); diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index 8bd1450874f..d5da8572d02 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -52,10 +52,34 @@ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_ 87 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, | ^^^^^^^^^^^^^^^^^ -error: use of deprecated method `pyo3::deprecations::GilRefs::>::is_python`: use `wrap_pyfunction_bound!` instead - --> tests/ui/deprecations.rs:94:13 - | -94 | let _ = wrap_pyfunction!(double, py); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor + --> tests/ui/deprecations.rs:97:27 | - = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) +97 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] + | ^^^^^^^^^^^^ + +error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor + --> tests/ui/deprecations.rs:107:27 + | +107 | #[pyo3(from_py_with = "PyAny::len")] usize, + | ^^^^^^^^^^^^ + +error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor + --> tests/ui/deprecations.rs:113:31 + | +113 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), + | ^^^^^^^^^^^^^^^^^ + +error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor + --> tests/ui/deprecations.rs:120:27 + | +120 | #[pyo3(from_py_with = "extract_gil_ref")] + | ^^^^^^^^^^^^^^^^^ + +error: use of deprecated method `pyo3::deprecations::GilRefs::>::is_python`: use `wrap_pyfunction_bound!` instead + --> tests/ui/deprecations.rs:133:13 + | +133 | let _ = wrap_pyfunction!(double, py); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From cedac43dbb90b191a6233d52493610ae521d8a14 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 20 Mar 2024 12:52:09 +0000 Subject: [PATCH 228/349] add Bound::as_unbound (#3973) * add Bound::as_unbound * Update src/instance.rs --- src/instance.rs | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/instance.rs b/src/instance.rs index 024a9fe5b51..5f054d0d2d4 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -465,6 +465,13 @@ impl<'py, T> Bound<'py, T> { unsafe { Py::from_non_null(non_null) } } + /// Removes the connection for this `Bound` from the GIL, allowing + /// it to cross thread boundaries, without transferring ownership. + #[inline] + pub fn as_unbound(&self) -> &Py { + &self.1 + } + /// Casts this `Bound` as the corresponding "GIL Ref" type. /// /// This is a helper to be used for migration from the deprecated "GIL Refs" API. @@ -521,11 +528,11 @@ impl<'py, T> Borrowed<'_, 'py, T> { /// # fn main() -> PyResult<()> { /// Python::with_gil(|py| -> PyResult<()> { /// let tuple = PyTuple::new_bound(py, [1, 2, 3]); - /// + /// /// // borrows from `tuple`, so can only be /// // used while `tuple` stays alive /// let borrowed = tuple.get_borrowed_item(0)?; - /// + /// /// // creates a new owned reference, which /// // can be used indendently of `tuple` /// let bound = borrowed.to_owned(); @@ -1960,8 +1967,8 @@ impl PyObject { mod tests { use super::{Bound, Py, PyObject}; use crate::types::any::PyAnyMethods; - use crate::types::PyCapsule; use crate::types::{dict::IntoPyDict, PyDict, PyString}; + use crate::types::{PyCapsule, PyStringMethods}; use crate::{ffi, Borrowed, PyAny, PyNativeType, PyResult, Python, ToPyObject}; #[test] @@ -2161,6 +2168,20 @@ a = A() }); } + #[test] + fn test_bound_py_conversions() { + Python::with_gil(|py| { + let obj: Bound<'_, PyString> = PyString::new_bound(py, "hello world"); + let obj_unbound: &Py = obj.as_unbound(); + let _: &Bound<'_, PyString> = obj_unbound.bind(py); + + let obj_unbound: Py = obj.unbind(); + let obj: Bound<'_, PyString> = obj_unbound.into_bound(py); + + assert_eq!(obj.to_cow().unwrap(), "hello world"); + }); + } + #[test] fn bound_from_borrowed_ptr_constructors() { // More detailed tests of the underlying semantics in pycell.rs From 870a4bb20dad883a50439b0e1b6cf64a7f15e049 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 20 Mar 2024 22:35:08 +0000 Subject: [PATCH 229/349] deprecate GIL refs in function argument (#3847) * deprecate GIL Refs in function arguments * fix deprecated gil refs in function arguments * add notes on deprecations limitations to migration guide * Apply suggestions from code review Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> * review: Icxolu * fix proto method extract failure for option * fix gil refs in examples --------- Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> --- examples/getitem/src/lib.rs | 9 +- guide/src/migration.md | 43 ++++++- pyo3-macros-backend/src/method.rs | 118 ++++++++---------- pyo3-macros-backend/src/module.rs | 7 +- pyo3-macros-backend/src/params.rs | 116 +++++++++++++---- pyo3-macros-backend/src/pyclass.rs | 3 +- pyo3-macros-backend/src/pymethod.rs | 94 +++++++++----- pyo3-macros-backend/src/quotes.rs | 4 +- pytests/src/buf_and_str.rs | 6 +- pytests/src/datetime.rs | 34 +++-- pytests/src/dict_iter.rs | 2 +- pytests/src/misc.rs | 4 +- pytests/src/objstore.rs | 2 +- pytests/src/pyclasses.rs | 35 +++--- src/coroutine.rs | 4 +- src/coroutine/waker.rs | 2 +- src/impl_/deprecations.rs | 4 +- src/impl_/extract_argument.rs | 25 +++- src/macros.rs | 6 +- src/pycell.rs | 2 +- src/tests/hygiene/pymethods.rs | 52 ++++---- src/types/bytearray.rs | 8 +- tests/test_arithmetics.rs | 86 ++++++------- tests/test_inheritance.rs | 4 +- tests/test_methods.rs | 81 ++++++------ tests/test_no_imports.rs | 2 +- tests/test_proto_methods.rs | 12 +- tests/test_pyfunction.rs | 6 +- tests/test_sequence.rs | 4 +- tests/test_text_signature.rs | 8 +- tests/test_variable_arguments.rs | 4 +- tests/ui/deprecations.rs | 6 + tests/ui/deprecations.stderr | 52 +++++--- tests/ui/invalid_cancel_handle.stderr | 1 + tests/ui/invalid_frozen_pyclass_borrow.stderr | 13 ++ tests/ui/invalid_pyfunctions.rs | 6 +- tests/ui/invalid_pyfunctions.stderr | 30 ++--- tests/ui/invalid_pymethod_enum.stderr | 13 ++ tests/ui/static_ref.rs | 7 +- tests/ui/static_ref.stderr | 34 +++++ 40 files changed, 597 insertions(+), 352 deletions(-) diff --git a/examples/getitem/src/lib.rs b/examples/getitem/src/lib.rs index eed60076bb8..c3c662ab92f 100644 --- a/examples/getitem/src/lib.rs +++ b/examples/getitem/src/lib.rs @@ -7,7 +7,7 @@ use std::os::raw::c_long; #[derive(FromPyObject)] enum IntOrSlice<'py> { Int(i32), - Slice(&'py PySlice), + Slice(Bound<'py, PySlice>), } #[pyclass] @@ -23,7 +23,7 @@ impl ExampleContainer { ExampleContainer { max_length: 100 } } - fn __getitem__(&self, key: &PyAny) -> PyResult { + fn __getitem__(&self, key: &Bound<'_, PyAny>) -> PyResult { if let Ok(position) = key.extract::() { return Ok(position); } else if let Ok(slice) = key.downcast::() { @@ -63,7 +63,10 @@ impl ExampleContainer { match idx { IntOrSlice::Slice(slice) => { let index = slice.indices(self.max_length as c_long).unwrap(); - println!("Got a slice! {}-{}, step: {}, value: {}", index.start, index.stop, index.step, value); + println!( + "Got a slice! {}-{}, step: {}, value: {}", + index.start, index.stop, index.step, value + ); } IntOrSlice::Int(index) => { println!("Got an index! {} : value: {}", index, value); diff --git a/guide/src/migration.md b/guide/src/migration.md index 16f9616e683..5a362ec448d 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -4,6 +4,8 @@ This guide can help you upgrade code through breaking changes from one PyO3 vers For a detailed list of all changes, see the [CHANGELOG](changelog.md). ## from 0.20.* to 0.21 +
+Click to expand PyO3 0.21 introduces a new `Bound<'py, T>` smart pointer which replaces the existing "GIL Refs" API to interact with Python objects. For example, in PyO3 0.20 the reference `&'py PyAny` would be used to interact with Python objects. In PyO3 0.21 the updated type is `Bound<'py, PyAny>`. Making this change moves Rust ownership semantics out of PyO3's internals and into user code. This change fixes [a known soundness edge case of interaction with gevent](https://github.com/PyO3/pyo3/issues/3668) as well as improves CPU and [memory performance](https://github.com/PyO3/pyo3/issues/1056). For a full history of discussion see https://github.com/PyO3/pyo3/issues/3382. @@ -17,8 +19,11 @@ The recommended steps to update to PyO3 0.21 is as follows: 3. Disable the `gil-refs` feature and migrate off the deprecated APIs The following sections are laid out in this order. +
### Enable the `gil-refs` feature +
+Click to expand To make the transition for the PyO3 ecosystem away from the GIL Refs API as smooth as possible, in PyO3 0.21 no APIs consuming or producing GIL Refs have been altered. Instead, variants using `Bound` smart pointers have been introduced, for example `PyTuple::new_bound` which returns `Bound` is the replacement form of `PyTuple::new`. The GIL Ref APIs have been deprecated, but to make migration easier it is possible to disable these deprecation warnings by enabling the `gil-refs` feature. @@ -41,8 +46,11 @@ After: [dependencies] pyo3 = { version = "0.21", features = ["gil-refs"] } ``` +
### `PyTypeInfo` and `PyTryFrom` have been adjusted +
+Click to expand The `PyTryFrom` trait has aged poorly, its `try_from` method now conflicts with `TryFrom::try_from` in the 2021 edition prelude. A lot of its functionality was also duplicated with `PyTypeInfo`. @@ -81,8 +89,11 @@ Python::with_gil(|py| { }) # } ``` +
### `Iter(A)NextOutput` are deprecated +
+Click to expand The `__next__` and `__anext__` magic methods can now return any type convertible into Python objects directly just like all other `#[pymethods]`. The `IterNextOutput` used by `__next__` and `IterANextOutput` used by `__anext__` are subsequently deprecated. Most importantly, this change allows returning an awaitable from `__anext__` without non-sensically wrapping it into `Yield` or `Some`. Only the return types `Option` and `Result, E>` are still handled in a special manner where `Some(val)` yields `val` and `None` stops iteration. @@ -200,20 +211,29 @@ impl PyClassAsyncIter { } } ``` +
### `PyType::name` has been renamed to `PyType::qualname` +
+Click to expand `PyType::name` has been renamed to `PyType::qualname` to indicate that it does indeed return the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name), matching the `__qualname__` attribute. The newly added `PyType::name` yields the full name including the module name now which corresponds to `__module__.__name__` on the level of attributes. +
### `PyCell` has been deprecated +
+Click to expand Interactions with Python objects implemented in Rust no longer need to go though `PyCell`. Instead iteractions with Python object now consistently go through `Bound` or `Py` independently of whether `T` is native Python object or a `#[pyclass]` implemented in Rust. Use `Bound::new` or `Py::new` respectively to create and `Bound::borrow(_mut)` / `Py::borrow(_mut)` to borrow the Rust object. +
-### Migrating from the GIL-Refs API to `Bound` +### Migrating from the GIL Refs API to `Bound` +
+Click to expand -To minimise breakage of code using the GIL-Refs API, the `Bound` smart pointer has been introduced by adding complements to all functions which accept or return GIL Refs. This allows code to migrate by replacing the deprecated APIs with the new ones. +To minimise breakage of code using the GIL Refs API, the `Bound` smart pointer has been introduced by adding complements to all functions which accept or return GIL Refs. This allows code to migrate by replacing the deprecated APIs with the new ones. -To identify what to migrate, temporarily switch off the `gil-refs` feature to see deprecation warnings on all uses of APIs accepting and producing GIL Refs. Over one or more PRs it should be possible to follow the deprecation hints to update code. Depending on your development environment, switching off the `gil-refs` feature may introduce [some very targeted breakages](#deactivating-the-gil-refs-feature), so you may need to fixup those first. +To identify what to migrate, temporarily switch off the `gil-refs` feature to see deprecation warnings on [almost](#cases-where-pyo3-cannot-emit-gil-ref-deprecation-warnings) all uses of APIs accepting and producing GIL Refs . Over one or more PRs it should be possible to follow the deprecation hints to update code. Depending on your development environment, switching off the `gil-refs` feature may introduce [some very targeted breakages](#deactivating-the-gil-refs-feature), so you may need to fixup those first. For example, the following APIs have gained updated variants: - `PyList::new`, `PyTyple::new` and similar constructors have replacements `PyList::new_bound`, `PyTuple::new_bound` etc. @@ -276,7 +296,19 @@ impl<'py> FromPyObject<'py> for MyType { The expectation is that in 0.22 `extract_bound` will have the default implementation removed and in 0.23 `extract` will be removed. +#### Cases where PyO3 cannot emit GIL Ref deprecation warnings + +Despite a large amount of deprecations warnings produced by PyO3 to aid with the transition from GIL Refs to the Bound API, there are a few cases where PyO3 cannot automatically warn on uses of GIL Refs. It is worth checking for these cases manually after the deprecation warnings have all been addressed: + +- Individual implementations of the `FromPyObject` trait cannot be deprecated, so PyO3 cannot warn about uses of code patterns like `.extract<&PyAny>()` which produce a GIL Ref. +- GIL Refs in `#[pyfunction]` arguments emit a warning, but if the GIL Ref is wrapped inside another container such as `Option<&PyAny>` or `Vec<&PyAny>` then PyO3 cannot warn against this. +- The `wrap_pyfunction!(function)(py)` deferred argument form of the `wrap_pyfunction` macro taking `py: Python<'py>` produces a GIL Ref, and due to limitations in type inference PyO3 cannot warn against this specific case. + +
+ ### Deactivating the `gil-refs` feature +
+Click to expand As a final step of migration, deactivating the `gil-refs` feature will set up code for best performance and is intended to set up a forward-compatible API for PyO3 0.22. @@ -284,9 +316,10 @@ At this point code which needed to manage GIL Ref memory can safely remove uses There is one notable API removed when this feature is disabled. `FromPyObject` trait implementations for types which borrow directly from the input data cannot be implemented by PyO3 without GIL Refs (while the migration is ongoing). These types are `&str`, `Cow<'_, str>`, `&[u8]`, `Cow<'_, u8>`. -To ease pain during migration, these types instead implement a new temporary trait `FromPyObjectBound` which is the expected future form of `FromPyObject`. The new temporary trait ensures is that `obj.extract::<&str>()` continues to work (with the new constraint that the extracted value now depends on the input `obj` lifetime), as well for these types in `#[pyfunction]` arguments. +To ease pain during migration, these types instead implement a new temporary trait `FromPyObjectBound` which is the expected future form of `FromPyObject`. The new temporary trait ensures is that `obj.extract::<&str>()` continues to work, as well for these types in `#[pyfunction]` arguments. -An unfortunate final point here is that PyO3 cannot offer this new implementation for `&str` on `abi3` builds for Python older than 3.10. On code which needs `abi3` builds for these older Python versions, many cases of `.extract::<&str>()` may need to be replaced with `.extract::()`, which is string data which borrows from the Python `str` object. Alternatively, use `.extract::>()`, `.extract::()` to copy the data into Rust for these versions. +An unfortunate final point here is that PyO3 cannot offer this new implementation for `&str` on `abi3` builds for Python older than 3.10. On code which needs `abi3` builds for these older Python versions, many cases of `.extract::<&str>()` may need to be replaced with `.extract::()`, which is string data which borrows from the Python `str` object. Alternatively, use `.extract::>()` ro `.extract::()` to copy the data into Rust for these versions. +
## from 0.19.* to 0.20 diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 93a54395c94..1a46d333c3b 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -8,7 +8,7 @@ use crate::utils::Ctx; use crate::{ attributes::{TextSignatureAttribute, TextSignatureAttributeValue}, deprecations::{Deprecation, Deprecations}, - params::impl_arg_params, + params::{impl_arg_params, Holders}, pyfunction::{ FunctionSignature, PyFunctionArgPyO3Attributes, PyFunctionOptions, SignatureAttribute, }, @@ -108,7 +108,7 @@ impl FnType { &self, cls: Option<&syn::Type>, error_mode: ExtractErrorMode, - holders: &mut Vec, + holders: &mut Holders, ctx: &Ctx, ) -> TokenStream { let Ctx { pyo3_path } = ctx; @@ -186,7 +186,7 @@ impl SelfType { &self, cls: &syn::Type, error_mode: ExtractErrorMode, - holders: &mut Vec, + holders: &mut Holders, ctx: &Ctx, ) -> TokenStream { // Due to use of quote_spanned in this function, need to bind these idents to the @@ -201,17 +201,12 @@ impl SelfType { } else { syn::Ident::new("extract_pyclass_ref", *span) }; - let holder = syn::Ident::new(&format!("holder_{}", holders.len()), *span); + let holder = holders.push_holder(*span); let pyo3_path = pyo3_path.to_tokens_spanned(*span); - holders.push(quote_spanned! { *span => - #[allow(clippy::let_unit_value)] - let mut #holder = #pyo3_path::impl_::extract_argument::FunctionArgumentHolder::INIT; - let mut #slf = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf); - }); error_mode.handle_error( quote_spanned! { *span => #pyo3_path::impl_::extract_argument::#method::<#cls>( - &#slf, + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(#py, &#slf).0, &mut #holder, ) }, @@ -519,10 +514,8 @@ impl<'a> FnSpec<'a> { ); } - let rust_call = |args: Vec, - self_e: &syn::Ident, - holders: &mut Vec| { - let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise, holders, ctx); + let rust_call = |args: Vec, holders: &mut Holders| { + let mut self_arg = || self.tp.self_arg(cls, ExtractErrorMode::Raise, holders, ctx); let call = if self.asyncness.is_some() { let throw_callback = if cancel_handle.is_some() { @@ -537,35 +530,30 @@ impl<'a> FnSpec<'a> { }; let future = match self.tp { FnType::Fn(SelfType::Receiver { mutable: false, .. }) => { - holders.pop().unwrap(); // does not actually use holder created by `self_arg` - quote! {{ - #self_e = #pyo3_path::impl_::deprecations::GilRefs::<()>::new(); let __guard = #pyo3_path::impl_::coroutine::RefGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; async move { function(&__guard, #(#args),*).await } }} } FnType::Fn(SelfType::Receiver { mutable: true, .. }) => { - holders.pop().unwrap(); // does not actually use holder created by `self_arg` - quote! {{ - #self_e = #pyo3_path::impl_::deprecations::GilRefs::<()>::new(); let mut __guard = #pyo3_path::impl_::coroutine::RefMutGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; async move { function(&mut __guard, #(#args),*).await } }} } _ => { + let self_arg = self_arg(); if self_arg.is_empty() { - quote! {{ - #self_e = #pyo3_path::impl_::deprecations::GilRefs::<()>::new(); - function(#(#args),*) - }} + quote! { function(#(#args),*) } } else { - quote! { function({ - let (self_arg, e) = #pyo3_path::impl_::deprecations::inspect_type(#self_arg); - #self_e = e; - self_arg - }, #(#args),*) } + let self_checker = holders.push_gil_refs_checker(self_arg.span()); + quote! { + function( + // NB #self_arg includes a comma, so none inserted here + #pyo3_path::impl_::deprecations::inspect_type(#self_arg &#self_checker), + #(#args),* + ) + } } } }; @@ -586,24 +574,22 @@ impl<'a> FnSpec<'a> { }}; } call - } else if self_arg.is_empty() { - quote! {{ - #self_e = #pyo3_path::impl_::deprecations::GilRefs::<()>::new(); - function(#(#args),*) - }} } else { - quote! { - function({ - let (self_arg, e) = #pyo3_path::impl_::deprecations::inspect_type(#self_arg); - #self_e = e; - self_arg - }, #(#args),*) + let self_arg = self_arg(); + if self_arg.is_empty() { + quote! { function(#(#args),*) } + } else { + let self_checker = holders.push_gil_refs_checker(self_arg.span()); + quote! { + function( + // NB #self_arg includes a comma, so none inserted here + #pyo3_path::impl_::deprecations::inspect_type(#self_arg &#self_checker), + #(#args),* + ) + } } }; - ( - quotes::map_result_into_ptr(quotes::ok_wrap(call, ctx), ctx), - self_arg.span(), - ) + quotes::map_result_into_ptr(quotes::ok_wrap(call, ctx), ctx) }; let func_name = &self.name; @@ -613,10 +599,9 @@ impl<'a> FnSpec<'a> { quote!(#func_name) }; - let self_e = syn::Ident::new("self_e", Span::call_site()); Ok(match self.convention { CallingConvention::Noargs => { - let mut holders = Vec::new(); + let mut holders = Holders::new(); let args = self .signature .arguments @@ -631,8 +616,9 @@ impl<'a> FnSpec<'a> { } }) .collect(); - let (call, self_arg_span) = rust_call(args, &self_e, &mut holders); - let function_arg = quote_spanned! { self_arg_span => #self_e.function_arg(); }; + let call = rust_call(args, &mut holders); + let check_gil_refs = holders.check_gil_refs(); + let init_holders = holders.init_holders(ctx); quote! { unsafe fn #ident<'py>( @@ -641,19 +627,19 @@ impl<'a> FnSpec<'a> { ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 - let #self_e; - #( #holders )* + #init_holders let result = #call; - #function_arg + #check_gil_refs result } } } CallingConvention::Fastcall => { - let mut holders = Vec::new(); + let mut holders = Holders::new(); let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders, ctx)?; - let (call, self_arg_span) = rust_call(args, &self_e, &mut holders); - let function_arg = quote_spanned! { self_arg_span => #self_e.function_arg(); }; + let call = rust_call(args, &mut holders); + let init_holders = holders.init_holders(ctx); + let check_gil_refs = holders.check_gil_refs(); quote! { unsafe fn #ident<'py>( @@ -665,20 +651,20 @@ impl<'a> FnSpec<'a> { ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 - let #self_e; #arg_convert - #( #holders )* + #init_holders let result = #call; - #function_arg + #check_gil_refs result } } } CallingConvention::Varargs => { - let mut holders = Vec::new(); + let mut holders = Holders::new(); let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx)?; - let (call, self_arg_span) = rust_call(args, &self_e, &mut holders); - let function_arg = quote_spanned! { self_arg_span => #self_e.function_arg(); }; + let call = rust_call(args, &mut holders); + let init_holders = holders.init_holders(ctx); + let check_gil_refs = holders.check_gil_refs(); quote! { unsafe fn #ident<'py>( @@ -689,22 +675,23 @@ impl<'a> FnSpec<'a> { ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 - let #self_e; #arg_convert - #( #holders )* + #init_holders let result = #call; - #function_arg + #check_gil_refs result } } } CallingConvention::TpNew => { - let mut holders = Vec::new(); + let mut holders = Holders::new(); let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx)?; let self_arg = self .tp .self_arg(cls, ExtractErrorMode::Raise, &mut holders, ctx); let call = quote! { #rust_name(#self_arg #(#args),*) }; + let init_holders = holders.init_holders(ctx); + let check_gil_refs = holders.check_gil_refs(); quote! { unsafe fn #ident( py: #pyo3_path::Python<'_>, @@ -716,9 +703,10 @@ impl<'a> FnSpec<'a> { let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert - #( #holders )* + #init_holders let result = #call; let initializer: #pyo3_path::PyClassInitializer::<#cls> = result.convert(py)?; + #check_gil_refs #pyo3_path::impl_::pymethods::tp_new_impl(py, initializer, _slf) } } diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 2515d519287..dc2b3bfc86b 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -305,10 +305,11 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result .filter_map(|param| { if let syn::FnArg::Typed(pat_type) = param { if let syn::Pat::Ident(pat_ident) = &*pat_type.pat { - let ident = &pat_ident.ident; + let ident: &syn::Ident = &pat_ident.ident; return Some([ - parse_quote! { let (#ident, e) = #pyo3_path::impl_::deprecations::inspect_type(#ident); }, - parse_quote_spanned! { pat_type.span() => e.function_arg(); }, + parse_quote!{ let check_gil_refs = #pyo3_path::impl_::deprecations::GilRefs::new(); }, + parse_quote! { let #ident = #pyo3_path::impl_::deprecations::inspect_type(#ident, &check_gil_refs); }, + parse_quote_spanned! { pat_type.span() => check_gil_refs.function_arg(); }, ]); } } diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index ae1661fe2c1..cab28698b71 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -9,6 +9,53 @@ use quote::{quote, quote_spanned}; use syn::spanned::Spanned; use syn::Result; +pub struct Holders { + holders: Vec, + gil_refs_checkers: Vec, +} + +impl Holders { + pub fn new() -> Self { + Holders { + holders: Vec::new(), + gil_refs_checkers: Vec::new(), + } + } + + pub fn push_holder(&mut self, span: Span) -> syn::Ident { + let holder = syn::Ident::new(&format!("holder_{}", self.holders.len()), span); + self.holders.push(holder.clone()); + holder + } + + pub fn push_gil_refs_checker(&mut self, span: Span) -> syn::Ident { + let gil_refs_checker = syn::Ident::new( + &format!("gil_refs_checker_{}", self.gil_refs_checkers.len()), + span, + ); + self.gil_refs_checkers.push(gil_refs_checker.clone()); + gil_refs_checker + } + + pub fn init_holders(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path } = ctx; + let holders = &self.holders; + let gil_refs_checkers = &self.gil_refs_checkers; + quote! { + #[allow(clippy::let_unit_value)] + #(let mut #holders = #pyo3_path::impl_::extract_argument::FunctionArgumentHolder::INIT;)* + #(let #gil_refs_checkers = #pyo3_path::impl_::deprecations::GilRefs::new();)* + } + } + + pub fn check_gil_refs(&self) -> TokenStream { + self.gil_refs_checkers + .iter() + .map(|e| quote_spanned! { e.span() => #e.function_arg(); }) + .collect() + } +} + /// Return true if the argument list is simply (*args, **kwds). pub fn is_forwarded_args(signature: &FunctionSignature<'_>) -> bool { matches!( @@ -26,11 +73,22 @@ pub fn is_forwarded_args(signature: &FunctionSignature<'_>) -> bool { ) } +fn check_arg_for_gil_refs( + tokens: TokenStream, + gil_refs_checker: syn::Ident, + ctx: &Ctx, +) -> TokenStream { + let Ctx { pyo3_path } = ctx; + quote! { + #pyo3_path::impl_::deprecations::inspect_type(#tokens, &#gil_refs_checker) + } +} + pub fn impl_arg_params( spec: &FnSpec<'_>, self_: Option<&syn::Type>, fastcall: bool, - holders: &mut Vec, + holders: &mut Holders, ctx: &Ctx, ) -> Result<(TokenStream, Vec)> { let args_array = syn::Ident::new("output", Span::call_site()); @@ -61,7 +119,15 @@ pub fn impl_arg_params( .arguments .iter() .enumerate() - .map(|(i, arg)| impl_arg_param(arg, i, &mut 0, &args_array, holders, ctx)) + .map(|(i, arg)| { + impl_arg_param(arg, i, &mut 0, &args_array, holders, ctx).map(|tokens| { + check_arg_for_gil_refs( + tokens, + holders.push_gil_refs_checker(arg.ty.span()), + ctx, + ) + }) + }) .collect::>()?; return Ok(( quote! { @@ -101,7 +167,11 @@ pub fn impl_arg_params( .arguments .iter() .enumerate() - .map(|(i, arg)| impl_arg_param(arg, i, &mut option_pos, &args_array, holders, ctx)) + .map(|(i, arg)| { + impl_arg_param(arg, i, &mut option_pos, &args_array, holders, ctx).map(|tokens| { + check_arg_for_gil_refs(tokens, holders.push_gil_refs_checker(arg.ty.span()), ctx) + }) + }) .collect::>()?; let args_handler = if spec.signature.python_signature.varargs.is_some() { @@ -169,7 +239,7 @@ fn impl_arg_param( pos: usize, option_pos: &mut usize, args_array: &syn::Ident, - holders: &mut Vec, + holders: &mut Holders, ctx: &Ctx, ) -> Result { let Ctx { pyo3_path } = ctx; @@ -192,21 +262,12 @@ fn impl_arg_param( let name = arg.name; let name_str = name.to_string(); - let mut push_holder = || { - let holder = syn::Ident::new(&format!("holder_{}", holders.len()), arg.ty.span()); - holders.push(quote_arg_span! { - #[allow(clippy::let_unit_value)] - let mut #holder = #pyo3_path::impl_::extract_argument::FunctionArgumentHolder::INIT; - }); - holder - }; - if arg.is_varargs { ensure_spanned!( arg.optional.is_none(), arg.name.span() => "args cannot be optional" ); - let holder = push_holder(); + let holder = holders.push_holder(arg.ty.span()); return Ok(quote_arg_span! { #pyo3_path::impl_::extract_argument::extract_argument( &_args, @@ -219,7 +280,7 @@ fn impl_arg_param( arg.optional.is_some(), arg.name.span() => "kwargs must be Option<_>" ); - let holder = push_holder(); + let holder = holders.push_holder(arg.name.span()); return Ok(quote_arg_span! { #pyo3_path::impl_::extract_argument::extract_optional_argument( _kwargs.as_deref(), @@ -254,12 +315,14 @@ fn impl_arg_param( let from_py_with = syn::Ident::new(&format!("from_py_with_{}", pos), Span::call_site()); if let Some(default) = default { quote_arg_span! { - #[allow(clippy::redundant_closure)] #pyo3_path::impl_::extract_argument::from_py_with_with_default( #arg_value.as_deref(), #name_str, #from_py_with as fn(_) -> _, - || #default + #[allow(clippy::redundant_closure)] + { + || #default + } )? } } else { @@ -272,29 +335,33 @@ fn impl_arg_param( } } } else if arg.optional.is_some() { - let holder = push_holder(); + let holder = holders.push_holder(arg.name.span()); quote_arg_span! { - #[allow(clippy::redundant_closure)] #pyo3_path::impl_::extract_argument::extract_optional_argument( #arg_value.as_deref(), &mut #holder, #name_str, - || #default + #[allow(clippy::redundant_closure)] + { + || #default + } )? } } else if let Some(default) = default { - let holder = push_holder(); + let holder = holders.push_holder(arg.name.span()); quote_arg_span! { - #[allow(clippy::redundant_closure)] #pyo3_path::impl_::extract_argument::extract_argument_with_default( #arg_value.as_deref(), &mut #holder, #name_str, - || #default + #[allow(clippy::redundant_closure)] + { + || #default + } )? } } else { - let holder = push_holder(); + let holder = holders.push_holder(arg.name.span()); quote_arg_span! { #pyo3_path::impl_::extract_argument::extract_argument( &#pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value), @@ -303,5 +370,6 @@ fn impl_arg_param( )? } }; + Ok(tokens) } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 27a7ee969d5..5122d205640 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -736,10 +736,11 @@ fn impl_simple_enum( fn __pyo3__richcmp__( &self, py: #pyo3_path::Python, - other: &#pyo3_path::PyAny, + other: &#pyo3_path::Bound<'_, #pyo3_path::PyAny>, op: #pyo3_path::basic::CompareOp ) -> #pyo3_path::PyResult<#pyo3_path::PyObject> { use #pyo3_path::conversion::ToPyObject; + use #pyo3_path::types::PyAnyMethods; use ::core::result::Result::*; match op { #pyo3_path::basic::CompareOp::Eq => { diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 1c19112d183..4e6f46d96d5 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; use crate::attributes::{NameAttribute, RenamingRule}; use crate::method::{CallingConvention, ExtractErrorMode}; +use crate::params::Holders; use crate::utils::Ctx; use crate::utils::PythonDoc; use crate::{ @@ -510,7 +511,7 @@ fn impl_call_setter( cls: &syn::Type, spec: &FnSpec<'_>, self_type: &SelfType, - holders: &mut Vec, + holders: &mut Holders, ctx: &Ctx, ) -> syn::Result { let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); @@ -544,7 +545,7 @@ pub fn impl_py_setter_def( let Ctx { pyo3_path } = ctx; let python_name = property_type.null_terminated_python_name()?; let doc = property_type.doc(); - let mut holders = Vec::new(); + let mut holders = Holders::new(); let setter_impl = match property_type { PropertyType::Descriptor { field_index, field, .. @@ -596,6 +597,8 @@ pub fn impl_py_setter_def( } } + let init_holders = holders.init_holders(ctx); + let check_gil_refs = holders.check_gil_refs(); let associated_method = quote! { #cfg_attrs unsafe fn #wrapper_ident( @@ -609,8 +612,10 @@ pub fn impl_py_setter_def( #pyo3_path::exceptions::PyAttributeError::new_err("can't delete attribute") })?; let _val = #pyo3_path::FromPyObject::extract_bound(_value.into())?; - #( #holders )* - #pyo3_path::callback::convert(py, #setter_impl) + #init_holders + let result = #setter_impl; + #check_gil_refs + #pyo3_path::callback::convert(py, result) } }; @@ -635,7 +640,7 @@ fn impl_call_getter( cls: &syn::Type, spec: &FnSpec<'_>, self_type: &SelfType, - holders: &mut Vec, + holders: &mut Holders, ctx: &Ctx, ) -> syn::Result { let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); @@ -665,7 +670,7 @@ pub fn impl_py_getter_def( let python_name = property_type.null_terminated_python_name()?; let doc = property_type.doc(); - let mut holders = Vec::new(); + let mut holders = Holders::new(); let body = match property_type { PropertyType::Descriptor { field_index, field, .. @@ -731,14 +736,17 @@ pub fn impl_py_getter_def( } } + let init_holders = holders.init_holders(ctx); + let check_gil_refs = holders.check_gil_refs(); let associated_method = quote! { #cfg_attrs unsafe fn #wrapper_ident( py: #pyo3_path::Python<'_>, _slf: *mut #pyo3_path::ffi::PyObject ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { - #( #holders )* + #init_holders let result = #body; + #check_gil_refs result } }; @@ -966,7 +974,7 @@ impl Ty { ident: &syn::Ident, arg: &FnArg<'_>, extract_error_mode: ExtractErrorMode, - holders: &mut Vec, + holders: &mut Holders, ctx: &Ctx, ) -> TokenStream { let Ctx { pyo3_path } = ctx; @@ -976,7 +984,9 @@ impl Ty { extract_error_mode, holders, &name_str, - quote! { #ident },ctx + quote! { #ident }, + arg.ty.span(), + ctx ), Ty::MaybeNullObject => extract_object( extract_error_mode, @@ -988,32 +998,40 @@ impl Ty { } else { #ident } - },ctx + }, + arg.ty.span(), + ctx ), Ty::NonNullObject => extract_object( extract_error_mode, holders, &name_str, - quote! { #ident.as_ptr() },ctx + quote! { #ident.as_ptr() }, + arg.ty.span(), + ctx ), Ty::IPowModulo => extract_object( extract_error_mode, holders, &name_str, - quote! { #ident.as_ptr() },ctx + quote! { #ident.as_ptr() }, + arg.ty.span(), + ctx ), Ty::CompareOp => extract_error_mode.handle_error( quote! { #pyo3_path::class::basic::CompareOp::from_raw(#ident) .ok_or_else(|| #pyo3_path::exceptions::PyValueError::new_err("invalid comparison operator")) - },ctx + }, + ctx ), Ty::PySsizeT => { let ty = arg.ty; extract_error_mode.handle_error( quote! { ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| #pyo3_path::exceptions::PyValueError::new_err(e.to_string())) - },ctx + }, + ctx ) } // Just pass other types through unmodified @@ -1024,27 +1042,28 @@ impl Ty { fn extract_object( extract_error_mode: ExtractErrorMode, - holders: &mut Vec, + holders: &mut Holders, name: &str, source_ptr: TokenStream, + span: Span, ctx: &Ctx, ) -> TokenStream { let Ctx { pyo3_path } = ctx; - let holder = syn::Ident::new(&format!("holder_{}", holders.len()), Span::call_site()); - holders.push(quote! { - #[allow(clippy::let_unit_value)] - let mut #holder = #pyo3_path::impl_::extract_argument::FunctionArgumentHolder::INIT; - }); - extract_error_mode.handle_error( + let holder = holders.push_holder(Span::call_site()); + let gil_refs_checker = holders.push_gil_refs_checker(span); + let extracted = extract_error_mode.handle_error( quote! { #pyo3_path::impl_::extract_argument::extract_argument( - &#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr), + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr).0, &mut #holder, #name ) }, ctx, - ) + ); + quote! { + #pyo3_path::impl_::deprecations::inspect_type(#extracted, &#gil_refs_checker) + } } enum ReturnMode { @@ -1054,13 +1073,15 @@ enum ReturnMode { } impl ReturnMode { - fn return_call_output(&self, call: TokenStream, ctx: &Ctx) -> TokenStream { + fn return_call_output(&self, call: TokenStream, ctx: &Ctx, holders: &Holders) -> TokenStream { let Ctx { pyo3_path } = ctx; + let check_gil_refs = holders.check_gil_refs(); match self { ReturnMode::Conversion(conversion) => { let conversion = TokenGeneratorCtx(*conversion, ctx); quote! { let _result: #pyo3_path::PyResult<#conversion> = #pyo3_path::callback::convert(py, #call); + #check_gil_refs #pyo3_path::callback::convert(py, _result) } } @@ -1070,12 +1091,14 @@ impl ReturnMode { quote! { let _result = #call; use #pyo3_path::impl_::pymethods::{#traits}; + #check_gil_refs (&_result).#tag().convert(py, _result) } } ReturnMode::ReturnSelf => quote! { let _result: #pyo3_path::PyResult<()> = #pyo3_path::callback::convert(py, #call); _result?; + #check_gil_refs #pyo3_path::ffi::Py_XINCREF(_raw_slf); ::std::result::Result::Ok(_raw_slf) }, @@ -1176,7 +1199,7 @@ impl SlotDef { .collect(); let wrapper_ident = format_ident!("__pymethod_{}__", method_name); let ret_ty = ret_ty.ffi_type(ctx); - let mut holders = Vec::new(); + let mut holders = Holders::new(); let body = generate_method_body( cls, spec, @@ -1187,6 +1210,7 @@ impl SlotDef { ctx, )?; let name = spec.name; + let holders = holders.init_holders(ctx); let associated_method = quote! { unsafe fn #wrapper_ident( py: #pyo3_path::Python<'_>, @@ -1195,7 +1219,7 @@ impl SlotDef { ) -> #pyo3_path::PyResult<#ret_ty> { let function = #cls::#name; // Shadow the method name to avoid #3017 let _slf = _raw_slf; - #( #holders )* + #holders #body } }; @@ -1229,7 +1253,7 @@ fn generate_method_body( spec: &FnSpec<'_>, arguments: &[Ty], extract_error_mode: ExtractErrorMode, - holders: &mut Vec, + holders: &mut Holders, return_mode: Option<&ReturnMode>, ctx: &Ctx, ) -> Result { @@ -1241,9 +1265,14 @@ fn generate_method_body( let args = extract_proto_arguments(spec, arguments, extract_error_mode, holders, ctx)?; let call = quote! { #cls::#rust_name(#self_arg #(#args),*) }; Ok(if let Some(return_mode) = return_mode { - return_mode.return_call_output(call, ctx) + return_mode.return_call_output(call, ctx, holders) } else { - quote! { #pyo3_path::callback::convert(py, #call) } + let check_gil_refs = holders.check_gil_refs(); + quote! { + let result = #call; + #check_gil_refs; + #pyo3_path::callback::convert(py, result) + } }) } @@ -1294,7 +1323,7 @@ impl SlotFragmentDef { let arg_idents: &Vec<_> = &(0..arguments.len()) .map(|i| format_ident!("arg{}", i)) .collect(); - let mut holders = Vec::new(); + let mut holders = Holders::new(); let body = generate_method_body( cls, spec, @@ -1305,6 +1334,7 @@ impl SlotFragmentDef { ctx, )?; let ret_ty = ret_ty.ffi_type(ctx); + let holders = holders.init_holders(ctx); Ok(quote! { impl #cls { unsafe fn #wrapper_ident( @@ -1313,7 +1343,7 @@ impl SlotFragmentDef { #(#arg_idents: #arg_types),* ) -> #pyo3_path::PyResult<#ret_ty> { let _slf = _raw_slf; - #( #holders )* + #holders #body } } @@ -1412,7 +1442,7 @@ fn extract_proto_arguments( spec: &FnSpec<'_>, proto_args: &[Ty], extract_error_mode: ExtractErrorMode, - holders: &mut Vec, + holders: &mut Holders, ctx: &Ctx, ) -> Result> { let mut args = Vec::with_capacity(spec.signature.arguments.len()); diff --git a/pyo3-macros-backend/src/quotes.rs b/pyo3-macros-backend/src/quotes.rs index 0219cb9ae7f..ceef23fb034 100644 --- a/pyo3-macros-backend/src/quotes.rs +++ b/pyo3-macros-backend/src/quotes.rs @@ -19,7 +19,5 @@ pub(crate) fn ok_wrap(obj: TokenStream, ctx: &Ctx) -> TokenStream { pub(crate) fn map_result_into_ptr(result: TokenStream, ctx: &Ctx) -> TokenStream { let Ctx { pyo3_path } = ctx; - quote! { - #pyo3_path::impl_::wrap::map_result_into_ptr(py, #result) - } + quote! { #pyo3_path::impl_::wrap::map_result_into_ptr(py, #result) } } diff --git a/pytests/src/buf_and_str.rs b/pytests/src/buf_and_str.rs index e9651e0cfd9..879d76af883 100644 --- a/pytests/src/buf_and_str.rs +++ b/pytests/src/buf_and_str.rs @@ -17,19 +17,19 @@ impl BytesExtractor { } #[staticmethod] - pub fn from_bytes(bytes: &PyBytes) -> PyResult { + pub fn from_bytes(bytes: &Bound<'_, PyBytes>) -> PyResult { let byte_vec: Vec = bytes.extract()?; Ok(byte_vec.len()) } #[staticmethod] - pub fn from_str(string: &PyString) -> PyResult { + pub fn from_str(string: &Bound<'_, PyString>) -> PyResult { let rust_string: String = string.extract()?; Ok(rust_string.len()) } #[staticmethod] - pub fn from_str_lossy(string: &PyString) -> usize { + pub fn from_str_lossy(string: &Bound<'_, PyString>) -> usize { let rust_string_lossy: String = string.to_string_lossy().to_string(); rust_string_lossy.len() } diff --git a/pytests/src/datetime.rs b/pytests/src/datetime.rs index aeb57240c15..d0de99ae406 100644 --- a/pytests/src/datetime.rs +++ b/pytests/src/datetime.rs @@ -12,8 +12,11 @@ fn make_date(py: Python<'_>, year: i32, month: u8, day: u8) -> PyResult(py: Python<'p>, d: &PyDate) -> Bound<'p, PyTuple> { - PyTuple::new_bound(py, [d.get_year(), d.get_month() as i32, d.get_day() as i32]) +fn get_date_tuple<'py>(d: &Bound<'py, PyDate>) -> Bound<'py, PyTuple> { + PyTuple::new_bound( + d.py(), + [d.get_year(), d.get_month() as i32, d.get_day() as i32], + ) } #[pyfunction] @@ -48,9 +51,9 @@ fn time_with_fold<'py>( } #[pyfunction] -fn get_time_tuple<'p>(py: Python<'p>, dt: &PyTime) -> Bound<'p, PyTuple> { +fn get_time_tuple<'py>(dt: &Bound<'py, PyTime>) -> Bound<'py, PyTuple> { PyTuple::new_bound( - py, + dt.py(), [ dt.get_hour() as u32, dt.get_minute() as u32, @@ -61,9 +64,9 @@ fn get_time_tuple<'p>(py: Python<'p>, dt: &PyTime) -> Bound<'p, PyTuple> { } #[pyfunction] -fn get_time_tuple_fold<'p>(py: Python<'p>, dt: &PyTime) -> Bound<'p, PyTuple> { +fn get_time_tuple_fold<'py>(dt: &Bound<'py, PyTime>) -> Bound<'py, PyTuple> { PyTuple::new_bound( - py, + dt.py(), [ dt.get_hour() as u32, dt.get_minute() as u32, @@ -123,9 +126,9 @@ fn make_datetime<'py>( } #[pyfunction] -fn get_datetime_tuple<'py>(py: Python<'py>, dt: &Bound<'py, PyDateTime>) -> Bound<'py, PyTuple> { +fn get_datetime_tuple<'py>(dt: &Bound<'py, PyDateTime>) -> Bound<'py, PyTuple> { PyTuple::new_bound( - py, + dt.py(), [ dt.get_year(), dt.get_month() as i32, @@ -139,12 +142,9 @@ fn get_datetime_tuple<'py>(py: Python<'py>, dt: &Bound<'py, PyDateTime>) -> Boun } #[pyfunction] -fn get_datetime_tuple_fold<'py>( - py: Python<'py>, - dt: &Bound<'py, PyDateTime>, -) -> Bound<'py, PyTuple> { +fn get_datetime_tuple_fold<'py>(dt: &Bound<'py, PyDateTime>) -> Bound<'py, PyTuple> { PyTuple::new_bound( - py, + dt.py(), [ dt.get_year(), dt.get_month() as i32, @@ -187,12 +187,8 @@ impl TzClass { TzClass {} } - fn utcoffset<'py>( - &self, - py: Python<'py>, - _dt: &Bound<'py, PyDateTime>, - ) -> PyResult> { - PyDelta::new_bound(py, 0, 3600, 0, true) + fn utcoffset<'py>(&self, dt: &Bound<'py, PyDateTime>) -> PyResult> { + PyDelta::new_bound(dt.py(), 0, 3600, 0, true) } fn tzname(&self, _dt: &Bound<'_, PyDateTime>) -> String { diff --git a/pytests/src/dict_iter.rs b/pytests/src/dict_iter.rs index 985c929792f..c312fbb5f83 100644 --- a/pytests/src/dict_iter.rs +++ b/pytests/src/dict_iter.rs @@ -20,7 +20,7 @@ impl DictSize { DictSize { expected } } - fn iter_dict(&mut self, _py: Python<'_>, dict: &PyDict) -> PyResult { + fn iter_dict(&mut self, _py: Python<'_>, dict: &Bound<'_, PyDict>) -> PyResult { let mut seen = 0u32; for (sym, values) in dict { seen += 1; diff --git a/pytests/src/misc.rs b/pytests/src/misc.rs index 3b893ccd1f6..7704098bd5b 100644 --- a/pytests/src/misc.rs +++ b/pytests/src/misc.rs @@ -8,8 +8,8 @@ fn issue_219() { } #[pyfunction] -fn get_type_full_name(obj: &PyAny) -> PyResult> { - obj.get_type().name() +fn get_type_full_name(obj: &Bound<'_, PyAny>) -> PyResult { + obj.get_type().name().map(Cow::into_owned) } #[pyfunction] diff --git a/pytests/src/objstore.rs b/pytests/src/objstore.rs index 440f29fad63..9a005c0ec97 100644 --- a/pytests/src/objstore.rs +++ b/pytests/src/objstore.rs @@ -13,7 +13,7 @@ impl ObjStore { ObjStore::default() } - fn push(&mut self, py: Python<'_>, obj: &PyAny) { + fn push(&mut self, py: Python<'_>, obj: &Bound<'_, PyAny>) { self.obj.push(obj.to_object(py)); } } diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index 1f3baec2755..8e957c77955 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -57,22 +57,27 @@ impl AssertingBaseClass { } } -#[pyclass(subclass)] -#[derive(Clone, Debug)] -struct AssertingBaseClassGilRef; +#[allow(deprecated)] +mod deprecated { + use super::*; -#[pymethods] -impl AssertingBaseClassGilRef { - #[new] - #[classmethod] - fn new(cls: &PyType, expected_type: &PyType) -> PyResult { - if !cls.is(expected_type) { - return Err(PyValueError::new_err(format!( - "{:?} != {:?}", - cls, expected_type - ))); + #[pyclass(subclass)] + #[derive(Clone, Debug)] + pub struct AssertingBaseClassGilRef; + + #[pymethods] + impl AssertingBaseClassGilRef { + #[new] + #[classmethod] + fn new(cls: &PyType, expected_type: &PyType) -> PyResult { + if !cls.is(expected_type) { + return Err(PyValueError::new_err(format!( + "{:?} != {:?}", + cls, expected_type + ))); + } + Ok(Self) } - Ok(Self) } } @@ -84,7 +89,7 @@ pub fn pyclasses(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; Ok(()) } diff --git a/src/coroutine.rs b/src/coroutine.rs index b24197ad593..f2feab4af16 100644 --- a/src/coroutine.rs +++ b/src/coroutine.rs @@ -15,7 +15,7 @@ use crate::{ exceptions::{PyAttributeError, PyRuntimeError, PyStopIteration}, panic::PanicException, types::{string::PyStringMethods, PyIterator, PyString}, - IntoPy, Py, PyAny, PyErr, PyObject, PyResult, Python, + Bound, IntoPy, Py, PyAny, PyErr, PyObject, PyResult, Python, }; pub(crate) mod cancel; @@ -143,7 +143,7 @@ impl Coroutine { } } - fn send(&mut self, py: Python<'_>, _value: &PyAny) -> PyResult { + fn send(&mut self, py: Python<'_>, _value: &Bound<'_, PyAny>) -> PyResult { self.poll(py, None) } diff --git a/src/coroutine/waker.rs b/src/coroutine/waker.rs index b524b6d7298..fc7c54e1f5a 100644 --- a/src/coroutine/waker.rs +++ b/src/coroutine/waker.rs @@ -97,7 +97,7 @@ impl LoopAndFuture { /// Future can be cancelled by the event loop before being waken. /// See #[pyfunction(crate = "crate")] -fn release_waiter(future: &PyAny) -> PyResult<()> { +fn release_waiter(future: &Bound<'_, PyAny>) -> PyResult<()> { let done = future.call_method0(intern!(future.py(), "done"))?; if !done.extract::()? { future.call_method1(intern!(future.py(), "set_result"), (future.py().None(),))?; diff --git a/src/impl_/deprecations.rs b/src/impl_/deprecations.rs index 0c749ff985b..459eba913d3 100644 --- a/src/impl_/deprecations.rs +++ b/src/impl_/deprecations.rs @@ -5,8 +5,8 @@ use crate::{PyResult, Python}; #[deprecated(since = "0.20.0", note = "use `#[new]` instead of `#[__new__]`")] pub const PYMETHODS_NEW_DEPRECATED_FORM: () = (); -pub fn inspect_type(t: T) -> (T, GilRefs) { - (t, GilRefs::new()) +pub fn inspect_type(t: T, _: &GilRefs) -> T { + t } pub fn inspect_fn(f: fn(A) -> PyResult, _: &GilRefs) -> fn(A) -> PyResult { diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index 82285b3d50c..4dcef02c5e6 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -41,14 +41,27 @@ impl<'a, 'py, T: 'py> PyFunctionArgument<'a, 'py> for &'a Bound<'py, T> where T: PyTypeCheck, { - type Holder = Option<&'a Bound<'py, T>>; + type Holder = Option<()>; #[inline] - fn extract( - obj: &'a Bound<'py, PyAny>, - holder: &'a mut Option<&'a Bound<'py, T>>, - ) -> PyResult { - Ok(holder.insert(obj.downcast()?)) + fn extract(obj: &'a Bound<'py, PyAny>, _: &'a mut Option<()>) -> PyResult { + obj.downcast().map_err(Into::into) + } +} + +impl<'a, 'py, T: 'py> PyFunctionArgument<'a, 'py> for Option<&'a Bound<'py, T>> +where + T: PyTypeCheck, +{ + type Holder = (); + + #[inline] + fn extract(obj: &'a Bound<'py, PyAny>, _: &'a mut ()) -> PyResult { + if obj.is_none() { + Ok(None) + } else { + Ok(Some(obj.downcast()?)) + } } } diff --git a/src/macros.rs b/src/macros.rs index db409515085..648bac180ff 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -144,8 +144,10 @@ macro_rules! wrap_pyfunction { }; ($function:path, $py_or_module:expr) => {{ use $function as wrapped_pyfunction; - let (py_or_module, e) = $crate::impl_::deprecations::inspect_type($py_or_module); - e.is_python(); + let check_gil_refs = $crate::impl_::deprecations::GilRefs::new(); + let py_or_module = + $crate::impl_::deprecations::inspect_type($py_or_module, &check_gil_refs); + check_gil_refs.is_python(); $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( py_or_module, &wrapped_pyfunction::DEF, diff --git a/src/pycell.rs b/src/pycell.rs index 636fb90cef6..ccc55a756d6 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -149,12 +149,12 @@ //! //! It is better to write that function like this: //! ```rust +//! # #![allow(deprecated)] //! # use pyo3::prelude::*; //! # #[pyclass] //! # pub struct Number { //! # inner: u32, //! # } -//! # #[allow(deprecated)] //! #[pyfunction] //! fn swap_numbers(a: &PyCell, b: &PyCell) { //! // Check that the pointers are unequal diff --git a/src/tests/hygiene/pymethods.rs b/src/tests/hygiene/pymethods.rs index 2ffaf68690f..020f983be31 100644 --- a/src/tests/hygiene/pymethods.rs +++ b/src/tests/hygiene/pymethods.rs @@ -28,7 +28,7 @@ impl Dummy { } fn __format__(&self, format_spec: ::std::string::String) -> ::std::string::String { - ::std::panic!("unimplemented isn't hygienic before 1.50") + ::std::unimplemented!() } fn __lt__(&self, other: &Self) -> bool { @@ -63,12 +63,12 @@ impl Dummy { // Customizing attribute access ////////////////////// - fn __getattr__(&self, name: ::std::string::String) -> &crate::PyAny { - ::std::panic!("unimplemented isn't hygienic before 1.50") + fn __getattr__(&self, name: ::std::string::String) -> &crate::Bound<'_, crate::PyAny> { + ::std::unimplemented!() } - fn __getattribute__(&self, name: ::std::string::String) -> &crate::PyAny { - ::std::panic!("unimplemented isn't hygienic before 1.50") + fn __getattribute__(&self, name: ::std::string::String) -> &crate::Bound<'_, crate::PyAny> { + ::std::unimplemented!() } fn __setattr__(&mut self, name: ::std::string::String, value: ::std::string::String) {} @@ -85,17 +85,27 @@ impl Dummy { fn __get__( &self, - instance: &crate::PyAny, - owner: &crate::PyAny, - ) -> crate::PyResult<&crate::PyAny> { - ::std::panic!("unimplemented isn't hygienic before 1.50") + instance: &crate::Bound<'_, crate::PyAny>, + owner: &crate::Bound<'_, crate::PyAny>, + ) -> crate::PyResult<&crate::Bound<'_, crate::PyAny>> { + ::std::unimplemented!() } - fn __set__(&self, instance: &crate::PyAny, owner: &crate::PyAny) {} + fn __set__( + &self, + instance: &crate::Bound<'_, crate::PyAny>, + owner: &crate::Bound<'_, crate::PyAny>, + ) { + } - fn __delete__(&self, instance: &crate::PyAny) {} + fn __delete__(&self, instance: &crate::Bound<'_, crate::PyAny>) {} - fn __set_name__(&self, owner: &crate::PyAny, name: &crate::PyAny) {} + fn __set_name__( + &self, + owner: &crate::Bound<'_, crate::PyAny>, + name: &crate::Bound<'_, crate::PyAny>, + ) { + } ////////////////////// // Implementing Descriptors @@ -323,9 +333,9 @@ impl Dummy { fn __exit__( &mut self, - exc_type: &crate::PyAny, - exc_value: &crate::PyAny, - traceback: &crate::PyAny, + exc_type: &crate::Bound<'_, crate::PyAny>, + exc_value: &crate::Bound<'_, crate::PyAny>, + traceback: &crate::Bound<'_, crate::PyAny>, ) { } @@ -361,9 +371,9 @@ impl Dummy { fn __aexit__( &mut self, - exc_type: &crate::PyAny, - exc_value: &crate::PyAny, - traceback: &crate::PyAny, + exc_type: &crate::Bound<'_, crate::PyAny>, + exc_value: &crate::Bound<'_, crate::PyAny>, + traceback: &crate::Bound<'_, crate::PyAny>, ) { } @@ -378,10 +388,10 @@ impl Dummy { #[pyo3(signature = (*_args, **_kwds))] fn __call__( &self, - _args: &crate::types::PyTuple, - _kwds: ::std::option::Option<&crate::types::PyDict>, + _args: &crate::Bound<'_, crate::types::PyTuple>, + _kwds: ::std::option::Option<&crate::Bound<'_, crate::types::PyDict>>, ) -> crate::PyResult { - ::std::panic!("unimplemented isn't hygienic before 1.50") + ::std::unimplemented!() } #[new] fn new(a: u8) -> Self { diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 55cda1a355a..7227519975b 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -171,7 +171,7 @@ impl PyByteArray { /// use pyo3::types::PyByteArray; /// /// #[pyfunction] - /// fn a_valid_function(bytes: &PyByteArray) -> PyResult<()> { + /// fn a_valid_function(bytes: &Bound<'_, PyByteArray>) -> PyResult<()> { /// let section = { /// // SAFETY: We promise to not let the interpreter regain control /// // or invoke any PyO3 APIs while using the slice. @@ -224,7 +224,7 @@ impl PyByteArray { /// /// # #[allow(dead_code)] /// #[pyfunction] - /// fn bug(py: Python<'_>, bytes: &PyByteArray) { + /// fn bug(py: Python<'_>, bytes: &Bound<'_, PyByteArray>) { /// let slice = unsafe { bytes.as_bytes() }; /// /// // This explicitly yields control back to the Python interpreter... @@ -333,7 +333,7 @@ pub trait PyByteArrayMethods<'py>: crate::sealed::Sealed { /// use pyo3::types::PyByteArray; /// /// #[pyfunction] - /// fn a_valid_function(bytes: &PyByteArray) -> PyResult<()> { + /// fn a_valid_function(bytes: &Bound<'_, PyByteArray>) -> PyResult<()> { /// let section = { /// // SAFETY: We promise to not let the interpreter regain control /// // or invoke any PyO3 APIs while using the slice. @@ -386,7 +386,7 @@ pub trait PyByteArrayMethods<'py>: crate::sealed::Sealed { /// /// # #[allow(dead_code)] /// #[pyfunction] - /// fn bug(py: Python<'_>, bytes: &PyByteArray) { + /// fn bug(py: Python<'_>, bytes: &Bound<'_, PyByteArray>) { /// let slice = unsafe { bytes.as_bytes() }; /// /// // This explicitly yields control back to the Python interpreter... diff --git a/tests/test_arithmetics.rs b/tests/test_arithmetics.rs index 88f9ca44c28..da8d72ea9cc 100644 --- a/tests/test_arithmetics.rs +++ b/tests/test_arithmetics.rs @@ -166,43 +166,43 @@ impl BinaryArithmetic { "BA" } - fn __add__(&self, rhs: &PyAny) -> String { + fn __add__(&self, rhs: &Bound<'_, PyAny>) -> String { format!("BA + {:?}", rhs) } - fn __sub__(&self, rhs: &PyAny) -> String { + fn __sub__(&self, rhs: &Bound<'_, PyAny>) -> String { format!("BA - {:?}", rhs) } - fn __mul__(&self, rhs: &PyAny) -> String { + fn __mul__(&self, rhs: &Bound<'_, PyAny>) -> String { format!("BA * {:?}", rhs) } - fn __truediv__(&self, rhs: &PyAny) -> String { + fn __truediv__(&self, rhs: &Bound<'_, PyAny>) -> String { format!("BA / {:?}", rhs) } - fn __lshift__(&self, rhs: &PyAny) -> String { + fn __lshift__(&self, rhs: &Bound<'_, PyAny>) -> String { format!("BA << {:?}", rhs) } - fn __rshift__(&self, rhs: &PyAny) -> String { + fn __rshift__(&self, rhs: &Bound<'_, PyAny>) -> String { format!("BA >> {:?}", rhs) } - fn __and__(&self, rhs: &PyAny) -> String { + fn __and__(&self, rhs: &Bound<'_, PyAny>) -> String { format!("BA & {:?}", rhs) } - fn __xor__(&self, rhs: &PyAny) -> String { + fn __xor__(&self, rhs: &Bound<'_, PyAny>) -> String { format!("BA ^ {:?}", rhs) } - fn __or__(&self, rhs: &PyAny) -> String { + fn __or__(&self, rhs: &Bound<'_, PyAny>) -> String { format!("BA | {:?}", rhs) } - fn __pow__(&self, rhs: &PyAny, mod_: Option) -> String { + fn __pow__(&self, rhs: &Bound<'_, PyAny>, mod_: Option) -> String { format!("BA ** {:?} (mod: {:?})", rhs, mod_) } } @@ -257,39 +257,39 @@ struct RhsArithmetic {} #[pymethods] impl RhsArithmetic { - fn __radd__(&self, other: &PyAny) -> String { + fn __radd__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} + RA", other) } - fn __rsub__(&self, other: &PyAny) -> String { + fn __rsub__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} - RA", other) } - fn __rmul__(&self, other: &PyAny) -> String { + fn __rmul__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} * RA", other) } - fn __rlshift__(&self, other: &PyAny) -> String { + fn __rlshift__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} << RA", other) } - fn __rrshift__(&self, other: &PyAny) -> String { + fn __rrshift__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} >> RA", other) } - fn __rand__(&self, other: &PyAny) -> String { + fn __rand__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} & RA", other) } - fn __rxor__(&self, other: &PyAny) -> String { + fn __rxor__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} ^ RA", other) } - fn __ror__(&self, other: &PyAny) -> String { + fn __ror__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} | RA", other) } - fn __rpow__(&self, other: &PyAny, _mod: Option<&PyAny>) -> String { + fn __rpow__(&self, other: &Bound<'_, PyAny>, _mod: Option<&PyAny>) -> String { format!("{:?} ** RA", other) } } @@ -334,91 +334,91 @@ impl LhsAndRhs { // "BA" // } - fn __add__(lhs: PyRef<'_, Self>, rhs: &PyAny) -> String { + fn __add__(lhs: PyRef<'_, Self>, rhs: &Bound<'_, PyAny>) -> String { format!("{:?} + {:?}", lhs, rhs) } - fn __sub__(lhs: PyRef<'_, Self>, rhs: &PyAny) -> String { + fn __sub__(lhs: PyRef<'_, Self>, rhs: &Bound<'_, PyAny>) -> String { format!("{:?} - {:?}", lhs, rhs) } - fn __mul__(lhs: PyRef<'_, Self>, rhs: &PyAny) -> String { + fn __mul__(lhs: PyRef<'_, Self>, rhs: &Bound<'_, PyAny>) -> String { format!("{:?} * {:?}", lhs, rhs) } - fn __lshift__(lhs: PyRef<'_, Self>, rhs: &PyAny) -> String { + fn __lshift__(lhs: PyRef<'_, Self>, rhs: &Bound<'_, PyAny>) -> String { format!("{:?} << {:?}", lhs, rhs) } - fn __rshift__(lhs: PyRef<'_, Self>, rhs: &PyAny) -> String { + fn __rshift__(lhs: PyRef<'_, Self>, rhs: &Bound<'_, PyAny>) -> String { format!("{:?} >> {:?}", lhs, rhs) } - fn __and__(lhs: PyRef<'_, Self>, rhs: &PyAny) -> String { + fn __and__(lhs: PyRef<'_, Self>, rhs: &Bound<'_, PyAny>) -> String { format!("{:?} & {:?}", lhs, rhs) } - fn __xor__(lhs: PyRef<'_, Self>, rhs: &PyAny) -> String { + fn __xor__(lhs: PyRef<'_, Self>, rhs: &Bound<'_, PyAny>) -> String { format!("{:?} ^ {:?}", lhs, rhs) } - fn __or__(lhs: PyRef<'_, Self>, rhs: &PyAny) -> String { + fn __or__(lhs: PyRef<'_, Self>, rhs: &Bound<'_, PyAny>) -> String { format!("{:?} | {:?}", lhs, rhs) } - fn __pow__(lhs: PyRef<'_, Self>, rhs: &PyAny, _mod: Option) -> String { + fn __pow__(lhs: PyRef<'_, Self>, rhs: &Bound<'_, PyAny>, _mod: Option) -> String { format!("{:?} ** {:?}", lhs, rhs) } - fn __matmul__(lhs: PyRef<'_, Self>, rhs: &PyAny) -> String { + fn __matmul__(lhs: PyRef<'_, Self>, rhs: &Bound<'_, PyAny>) -> String { format!("{:?} @ {:?}", lhs, rhs) } - fn __radd__(&self, other: &PyAny) -> String { + fn __radd__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} + RA", other) } - fn __rsub__(&self, other: &PyAny) -> String { + fn __rsub__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} - RA", other) } - fn __rmul__(&self, other: &PyAny) -> String { + fn __rmul__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} * RA", other) } - fn __rlshift__(&self, other: &PyAny) -> String { + fn __rlshift__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} << RA", other) } - fn __rrshift__(&self, other: &PyAny) -> String { + fn __rrshift__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} >> RA", other) } - fn __rand__(&self, other: &PyAny) -> String { + fn __rand__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} & RA", other) } - fn __rxor__(&self, other: &PyAny) -> String { + fn __rxor__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} ^ RA", other) } - fn __ror__(&self, other: &PyAny) -> String { + fn __ror__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} | RA", other) } - fn __rpow__(&self, other: &PyAny, _mod: Option<&PyAny>) -> String { + fn __rpow__(&self, other: &Bound<'_, PyAny>, _mod: Option<&PyAny>) -> String { format!("{:?} ** RA", other) } - fn __rmatmul__(&self, other: &PyAny) -> String { + fn __rmatmul__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} @ RA", other) } - fn __rtruediv__(&self, other: &PyAny) -> String { + fn __rtruediv__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} / RA", other) } - fn __rfloordiv__(&self, other: &PyAny) -> String { + fn __rfloordiv__(&self, other: &Bound<'_, PyAny>) -> String { format!("{:?} // RA", other) } } @@ -461,7 +461,7 @@ impl RichComparisons { "RC" } - fn __richcmp__(&self, other: &PyAny, op: CompareOp) -> String { + fn __richcmp__(&self, other: &Bound<'_, PyAny>, op: CompareOp) -> String { match op { CompareOp::Lt => format!("{} < {:?}", self.__repr__(), other), CompareOp::Le => format!("{} <= {:?}", self.__repr__(), other), @@ -482,7 +482,7 @@ impl RichComparisons2 { "RC2" } - fn __richcmp__(&self, other: &PyAny, op: CompareOp) -> PyObject { + fn __richcmp__(&self, other: &Bound<'_, PyAny>, op: CompareOp) -> PyObject { match op { CompareOp::Eq => true.into_py(other.py()), CompareOp::Ne => false.into_py(other.py()), diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 1209713e487..fc9a7699c1b 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -41,7 +41,7 @@ impl BaseClass { fn base_method(&self, x: usize) -> usize { x * self.val1 } - fn base_set(&mut self, fn_: &pyo3::PyAny) -> PyResult<()> { + fn base_set(&mut self, fn_: &Bound<'_, PyAny>) -> PyResult<()> { let value: usize = fn_.call0()?.extract()?; self.val1 = value; Ok(()) @@ -264,7 +264,7 @@ mod inheriting_native_type { #[pymethods] impl CustomException { #[new] - fn new(_exc_arg: &PyAny) -> Self { + fn new(_exc_arg: &Bound<'_, PyAny>) -> Self { CustomException { context: "Hello :)", } diff --git a/tests/test_methods.rs b/tests/test_methods.rs index 7ca06b27c05..fa6813379bf 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -112,12 +112,8 @@ struct ClassMethodWithArgs {} #[pymethods] impl ClassMethodWithArgs { #[classmethod] - fn method(cls: &Bound<'_, PyType>, input: &PyString) -> PyResult { - Ok(format!( - "{}.method({})", - cls.as_gil_ref().qualname()?, - input - )) + fn method(cls: &Bound<'_, PyType>, input: &Bound<'_, PyString>) -> PyResult { + Ok(format!("{}.method({})", cls.qualname()?, input)) } } @@ -215,8 +211,13 @@ impl MethSignature { test } #[pyo3(signature = (*args, **kwargs))] - fn get_kwargs(&self, py: Python<'_>, args: &PyTuple, kwargs: Option<&PyDict>) -> PyObject { - [args.into(), kwargs.to_object(py)].to_object(py) + fn get_kwargs( + &self, + py: Python<'_>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyObject { + [args.to_object(py), kwargs.to_object(py)].to_object(py) } #[pyo3(signature = (a, *args, **kwargs))] @@ -224,10 +225,10 @@ impl MethSignature { &self, py: Python<'_>, a: i32, - args: &PyTuple, - kwargs: Option<&PyDict>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyObject { - [a.to_object(py), args.into(), kwargs.to_object(py)].to_object(py) + [a.to_object(py), args.to_object(py), kwargs.to_object(py)].to_object(py) } #[pyo3(signature = (a, b, /))] @@ -270,7 +271,7 @@ impl MethSignature { &self, py: Python<'_>, a: i32, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyObject { [a.to_object(py), kwargs.to_object(py)].to_object(py) } @@ -280,7 +281,7 @@ impl MethSignature { &self, py: Python<'_>, a: i32, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyObject { [a.to_object(py), kwargs.to_object(py)].to_object(py) } @@ -301,7 +302,12 @@ impl MethSignature { } #[pyo3(signature = (*args, a))] - fn get_args_and_required_keyword(&self, py: Python<'_>, args: &PyTuple, a: i32) -> PyObject { + fn get_args_and_required_keyword( + &self, + py: Python<'_>, + args: &Bound<'_, PyTuple>, + a: i32, + ) -> PyObject { (args, a).to_object(py) } @@ -316,7 +322,7 @@ impl MethSignature { } #[pyo3(signature = (a, **kwargs))] - fn get_pos_kw(&self, py: Python<'_>, a: i32, kwargs: Option<&PyDict>) -> PyObject { + fn get_pos_kw(&self, py: Python<'_>, a: i32, kwargs: Option<&Bound<'_, PyDict>>) -> PyObject { [a.to_object(py), kwargs.to_object(py)].to_object(py) } @@ -697,7 +703,8 @@ struct MethodWithLifeTime {} #[pymethods] impl MethodWithLifeTime { - fn set_to_list<'py>(&self, py: Python<'py>, set: &'py PySet) -> PyResult> { + fn set_to_list<'py>(&self, set: &Bound<'py, PySet>) -> PyResult> { + let py = set.py(); let mut items = vec![]; for _ in 0..set.len() { items.push(set.pop().unwrap()); @@ -1028,45 +1035,45 @@ issue_1506!( fn issue_1506( &self, _py: Python<'_>, - _arg: &PyAny, - _args: &PyTuple, - _kwargs: Option<&PyDict>, + _arg: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, ) { } fn issue_1506_mut( &mut self, _py: Python<'_>, - _arg: &PyAny, - _args: &PyTuple, - _kwargs: Option<&PyDict>, + _arg: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, ) { } fn issue_1506_custom_receiver( _slf: Py, _py: Python<'_>, - _arg: &PyAny, - _args: &PyTuple, - _kwargs: Option<&PyDict>, + _arg: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, ) { } fn issue_1506_custom_receiver_explicit( _slf: Py, _py: Python<'_>, - _arg: &PyAny, - _args: &PyTuple, - _kwargs: Option<&PyDict>, + _arg: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, ) { } #[new] fn issue_1506_new( _py: Python<'_>, - _arg: &PyAny, - _args: &PyTuple, - _kwargs: Option<&PyDict>, + _arg: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, ) -> Self { Issue1506 {} } @@ -1082,9 +1089,9 @@ issue_1506!( #[staticmethod] fn issue_1506_static( _py: Python<'_>, - _arg: &PyAny, - _args: &PyTuple, - _kwargs: Option<&PyDict>, + _arg: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, ) { } @@ -1092,9 +1099,9 @@ issue_1506!( fn issue_1506_class( _cls: &Bound<'_, PyType>, _py: Python<'_>, - _arg: &PyAny, - _args: &PyTuple, - _kwargs: Option<&PyDict>, + _arg: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, ) { } } diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index 4c77cc8e2a0..35c978b0da5 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -49,7 +49,7 @@ impl BasicClass { const OKAY: bool = true; #[new] - fn new(arg: &pyo3::PyAny) -> pyo3::PyResult { + fn new(arg: &pyo3::Bound<'_, pyo3::PyAny>) -> pyo3::PyResult { if let Ok(v) = arg.extract::() { Ok(Self { v, diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index a1cfde2badf..dd707990c0b 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -28,7 +28,7 @@ impl ExampleClass { } } - fn __setattr__(&mut self, attr: &str, value: &PyAny) -> PyResult<()> { + fn __setattr__(&mut self, attr: &str, value: &Bound<'_, PyAny>) -> PyResult<()> { if attr == "special_custom_attr" { self.custom_attr = Some(value.extract()?); Ok(()) @@ -528,7 +528,7 @@ struct GetItem {} #[pymethods] impl GetItem { - fn __getitem__(&self, idx: &PyAny) -> PyResult<&'static str> { + fn __getitem__(&self, idx: &Bound<'_, PyAny>) -> PyResult<&'static str> { if let Ok(slice) = idx.downcast::() { let indices = slice.indices(1000)?; if indices.start == 100 && indices.stop == 200 && indices.step == 1 { @@ -767,18 +767,18 @@ impl DescrCounter { /// Each access will increase the count fn __get__<'a>( mut slf: PyRefMut<'a, Self>, - _instance: &PyAny, - _owner: Option<&PyType>, + _instance: &Bound<'_, PyAny>, + _owner: Option<&Bound<'_, PyType>>, ) -> PyRefMut<'a, Self> { slf.count += 1; slf } /// Allow assigning a new counter to the descriptor, copying the count across - fn __set__(&self, _instance: &PyAny, new_value: &mut Self) { + fn __set__(&self, _instance: &Bound<'_, PyAny>, new_value: &mut Self) { new_value.count = self.count; } /// Delete to reset the counter - fn __delete__(&mut self, _instance: &PyAny) { + fn __delete__(&mut self, _instance: &Bound<'_, PyAny>) { self.count = 0; } } diff --git a/tests/test_pyfunction.rs b/tests/test_pyfunction.rs index 5221b0f85a5..8a57e2707c9 100644 --- a/tests/test_pyfunction.rs +++ b/tests/test_pyfunction.rs @@ -77,12 +77,14 @@ assert a, array.array("i", [2, 4, 6, 8]) #[cfg(not(any(Py_LIMITED_API, PyPy)))] #[pyfunction] -fn function_with_pyfunction_arg(fun: &PyFunction) -> PyResult<&PyAny> { +fn function_with_pyfunction_arg<'py>(fun: &Bound<'py, PyFunction>) -> PyResult> { fun.call((), None) } #[pyfunction] -fn function_with_pycfunction_arg(fun: &PyCFunction) -> PyResult<&PyAny> { +fn function_with_pycfunction_arg<'py>( + fun: &Bound<'py, PyCFunction>, +) -> PyResult> { fun.call((), None) } diff --git a/tests/test_sequence.rs b/tests/test_sequence.rs index 363c27f5eb5..4e128a0bbff 100644 --- a/tests/test_sequence.rs +++ b/tests/test_sequence.rs @@ -60,8 +60,8 @@ impl ByteSequence { } } - fn __contains__(&self, other: &PyAny) -> bool { - match u8::extract(other) { + fn __contains__(&self, other: &Bound<'_, PyAny>) -> bool { + match other.extract::() { Ok(x) => self.elements.contains(&x), Err(_) => false, } diff --git a/tests/test_text_signature.rs b/tests/test_text_signature.rs index 32a78346e9a..a9f5a041596 100644 --- a/tests/test_text_signature.rs +++ b/tests/test_text_signature.rs @@ -128,10 +128,10 @@ fn test_auto_test_signature_function() { fn my_function_4( a: i32, b: Option, - args: &PyTuple, + args: &Bound<'_, PyTuple>, c: i32, d: i32, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) { let _ = (a, b, args, c, d, kwargs); } @@ -218,10 +218,10 @@ fn test_auto_test_signature_method() { &self, a: i32, b: Option, - args: &PyTuple, + args: &Bound<'_, PyTuple>, c: i32, d: i32, - kwargs: Option<&PyDict>, + kwargs: Option<&Bound<'_, PyDict>>, ) { let _ = (a, b, args, c, d, kwargs); } diff --git a/tests/test_variable_arguments.rs b/tests/test_variable_arguments.rs index 8d3cd5d3935..3724689d836 100644 --- a/tests/test_variable_arguments.rs +++ b/tests/test_variable_arguments.rs @@ -13,13 +13,13 @@ struct MyClass {} impl MyClass { #[staticmethod] #[pyo3(signature = (*args))] - fn test_args(args: &PyTuple) -> &PyTuple { + fn test_args(args: Bound<'_, PyTuple>) -> Bound<'_, PyTuple> { args } #[staticmethod] #[pyo3(signature = (**kwargs))] - fn test_kwargs(kwargs: Option<&PyDict>) -> Option<&PyDict> { + fn test_kwargs(kwargs: Option>) -> Option> { kwargs } } diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index cbaba2fa4ff..f623215f531 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -23,6 +23,9 @@ impl MyClass { fn method_gil_ref(_slf: &PyCell) {} fn method_bound(_slf: &Bound<'_, Self>) {} + + #[staticmethod] + fn static_method_gil_ref(_any: &PyAny) {} } fn main() {} @@ -89,6 +92,9 @@ fn pyfunction_from_py_with( ) { } +#[pyfunction] +fn pyfunction_gil_ref(_any: &PyAny) {} + #[derive(Debug, FromPyObject)] pub struct Zap { #[pyo3(item)] diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index d5da8572d02..b133b21486c 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -29,57 +29,69 @@ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg` | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:38:43 + --> tests/ui/deprecations.rs:28:36 | -38 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { +28 | fn static_method_gil_ref(_any: &PyAny) {} + | ^ + +error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument + --> tests/ui/deprecations.rs:41:43 + | +41 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:48:19 + --> tests/ui/deprecations.rs:51:19 | -48 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { +51 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:54:57 + --> tests/ui/deprecations.rs:57:57 | -54 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +57 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:87:27 + --> tests/ui/deprecations.rs:90:27 | -87 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, +90 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, | ^^^^^^^^^^^^^^^^^ -error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:97:27 +error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument + --> tests/ui/deprecations.rs:96:29 | -97 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] - | ^^^^^^^^^^^^ +96 | fn pyfunction_gil_ref(_any: &PyAny) {} + | ^ + +error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor + --> tests/ui/deprecations.rs:103:27 + | +103 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] + | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:107:27 + --> tests/ui/deprecations.rs:113:27 | -107 | #[pyo3(from_py_with = "PyAny::len")] usize, +113 | #[pyo3(from_py_with = "PyAny::len")] usize, | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:113:31 + --> tests/ui/deprecations.rs:119:31 | -113 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), +119 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:120:27 + --> tests/ui/deprecations.rs:126:27 | -120 | #[pyo3(from_py_with = "extract_gil_ref")] +126 | #[pyo3(from_py_with = "extract_gil_ref")] | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::>::is_python`: use `wrap_pyfunction_bound!` instead - --> tests/ui/deprecations.rs:133:13 + --> tests/ui/deprecations.rs:139:13 | -133 | let _ = wrap_pyfunction!(double, py); +139 | let _ = wrap_pyfunction!(double, py); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/invalid_cancel_handle.stderr b/tests/ui/invalid_cancel_handle.stderr index 9e617bca988..04f55ede4a5 100644 --- a/tests/ui/invalid_cancel_handle.stderr +++ b/tests/ui/invalid_cancel_handle.stderr @@ -58,6 +58,7 @@ error[E0277]: the trait bound `CancelHandle: Clone` is not satisfied | ^^^^ the trait `Clone` is not implemented for `CancelHandle` | = help: the following other types implement trait `PyFunctionArgument<'a, 'py>`: + Option<&'a pyo3::Bound<'py, T>> &'a pyo3::Bound<'py, T> &'a pyo3::coroutine::Coroutine &'a mut pyo3::coroutine::Coroutine diff --git a/tests/ui/invalid_frozen_pyclass_borrow.stderr b/tests/ui/invalid_frozen_pyclass_borrow.stderr index 3acfbeb1823..52a0623f282 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.stderr +++ b/tests/ui/invalid_frozen_pyclass_borrow.stderr @@ -16,6 +16,19 @@ note: required by a bound in `extract_pyclass_ref_mut` | pub fn extract_pyclass_ref_mut<'a, 'py: 'a, T: PyClass>( | ^^^^^^^^^^^^^^ required by this bound in `extract_pyclass_ref_mut` +error[E0271]: type mismatch resolving `::Frozen == False` + --> tests/ui/invalid_frozen_pyclass_borrow.rs:9:1 + | +9 | #[pymethods] + | ^^^^^^^^^^^^ expected `False`, found `True` + | +note: required by a bound in `PyRefMut` + --> src/pycell.rs + | + | pub struct PyRefMut<'p, T: PyClass> { + | ^^^^^^^^^^^^^^ required by this bound in `PyRefMut` + = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) + error[E0271]: type mismatch resolving `::Frozen == False` --> tests/ui/invalid_frozen_pyclass_borrow.rs:15:31 | diff --git a/tests/ui/invalid_pyfunctions.rs b/tests/ui/invalid_pyfunctions.rs index fba7d8b5016..eaa241c074b 100644 --- a/tests/ui/invalid_pyfunctions.rs +++ b/tests/ui/invalid_pyfunctions.rs @@ -1,4 +1,5 @@ use pyo3::prelude::*; +use pyo3::types::PyString; #[pyfunction] fn generic_function(value: T) {} @@ -19,7 +20,10 @@ fn function_with_required_after_option(_opt: Option, _x: i32) {} fn pass_module_but_no_arguments<'py>() {} #[pyfunction(pass_module)] -fn first_argument_not_module<'py>(string: &str, module: &'py PyModule) -> PyResult<&'py str> { +fn first_argument_not_module<'a, 'py>( + string: &str, + module: &'a Bound<'_, PyModule>, +) -> PyResult> { module.name() } diff --git a/tests/ui/invalid_pyfunctions.stderr b/tests/ui/invalid_pyfunctions.stderr index 6576997a3eb..8ae40243ba6 100644 --- a/tests/ui/invalid_pyfunctions.stderr +++ b/tests/ui/invalid_pyfunctions.stderr @@ -1,45 +1,45 @@ error: Python functions cannot have generic type parameters - --> tests/ui/invalid_pyfunctions.rs:4:21 + --> tests/ui/invalid_pyfunctions.rs:5:21 | -4 | fn generic_function(value: T) {} +5 | fn generic_function(value: T) {} | ^ error: Python functions cannot have `impl Trait` arguments - --> tests/ui/invalid_pyfunctions.rs:7:36 + --> tests/ui/invalid_pyfunctions.rs:8:36 | -7 | fn impl_trait_function(impl_trait: impl AsRef) {} +8 | fn impl_trait_function(impl_trait: impl AsRef) {} | ^^^^ error: wildcard argument names are not supported - --> tests/ui/invalid_pyfunctions.rs:10:22 + --> tests/ui/invalid_pyfunctions.rs:11:22 | -10 | fn wildcard_argument(_: i32) {} +11 | fn wildcard_argument(_: i32) {} | ^ error: destructuring in arguments is not supported - --> tests/ui/invalid_pyfunctions.rs:13:26 + --> tests/ui/invalid_pyfunctions.rs:14:26 | -13 | fn destructured_argument((a, b): (i32, i32)) {} +14 | fn destructured_argument((a, b): (i32, i32)) {} | ^^^^^^ error: required arguments after an `Option<_>` argument are ambiguous = help: add a `#[pyo3(signature)]` annotation on this function to unambiguously specify the default values for all optional parameters - --> tests/ui/invalid_pyfunctions.rs:16:63 + --> tests/ui/invalid_pyfunctions.rs:17:63 | -16 | fn function_with_required_after_option(_opt: Option, _x: i32) {} +17 | fn function_with_required_after_option(_opt: Option, _x: i32) {} | ^^^ error: expected `&PyModule` or `Py` as first argument with `pass_module` - --> tests/ui/invalid_pyfunctions.rs:19:37 + --> tests/ui/invalid_pyfunctions.rs:20:37 | -19 | fn pass_module_but_no_arguments<'py>() {} +20 | fn pass_module_but_no_arguments<'py>() {} | ^^ error[E0277]: the trait bound `&str: From>` is not satisfied - --> tests/ui/invalid_pyfunctions.rs:22:43 + --> tests/ui/invalid_pyfunctions.rs:24:13 | -22 | fn first_argument_not_module<'py>(string: &str, module: &'py PyModule) -> PyResult<&'py str> { - | ^ the trait `From>` is not implemented for `&str` +24 | string: &str, + | ^ the trait `From>` is not implemented for `&str` | = help: the following other types implement trait `From`: > diff --git a/tests/ui/invalid_pymethod_enum.stderr b/tests/ui/invalid_pymethod_enum.stderr index bb327dccb5d..6cf6fe89bdf 100644 --- a/tests/ui/invalid_pymethod_enum.stderr +++ b/tests/ui/invalid_pymethod_enum.stderr @@ -9,3 +9,16 @@ note: required by a bound in `extract_pyclass_ref_mut` | | pub fn extract_pyclass_ref_mut<'a, 'py: 'a, T: PyClass>( | ^^^^^^^^^^^^^^ required by this bound in `extract_pyclass_ref_mut` + +error[E0271]: type mismatch resolving `::Frozen == False` + --> tests/ui/invalid_pymethod_enum.rs:9:1 + | +9 | #[pymethods] + | ^^^^^^^^^^^^ expected `False`, found `True` + | +note: required by a bound in `PyRefMut` + --> src/pycell.rs + | + | pub struct PyRefMut<'p, T: PyClass> { + | ^^^^^^^^^^^^^^ required by this bound in `PyRefMut` + = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/static_ref.rs b/tests/ui/static_ref.rs index a426536d499..e8015db2f64 100644 --- a/tests/ui/static_ref.rs +++ b/tests/ui/static_ref.rs @@ -2,7 +2,12 @@ use pyo3::prelude::*; use pyo3::types::PyList; #[pyfunction] -fn static_ref(list: &'static PyList) -> usize { +fn static_ref(list: &'static Bound<'_, PyList>) -> usize { + list.len() +} + +#[pyfunction] +fn static_py(list: &Bound<'static, PyList>) -> usize { list.len() } diff --git a/tests/ui/static_ref.stderr b/tests/ui/static_ref.stderr index 2dd3342e5ba..50b054f6245 100644 --- a/tests/ui/static_ref.stderr +++ b/tests/ui/static_ref.stderr @@ -8,3 +8,37 @@ error: lifetime may not live long enough | cast requires that `'py` must outlive `'static` | = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0597]: `holder_0` does not live long enough + --> tests/ui/static_ref.rs:5:15 + | +4 | #[pyfunction] + | ------------- + | | | + | | `holder_0` dropped here while still borrowed + | binding `holder_0` declared here + | argument requires that `holder_0` is borrowed for `'static` +5 | fn static_ref(list: &'static Bound<'_, PyList>) -> usize { + | ^^^^^^^ borrowed value does not live long enough + +error[E0716]: temporary value dropped while borrowed + --> tests/ui/static_ref.rs:5:21 + | +4 | #[pyfunction] + | ------------- + | | | + | | temporary value is freed at the end of this statement + | argument requires that borrow lasts for `'static` +5 | fn static_ref(list: &'static Bound<'_, PyList>) -> usize { + | ^ creates a temporary value which is freed while still in use + +error: lifetime may not live long enough + --> tests/ui/static_ref.rs:9:1 + | +9 | #[pyfunction] + | ^^^^^^^^^^^^^ + | | + | lifetime `'py` defined here + | cast requires that `'py` must outlive `'static` + | + = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From 351c6a0a49d67b21837487e12e0375ed1ab75144 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 21 Mar 2024 07:24:40 +0000 Subject: [PATCH 230/349] deprecate optional GIL Ref in function argument (#3975) --- guide/src/migration.md | 2 +- src/conversions/chrono.rs | 2 +- src/impl_/deprecations.rs | 23 +++++++++++++++++++++-- tests/test_arithmetics.rs | 4 ++-- tests/test_mapping.rs | 4 ++-- tests/test_methods.rs | 3 ++- tests/test_sequence.rs | 4 ++-- tests/ui/deprecations.rs | 3 +++ tests/ui/deprecations.stderr | 26 ++++++++++++++++---------- 9 files changed, 50 insertions(+), 21 deletions(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index 5a362ec448d..ac20c775d00 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -301,7 +301,7 @@ The expectation is that in 0.22 `extract_bound` will have the default implementa Despite a large amount of deprecations warnings produced by PyO3 to aid with the transition from GIL Refs to the Bound API, there are a few cases where PyO3 cannot automatically warn on uses of GIL Refs. It is worth checking for these cases manually after the deprecation warnings have all been addressed: - Individual implementations of the `FromPyObject` trait cannot be deprecated, so PyO3 cannot warn about uses of code patterns like `.extract<&PyAny>()` which produce a GIL Ref. -- GIL Refs in `#[pyfunction]` arguments emit a warning, but if the GIL Ref is wrapped inside another container such as `Option<&PyAny>` or `Vec<&PyAny>` then PyO3 cannot warn against this. +- GIL Refs in `#[pyfunction]` arguments emit a warning, but if the GIL Ref is wrapped inside another container such as `Vec<&PyAny>` then PyO3 cannot warn against this. - The `wrap_pyfunction!(function)(py)` deferred argument form of the `wrap_pyfunction` macro taking `py: Python<'py>` produces a GIL Ref, and due to limitations in type inference PyO3 cannot warn against this specific case. diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 2e46a9e54ff..544d1cf2663 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -292,7 +292,7 @@ impl FromPyObject<'py>> FromPyObject<'_> for DateTime = dt.getattr(intern!(dt.py(), "tzinfo"))?.extract()?; + let tzinfo: Option> = dt.getattr(intern!(dt.py(), "tzinfo"))?.extract()?; let tz = if let Some(tzinfo) = tzinfo { tzinfo.extract()? diff --git a/src/impl_/deprecations.rs b/src/impl_/deprecations.rs index 459eba913d3..9eb1da05c92 100644 --- a/src/impl_/deprecations.rs +++ b/src/impl_/deprecations.rs @@ -13,7 +13,8 @@ pub fn inspect_fn(f: fn(A) -> PyResult, _: &GilRefs) -> fn(A) -> PyR f } -pub struct GilRefs(NotAGilRef); +pub struct GilRefs(OptionGilRefs); +pub struct OptionGilRefs(NotAGilRef); pub struct NotAGilRef(std::marker::PhantomData); pub trait IsGilRef {} @@ -23,7 +24,7 @@ impl IsGilRef for &'_ T {} impl GilRefs { #[allow(clippy::new_without_default)] pub fn new() -> Self { - GilRefs(NotAGilRef(std::marker::PhantomData)) + GilRefs(OptionGilRefs(NotAGilRef(std::marker::PhantomData))) } } @@ -54,6 +55,17 @@ impl GilRefs { pub fn from_py_with_arg(&self) {} } +impl OptionGilRefs> { + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.0", + note = "use `Option<&Bound<'_, T>>` instead for this function argument" + ) + )] + pub fn function_arg(&self) {} +} + impl NotAGilRef { pub fn function_arg(&self) {} pub fn from_py_with_arg(&self) {} @@ -61,6 +73,13 @@ impl NotAGilRef { } impl std::ops::Deref for GilRefs { + type Target = OptionGilRefs; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::Deref for OptionGilRefs { type Target = NotAGilRef; fn deref(&self) -> &Self::Target { &self.0 diff --git a/tests/test_arithmetics.rs b/tests/test_arithmetics.rs index da8d72ea9cc..b1efcd0ac55 100644 --- a/tests/test_arithmetics.rs +++ b/tests/test_arithmetics.rs @@ -289,7 +289,7 @@ impl RhsArithmetic { format!("{:?} | RA", other) } - fn __rpow__(&self, other: &Bound<'_, PyAny>, _mod: Option<&PyAny>) -> String { + fn __rpow__(&self, other: &Bound<'_, PyAny>, _mod: Option<&Bound<'_, PyAny>>) -> String { format!("{:?} ** RA", other) } } @@ -406,7 +406,7 @@ impl LhsAndRhs { format!("{:?} | RA", other) } - fn __rpow__(&self, other: &Bound<'_, PyAny>, _mod: Option<&PyAny>) -> String { + fn __rpow__(&self, other: &Bound<'_, PyAny>, _mod: Option<&Bound<'_, PyAny>>) -> String { format!("{:?} ** RA", other) } diff --git a/tests/test_mapping.rs b/tests/test_mapping.rs index 4e24db97793..675dd14d63f 100644 --- a/tests/test_mapping.rs +++ b/tests/test_mapping.rs @@ -21,11 +21,11 @@ struct Mapping { #[pymethods] impl Mapping { #[new] - fn new(elements: Option<&PyList>) -> PyResult { + fn new(elements: Option<&Bound<'_, PyList>>) -> PyResult { if let Some(pylist) = elements { let mut elems = HashMap::with_capacity(pylist.len()); for (i, pyelem) in pylist.into_iter().enumerate() { - let elem = String::extract(pyelem)?; + let elem = pyelem.extract()?; elems.insert(elem, i); } Ok(Self { index: elems }) diff --git a/tests/test_methods.rs b/tests/test_methods.rs index fa6813379bf..2b5396e9ee4 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -2,6 +2,7 @@ use pyo3::prelude::*; use pyo3::py_run; +use pyo3::types::PySequence; use pyo3::types::{IntoPyDict, PyDict, PyList, PySet, PyString, PyTuple, PyType}; #[path = "../src/tests/common.rs"] @@ -857,7 +858,7 @@ struct FromSequence { #[pymethods] impl FromSequence { #[new] - fn new(seq: Option<&pyo3::types::PySequence>) -> PyResult { + fn new(seq: Option<&Bound<'_, PySequence>>) -> PyResult { if let Some(seq) = seq { Ok(FromSequence { numbers: seq.as_ref().extract::>()?, diff --git a/tests/test_sequence.rs b/tests/test_sequence.rs index 4e128a0bbff..3715affc05b 100644 --- a/tests/test_sequence.rs +++ b/tests/test_sequence.rs @@ -17,11 +17,11 @@ struct ByteSequence { #[pymethods] impl ByteSequence { #[new] - fn new(elements: Option<&PyList>) -> PyResult { + fn new(elements: Option<&Bound<'_, PyList>>) -> PyResult { if let Some(pylist) = elements { let mut elems = Vec::with_capacity(pylist.len()); for pyelem in pylist { - let elem = u8::extract(pyelem)?; + let elem = pyelem.extract()?; elems.push(elem); } Ok(Self { elements: elems }) diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index f623215f531..6f3a1feda50 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -95,6 +95,9 @@ fn pyfunction_from_py_with( #[pyfunction] fn pyfunction_gil_ref(_any: &PyAny) {} +#[pyfunction] +fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} + #[derive(Debug, FromPyObject)] pub struct Zap { #[pyo3(item)] diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index b133b21486c..1d87f31c3f8 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -64,34 +64,40 @@ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg` 96 | fn pyfunction_gil_ref(_any: &PyAny) {} | ^ +error: use of deprecated method `pyo3::deprecations::OptionGilRefs::>::function_arg`: use `Option<&Bound<'_, T>>` instead for this function argument + --> tests/ui/deprecations.rs:99:36 + | +99 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} + | ^^^^^^ + error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:103:27 + --> tests/ui/deprecations.rs:106:27 | -103 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] +106 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:113:27 + --> tests/ui/deprecations.rs:116:27 | -113 | #[pyo3(from_py_with = "PyAny::len")] usize, +116 | #[pyo3(from_py_with = "PyAny::len")] usize, | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:119:31 + --> tests/ui/deprecations.rs:122:31 | -119 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), +122 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:126:27 + --> tests/ui/deprecations.rs:129:27 | -126 | #[pyo3(from_py_with = "extract_gil_ref")] +129 | #[pyo3(from_py_with = "extract_gil_ref")] | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::>::is_python`: use `wrap_pyfunction_bound!` instead - --> tests/ui/deprecations.rs:139:13 + --> tests/ui/deprecations.rs:142:13 | -139 | let _ = wrap_pyfunction!(double, py); +142 | let _ = wrap_pyfunction!(double, py); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From d0f5b6af465374e125cae56f12e6d798a83180ef Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 22 Mar 2024 20:43:23 +0000 Subject: [PATCH 231/349] ci: updates for Rust 1.77 (#3978) * ci: updates for Rust 1.77 * move `SendablePtr` inside of test which uses it --- tests/test_gc.rs | 10 +++++----- tests/ui/abi3_nativetype_inheritance.stderr | 20 ++++++++++---------- tests/ui/invalid_cancel_handle.stderr | 4 ++-- tests/ui/invalid_pyfunctions.stderr | 2 +- tests/ui/invalid_pymethod_receiver.stderr | 4 ++-- tests/ui/invalid_pymethods.stderr | 4 ++-- tests/ui/invalid_result_conversion.stderr | 2 +- tests/ui/missing_intopy.stderr | 2 +- tests/ui/not_send.stderr | 2 +- tests/ui/not_send2.stderr | 2 +- tests/ui/pyclass_send.stderr | 20 ++++++++++---------- 11 files changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/test_gc.rs b/tests/test_gc.rs index 7a8bc9ded2d..11d9221c7ad 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -528,6 +528,11 @@ impl UnsendableTraversal { #[test] #[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled fn unsendable_are_not_traversed_on_foreign_thread() { + #[derive(Clone, Copy)] + struct SendablePtr(*mut pyo3::ffi::PyObject); + + unsafe impl Send for SendablePtr {} + Python::with_gil(|py| unsafe { let ty = py.get_type_bound::(); let traverse = get_type_traverse(ty.as_type_ptr()).unwrap(); @@ -579,8 +584,3 @@ extern "C" fn visit_error( ) -> std::os::raw::c_int { -1 } - -#[derive(Clone, Copy)] -struct SendablePtr(*mut pyo3::ffi::PyObject); - -unsafe impl Send for SendablePtr {} diff --git a/tests/ui/abi3_nativetype_inheritance.stderr b/tests/ui/abi3_nativetype_inheritance.stderr index f0672fbf0a7..784da4f55ba 100644 --- a/tests/ui/abi3_nativetype_inheritance.stderr +++ b/tests/ui/abi3_nativetype_inheritance.stderr @@ -1,27 +1,27 @@ error[E0277]: the trait bound `PyDict: PyClass` is not satisfied - --> tests/ui/abi3_nativetype_inheritance.rs:5:1 + --> tests/ui/abi3_nativetype_inheritance.rs:5:19 | 5 | #[pyclass(extends=PyDict)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `PyClass` is not implemented for `PyDict` + | ^^^^^^ the trait `PyClass` is not implemented for `PyDict`, which is required by `PyDict: PyClassBaseType` | = help: the following other types implement trait `PyClass`: TestClass pyo3::coroutine::Coroutine = note: required for `PyDict` to implement `PyClassBaseType` - = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) +note: required by a bound in `PyClassImpl::BaseType` + --> src/impl_/pyclass.rs + | + | type BaseType: PyTypeInfo + PyClassBaseType; + | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` error[E0277]: the trait bound `PyDict: PyClass` is not satisfied - --> tests/ui/abi3_nativetype_inheritance.rs:5:19 + --> tests/ui/abi3_nativetype_inheritance.rs:5:1 | 5 | #[pyclass(extends=PyDict)] - | ^^^^^^ the trait `PyClass` is not implemented for `PyDict` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `PyClass` is not implemented for `PyDict`, which is required by `PyDict: PyClassBaseType` | = help: the following other types implement trait `PyClass`: TestClass pyo3::coroutine::Coroutine = note: required for `PyDict` to implement `PyClassBaseType` -note: required by a bound in `PyClassImpl::BaseType` - --> src/impl_/pyclass.rs - | - | type BaseType: PyTypeInfo + PyClassBaseType; - | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/invalid_cancel_handle.stderr b/tests/ui/invalid_cancel_handle.stderr index 04f55ede4a5..ffd0b3fd0da 100644 --- a/tests/ui/invalid_cancel_handle.stderr +++ b/tests/ui/invalid_cancel_handle.stderr @@ -36,7 +36,7 @@ error[E0277]: the trait bound `CancelHandle: PyClass` is not satisfied --> tests/ui/invalid_cancel_handle.rs:20:50 | 20 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} - | ^^^^ the trait `PyClass` is not implemented for `CancelHandle` + | ^^^^ the trait `PyClass` is not implemented for `CancelHandle`, which is required by `CancelHandle: PyFunctionArgument<'_, '_>` | = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` = note: required for `CancelHandle` to implement `FromPyObject<'_>` @@ -55,7 +55,7 @@ error[E0277]: the trait bound `CancelHandle: Clone` is not satisfied --> tests/ui/invalid_cancel_handle.rs:20:50 | 20 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} - | ^^^^ the trait `Clone` is not implemented for `CancelHandle` + | ^^^^ the trait `Clone` is not implemented for `CancelHandle`, which is required by `CancelHandle: PyFunctionArgument<'_, '_>` | = help: the following other types implement trait `PyFunctionArgument<'a, 'py>`: Option<&'a pyo3::Bound<'py, T>> diff --git a/tests/ui/invalid_pyfunctions.stderr b/tests/ui/invalid_pyfunctions.stderr index 8ae40243ba6..299b20687cb 100644 --- a/tests/ui/invalid_pyfunctions.stderr +++ b/tests/ui/invalid_pyfunctions.stderr @@ -39,7 +39,7 @@ error[E0277]: the trait bound `&str: From tests/ui/invalid_pyfunctions.rs:24:13 | 24 | string: &str, - | ^ the trait `From>` is not implemented for `&str` + | ^ the trait `From>` is not implemented for `&str`, which is required by `BoundRef<'_, '_, pyo3::prelude::PyModule>: Into<_>` | = help: the following other types implement trait `From`: > diff --git a/tests/ui/invalid_pymethod_receiver.stderr b/tests/ui/invalid_pymethod_receiver.stderr index b6dd44bd9bf..a79e289716a 100644 --- a/tests/ui/invalid_pymethod_receiver.stderr +++ b/tests/ui/invalid_pymethod_receiver.stderr @@ -2,7 +2,7 @@ error[E0277]: the trait bound `i32: From>` is not sati --> tests/ui/invalid_pymethod_receiver.rs:8:43 | 8 | fn method_with_invalid_self_type(slf: i32, py: Python<'_>, index: u32) {} - | ^^^ the trait `From>` is not implemented for `i32` + | ^^^ the trait `From>` is not implemented for `i32`, which is required by `i32: TryFrom>` | = help: the following other types implement trait `From`: > @@ -10,6 +10,6 @@ error[E0277]: the trait bound `i32: From>` is not sati > > > - > + >> = note: required for `BoundRef<'_, '_, MyClass>` to implement `Into` = note: required for `i32` to implement `TryFrom>` diff --git a/tests/ui/invalid_pymethods.stderr b/tests/ui/invalid_pymethods.stderr index 6a8d6ecab78..5bcb6aa1be6 100644 --- a/tests/ui/invalid_pymethods.stderr +++ b/tests/ui/invalid_pymethods.stderr @@ -183,7 +183,7 @@ error[E0277]: the trait bound `i32: From>` is not satis --> tests/ui/invalid_pymethods.rs:46:45 | 46 | fn classmethod_wrong_first_argument(_x: i32) -> Self { - | ^^^ the trait `From>` is not implemented for `i32` + | ^^^ the trait `From>` is not implemented for `i32`, which is required by `BoundRef<'_, '_, PyType>: Into<_>` | = help: the following other types implement trait `From`: > @@ -191,5 +191,5 @@ error[E0277]: the trait bound `i32: From>` is not satis > > > - > + >> = note: required for `BoundRef<'_, '_, PyType>` to implement `Into` diff --git a/tests/ui/invalid_result_conversion.stderr b/tests/ui/invalid_result_conversion.stderr index 73ed2cba1aa..9695c5e6a19 100644 --- a/tests/ui/invalid_result_conversion.stderr +++ b/tests/ui/invalid_result_conversion.stderr @@ -2,7 +2,7 @@ error[E0277]: the trait bound `PyErr: From` is not satisfied --> tests/ui/invalid_result_conversion.rs:21:1 | 21 | #[pyfunction] - | ^^^^^^^^^^^^^ the trait `From` is not implemented for `PyErr` + | ^^^^^^^^^^^^^ the trait `From` is not implemented for `PyErr`, which is required by `MyError: Into` | = help: the following other types implement trait `From`: >> diff --git a/tests/ui/missing_intopy.stderr b/tests/ui/missing_intopy.stderr index 26b32b3e742..1d233b28b6c 100644 --- a/tests/ui/missing_intopy.stderr +++ b/tests/ui/missing_intopy.stderr @@ -2,7 +2,7 @@ error[E0277]: the trait bound `Blah: IntoPy>` is not satisfied --> tests/ui/missing_intopy.rs:3:1 | 3 | #[pyo3::pyfunction] - | ^^^^^^^^^^^^^^^^^^^ the trait `IntoPy>` is not implemented for `Blah` + | ^^^^^^^^^^^^^^^^^^^ the trait `IntoPy>` is not implemented for `Blah`, which is required by `Blah: OkWrap<_>` | = help: the following other types implement trait `IntoPy`: >> diff --git a/tests/ui/not_send.stderr b/tests/ui/not_send.stderr index 5d05b3de059..8007c2f4e54 100644 --- a/tests/ui/not_send.stderr +++ b/tests/ui/not_send.stderr @@ -6,7 +6,7 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe | | | required by a bound introduced by this call | - = help: within `pyo3::Python<'_>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` + = help: within `pyo3::Python<'_>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>`, which is required by `{closure@$DIR/tests/ui/not_send.rs:4:22: 4:24}: Ungil` note: required because it appears within the type `PhantomData<*mut pyo3::Python<'static>>` --> $RUST/core/src/marker.rs | diff --git a/tests/ui/not_send2.stderr b/tests/ui/not_send2.stderr index eec2b644e9a..52a4e6a7208 100644 --- a/tests/ui/not_send2.stderr +++ b/tests/ui/not_send2.stderr @@ -9,7 +9,7 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe 10 | | }); | |_________^ `*mut pyo3::Python<'static>` cannot be shared between threads safely | - = help: within `pyo3::Bound<'_, PyString>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` + = help: within `pyo3::Bound<'_, PyString>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>`, which is required by `{closure@$DIR/tests/ui/not_send2.rs:8:26: 8:28}: Ungil` note: required because it appears within the type `PhantomData<*mut pyo3::Python<'static>>` --> $RUST/core/src/marker.rs | diff --git a/tests/ui/pyclass_send.stderr b/tests/ui/pyclass_send.stderr index 7a80989b63f..0db9106ab42 100644 --- a/tests/ui/pyclass_send.stderr +++ b/tests/ui/pyclass_send.stderr @@ -4,17 +4,19 @@ error[E0277]: `Rc` cannot be sent between threads safely 4 | #[pyclass] | ^^^^^^^^^^ `Rc` cannot be sent between threads safely | - = help: within `NotThreadSafe`, the trait `Send` is not implemented for `Rc` + = help: within `NotThreadSafe`, the trait `Send` is not implemented for `Rc`, which is required by `SendablePyClass: pyo3::impl_::pyclass::PyClassThreadChecker` + = help: the trait `pyo3::impl_::pyclass::PyClassThreadChecker` is implemented for `SendablePyClass` note: required because it appears within the type `NotThreadSafe` --> tests/ui/pyclass_send.rs:5:8 | 5 | struct NotThreadSafe { | ^^^^^^^^^^^^^ -note: required by a bound in `SendablePyClass` + = note: required for `SendablePyClass` to implement `pyo3::impl_::pyclass::PyClassThreadChecker` +note: required by a bound in `PyClassImpl::ThreadChecker` --> src/impl_/pyclass.rs | - | pub struct SendablePyClass(PhantomData); - | ^^^^ required by this bound in `SendablePyClass` + | type ThreadChecker: PyClassThreadChecker; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::ThreadChecker` = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0277]: `Rc` cannot be sent between threads safely @@ -23,17 +25,15 @@ error[E0277]: `Rc` cannot be sent between threads safely 4 | #[pyclass] | ^^^^^^^^^^ `Rc` cannot be sent between threads safely | - = help: within `NotThreadSafe`, the trait `Send` is not implemented for `Rc` - = help: the trait `pyo3::impl_::pyclass::PyClassThreadChecker` is implemented for `SendablePyClass` + = help: within `NotThreadSafe`, the trait `Send` is not implemented for `Rc`, which is required by `NotThreadSafe: Send` note: required because it appears within the type `NotThreadSafe` --> tests/ui/pyclass_send.rs:5:8 | 5 | struct NotThreadSafe { | ^^^^^^^^^^^^^ - = note: required for `SendablePyClass` to implement `pyo3::impl_::pyclass::PyClassThreadChecker` -note: required by a bound in `PyClassImpl::ThreadChecker` +note: required by a bound in `SendablePyClass` --> src/impl_/pyclass.rs | - | type ThreadChecker: PyClassThreadChecker; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::ThreadChecker` + | pub struct SendablePyClass(PhantomData); + | ^^^^ required by this bound in `SendablePyClass` = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) From 990886efdad1bad0e772ea5f9439fb95c7335961 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 22 Mar 2024 22:16:28 +0000 Subject: [PATCH 232/349] docs: better document `FromPyObject` for `&str` changes in migration guide (#3974) * docs: better document `FromPyObject` for `&str` changes in migration guide * review: LilyFoote --- guide/src/index.md | 1 + guide/src/memory.md | 5 ++- guide/src/migration.md | 71 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/guide/src/index.md b/guide/src/index.md index 87914975afe..fe8b76b69c9 100644 --- a/guide/src/index.md +++ b/guide/src/index.md @@ -11,6 +11,7 @@ The rough order of material in this user guide is as follows: Please choose from the chapters on the left to jump to individual topics, or continue below to start with PyO3's README.
+ ⚠️ Warning: API update in progress 🛠️ PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the new smart pointer `Bound`. diff --git a/guide/src/memory.md b/guide/src/memory.md index 78f9f348f40..cd4af4f8f13 100644 --- a/guide/src/memory.md +++ b/guide/src/memory.md @@ -1,6 +1,7 @@ # Memory management
+ ⚠️ Warning: API update in progress 🛠️ PyO3 0.21 has introduced a significant new API, termed the "Bound" API after the new smart pointer `Bound`. @@ -84,6 +85,7 @@ the end of the `with_gil()` closure, at which point the 10 copies of `hello` are finally released to the Python garbage collector.
+ ⚠️ Warning: `GILPool` is no longer the preferred way to manage memory with PyO3 🛠️ PyO3 0.21 has introduced a new API known as the Bound API, which doesn't have the same surprising results. Instead, each `Bound` smart pointer releases the Python reference immediately on drop. See [the smart pointer types](./types.md#pyo3s-smart-pointers) for more details. @@ -121,7 +123,7 @@ this is unsafe. # fn main() -> PyResult<()> { Python::with_gil(|py| -> PyResult<()> { for _ in 0..10 { - #[allow(deprecated)] // `new_pool` is not needed in code not using the GIL Refs API + #[allow(deprecated)] // `new_pool` is not needed in code not using the GIL Refs API let pool = unsafe { py.new_pool() }; let py = pool.python(); #[allow(deprecated)] // py.eval() is part of the GIL Refs API @@ -155,6 +157,7 @@ a few objects, meaning this doesn't have a significant impact. Occasionally func with long complex loops may need to use `Python::new_pool` as shown above.
+ ⚠️ Warning: `GILPool` is no longer the preferred way to manage memory with PyO3 🛠️ PyO3 0.21 has introduced a new API known as the Bound API, which doesn't have the same surprising results. Instead, each `Bound` smart pointer releases the Python reference immediately on drop. See [the smart pointer types](./types.md#pyo3s-smart-pointers) for more details. diff --git a/guide/src/migration.md b/guide/src/migration.md index ac20c775d00..69cf8922255 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -245,13 +245,19 @@ Because the new `Bound` API brings ownership out of the PyO3 framework and in - `Bound` and `Bound` cannot support indexing with `list[0]`, you should use `list.get_item(0)` instead. - `Bound::iter_borrowed` is slightly more efficient than `Bound::iter`. The default iteration of `Bound` cannot return borrowed references because Rust does not (yet) have "lending iterators". Similarly `Bound::get_borrowed_item` is more efficient than `Bound::get_item` for the same reason. - `&Bound` does not implement `FromPyObject` (although it might be possible to do this in the future once the GIL Refs API is completely removed). Use `bound_any.downcast::()` instead of `bound_any.extract::<&Bound>()`. -- To convert between `&PyAny` and `&Bound` you can use the `as_borrowed()` method: +- `Bound::to_str` now borrows from the `Bound` rather than from the `'py` lifetime, so code will need to store the smart pointer as a value in some cases where previously `&PyString` was just used as a temporary. (There are some more details relating to this in [the section below](#deactivating-the-gil-refs-feature).) + +To convert between `&PyAny` and `&Bound` you can use the `as_borrowed()` method: ```rust,ignore let gil_ref: &PyAny = ...; let bound: &Bound = &gil_ref.as_borrowed(); ``` +
+ +⚠️ Warning: dangling pointer trap 💣 + > Because of the ownership changes, code which uses `.as_ptr()` to convert `&PyAny` and other GIL Refs to a `*mut pyo3_ffi::PyObject` should take care to avoid creating dangling pointers now that `Bound` carries ownership. > > For example, the following pattern with `Option<&PyAny>` can easily create a dangling pointer when migrating to the `Bound` smart pointer: @@ -267,6 +273,7 @@ let bound: &Bound = &gil_ref.as_borrowed(); > let opt: Option> = ...; > let p: *mut ffi::PyObject = opt.as_ref().map_or(std::ptr::null_mut(), Bound::as_ptr); > ``` +
#### Migrating `FromPyObject` implementations @@ -312,14 +319,66 @@ Despite a large amount of deprecations warnings produced by PyO3 to aid with the As a final step of migration, deactivating the `gil-refs` feature will set up code for best performance and is intended to set up a forward-compatible API for PyO3 0.22. -At this point code which needed to manage GIL Ref memory can safely remove uses of `GILPool` (which are constructed by calls to `Python::new_pool` and `Python::with_pool`). Deprecation warnings will highlight these cases. +At this point code that needed to manage GIL Ref memory can safely remove uses of `GILPool` (which are constructed by calls to `Python::new_pool` and `Python::with_pool`). Deprecation warnings will highlight these cases. -There is one notable API removed when this feature is disabled. `FromPyObject` trait implementations for types which borrow directly from the input data cannot be implemented by PyO3 without GIL Refs (while the migration is ongoing). These types are `&str`, `Cow<'_, str>`, `&[u8]`, `Cow<'_, u8>`. +There is just one case of code that changes upon disabling these features: `FromPyObject` trait implementations for types that borrow directly from the input data cannot be implemented by PyO3 without GIL Refs (while the GIL Refs API is in the process of being removed). The main types affected are `&str`, `Cow<'_, str>`, `&[u8]`, `Cow<'_, u8>`. -To ease pain during migration, these types instead implement a new temporary trait `FromPyObjectBound` which is the expected future form of `FromPyObject`. The new temporary trait ensures is that `obj.extract::<&str>()` continues to work, as well for these types in `#[pyfunction]` arguments. +To make PyO3's core functionality continue to work while the GIL Refs API is in the process of being removed, disabling the `gil-refs` feature moves the implementations of `FromPyObject` for `&str`, `Cow<'_, str>`, `&[u8]`, `Cow<'_, u8>` to a new temporary trait `FromPyObjectBound`. This trait is the expected future form of `FromPyObject` and has an additional lifetime `'a` to enable these types to borrow data from Python objects. -An unfortunate final point here is that PyO3 cannot offer this new implementation for `&str` on `abi3` builds for Python older than 3.10. On code which needs `abi3` builds for these older Python versions, many cases of `.extract::<&str>()` may need to be replaced with `.extract::()`, which is string data which borrows from the Python `str` object. Alternatively, use `.extract::>()` ro `.extract::()` to copy the data into Rust for these versions. - +A key thing to note here is because extracting to these types now ties them to the input lifetime, some extremely common patterns may need to be split into multiple Rust lines. For example, the following snippet of calling `.extract::<&str>()` directly on the result of `.getattr()` needs to be adjusted when deactivating the `gil-refs-migration` feature. + +Before: + +```rust +# #[cfg(feature = "gil-refs-migration")] { +# use pyo3::prelude::*; +# use pyo3::types::{PyList, PyType}; +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +#[allow(deprecated)] // GIL Ref API +let obj: &'py PyType = py.get_type::(); +let name: &'py str = obj.getattr("__name__")?.extract()?; +assert_eq!(name, "list"); +# Ok(()) +# } +# Python::with_gil(example).unwrap(); +# } +``` + +After: + +```rust +# #[cfg(any(not(Py_LIMITED_API), Py_3_10))] { +# use pyo3::prelude::*; +# use pyo3::types::{PyList, PyType}; +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +let obj: Bound<'py, PyType> = py.get_type_bound::(); +let name_obj: Bound<'py, PyAny> = obj.getattr("__name__")?; +// the lifetime of the data is no longer `'py` but the much shorter +// lifetime of the `name_obj` smart pointer above +let name: &'_ str = name_obj.extract()?; +assert_eq!(name, "list"); +# Ok(()) +# } +# Python::with_gil(example).unwrap(); +# } +``` + +To avoid needing to worry about lifetimes at all, it is also possible to use the new `PyBackedStr` type, which stores a reference to the Python `str` without a lifetime attachment. In particular, `PyBackedStr` helps for `abi3` builds for Python older than 3.10. Due to limitations in the `abi3` CPython API for those older versions, PyO3 cannot offer a `FromPyObjectBound` implementation for `&str` on those versions. The easiest way to migrate for older `abi3` builds is to replace any cases of `.extract::<&str>()` with `.extract::()`. Alternatively, use `.extract::>()`, `.extract::()` to copy the data into Rust. + +The following example uses the same snippet as those just above, but this time the final extracted type is `PyBackedStr`: + +```rust +# use pyo3::prelude::*; +# use pyo3::types::{PyList, PyType}; +# fn example<'py>(py: Python<'py>) -> PyResult<()> { +use pyo3::pybacked::PyBackedStr; +let obj: Bound<'py, PyType> = py.get_type_bound::(); +let name: PyBackedStr = obj.getattr("__name__")?.extract()?; +assert_eq!(&*name, "list"); +# Ok(()) +# } +# Python::with_gil(example).unwrap(); +``` ## from 0.19.* to 0.20 From 20e477a7cd19c5bfeec1ebe813d2df2cea0f8a2b Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 22 Mar 2024 22:43:05 +0000 Subject: [PATCH 233/349] avoid reference count cycle in tuple extraction (#3976) --- src/types/tuple.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types/tuple.rs b/src/types/tuple.rs index d8053e1441a..d89830daa97 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -690,10 +690,10 @@ fn type_output() -> TypeInfo { let t = obj.downcast::()?; if t.len() == $length { #[cfg(any(Py_LIMITED_API, PyPy))] - return Ok(($(t.get_item($n)?.extract::<$T>()?,)+)); + return Ok(($(t.get_borrowed_item($n)?.extract::<$T>()?,)+)); #[cfg(not(any(Py_LIMITED_API, PyPy)))] - unsafe {return Ok(($(t.get_item_unchecked($n).extract::<$T>()?,)+));} + unsafe {return Ok(($(t.get_borrowed_item_unchecked($n).extract::<$T>()?,)+));} } else { Err(wrong_tuple_length(t, $length)) } From 9808f7111c5ec11725cb02a963475f6cf352eed8 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 22 Mar 2024 22:43:08 +0000 Subject: [PATCH 234/349] ci: add `update-ui-tests` nox session (#3979) * ci: add `update-ui-tests` nox session * defer error messages after the group closes * fix syntax warnings --- Contributing.md | 15 +++++++++++---- noxfile.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/Contributing.md b/Contributing.md index 2abdd80d47f..29d1bb758a7 100644 --- a/Contributing.md +++ b/Contributing.md @@ -101,10 +101,17 @@ Tests run with all supported Python versions with the latest stable Rust compile If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI. You can run these tests yourself with -```nox``` -and -```nox -l``` -lists further commands you can run. +`nox`. Use `nox -l` to list the full set of subcommands you can run. + +#### UI Tests + +PyO3 uses [`trybuild`][trybuild] to develop UI tests to capture error messages from the Rust compiler for some of the macro functionality. + +Because there are several feature combinations for these UI tests, when updating them all (e.g. for a new Rust compiler version) it may be helpful to use the `update-ui-tests` nox session: + +```bash +nox -s update-ui-tests +``` ### Documenting changes diff --git a/noxfile.py b/noxfile.py index f70fced76d6..3ee76da5f08 100644 --- a/noxfile.py +++ b/noxfile.py @@ -395,8 +395,8 @@ def check_guide(session: nox.Session): session.posargs.extend(posargs) remaps = { - f"file://{PYO3_GUIDE_SRC}/([^/]*/)*?%7B%7B#PYO3_DOCS_URL\}}\}}": f"file://{PYO3_DOCS_TARGET}", - "%7B%7B#PYO3_DOCS_VERSION\}\}": "latest", + f"file://{PYO3_GUIDE_SRC}/([^/]*/)*?%7B%7B#PYO3_DOCS_URL}}}}": f"file://{PYO3_DOCS_TARGET}", + "%7B%7B#PYO3_DOCS_VERSION}}": "latest", } remap_args = [] for key, value in remaps.items(): @@ -732,6 +732,16 @@ def check_feature_powerset(session: nox.Session): ) +@nox.session(name="update-ui-tests", venv_backend="none") +def update_ui_tests(session: nox.Session): + env = os.environ.copy() + env["TRYBUILD"] = "overwrite" + command = ["test", "--test", "test_compile_error"] + _run_cargo(session, *command, env=env) + _run_cargo(session, *command, "--features=full", env=env) + _run_cargo(session, *command, "--features=abi3,full", env=env) + + def _build_docs_for_ffi_check(session: nox.Session) -> None: # pyo3-ffi-check needs to scrape docs of pyo3-ffi _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") @@ -813,12 +823,23 @@ def _get_coverage_env() -> Dict[str, str]: def _run(session: nox.Session, *args: str, **kwargs: Any) -> None: """Wrapper for _run(session, which creates nice groups on GitHub Actions.""" is_github_actions = _is_github_actions() + failed = False if is_github_actions: # Insert ::group:: at the start of nox's command line output print("::group::", end="", flush=True, file=sys.stderr) - session.run(*args, **kwargs) - if is_github_actions: - print("::endgroup::", file=sys.stderr) + try: + session.run(*args, **kwargs) + except nox.command.CommandFailed: + failed = True + raise + finally: + if is_github_actions: + print("::endgroup::", file=sys.stderr) + # Defer the error message until after the group to make them easier + # to find in the log + if failed: + command = " ".join(args) + print(f"::error::`{command}` failed", file=sys.stderr) def _run_cargo( From 009cd32a444ceacd6142f8a5195d9b38ccfa5883 Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Sat, 23 Mar 2024 01:13:24 +0000 Subject: [PATCH 235/349] Add into_mapping and into_sequence methods to Bound types (#3982) * Add Bound<'py, PyDict>.into_mapping method * Add Bound<'py, PyTuple>.into_sequence method * Add Bound<'py, PyList>.into_sequence method * Add newsfragment * Add tests for into_mapping and into_sequence --- newsfragments/3982.added.md | 1 + src/types/dict.rs | 21 +++++++++++++++++++ src/types/list.rs | 40 +++++++++++++++++++++++++++++++++++++ src/types/tuple.rs | 19 +++++++++++++++++- 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3982.added.md diff --git a/newsfragments/3982.added.md b/newsfragments/3982.added.md new file mode 100644 index 00000000000..abc89574d38 --- /dev/null +++ b/newsfragments/3982.added.md @@ -0,0 +1 @@ +Added `Bound<'py, PyDict>::into_mapping`, `Bound<'py, PyList>::into_sequence` and `Bound<'py, PyTuple>::into_sequence`. diff --git a/src/types/dict.rs b/src/types/dict.rs index 2302004d238..43bade5a80c 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -365,6 +365,9 @@ pub trait PyDictMethods<'py>: crate::sealed::Sealed { /// Returns `self` cast as a `PyMapping`. fn as_mapping(&self) -> &Bound<'py, PyMapping>; + /// Returns `self` cast as a `PyMapping`. + fn into_mapping(self) -> Bound<'py, PyMapping>; + /// Update this dictionary with the key/value pairs from another. /// /// This is equivalent to the Python expression `self.update(other)`. If `other` is a `PyDict`, you may want @@ -511,6 +514,10 @@ impl<'py> PyDictMethods<'py> for Bound<'py, PyDict> { unsafe { self.downcast_unchecked() } } + fn into_mapping(self) -> Bound<'py, PyMapping> { + unsafe { self.into_any().downcast_into_unchecked() } + } + fn update(&self, other: &Bound<'_, PyMapping>) -> PyResult<()> { err::error_on_minusone(self.py(), unsafe { ffi::PyDict_Update(self.as_ptr(), other.as_ptr()) @@ -1367,6 +1374,20 @@ mod tests { }); } + #[test] + fn dict_into_mapping() { + Python::with_gil(|py| { + let mut map = HashMap::::new(); + map.insert(1, 1); + + let py_map = map.into_py_dict_bound(py); + + let py_mapping = py_map.into_mapping(); + assert_eq!(py_mapping.len().unwrap(), 1); + assert_eq!(py_mapping.get_item(1).unwrap().extract::().unwrap(), 1); + }); + } + #[cfg(not(PyPy))] fn abc_dict(py: Python<'_>) -> Bound<'_, PyDict> { let mut map = HashMap::<&'static str, i32>::new(); diff --git a/src/types/list.rs b/src/types/list.rs index 3aa7ddca3f0..19dbe59510f 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -295,6 +295,9 @@ pub trait PyListMethods<'py>: crate::sealed::Sealed { /// Returns `self` cast as a `PySequence`. fn as_sequence(&self) -> &Bound<'py, PySequence>; + /// Returns `self` cast as a `PySequence`. + fn into_sequence(self) -> Bound<'py, PySequence>; + /// Gets the list item at the specified index. /// # Example /// ``` @@ -408,6 +411,11 @@ impl<'py> PyListMethods<'py> for Bound<'py, PyList> { unsafe { self.downcast_unchecked() } } + /// Returns `self` cast as a `PySequence`. + fn into_sequence(self) -> Bound<'py, PySequence> { + unsafe { self.into_any().downcast_into_unchecked() } + } + /// Gets the list item at the specified index. /// # Example /// ``` @@ -715,6 +723,9 @@ impl<'py> IntoIterator for &Bound<'py, PyList> { #[cfg(test)] #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { + use crate::types::any::PyAnyMethods; + use crate::types::list::PyListMethods; + use crate::types::sequence::PySequenceMethods; use crate::types::{PyList, PyTuple}; use crate::Python; use crate::{IntoPy, PyObject, ToPyObject}; @@ -934,6 +945,35 @@ mod tests { }); } + #[test] + fn test_as_sequence() { + Python::with_gil(|py| { + let list = PyList::new_bound(py, [1, 2, 3, 4]); + + assert_eq!(list.as_sequence().len().unwrap(), 4); + assert_eq!( + list.as_sequence() + .get_item(1) + .unwrap() + .extract::() + .unwrap(), + 2 + ); + }); + } + + #[test] + fn test_into_sequence() { + Python::with_gil(|py| { + let list = PyList::new_bound(py, [1, 2, 3, 4]); + + let sequence = list.into_sequence(); + + assert_eq!(sequence.len().unwrap(), 4); + assert_eq!(sequence.get_item(1).unwrap().extract::().unwrap(), 2); + }); + } + #[test] fn test_extract() { Python::with_gil(|py| { diff --git a/src/types/tuple.rs b/src/types/tuple.rs index d89830daa97..4dfe4436bf0 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -258,6 +258,9 @@ pub trait PyTupleMethods<'py>: crate::sealed::Sealed { /// Returns `self` cast as a `PySequence`. fn as_sequence(&self) -> &Bound<'py, PySequence>; + /// Returns `self` cast as a `PySequence`. + fn into_sequence(self) -> Bound<'py, PySequence>; + /// Takes the slice `self[low:high]` and returns it as a new tuple. /// /// Indices must be nonnegative, and out-of-range indices are clipped to @@ -353,6 +356,10 @@ impl<'py> PyTupleMethods<'py> for Bound<'py, PyTuple> { unsafe { self.downcast_unchecked() } } + fn into_sequence(self) -> Bound<'py, PySequence> { + unsafe { self.into_any().downcast_into_unchecked() } + } + fn get_slice(&self, low: usize, high: usize) -> Bound<'py, PyTuple> { unsafe { ffi::PyTuple_GetSlice(self.as_ptr(), get_ssize_index(low), get_ssize_index(high)) @@ -1415,7 +1422,7 @@ mod tests { #[test] fn test_tuple_as_sequence() { Python::with_gil(|py| { - let tuple = PyTuple::new(py, vec![1, 2, 3]); + let tuple = PyTuple::new_bound(py, vec![1, 2, 3]); let sequence = tuple.as_sequence(); assert!(tuple.get_item(0).unwrap().eq(1).unwrap()); assert!(sequence.get_item(0).unwrap().eq(1).unwrap()); @@ -1425,6 +1432,16 @@ mod tests { }) } + #[test] + fn test_tuple_into_sequence() { + Python::with_gil(|py| { + let tuple = PyTuple::new_bound(py, vec![1, 2, 3]); + let sequence = tuple.into_sequence(); + assert!(sequence.get_item(0).unwrap().eq(1).unwrap()); + assert_eq!(sequence.len().unwrap(), 3); + }) + } + #[test] fn test_bound_tuple_get_item() { Python::with_gil(|py| { From aeb74c7093a0979c2ff7436150b85d8849c5bd1c Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 23 Mar 2024 20:16:37 +0000 Subject: [PATCH 236/349] ci: use macos-14 runners (#3985) * ci: use macos-14 runners * use arm64 python architecture --- .github/workflows/ci.yml | 42 ++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 148938fe016..5ee0776ebed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,9 +74,9 @@ jobs: rust: [stable] platform: [ { - os: "macos-latest", - python-architecture: "x64", - rust-target: "x86_64-apple-darwin", + os: "macos-14", # first available arm macos runner + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", }, { os: "ubuntu-latest", @@ -119,7 +119,7 @@ jobs: rust-target: "x86_64-unknown-linux-gnu", } name: clippy/${{ matrix.platform.rust-target }}/${{ matrix.rust }} - continue-on-error: ${{ matrix.platform.rust != 'stable' }} + continue-on-error: ${{ matrix.rust != 'stable' }} steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master @@ -161,7 +161,12 @@ jobs: platform: [ { - os: "macos-latest", + os: "macos-14", # first available arm macos runner + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + }, + { + os: "macos-13", # last available x86_64 macos runner python-architecture: "x64", rust-target: "x86_64-apple-darwin", }, @@ -226,8 +231,13 @@ jobs: ] platform: [ + # for the full matrix, use x86_64 macos runners because not all Python versions + # PyO3 supports are available for arm on GitHub Actions. (Availability starts + # around Python 3.10, can switch the full matrix to arm once earlier versions + # are dropped.) + # NB: if this switches to arm, switch the arm job below in the `include` to x86_64 { - os: "macos-latest", + os: "macos-13", python-architecture: "x64", rust-target: "x86_64-apple-darwin", }, @@ -287,6 +297,18 @@ jobs: } extra-features: "multiple-pymethods" + # test arm macos runner with the latest Python version + # NB: if the full matrix switchess to arm, switch to x86_64 here + - rust: stable + python-version: "3.12" + platform: + { + os: "macos-14", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + } + extra-features: "multiple-pymethods" + valgrind: if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] @@ -343,13 +365,13 @@ jobs: coverage: needs: [fmt] - name: coverage-${{ matrix.os }} + name: coverage ${{ matrix.os }} strategy: matrix: - os: ["windows", "macos", "ubuntu"] - runs-on: ${{ matrix.os }}-latest + os: ["windows-latest", "macos-14", "ubuntu-latest"] # first available arm macos runner + runs-on: ${{ matrix.os }} steps: - - if: ${{ github.event_name == 'pull_request' && matrix.os != 'ubuntu' }} + - if: ${{ github.event_name == 'pull_request' && matrix.os != 'ubuntu-latest' }} id: should-skip shell: bash run: echo 'skip=true' >> $GITHUB_OUTPUT From 7d319a906e3a5173b2db1fddf239c21921297895 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 23 Mar 2024 20:17:33 +0000 Subject: [PATCH 237/349] ci: tidy up benchmarks a little (#3986) --- pyo3-benches/benches/bench_bigint.rs | 34 ++++-------- pyo3-benches/benches/bench_decimal.rs | 10 ++-- pyo3-benches/benches/bench_dict.rs | 10 ++-- pyo3-benches/benches/bench_extract.rs | 63 ++++++++-------------- pyo3-benches/benches/bench_frompyobject.rs | 50 +++++++---------- pyo3-benches/benches/bench_list.rs | 2 - pyo3-benches/benches/bench_set.rs | 23 +++++--- pyo3-benches/benches/bench_tuple.rs | 8 +-- 8 files changed, 81 insertions(+), 119 deletions(-) diff --git a/pyo3-benches/benches/bench_bigint.rs b/pyo3-benches/benches/bench_bigint.rs index 4a50b437d95..99635a70279 100644 --- a/pyo3-benches/benches/bench_bigint.rs +++ b/pyo3-benches/benches/bench_bigint.rs @@ -1,17 +1,18 @@ -use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use std::hint::black_box; + +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; +use num_bigint::BigInt; use pyo3::prelude::*; use pyo3::types::PyDict; -use num_bigint::BigInt; - fn extract_bigint_extract_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let d = PyDict::new_bound(py).into_any(); bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } @@ -20,10 +21,7 @@ fn extract_bigint_small(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let int = py.eval_bound("-42", None, None).unwrap(); - bench.iter(|| { - let v = black_box(&int).extract::().unwrap(); - black_box(v); - }); + bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); } @@ -31,10 +29,7 @@ fn extract_bigint_big_negative(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let int = py.eval_bound("-10**300", None, None).unwrap(); - bench.iter(|| { - let v = black_box(&int).extract::().unwrap(); - black_box(v); - }); + bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); } @@ -42,10 +37,7 @@ fn extract_bigint_big_positive(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let int = py.eval_bound("10**300", None, None).unwrap(); - bench.iter(|| { - let v = black_box(&int).extract::().unwrap(); - black_box(v); - }); + bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); } @@ -53,10 +45,7 @@ fn extract_bigint_huge_negative(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let int = py.eval_bound("-10**3000", None, None).unwrap(); - bench.iter(|| { - let v = black_box(&int).extract::().unwrap(); - black_box(v); - }); + bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); } @@ -64,10 +53,7 @@ fn extract_bigint_huge_positive(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let int = py.eval_bound("10**3000", None, None).unwrap(); - bench.iter(|| { - let v = black_box(&int).extract::().unwrap(); - black_box(v); - }); + bench.iter_with_large_drop(|| black_box(&int).extract::().unwrap()); }); } diff --git a/pyo3-benches/benches/bench_decimal.rs b/pyo3-benches/benches/bench_decimal.rs index 6db6704bf8e..53b79abbd38 100644 --- a/pyo3-benches/benches/bench_decimal.rs +++ b/pyo3-benches/benches/bench_decimal.rs @@ -1,8 +1,10 @@ -use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use std::hint::black_box; + +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; +use rust_decimal::Decimal; use pyo3::prelude::*; use pyo3::types::PyDict; -use rust_decimal::Decimal; fn decimal_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { @@ -18,9 +20,7 @@ py_dec = decimal.Decimal("0.0") .unwrap(); let py_dec = locals.get_item("py_dec").unwrap().unwrap(); - b.iter(|| { - let _: Decimal = black_box(&py_dec).extract().unwrap(); - }); + b.iter(|| black_box(&py_dec).extract::().unwrap()); }) } diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs index e6a1e2e6b0b..8c3dfe023c8 100644 --- a/pyo3-benches/benches/bench_dict.rs +++ b/pyo3-benches/benches/bench_dict.rs @@ -1,9 +1,10 @@ +use std::collections::{BTreeMap, HashMap}; +use std::hint::black_box; + use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::types::IntoPyDict; use pyo3::{prelude::*, types::PyMapping}; -use std::collections::{BTreeMap, HashMap}; -use std::hint::black_box; fn iter_dict(b: &mut Bencher<'_>) { Python::with_gil(|py| { @@ -69,7 +70,6 @@ fn extract_hashbrown_map(b: &mut Bencher<'_>) { }); } -#[cfg(not(codspeed))] fn mapping_from_dict(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; @@ -84,12 +84,10 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("dict_get_item", dict_get_item); c.bench_function("extract_hashmap", extract_hashmap); c.bench_function("extract_btreemap", extract_btreemap); + c.bench_function("mapping_from_dict", mapping_from_dict); #[cfg(feature = "hashbrown")] c.bench_function("extract_hashbrown_map", extract_hashbrown_map); - - #[cfg(not(codspeed))] - c.bench_function("mapping_from_dict", mapping_from_dict); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs index 434d8eb5b33..9bb7ef60ab4 100644 --- a/pyo3-benches/benches/bench_extract.rs +++ b/pyo3-benches/benches/bench_extract.rs @@ -1,4 +1,6 @@ -use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use std::hint::black_box; + +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ prelude::*, @@ -7,9 +9,9 @@ use pyo3::{ fn extract_str_extract_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let s = &PyString::new_bound(py, "Hello, World!"); + let s = PyString::new_bound(py, "Hello, World!").into_any(); - bench.iter(|| black_box(s).extract::<&str>().unwrap()); + bench.iter(|| black_box(&s).extract::<&str>().unwrap()); }); } @@ -19,7 +21,7 @@ fn extract_str_extract_fail(bench: &mut Bencher<'_>) { bench.iter(|| match black_box(&d).extract::<&str>() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } @@ -27,10 +29,10 @@ fn extract_str_extract_fail(bench: &mut Bencher<'_>) { #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] fn extract_str_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let s = &PyString::new_bound(py, "Hello, World!"); + let s = PyString::new_bound(py, "Hello, World!").into_any(); bench.iter(|| { - let py_str = black_box(s).downcast::().unwrap(); + let py_str = black_box(&s).downcast::().unwrap(); py_str.to_str().unwrap() }); }); @@ -42,20 +44,16 @@ fn extract_str_downcast_fail(bench: &mut Bencher<'_>) { bench.iter(|| match black_box(&d).downcast::() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } fn extract_int_extract_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int_obj: PyObject = 123.into_py(py); - let int = int_obj.as_ref(py); + let int = 123.to_object(py).into_bound(py); - bench.iter(|| { - let v = black_box(int).extract::().unwrap(); - black_box(v); - }); + bench.iter(|| black_box(&int).extract::().unwrap()); }); } @@ -65,26 +63,22 @@ fn extract_int_extract_fail(bench: &mut Bencher<'_>) { bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } -#[cfg(not(codspeed))] fn extract_int_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int_obj: PyObject = 123.into_py(py); - let int = int_obj.as_ref(py); + let int = 123.to_object(py).into_bound(py); bench.iter(|| { - let py_int = black_box(int).downcast::().unwrap(); - let v = py_int.extract::().unwrap(); - black_box(v); + let py_int = black_box(&int).downcast::().unwrap(); + py_int.extract::().unwrap() }); }); } -#[cfg(not(codspeed))] fn extract_int_downcast_fail(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let d = PyDict::new_bound(py).into_any(); @@ -96,16 +90,11 @@ fn extract_int_downcast_fail(bench: &mut Bencher<'_>) { }); } -#[cfg(not(codspeed))] fn extract_float_extract_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let float_obj: PyObject = 23.42.into_py(py); - let float = float_obj.as_ref(py); + let float = 23.42.to_object(py).into_bound(py); - bench.iter(|| { - let v = black_box(float).extract::().unwrap(); - black_box(v); - }); + bench.iter(|| black_box(&float).extract::().unwrap()); }); } @@ -115,21 +104,18 @@ fn extract_float_extract_fail(bench: &mut Bencher<'_>) { bench.iter(|| match black_box(&d).extract::() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } -#[cfg(not(codspeed))] fn extract_float_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let float_obj: PyObject = 23.42.into_py(py); - let float = float_obj.as_ref(py); + let float = 23.42.to_object(py).into_bound(py); bench.iter(|| { - let py_int = black_box(float).downcast::().unwrap(); - let v = py_int.extract::().unwrap(); - black_box(v); + let py_float = black_box(&float).downcast::().unwrap(); + py_float.value() }); }); } @@ -140,7 +126,7 @@ fn extract_float_downcast_fail(bench: &mut Bencher<'_>) { bench.iter(|| match black_box(&d).downcast::() { Ok(v) => panic!("should err {}", v), - Err(e) => black_box(e), + Err(e) => e, }); }); } @@ -151,20 +137,15 @@ fn criterion_benchmark(c: &mut Criterion) { #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] c.bench_function("extract_str_downcast_success", extract_str_downcast_success); c.bench_function("extract_str_downcast_fail", extract_str_downcast_fail); - #[cfg(not(codspeed))] c.bench_function("extract_int_extract_success", extract_int_extract_success); c.bench_function("extract_int_extract_fail", extract_int_extract_fail); - #[cfg(not(codspeed))] c.bench_function("extract_int_downcast_success", extract_int_downcast_success); - #[cfg(not(codspeed))] c.bench_function("extract_int_downcast_fail", extract_int_downcast_fail); - #[cfg(not(codspeed))] c.bench_function( "extract_float_extract_success", extract_float_extract_success, ); c.bench_function("extract_float_extract_fail", extract_float_extract_fail); - #[cfg(not(codspeed))] c.bench_function( "extract_float_downcast_success", extract_float_downcast_success, diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs index e78c48fcaa1..f53f116a154 100644 --- a/pyo3-benches/benches/bench_frompyobject.rs +++ b/pyo3-benches/benches/bench_frompyobject.rs @@ -1,11 +1,14 @@ -use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use std::hint::black_box; + +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ prelude::*, - types::{PyFloat, PyList, PyString}, + types::{PyList, PyString}, }; #[derive(FromPyObject)] +#[allow(dead_code)] enum ManyTypes { Int(i32), Bytes(Vec), @@ -14,44 +17,41 @@ enum ManyTypes { fn enum_from_pyobject(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "hello world"); + let any = PyString::new_bound(py, "hello world").into_any(); - b.iter(|| any.extract::().unwrap()); + b.iter(|| black_box(&any).extract::().unwrap()); }) } -#[cfg(not(codspeed))] fn list_via_downcast(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyList::empty_bound(py); + let any = PyList::empty_bound(py).into_any(); - b.iter(|| black_box(any).downcast::().unwrap()); + b.iter(|| black_box(&any).downcast::().unwrap()); }) } -#[cfg(not(codspeed))] fn list_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyList::empty_bound(py); + let any = PyList::empty_bound(py).into_any(); - b.iter(|| black_box(any).extract::>().unwrap()); + b.iter(|| black_box(&any).extract::>().unwrap()); }) } -#[cfg(not(codspeed))] fn not_a_list_via_downcast(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "foobar"); + let any = PyString::new_bound(py, "foobar").into_any(); - b.iter(|| black_box(any).downcast::().unwrap_err()); + b.iter(|| black_box(&any).downcast::().unwrap_err()); }) } fn not_a_list_via_extract(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "foobar"); + let any = PyString::new_bound(py, "foobar").into_any(); - b.iter(|| black_box(any).extract::>().unwrap_err()); + b.iter(|| black_box(&any).extract::>().unwrap_err()); }) } @@ -63,9 +63,9 @@ enum ListOrNotList<'a> { fn not_a_list_via_extract_enum(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let any: &Bound<'_, PyAny> = &PyString::new_bound(py, "foobar"); + let any = PyString::new_bound(py, "foobar").into_any(); - b.iter(|| match black_box(any).extract::>() { + b.iter(|| match black_box(&any).extract::>() { Ok(ListOrNotList::List(_list)) => panic!(), Ok(ListOrNotList::NotList(any)) => any, Err(_) => panic!(), @@ -73,26 +73,16 @@ fn not_a_list_via_extract_enum(b: &mut Bencher<'_>) { }) } -#[cfg(not(codspeed))] -fn f64_from_pyobject(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let obj = &PyFloat::new_bound(py, 1.234); - b.iter(|| black_box(obj).extract::().unwrap()); - }) -} - fn criterion_benchmark(c: &mut Criterion) { c.bench_function("enum_from_pyobject", enum_from_pyobject); - #[cfg(not(codspeed))] + c.bench_function("list_via_downcast", list_via_downcast); - #[cfg(not(codspeed))] + c.bench_function("list_via_extract", list_via_extract); - #[cfg(not(codspeed))] + c.bench_function("not_a_list_via_downcast", not_a_list_via_downcast); c.bench_function("not_a_list_via_extract", not_a_list_via_extract); c.bench_function("not_a_list_via_extract_enum", not_a_list_via_extract_enum); - #[cfg(not(codspeed))] - c.bench_function("f64_from_pyobject", f64_from_pyobject); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs index 774dddcea38..dcbdb4779cb 100644 --- a/pyo3-benches/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -55,7 +55,6 @@ fn list_get_item_unchecked(b: &mut Bencher<'_>) { }); } -#[cfg(not(codspeed))] fn sequence_from_list(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; @@ -70,7 +69,6 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("list_get_item", list_get_item); #[cfg(not(Py_LIMITED_API))] c.bench_function("list_get_item_unchecked", list_get_item_unchecked); - #[cfg(not(codspeed))] c.bench_function("sequence_from_list", sequence_from_list); } diff --git a/pyo3-benches/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs index b04cb491304..18134a15bd5 100644 --- a/pyo3-benches/benches/bench_set.rs +++ b/pyo3-benches/benches/bench_set.rs @@ -2,7 +2,10 @@ use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criter use pyo3::prelude::*; use pyo3::types::PySet; -use std::collections::{BTreeSet, HashSet}; +use std::{ + collections::{BTreeSet, HashSet}, + hint::black_box, +}; fn set_new(b: &mut Bencher<'_>) { Python::with_gil(|py| { @@ -31,16 +34,20 @@ fn iter_set(b: &mut Bencher<'_>) { fn extract_hashset(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); - b.iter_with_large_drop(|| HashSet::::extract(set.as_gil_ref())); + let any = PySet::new_bound(py, &(0..LEN).collect::>()) + .unwrap() + .into_any(); + b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } fn extract_btreeset(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); - b.iter_with_large_drop(|| BTreeSet::::extract(set.as_gil_ref())); + let any = PySet::new_bound(py, &(0..LEN).collect::>()) + .unwrap() + .into_any(); + b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } @@ -48,8 +55,10 @@ fn extract_btreeset(b: &mut Bencher<'_>) { fn extract_hashbrown_set(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let set = PySet::new_bound(py, &(0..LEN).collect::>()).unwrap(); - b.iter_with_large_drop(|| hashbrown::HashSet::::extract(set.as_gil_ref())); + let any = PySet::new_bound(py, &(0..LEN).collect::>()) + .unwrap() + .into_any(); + b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs index 22bf7588ed1..3c0b56a0234 100644 --- a/pyo3-benches/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -1,3 +1,5 @@ +use std::hint::black_box; + use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; @@ -89,12 +91,11 @@ fn tuple_get_borrowed_item_unchecked(b: &mut Bencher<'_>) { }); } -#[cfg(not(codspeed))] fn sequence_from_tuple(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50_000; - let tuple = PyTuple::new_bound(py, 0..LEN).to_object(py); - b.iter(|| tuple.downcast::(py).unwrap()); + let tuple = PyTuple::new_bound(py, 0..LEN).into_any(); + b.iter(|| black_box(&tuple).downcast::().unwrap()); }); } @@ -132,7 +133,6 @@ fn criterion_benchmark(c: &mut Criterion) { "tuple_get_borrowed_item_unchecked", tuple_get_borrowed_item_unchecked, ); - #[cfg(not(codspeed))] c.bench_function("sequence_from_tuple", sequence_from_tuple); c.bench_function("tuple_new_list", tuple_new_list); c.bench_function("tuple_to_list", tuple_to_list); From 54ffaecd65a746e702e8ec904d2af4f3a8aeaee9 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:21:27 +0100 Subject: [PATCH 238/349] add `AsRef` impls for `PyBackedStr` and `PyBackedBytes` (#3991) * add `AsRef` impls for `PyBackedStr` and `PyBackedBytes` * add newsfragment --- newsfragments/3991.added.md | 1 + src/pybacked.rs | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 newsfragments/3991.added.md diff --git a/newsfragments/3991.added.md b/newsfragments/3991.added.md new file mode 100644 index 00000000000..1a59ecee79c --- /dev/null +++ b/newsfragments/3991.added.md @@ -0,0 +1 @@ +implemented `AsRef` and `AsRef<[u8]>` for `PyBackedStr`, `AsRef<[u8]>` for `PyBackedBytes`. diff --git a/src/pybacked.rs b/src/pybacked.rs index bd29c830e65..dbb0865285e 100644 --- a/src/pybacked.rs +++ b/src/pybacked.rs @@ -27,6 +27,18 @@ impl Deref for PyBackedStr { } } +impl AsRef for PyBackedStr { + fn as_ref(&self) -> &str { + self + } +} + +impl AsRef<[u8]> for PyBackedStr { + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + impl TryFrom> for PyBackedStr { type Error = PyErr; fn try_from(py_string: Bound<'_, PyString>) -> Result { @@ -82,6 +94,12 @@ impl Deref for PyBackedBytes { } } +impl AsRef<[u8]> for PyBackedBytes { + fn as_ref(&self) -> &[u8] { + self + } +} + impl From> for PyBackedBytes { fn from(py_bytes: Bound<'_, PyBytes>) -> Self { let b = py_bytes.as_bytes(); From 9a38e709bb43bdd8e43aa7c2fce603ed30ed469f Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 25 Mar 2024 19:54:52 +0100 Subject: [PATCH 239/349] Basic GraalPy Support (#3247) * graalpy: recognize graalpy implementation when building * graalpy: global Ellipse, None, NotImplemented, True, and False are only available as pointers * graalpy: PyObject struct is opaque, use functions for everything * graalpy: missing many of the same functions as pypy * graalpy: do not have 128bit conversion functions * graalpy: add functions for datetime accessor macros * graalpy: add implementations for list macro functions * graalpy: skip tuple macros * graalpy: always use extern Py_CompileString function * graalpy: disable assertion that does not apply to graalpy * graalpy: floatobject structure is opaque on graalpy * graalpy: ignore gc dependent test * graalpy: add CI config * graalpy: run rust fmt * graalpy: add changelog entry * graalpy: discover interpreter on PATH * graalpy: interpreter id is not applicable to graalpy (just like pypy) * graalpy: skip tests that cannot work on GraalPy * graalpy: fix constructing normalized Err instances Co-authored-by: David Hewitt * graalpy: correct capi library name, but skip rust tests due to missing symbols * graalpy: no support for C extensions on windows in latest release * graalpy: declare support versions * graalpy: frame, code, method, and function objects access from C API is mostly missing * graalpy: take care only to expose C structure that GraalPy allocates * graalpy: Bail out if graalpy version is less than what we support --------- Co-authored-by: David Hewitt --- .github/workflows/build.yml | 15 +- .github/workflows/ci.yml | 1 + README.md | 2 +- newsfragments/3247.added.md | 1 + pyo3-build-config/src/impl_.rs | 69 +++++++- pyo3-build-config/src/import_lib.rs | 5 +- pyo3-build-config/src/lib.rs | 1 + pyo3-ffi/build.rs | 29 ++++ pyo3-ffi/src/abstract_.rs | 1 + pyo3-ffi/src/boolobject.rs | 18 ++- pyo3-ffi/src/bytearrayobject.rs | 4 +- pyo3-ffi/src/complexobject.rs | 1 + pyo3-ffi/src/cpython/abstract_.rs | 39 ++--- pyo3-ffi/src/cpython/bytesobject.rs | 6 +- pyo3-ffi/src/cpython/code.rs | 24 +-- pyo3-ffi/src/cpython/compile.rs | 4 +- pyo3-ffi/src/cpython/dictobject.rs | 1 + pyo3-ffi/src/cpython/floatobject.rs | 8 +- pyo3-ffi/src/cpython/frameobject.rs | 15 +- pyo3-ffi/src/cpython/funcobject.rs | 9 +- pyo3-ffi/src/cpython/genobject.rs | 4 +- pyo3-ffi/src/cpython/listobject.rs | 8 +- pyo3-ffi/src/cpython/methodobject.rs | 6 + pyo3-ffi/src/cpython/mod.rs | 2 +- pyo3-ffi/src/cpython/object.rs | 2 + pyo3-ffi/src/cpython/objimpl.rs | 10 +- pyo3-ffi/src/cpython/pyerrors.rs | 42 ++--- pyo3-ffi/src/cpython/pymem.rs | 8 +- pyo3-ffi/src/cpython/pythonrun.rs | 43 ++--- pyo3-ffi/src/cpython/tupleobject.rs | 5 +- pyo3-ffi/src/cpython/unicodeobject.rs | 51 ++++-- pyo3-ffi/src/cpython/weakrefobject.rs | 2 +- pyo3-ffi/src/datetime.rs | 224 ++++++++++++++++++++------ pyo3-ffi/src/dictobject.rs | 2 +- pyo3-ffi/src/listobject.rs | 8 +- pyo3-ffi/src/methodobject.rs | 2 +- pyo3-ffi/src/object.rs | 55 ++++++- pyo3-ffi/src/pyframe.rs | 2 + pyo3-ffi/src/pyhash.rs | 12 +- pyo3-ffi/src/pythonrun.rs | 4 +- pyo3-ffi/src/setobject.rs | 10 +- pyo3-ffi/src/sliceobject.rs | 12 +- pyo3-ffi/src/structseq.rs | 4 +- pyo3-ffi/src/weakrefobject.rs | 4 +- pytests/tests/test_awaitable.py | 9 ++ pytests/tests/test_misc.py | 4 +- pytests/tests/test_objstore.py | 5 + src/conversions/std/num.rs | 4 +- src/err/mod.rs | 13 +- src/exceptions.rs | 16 +- src/gil.rs | 8 +- src/impl_/pymodule.rs | 22 ++- src/impl_/trampoline.rs | 6 +- src/lib.rs | 2 +- src/macros.rs | 2 +- src/marker.rs | 2 +- src/types/any.rs | 11 +- src/types/complex.rs | 10 +- src/types/datetime.rs | 32 ++++ src/types/dict.rs | 38 ++--- src/types/frozenset.rs | 4 +- src/types/mod.rs | 16 +- src/types/sequence.rs | 4 +- src/types/set.rs | 4 +- src/types/string.rs | 8 +- src/types/tuple.rs | 44 ++--- 66 files changed, 731 insertions(+), 308 deletions(-) create mode 100644 newsfragments/3247.added.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 25c8014f762..d2f74401dd6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,6 +24,7 @@ jobs: build: continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "pypy3.7"]'), inputs.python-version) || inputs.rust == 'beta' || inputs.rust == 'nightly' }} runs-on: ${{ inputs.os }} + if: ${{ !(startsWith(inputs.python-version, 'graalpy') && startsWith(inputs.os, 'windows')) }} steps: - uses: actions/checkout@v4 @@ -65,6 +66,7 @@ jobs: run: nox -s docs - name: Build (no features) + if: ${{ !startsWith(inputs.python-version, 'graalpy') }} run: cargo build --lib --tests --no-default-features # --no-default-features when used with `cargo build/test -p` doesn't seem to work! @@ -74,7 +76,7 @@ jobs: cargo build --no-default-features # Run tests (except on PyPy, because no embedding API). - - if: ${{ !startsWith(inputs.python-version, 'pypy') }} + - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} name: Test (no features) run: cargo test --no-default-features --lib --tests @@ -85,6 +87,7 @@ jobs: cargo test --no-default-features - name: Build (all additive features) + if: ${{ !startsWith(inputs.python-version, 'graalpy') }} run: cargo build --lib --tests --no-default-features --features "full ${{ inputs.extra-features }}" - if: ${{ startsWith(inputs.python-version, 'pypy') }} @@ -92,17 +95,17 @@ jobs: run: cargo build --lib --tests --no-default-features --features "abi3-py37 full ${{ inputs.extra-features }}" # Run tests (except on PyPy, because no embedding API). - - if: ${{ !startsWith(inputs.python-version, 'pypy') }} + - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} name: Test run: cargo test --no-default-features --features "full ${{ inputs.extra-features }}" # Run tests again, but in abi3 mode - - if: ${{ !startsWith(inputs.python-version, 'pypy') }} + - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} name: Test (abi3) run: cargo test --no-default-features --features "abi3 full ${{ inputs.extra-features }}" # Run tests again, for abi3-py37 (the minimal Python version) - - if: ${{ (!startsWith(inputs.python-version, 'pypy')) && (inputs.python-version != '3.7') }} + - if: ${{ (!startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy')) && (inputs.python-version != '3.7') }} name: Test (abi3-py37) run: cargo test --no-default-features --features "abi3-py37 full ${{ inputs.extra-features }}" @@ -120,7 +123,7 @@ jobs: - uses: dorny/paths-filter@v3 # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here - if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' }} + if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !startsWith(inputs.python-version, 'graalpy') }} id: ffi-changes with: base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} @@ -135,7 +138,7 @@ jobs: - name: Run pyo3-ffi-check # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor # is pypy 3.9 on windows - if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }} + if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !startsWith(inputs.python-version, 'graalpy') && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }} run: nox -s ffi-check env: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ee0776ebed..63f6c31a599 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -228,6 +228,7 @@ jobs: "pypy3.8", "pypy3.9", "pypy3.10", + "graalpy24.0", ] platform: [ diff --git a/README.md b/README.md index c11754c3ca1..461e56612ad 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ ## Usage PyO3 supports the following software versions: - - Python 3.7 and up (CPython and PyPy) + - Python 3.7 and up (CPython, PyPy, and GraalPy) - Rust 1.56 and up You can use PyO3 to write a native Python module in Rust, or to embed Python in a Rust binary. The following sections explain each of these in turn. diff --git a/newsfragments/3247.added.md b/newsfragments/3247.added.md new file mode 100644 index 00000000000..dd2641c3194 --- /dev/null +++ b/newsfragments/3247.added.md @@ -0,0 +1 @@ +Added support for running PyO3 extensions on GraalPy. diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 7f02f744fc2..d5373db9655 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -32,6 +32,12 @@ use crate::{ /// Minimum Python version PyO3 supports. const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; +/// GraalPy may implement the same CPython version over multiple releases. +const MINIMUM_SUPPORTED_VERSION_GRAALPY: PythonVersion = PythonVersion { + major: 24, + minor: 0, +}; + /// Maximum Python version that can be used as minimum required Python version with abi3. const ABI3_MAX_MINOR: u8 = 12; @@ -173,6 +179,11 @@ impl InterpreterConfig { See https://foss.heptapod.net/pypy/pypy/-/issues/3397 for more information." )); } + } else if self.implementation.is_graalpy() { + println!("cargo:rustc-cfg=GraalPy"); + if self.abi3 { + warn!("GraalPy does not support abi3 so the build artifacts will be version-specific."); + } } else if self.abi3 { out.push("cargo:rustc-cfg=Py_LIMITED_API".to_owned()); } @@ -197,6 +208,12 @@ import sys from sysconfig import get_config_var, get_platform PYPY = platform.python_implementation() == "PyPy" +GRAALPY = platform.python_implementation() == "GraalVM" + +if GRAALPY: + graalpy_ver = map(int, __graalpython__.get_graalvm_version().split('.')); + print("graalpy_major", next(graalpy_ver)) + print("graalpy_minor", next(graalpy_ver)) # sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue # so that the version mismatch can be reported in a nicer way later. @@ -226,7 +243,7 @@ SHARED = bool(get_config_var("Py_ENABLE_SHARED")) print("implementation", platform.python_implementation()) print("version_major", sys.version_info[0]) print("version_minor", sys.version_info[1]) -print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) +print("shared", PYPY or GRAALPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) print_if_set("ld_version", get_config_var("LDVERSION")) print_if_set("libdir", get_config_var("LIBDIR")) print_if_set("base_prefix", base_prefix) @@ -244,6 +261,23 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) interpreter.as_ref().display() ); + if let Some(value) = map.get("graalpy_major") { + let graalpy_version = PythonVersion { + major: value + .parse() + .context("failed to parse GraalPy major version")?, + minor: map["graalpy_minor"] + .parse() + .context("failed to parse GraalPy minor version")?, + }; + ensure!( + graalpy_version >= MINIMUM_SUPPORTED_VERSION_GRAALPY, + "At least GraalPy version {} needed, got {}", + MINIMUM_SUPPORTED_VERSION_GRAALPY, + graalpy_version + ); + }; + let shared = map["shared"].as_str() == "True"; let version = PythonVersion { @@ -588,7 +622,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) /// Lowers the configured version to the abi3 version, if set. fn fixup_for_abi3_version(&mut self, abi3_version: Option) -> Result<()> { // PyPy doesn't support abi3; don't adjust the version - if self.implementation.is_pypy() { + if self.implementation.is_pypy() || self.implementation.is_graalpy() { return Ok(()); } @@ -647,6 +681,7 @@ impl FromStr for PythonVersion { pub enum PythonImplementation { CPython, PyPy, + GraalPy, } impl PythonImplementation { @@ -655,12 +690,19 @@ impl PythonImplementation { self == PythonImplementation::PyPy } + #[doc(hidden)] + pub fn is_graalpy(self) -> bool { + self == PythonImplementation::GraalPy + } + #[doc(hidden)] pub fn from_soabi(soabi: &str) -> Result { if soabi.starts_with("pypy") { Ok(PythonImplementation::PyPy) } else if soabi.starts_with("cpython") { Ok(PythonImplementation::CPython) + } else if soabi.starts_with("graalpy") { + Ok(PythonImplementation::GraalPy) } else { bail!("unsupported Python interpreter"); } @@ -672,6 +714,7 @@ impl Display for PythonImplementation { match self { PythonImplementation::CPython => write!(f, "CPython"), PythonImplementation::PyPy => write!(f, "PyPy"), + PythonImplementation::GraalPy => write!(f, "GraalVM"), } } } @@ -682,6 +725,7 @@ impl FromStr for PythonImplementation { match s { "CPython" => Ok(PythonImplementation::CPython), "PyPy" => Ok(PythonImplementation::PyPy), + "GraalVM" => Ok(PythonImplementation::GraalPy), _ => bail!("unknown interpreter: {}", s), } } @@ -760,7 +804,7 @@ pub struct CrossCompileConfig { /// The version of the Python library to link against. version: Option, - /// The target Python implementation hint (CPython or PyPy) + /// The target Python implementation hint (CPython, PyPy, GraalPy, ...) implementation: Option, /// The compile target triple (e.g. aarch64-unknown-linux-gnu) @@ -1264,6 +1308,15 @@ fn is_pypy_lib_dir(path: &str, v: &Option) -> bool { path == "lib_pypy" || path.starts_with(&pypy_version_pat) } +fn is_graalpy_lib_dir(path: &str, v: &Option) -> bool { + let graalpy_version_pat = if let Some(v) = v { + format!("graalpy{}", v) + } else { + "graalpy2".into() + }; + path == "lib_graalpython" || path.starts_with(&graalpy_version_pat) +} + fn is_cpython_lib_dir(path: &str, v: &Option) -> bool { let cpython_version_pat = if let Some(v) = v { format!("python{}", v) @@ -1297,6 +1350,7 @@ fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec Result InterpreterConfig { - // FIXME: PyPy does not support the Stable ABI yet. + // FIXME: PyPy & GraalPy do not support the Stable ABI. let implementation = PythonImplementation::CPython; let abi3 = true; @@ -1524,7 +1578,7 @@ fn default_lib_name_windows( // CPython bug: linking against python3_d.dll raises error // https://github.com/python/cpython/issues/101614 format!("python{}{}_d", version.major, version.minor) - } else if abi3 && !implementation.is_pypy() { + } else if abi3 && !(implementation.is_pypy() || implementation.is_graalpy()) { WINDOWS_ABI3_LIB_NAME.to_owned() } else if mingw { // https://packages.msys2.org/base/mingw-w64-python @@ -1562,6 +1616,7 @@ fn default_lib_name_unix( format!("pypy{}-c", version.major) } } + PythonImplementation::GraalPy => "python-native".to_string(), } } @@ -1662,7 +1717,9 @@ pub fn find_interpreter() -> Result { .find(|bin| { if let Ok(out) = Command::new(bin).arg("--version").output() { // begin with `Python 3.X.X :: additional info` - out.stdout.starts_with(b"Python 3") || out.stderr.starts_with(b"Python 3") + out.stdout.starts_with(b"Python 3") + || out.stderr.starts_with(b"Python 3") + || out.stdout.starts_with(b"GraalPy 3") } else { false } diff --git a/pyo3-build-config/src/import_lib.rs b/pyo3-build-config/src/import_lib.rs index dc21638c3db..0925a861b5b 100644 --- a/pyo3-build-config/src/import_lib.rs +++ b/pyo3-build-config/src/import_lib.rs @@ -7,7 +7,7 @@ use python3_dll_a::ImportLibraryGenerator; use target_lexicon::{Architecture, OperatingSystem, Triple}; use super::{PythonImplementation, PythonVersion}; -use crate::errors::{Context, Result}; +use crate::errors::{Context, Error, Result}; /// Generates the `python3.dll` or `pythonXY.dll` import library for Windows targets. /// @@ -42,6 +42,9 @@ pub(super) fn generate_import_lib( let implementation = match py_impl { PythonImplementation::CPython => python3_dll_a::PythonImplementation::CPython, PythonImplementation::PyPy => python3_dll_a::PythonImplementation::PyPy, + PythonImplementation::GraalPy => { + return Err(Error::from("No support for GraalPy on Windows")) + } }; ImportLibraryGenerator::new(&arch, &env) diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index e7e59d3b79d..eab7376d0ea 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -38,6 +38,7 @@ use target_lexicon::OperatingSystem; /// | `#[cfg(Py_3_7)]`, `#[cfg(Py_3_8)]`, `#[cfg(Py_3_9)]`, `#[cfg(Py_3_10)]` | These attributes mark code only for a given Python version and up. For example, `#[cfg(Py_3_7)]` marks code which can run on Python 3.7 **and newer**. | /// | `#[cfg(Py_LIMITED_API)]` | This marks code which is run when compiling with PyO3's `abi3` feature enabled. | /// | `#[cfg(PyPy)]` | This marks code which is run when compiling for PyPy. | +/// | `#[cfg(GraalPy)]` | This marks code which is run when compiling for GraalPy. | /// /// For examples of how to use these attributes, [see PyO3's guide](https://pyo3.rs/latest/building-and-distribution/multiple_python_versions.html). #[cfg(feature = "resolve-config")] diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index 286767d8f25..a5ab352b6a7 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -29,6 +29,17 @@ const SUPPORTED_VERSIONS_PYPY: SupportedVersions = SupportedVersions { }, }; +const SUPPORTED_VERSIONS_GRAALPY: SupportedVersions = SupportedVersions { + min: PythonVersion { + major: 3, + minor: 10, + }, + max: PythonVersion { + major: 3, + minor: 11, + }, +}; + fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { // This is an undocumented env var which is only really intended to be used in CI / for testing // and development. @@ -73,6 +84,24 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { std::env::var("CARGO_PKG_VERSION").unwrap() ); } + PythonImplementation::GraalPy => { + let versions = SUPPORTED_VERSIONS_GRAALPY; + ensure!( + interpreter_config.version >= versions.min, + "the configured GraalPy interpreter version ({}) is lower than PyO3's minimum supported version ({})", + interpreter_config.version, + versions.min, + ); + // GraalPy does not support abi3, so we cannot offer forward compatibility + ensure!( + interpreter_config.version <= versions.max, + "the configured GraalPy interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ + = help: please check if an updated version of PyO3 is available. Current version: {}", + interpreter_config.version, + versions.max, + std::env::var("CARGO_PKG_VERSION").unwrap() + ); + } } Ok(()) diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs index 0b3b7dbb3c2..b5bf9cc3d35 100644 --- a/pyo3-ffi/src/abstract_.rs +++ b/pyo3-ffi/src/abstract_.rs @@ -23,6 +23,7 @@ pub unsafe fn PyObject_DelAttr(o: *mut PyObject, attr_name: *mut PyObject) -> c_ extern "C" { #[cfg(all( not(PyPy), + not(GraalPy), any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // Added to python in 3.9 but to limited API in 3.10 ))] #[cfg_attr(PyPy, link_name = "PyPyObject_CallNoArgs")] diff --git a/pyo3-ffi/src/boolobject.rs b/pyo3-ffi/src/boolobject.rs index 78972ff0835..10b5969fa4f 100644 --- a/pyo3-ffi/src/boolobject.rs +++ b/pyo3-ffi/src/boolobject.rs @@ -1,3 +1,4 @@ +#[cfg(not(GraalPy))] use crate::longobject::PyLongObject; use crate::object::*; use std::os::raw::{c_int, c_long}; @@ -16,20 +17,33 @@ pub unsafe fn PyBool_Check(op: *mut PyObject) -> c_int { #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "_PyPy_FalseStruct")] static mut _Py_FalseStruct: PyLongObject; + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "_PyPy_TrueStruct")] static mut _Py_TrueStruct: PyLongObject; + + #[cfg(GraalPy)] + static mut _Py_FalseStructReference: *mut PyObject; + #[cfg(GraalPy)] + static mut _Py_TrueStructReference: *mut PyObject; } #[inline] pub unsafe fn Py_False() -> *mut PyObject { - addr_of_mut!(_Py_FalseStruct) as *mut PyObject + #[cfg(not(GraalPy))] + return addr_of_mut!(_Py_FalseStruct) as *mut PyObject; + #[cfg(GraalPy)] + return _Py_FalseStructReference; } #[inline] pub unsafe fn Py_True() -> *mut PyObject { - addr_of_mut!(_Py_TrueStruct) as *mut PyObject + #[cfg(not(GraalPy))] + return addr_of_mut!(_Py_TrueStruct) as *mut PyObject; + #[cfg(GraalPy)] + return _Py_TrueStructReference; } #[inline] diff --git a/pyo3-ffi/src/bytearrayobject.rs b/pyo3-ffi/src/bytearrayobject.rs index a37deb410f7..c09eac5b22c 100644 --- a/pyo3-ffi/src/bytearrayobject.rs +++ b/pyo3-ffi/src/bytearrayobject.rs @@ -3,7 +3,7 @@ use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; use std::ptr::addr_of_mut; -#[cfg(not(any(PyPy, Py_LIMITED_API)))] +#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyByteArrayObject { @@ -17,7 +17,7 @@ pub struct PyByteArrayObject { pub ob_exports: c_int, } -#[cfg(any(PyPy, Py_LIMITED_API))] +#[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] opaque_struct!(PyByteArrayObject); #[cfg_attr(windows, link(name = "pythonXY"))] diff --git a/pyo3-ffi/src/complexobject.rs b/pyo3-ffi/src/complexobject.rs index 339f5d8c81a..a03d9b00932 100644 --- a/pyo3-ffi/src/complexobject.rs +++ b/pyo3-ffi/src/complexobject.rs @@ -30,6 +30,7 @@ extern "C" { // non-limited pub struct PyComplexObject { pub ob_base: PyObject, + #[cfg(not(GraalPy))] pub cval: Py_complex, } diff --git a/pyo3-ffi/src/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs index d2e3ca9d67a..cf95f6711d4 100644 --- a/pyo3-ffi/src/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -6,7 +6,7 @@ use crate::Py_buffer; #[cfg(Py_3_8)] use crate::pyport::PY_SSIZE_T_MAX; -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] use crate::{ vectorcallfunc, PyCallable_Check, PyThreadState, PyThreadState_GET, PyTuple_Check, PyType_HasFeature, Py_TPFLAGS_HAVE_VECTORCALL, @@ -15,15 +15,15 @@ use crate::{ use libc::size_t; extern "C" { - #[cfg(all(Py_3_8, not(PyPy)))] + #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] pub fn _PyStack_AsDict(values: *const *mut PyObject, kwnames: *mut PyObject) -> *mut PyObject; } -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] const _PY_FASTCALL_SMALL_STACK: size_t = 5; extern "C" { - #[cfg(all(Py_3_8, not(PyPy)))] + #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] pub fn _Py_CheckFunctionResult( tstate: *mut PyThreadState, callable: *mut PyObject, @@ -31,7 +31,7 @@ extern "C" { where_: *const c_char, ) -> *mut PyObject; - #[cfg(all(Py_3_8, not(PyPy)))] + #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] pub fn _PyObject_MakeTpCall( tstate: *mut PyThreadState, callable: *mut PyObject, @@ -52,7 +52,7 @@ pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t { (n as Py_ssize_t) & !PY_VECTORCALL_ARGUMENTS_OFFSET } -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] #[inline(always)] pub unsafe fn PyVectorcall_Function(callable: *mut PyObject) -> Option { assert!(!callable.is_null()); @@ -67,7 +67,7 @@ pub unsafe fn PyVectorcall_Function(callable: *mut PyObject) -> Option *mut PyObject; #[cfg(Py_3_8)] - #[cfg_attr(all(not(PyPy), not(Py_3_9)), link_name = "_PyObject_VectorcallDict")] + #[cfg_attr( + all(not(any(PyPy, GraalPy)), not(Py_3_9)), + link_name = "_PyObject_VectorcallDict" + )] #[cfg_attr(all(PyPy, not(Py_3_9)), link_name = "_PyPyObject_VectorcallDict")] #[cfg_attr(all(PyPy, Py_3_9), link_name = "PyPyObject_VectorcallDict")] pub fn PyObject_VectorcallDict( @@ -134,7 +137,7 @@ extern "C" { ) -> *mut PyObject; } -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] #[inline(always)] pub unsafe fn _PyObject_FastCallTstate( tstate: *mut PyThreadState, @@ -145,7 +148,7 @@ pub unsafe fn _PyObject_FastCallTstate( _PyObject_VectorcallTstate(tstate, func, args, nargs as size_t, std::ptr::null_mut()) } -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] #[inline(always)] pub unsafe fn _PyObject_FastCall( func: *mut PyObject, @@ -155,7 +158,7 @@ pub unsafe fn _PyObject_FastCall( _PyObject_FastCallTstate(PyThreadState_GET(), func, args, nargs) } -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] #[inline(always)] pub unsafe fn _PyObject_CallNoArg(func: *mut PyObject) -> *mut PyObject { _PyObject_VectorcallTstate( @@ -173,7 +176,7 @@ extern "C" { pub fn _PyObject_CallNoArg(func: *mut PyObject) -> *mut PyObject; } -#[cfg(all(Py_3_8, not(PyPy)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] #[inline(always)] pub unsafe fn PyObject_CallOneArg(func: *mut PyObject, arg: *mut PyObject) -> *mut PyObject { assert!(!arg.is_null()); @@ -185,7 +188,7 @@ pub unsafe fn PyObject_CallOneArg(func: *mut PyObject, arg: *mut PyObject) -> *m } extern "C" { - #[cfg(all(Py_3_9, not(PyPy)))] + #[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] pub fn PyObject_VectorcallMethod( name: *mut PyObject, args: *const *mut PyObject, @@ -194,7 +197,7 @@ extern "C" { ) -> *mut PyObject; } -#[cfg(all(Py_3_9, not(PyPy)))] +#[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] #[inline(always)] pub unsafe fn PyObject_CallMethodNoArgs( self_: *mut PyObject, @@ -208,7 +211,7 @@ pub unsafe fn PyObject_CallMethodNoArgs( ) } -#[cfg(all(Py_3_9, not(PyPy)))] +#[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] #[inline(always)] pub unsafe fn PyObject_CallMethodOneArg( self_: *mut PyObject, @@ -236,7 +239,7 @@ extern "C" { pub fn PyObject_LengthHint(o: *mut PyObject, arg1: Py_ssize_t) -> Py_ssize_t; #[cfg(not(Py_3_11))] // moved to src/buffer.rs from 3.11 - #[cfg(all(Py_3_9, not(PyPy)))] + #[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] pub fn PyObject_CheckBuffer(obj: *mut PyObject) -> c_int; } @@ -308,7 +311,7 @@ pub const PY_ITERSEARCH_INDEX: c_int = 2; pub const PY_ITERSEARCH_CONTAINS: c_int = 3; extern "C" { - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn _PySequence_IterSearch( seq: *mut PyObject, obj: *mut PyObject, diff --git a/pyo3-ffi/src/cpython/bytesobject.rs b/pyo3-ffi/src/cpython/bytesobject.rs index 912fc0ac427..fb0b38cf1d8 100644 --- a/pyo3-ffi/src/cpython/bytesobject.rs +++ b/pyo3-ffi/src/cpython/bytesobject.rs @@ -1,10 +1,10 @@ use crate::object::*; use crate::Py_ssize_t; -#[cfg(not(any(PyPy, Py_LIMITED_API)))] +#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] use std::os::raw::c_char; use std::os::raw::c_int; -#[cfg(not(any(PyPy, Py_LIMITED_API)))] +#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyBytesObject { @@ -13,7 +13,7 @@ pub struct PyBytesObject { pub ob_sval: [c_char; 1], } -#[cfg(any(PyPy, Py_LIMITED_API))] +#[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] opaque_struct!(PyBytesObject); extern "C" { diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index 498eab59cce..74586eac595 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -3,10 +3,10 @@ use crate::pyport::Py_ssize_t; #[allow(unused_imports)] use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void}; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] use std::ptr::addr_of_mut; -#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))] +#[cfg(all(Py_3_8, not(any(PyPy, GraalPy)), not(Py_3_11)))] opaque_struct!(_PyOpcache); #[cfg(Py_3_12)] @@ -73,10 +73,10 @@ pub struct _PyCoMonitoringData { pub per_instruction_tools: *mut u8, } -#[cfg(all(not(PyPy), not(Py_3_7)))] +#[cfg(all(not(any(PyPy, GraalPy)), not(Py_3_7)))] opaque_struct!(PyCodeObject); -#[cfg(all(not(PyPy), Py_3_7, not(Py_3_8)))] +#[cfg(all(not(any(PyPy, GraalPy)), Py_3_7, not(Py_3_8)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyCodeObject { @@ -102,7 +102,7 @@ pub struct PyCodeObject { pub co_extra: *mut c_void, } -#[cfg(all(not(PyPy), Py_3_8, not(Py_3_11)))] +#[cfg(all(not(any(PyPy, GraalPy)), Py_3_8, not(Py_3_11)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyCodeObject { @@ -136,7 +136,7 @@ pub struct PyCodeObject { pub co_opcache_size: c_uchar, } -#[cfg(all(not(PyPy), Py_3_11))] +#[cfg(all(not(any(PyPy, GraalPy)), Py_3_11))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyCodeObject { @@ -230,26 +230,26 @@ pub const CO_FUTURE_GENERATOR_STOP: c_int = 0x8_0000; pub const CO_MAXBLOCKS: usize = 20; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { pub static mut PyCode_Type: PyTypeObject; } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyCode_Check(op: *mut PyObject) -> c_int { (Py_TYPE(op) == addr_of_mut!(PyCode_Type)) as c_int } #[inline] -#[cfg(all(not(PyPy), Py_3_10, not(Py_3_11)))] +#[cfg(all(not(any(PyPy, GraalPy)), Py_3_10, not(Py_3_11)))] pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t { crate::PyTuple_GET_SIZE((*op).co_freevars) } #[inline] -#[cfg(all(not(Py_3_10), Py_3_11, not(PyPy)))] +#[cfg(all(not(Py_3_10), Py_3_11, not(any(PyPy, GraalPy))))] pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> c_int { (*op).co_nfreevars } @@ -265,6 +265,7 @@ extern "C" { } extern "C" { + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyCode_New")] pub fn PyCode_New( argcount: c_int, @@ -283,6 +284,7 @@ extern "C" { firstlineno: c_int, lnotab: *mut PyObject, ) -> *mut PyCodeObject; + #[cfg(not(GraalPy))] #[cfg(Py_3_8)] pub fn PyCode_NewWithPosOnlyArgs( argcount: c_int, @@ -302,12 +304,14 @@ extern "C" { firstlineno: c_int, lnotab: *mut PyObject, ) -> *mut PyCodeObject; + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyCode_NewEmpty")] pub fn PyCode_NewEmpty( filename: *const c_char, funcname: *const c_char, firstlineno: c_int, ) -> *mut PyCodeObject; + #[cfg(not(GraalPy))] pub fn PyCode_Addr2Line(arg1: *mut PyCodeObject, arg2: c_int) -> c_int; // skipped PyCodeAddressRange "for internal use only" // skipped _PyCode_CheckLineNumber diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 71af81e83e5..8bce9dacb3b 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -30,7 +30,7 @@ pub struct PyCompilerFlags { // skipped non-limited _PyCompilerFlags_INIT -#[cfg(all(Py_3_12, not(PyPy)))] +#[cfg(all(Py_3_12, not(any(PyPy, GraalPy))))] #[repr(C)] #[derive(Copy, Clone)] pub struct _PyCompilerSrcLocation { @@ -42,7 +42,7 @@ pub struct _PyCompilerSrcLocation { // skipped SRC_LOCATION_FROM_AST -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyFutureFeatures { diff --git a/pyo3-ffi/src/cpython/dictobject.rs b/pyo3-ffi/src/cpython/dictobject.rs index 4af990a2d9a..74b970ebac2 100644 --- a/pyo3-ffi/src/cpython/dictobject.rs +++ b/pyo3-ffi/src/cpython/dictobject.rs @@ -7,6 +7,7 @@ opaque_struct!(PyDictKeysObject); #[cfg(Py_3_11)] opaque_struct!(PyDictValues); +#[cfg(not(GraalPy))] #[repr(C)] #[derive(Debug)] pub struct PyDictObject { diff --git a/pyo3-ffi/src/cpython/floatobject.rs b/pyo3-ffi/src/cpython/floatobject.rs index e33da0b91b9..8c7ee88543d 100644 --- a/pyo3-ffi/src/cpython/floatobject.rs +++ b/pyo3-ffi/src/cpython/floatobject.rs @@ -1,9 +1,12 @@ +#[cfg(GraalPy)] +use crate::PyFloat_AsDouble; use crate::{PyFloat_Check, PyObject}; use std::os::raw::c_double; #[repr(C)] pub struct PyFloatObject { pub ob_base: PyObject, + #[cfg(not(GraalPy))] pub ob_fval: c_double, } @@ -15,7 +18,10 @@ pub unsafe fn _PyFloat_CAST(op: *mut PyObject) -> *mut PyFloatObject { #[inline] pub unsafe fn PyFloat_AS_DOUBLE(op: *mut PyObject) -> c_double { - (*_PyFloat_CAST(op)).ob_fval + #[cfg(not(GraalPy))] + return (*_PyFloat_CAST(op)).ob_fval; + #[cfg(GraalPy)] + return PyFloat_AsDouble(op); } // skipped PyFloat_Pack2 diff --git a/pyo3-ffi/src/cpython/frameobject.rs b/pyo3-ffi/src/cpython/frameobject.rs index 7410000ef45..a85818ace0a 100644 --- a/pyo3-ffi/src/cpython/frameobject.rs +++ b/pyo3-ffi/src/cpython/frameobject.rs @@ -1,17 +1,19 @@ +#[cfg(not(GraalPy))] use crate::cpython::code::PyCodeObject; use crate::object::*; +#[cfg(not(GraalPy))] use crate::pystate::PyThreadState; -#[cfg(not(any(PyPy, Py_3_11)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_11)))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr::addr_of_mut; -#[cfg(not(any(PyPy, Py_3_11)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_11)))] pub type PyFrameState = c_char; #[repr(C)] #[derive(Copy, Clone)] -#[cfg(not(any(PyPy, Py_3_11)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_11)))] pub struct PyTryBlock { pub b_type: c_int, pub b_handler: c_int, @@ -20,7 +22,7 @@ pub struct PyTryBlock { #[repr(C)] #[derive(Copy, Clone)] -#[cfg(not(any(PyPy, Py_3_11)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_11)))] pub struct PyFrameObject { pub ob_base: PyVarObject, pub f_back: *mut PyFrameObject, @@ -51,7 +53,7 @@ pub struct PyFrameObject { pub f_localsplus: [*mut PyObject; 1], } -#[cfg(any(PyPy, Py_3_11))] +#[cfg(any(PyPy, GraalPy, Py_3_11))] opaque_struct!(PyFrameObject); // skipped _PyFrame_IsRunnable @@ -69,6 +71,7 @@ pub unsafe fn PyFrame_Check(op: *mut PyObject) -> c_int { } extern "C" { + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyFrame_New")] pub fn PyFrame_New( tstate: *mut PyThreadState, @@ -79,7 +82,7 @@ extern "C" { // skipped _PyFrame_New_NoTrack pub fn PyFrame_BlockSetup(f: *mut PyFrameObject, _type: c_int, handler: c_int, level: c_int); - #[cfg(not(any(PyPy, Py_3_11)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_11)))] pub fn PyFrame_BlockPop(f: *mut PyFrameObject) -> *mut PyTryBlock; pub fn PyFrame_LocalsToFast(f: *mut PyFrameObject, clear: c_int); diff --git a/pyo3-ffi/src/cpython/funcobject.rs b/pyo3-ffi/src/cpython/funcobject.rs index 1e9ee0cc18c..25de30d57f7 100644 --- a/pyo3-ffi/src/cpython/funcobject.rs +++ b/pyo3-ffi/src/cpython/funcobject.rs @@ -4,7 +4,7 @@ use std::ptr::addr_of_mut; use crate::PyObject; -#[cfg(all(not(PyPy), not(Py_3_10)))] +#[cfg(all(not(any(PyPy, GraalPy)), not(Py_3_10)))] #[repr(C)] pub struct PyFunctionObject { pub ob_base: PyObject, @@ -24,7 +24,7 @@ pub struct PyFunctionObject { pub vectorcall: Option, } -#[cfg(all(not(PyPy), Py_3_10))] +#[cfg(all(not(any(PyPy, GraalPy)), Py_3_10))] #[repr(C)] pub struct PyFunctionObject { pub ob_base: PyObject, @@ -55,6 +55,11 @@ pub struct PyFunctionObject { pub func_name: *mut PyObject, } +#[cfg(GraalPy)] +pub struct PyFunctionObject { + pub ob_base: PyObject, +} + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { #[cfg(not(all(PyPy, not(Py_3_8))))] diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index aaa03f82eef..73ebdb491ff 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -1,13 +1,13 @@ use crate::object::*; use crate::PyFrameObject; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] use crate::_PyErr_StackItem; #[cfg(Py_3_11)] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr::addr_of_mut; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyGenObject { diff --git a/pyo3-ffi/src/cpython/listobject.rs b/pyo3-ffi/src/cpython/listobject.rs index 7fb2228fadd..ea15cfc1ff5 100644 --- a/pyo3-ffi/src/cpython/listobject.rs +++ b/pyo3-ffi/src/cpython/listobject.rs @@ -2,7 +2,7 @@ use crate::object::*; #[cfg(not(PyPy))] use crate::pyport::Py_ssize_t; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyListObject { @@ -11,7 +11,7 @@ pub struct PyListObject { pub allocated: Py_ssize_t, } -#[cfg(PyPy)] +#[cfg(any(PyPy, GraalPy))] pub struct PyListObject { pub ob_base: PyObject, } @@ -22,14 +22,14 @@ pub struct PyListObject { /// Macro, trading safety for speed #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyList_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject { *(*(op as *mut PyListObject)).ob_item.offset(i) } /// Macro, *only* to be used to fill in brand new lists #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyList_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) { *(*(op as *mut PyListObject)).ob_item.offset(i) = v; } diff --git a/pyo3-ffi/src/cpython/methodobject.rs b/pyo3-ffi/src/cpython/methodobject.rs index 7d9659785ba..97ad9ce35f0 100644 --- a/pyo3-ffi/src/cpython/methodobject.rs +++ b/pyo3-ffi/src/cpython/methodobject.rs @@ -1,8 +1,10 @@ use crate::object::*; +#[cfg(not(GraalPy))] use crate::{PyCFunctionObject, PyMethodDefPointer, METH_METHOD, METH_STATIC}; use std::os::raw::c_int; use std::ptr::addr_of_mut; +#[cfg(not(GraalPy))] pub struct PyCMethodObject { pub func: PyCFunctionObject, pub mm_class: *mut PyTypeObject, @@ -23,6 +25,7 @@ pub unsafe fn PyCMethod_Check(op: *mut PyObject) -> c_int { PyObject_TypeCheck(op, addr_of_mut!(PyCMethod_Type)) } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyCFunction_GET_FUNCTION(func: *mut PyObject) -> PyMethodDefPointer { debug_assert_eq!(PyCMethod_Check(func), 1); @@ -31,6 +34,7 @@ pub unsafe fn PyCFunction_GET_FUNCTION(func: *mut PyObject) -> PyMethodDefPointe (*(*func).m_ml).ml_meth } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyCFunction_GET_SELF(func: *mut PyObject) -> *mut PyObject { debug_assert_eq!(PyCMethod_Check(func), 1); @@ -43,6 +47,7 @@ pub unsafe fn PyCFunction_GET_SELF(func: *mut PyObject) -> *mut PyObject { } } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyCFunction_GET_FLAGS(func: *mut PyObject) -> c_int { debug_assert_eq!(PyCMethod_Check(func), 1); @@ -51,6 +56,7 @@ pub unsafe fn PyCFunction_GET_FLAGS(func: *mut PyObject) -> c_int { (*(*func).m_ml).ml_flags } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyCFunction_GET_CLASS(func: *mut PyObject) -> *mut PyTypeObject { debug_assert_eq!(PyCMethod_Check(func), 1); diff --git a/pyo3-ffi/src/cpython/mod.rs b/pyo3-ffi/src/cpython/mod.rs index 738ba37652e..1ab0e3c893f 100644 --- a/pyo3-ffi/src/cpython/mod.rs +++ b/pyo3-ffi/src/cpython/mod.rs @@ -68,5 +68,5 @@ pub use self::pystate::*; pub use self::pythonrun::*; pub use self::tupleobject::*; pub use self::unicodeobject::*; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub use self::weakrefobject::*; diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index d0c1634082b..0f1778f6a3d 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -217,6 +217,8 @@ pub struct PyTypeObject { pub ob_size: Py_ssize_t, #[cfg(not(all(PyPy, not(Py_3_9))))] pub ob_base: object::PyVarObject, + #[cfg(GraalPy)] + pub ob_size: Py_ssize_t, pub tp_name: *const c_char, pub tp_basicsize: Py_ssize_t, pub tp_itemsize: Py_ssize_t, diff --git a/pyo3-ffi/src/cpython/objimpl.rs b/pyo3-ffi/src/cpython/objimpl.rs index 36a4380d122..3e0270ddc8f 100644 --- a/pyo3-ffi/src/cpython/objimpl.rs +++ b/pyo3-ffi/src/cpython/objimpl.rs @@ -1,7 +1,7 @@ use libc::size_t; use std::os::raw::c_int; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] use std::os::raw::c_void; use crate::object::*; @@ -14,7 +14,7 @@ extern "C" { pub fn _Py_GetAllocatedBlocks() -> crate::Py_ssize_t; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyObjectArenaAllocator { @@ -23,7 +23,7 @@ pub struct PyObjectArenaAllocator { pub free: Option, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] impl Default for PyObjectArenaAllocator { #[inline] fn default() -> Self { @@ -32,9 +32,9 @@ impl Default for PyObjectArenaAllocator { } extern "C" { - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyObject_GetArenaAllocator(allocator: *mut PyObjectArenaAllocator); - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyObject_SetArenaAllocator(allocator: *mut PyObjectArenaAllocator); #[cfg(Py_3_9)] diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs index fe7b4d4b045..6d17ebc8124 100644 --- a/pyo3-ffi/src/cpython/pyerrors.rs +++ b/pyo3-ffi/src/cpython/pyerrors.rs @@ -1,28 +1,28 @@ use crate::PyObject; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] use crate::Py_ssize_t; #[repr(C)] #[derive(Debug)] pub struct PyBaseExceptionObject { pub ob_base: PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub dict: *mut PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub args: *mut PyObject, - #[cfg(all(Py_3_11, not(PyPy)))] + #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] pub notes: *mut PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub traceback: *mut PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub context: *mut PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub cause: *mut PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub suppress_context: char, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PySyntaxErrorObject { @@ -48,7 +48,7 @@ pub struct PySyntaxErrorObject { pub print_file_and_line: *mut PyObject, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PyImportErrorObject { @@ -69,7 +69,7 @@ pub struct PyImportErrorObject { pub name_from: *mut PyObject, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PyUnicodeErrorObject { @@ -90,7 +90,7 @@ pub struct PyUnicodeErrorObject { pub reason: *mut PyObject, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PySystemExitObject { @@ -107,7 +107,7 @@ pub struct PySystemExitObject { pub code: *mut PyObject, } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PyOSErrorObject { @@ -134,26 +134,26 @@ pub struct PyOSErrorObject { #[derive(Debug)] pub struct PyStopIterationObject { pub ob_base: PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub dict: *mut PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub args: *mut PyObject, - #[cfg(all(Py_3_11, not(PyPy)))] + #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] pub notes: *mut PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub traceback: *mut PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub context: *mut PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub cause: *mut PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub suppress_context: char, pub value: *mut PyObject, } extern "C" { - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn _PyErr_ChainExceptions(typ: *mut PyObject, val: *mut PyObject, tb: *mut PyObject); } diff --git a/pyo3-ffi/src/cpython/pymem.rs b/pyo3-ffi/src/cpython/pymem.rs index 2dfb3f3bcfa..c400fa30b05 100644 --- a/pyo3-ffi/src/cpython/pymem.rs +++ b/pyo3-ffi/src/cpython/pymem.rs @@ -26,7 +26,7 @@ pub enum PyMemAllocatorDomain { } // skipped PyMemAllocatorName -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyMemAllocatorEx { @@ -40,10 +40,10 @@ pub struct PyMemAllocatorEx { } extern "C" { - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyMem_GetAllocator(domain: PyMemAllocatorDomain, allocator: *mut PyMemAllocatorEx); - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyMem_SetAllocator(domain: PyMemAllocatorDomain, allocator: *mut PyMemAllocatorEx); - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyMem_SetupDebugHooks(); } diff --git a/pyo3-ffi/src/cpython/pythonrun.rs b/pyo3-ffi/src/cpython/pythonrun.rs index a92528b7fdc..94863166e11 100644 --- a/pyo3-ffi/src/cpython/pythonrun.rs +++ b/pyo3-ffi/src/cpython/pythonrun.rs @@ -1,8 +1,8 @@ use crate::object::*; -#[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] +#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API, Py_3_10)))] use crate::pyarena::PyArena; use crate::PyCompilerFlags; -#[cfg(not(any(PyPy, Py_3_10)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_10)))] use crate::{_mod, _node}; use libc::FILE; use std::os::raw::{c_char, c_int}; @@ -54,7 +54,7 @@ extern "C" { flags: *mut PyCompilerFlags, ) -> c_int; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] pub fn PyParser_ASTFromString( s: *const c_char, filename: *const c_char, @@ -62,7 +62,7 @@ extern "C" { flags: *mut PyCompilerFlags, arena: *mut PyArena, ) -> *mut _mod; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] pub fn PyParser_ASTFromStringObject( s: *const c_char, filename: *mut PyObject, @@ -70,7 +70,7 @@ extern "C" { flags: *mut PyCompilerFlags, arena: *mut PyArena, ) -> *mut _mod; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] pub fn PyParser_ASTFromFile( fp: *mut FILE, filename: *const c_char, @@ -82,7 +82,7 @@ extern "C" { errcode: *mut c_int, arena: *mut PyArena, ) -> *mut _mod; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] pub fn PyParser_ASTFromFileObject( fp: *mut FILE, filename: *mut PyObject, @@ -105,7 +105,7 @@ extern "C" { arg4: *mut PyObject, arg5: *mut PyCompilerFlags, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_FileExFlags( fp: *mut FILE, filename: *const c_char, @@ -116,7 +116,7 @@ extern "C" { flags: *mut PyCompilerFlags, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn Py_CompileStringExFlags( str: *const c_char, filename: *const c_char, @@ -135,6 +135,7 @@ extern "C" { } #[inline] +#[cfg(not(GraalPy))] pub unsafe fn Py_CompileString(string: *const c_char, p: *const c_char, s: c_int) -> *mut PyObject { #[cfg(not(PyPy))] return Py_CompileStringExFlags(string, p, s, std::ptr::null_mut(), -1); @@ -144,7 +145,7 @@ pub unsafe fn Py_CompileString(string: *const c_char, p: *const c_char, s: c_int } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn Py_CompileStringFlags( string: *const c_char, p: *const c_char, @@ -164,11 +165,11 @@ extern "C" { g: *mut PyObject, l: *mut PyObject, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_AnyFile(fp: *mut FILE, name: *const c_char) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_AnyFileEx(fp: *mut FILE, name: *const c_char, closeit: c_int) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_AnyFileFlags( arg1: *mut FILE, arg2: *const c_char, @@ -176,13 +177,13 @@ extern "C" { ) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyRun_SimpleString")] pub fn PyRun_SimpleString(s: *const c_char) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_SimpleFile(f: *mut FILE, p: *const c_char) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_SimpleFileEx(f: *mut FILE, p: *const c_char, c: c_int) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_InteractiveOne(f: *mut FILE, p: *const c_char) -> c_int; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_InteractiveLoop(f: *mut FILE, p: *const c_char) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyRun_File")] pub fn PyRun_File( @@ -192,7 +193,7 @@ extern "C" { g: *mut PyObject, l: *mut PyObject, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_FileEx( fp: *mut FILE, p: *const c_char, @@ -201,7 +202,7 @@ extern "C" { l: *mut PyObject, c: c_int, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn PyRun_FileFlags( fp: *mut FILE, p: *const c_char, @@ -218,14 +219,14 @@ extern "C" { // skipped macro PyRun_AnyFileFlags extern "C" { - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] pub fn PyParser_SimpleParseStringFlags( arg1: *const c_char, arg2: c_int, arg3: c_int, ) -> *mut _node; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] pub fn PyParser_SimpleParseStringFlagsFilename( arg1: *const c_char, @@ -233,7 +234,7 @@ extern "C" { arg3: c_int, arg4: c_int, ) -> *mut _node; - #[cfg(not(any(PyPy, Py_3_10)))] + #[cfg(not(any(PyPy, GraalPy, Py_3_10)))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] pub fn PyParser_SimpleParseFileFlags( arg1: *mut FILE, diff --git a/pyo3-ffi/src/cpython/tupleobject.rs b/pyo3-ffi/src/cpython/tupleobject.rs index 24dde268526..4ed8520daf3 100644 --- a/pyo3-ffi/src/cpython/tupleobject.rs +++ b/pyo3-ffi/src/cpython/tupleobject.rs @@ -5,6 +5,7 @@ use crate::pyport::Py_ssize_t; #[repr(C)] pub struct PyTupleObject { pub ob_base: PyVarObject, + #[cfg(not(GraalPy))] pub ob_item: [*mut PyObject; 1], } @@ -22,14 +23,14 @@ pub unsafe fn PyTuple_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyTuple_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject { *(*(op as *mut PyTupleObject)).ob_item.as_ptr().offset(i) } /// Macro, *only* to be used to fill in brand new tuples #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyTuple_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) { *(*(op as *mut PyTupleObject)).ob_item.as_mut_ptr().offset(i) = v; } diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index f618ecf0a84..9ab523a2d7f 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -1,7 +1,7 @@ -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] use crate::Py_hash_t; use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_UNICODE, Py_ssize_t}; -#[cfg(not(Py_3_12))] +#[cfg(not(any(Py_3_12, GraalPy)))] use libc::wchar_t; use std::os::raw::{c_char, c_int, c_uint, c_void}; @@ -44,6 +44,7 @@ impl BitfieldUnit { } } +#[cfg(not(GraalPy))] impl BitfieldUnit where Storage: AsRef<[u8]> + AsMut<[u8]>, @@ -117,23 +118,31 @@ where } } +#[cfg(not(GraalPy))] const STATE_INTERNED_INDEX: usize = 0; +#[cfg(not(GraalPy))] const STATE_INTERNED_WIDTH: u8 = 2; +#[cfg(not(GraalPy))] const STATE_KIND_INDEX: usize = STATE_INTERNED_WIDTH as usize; +#[cfg(not(GraalPy))] const STATE_KIND_WIDTH: u8 = 3; +#[cfg(not(GraalPy))] const STATE_COMPACT_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH) as usize; +#[cfg(not(GraalPy))] const STATE_COMPACT_WIDTH: u8 = 1; +#[cfg(not(GraalPy))] const STATE_ASCII_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH) as usize; +#[cfg(not(GraalPy))] const STATE_ASCII_WIDTH: u8 = 1; -#[cfg(not(Py_3_12))] +#[cfg(not(any(Py_3_12, GraalPy)))] const STATE_READY_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH + STATE_ASCII_WIDTH) as usize; -#[cfg(not(Py_3_12))] +#[cfg(not(any(Py_3_12, GraalPy)))] const STATE_READY_WIDTH: u8 = 1; // generated by bindgen v0.63.0 (with small adaptations) @@ -153,6 +162,7 @@ struct PyASCIIObjectState { } // c_uint and u32 are not necessarily the same type on all targets / architectures +#[cfg(not(GraalPy))] #[allow(clippy::useless_transmute)] impl PyASCIIObjectState { #[inline] @@ -241,8 +251,9 @@ impl From for u32 { #[repr(C)] pub struct PyASCIIObject { pub ob_base: PyObject, + #[cfg(not(GraalPy))] pub length: Py_ssize_t, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub hash: Py_hash_t, /// A bit field with various properties. /// @@ -255,12 +266,14 @@ pub struct PyASCIIObject { /// unsigned int ascii:1; /// unsigned int ready:1; /// unsigned int :24; + #[cfg(not(GraalPy))] pub state: u32, - #[cfg(not(Py_3_12))] + #[cfg(not(any(Py_3_12, GraalPy)))] pub wstr: *mut wchar_t, } /// Interacting with the bitfield is not actually well-defined, so we mark these APIs unsafe. +#[cfg(not(GraalPy))] impl PyASCIIObject { #[cfg_attr(not(Py_3_12), allow(rustdoc::broken_intra_doc_links))] // SSTATE_INTERNED_IMMORTAL_STATIC requires 3.12 /// Get the `interned` field of the [`PyASCIIObject`] state bitfield. @@ -367,9 +380,11 @@ impl PyASCIIObject { #[repr(C)] pub struct PyCompactUnicodeObject { pub _base: PyASCIIObject, + #[cfg(not(GraalPy))] pub utf8_length: Py_ssize_t, + #[cfg(not(GraalPy))] pub utf8: *mut c_char, - #[cfg(not(Py_3_12))] + #[cfg(not(any(Py_3_12, GraalPy)))] pub wstr_length: Py_ssize_t, } @@ -384,11 +399,12 @@ pub union PyUnicodeObjectData { #[repr(C)] pub struct PyUnicodeObject { pub _base: PyCompactUnicodeObject, + #[cfg(not(GraalPy))] pub data: PyUnicodeObjectData, } extern "C" { - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn _PyUnicode_CheckConsistency(op: *mut PyObject, check_content: c_int) -> c_int; } @@ -403,6 +419,7 @@ pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2; #[cfg(Py_3_12)] pub const SSTATE_INTERNED_IMMORTAL_STATIC: c_uint = 3; +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -412,11 +429,13 @@ pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).ascii() } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyUnicode_IS_COMPACT(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).compact() } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint { ((*(op as *mut PyASCIIObject)).ascii() != 0 && PyUnicode_IS_COMPACT(op) != 0).into() @@ -430,21 +449,25 @@ pub const PyUnicode_1BYTE_KIND: c_uint = 1; pub const PyUnicode_2BYTE_KIND: c_uint = 2; pub const PyUnicode_4BYTE_KIND: c_uint = 4; +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyUnicode_1BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS1 { PyUnicode_DATA(op) as *mut Py_UCS1 } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyUnicode_2BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS2 { PyUnicode_DATA(op) as *mut Py_UCS2 } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { PyUnicode_DATA(op) as *mut Py_UCS4 } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -454,6 +477,7 @@ pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).kind() } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void { if PyUnicode_IS_ASCII(op) != 0 { @@ -463,6 +487,7 @@ pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void { } } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { debug_assert!(!(*(op as *mut PyUnicodeObject)).data.any.is_null()); @@ -470,6 +495,7 @@ pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { (*(op as *mut PyUnicodeObject)).data.any } +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -485,6 +511,7 @@ pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { // skipped PyUnicode_READ // skipped PyUnicode_READ_CHAR +#[cfg(not(GraalPy))] #[inline] pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -494,26 +521,26 @@ pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t { (*(op as *mut PyASCIIObject)).length } -#[cfg(Py_3_12)] +#[cfg(any(Py_3_12, GraalPy))] #[inline] pub unsafe fn PyUnicode_IS_READY(_op: *mut PyObject) -> c_uint { // kept in CPython for backwards compatibility 1 } -#[cfg(not(Py_3_12))] +#[cfg(not(any(GraalPy, Py_3_12)))] #[inline] pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).ready() } -#[cfg(Py_3_12)] +#[cfg(any(Py_3_12, GraalPy))] #[inline] pub unsafe fn PyUnicode_READY(_op: *mut PyObject) -> c_int { 0 } -#[cfg(not(Py_3_12))] +#[cfg(not(any(Py_3_12, GraalPy)))] #[inline] pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int { debug_assert!(crate::PyUnicode_Check(op) != 0); diff --git a/pyo3-ffi/src/cpython/weakrefobject.rs b/pyo3-ffi/src/cpython/weakrefobject.rs index 5a5f85c5f0c..3a232c7ed38 100644 --- a/pyo3-ffi/src/cpython/weakrefobject.rs +++ b/pyo3-ffi/src/cpython/weakrefobject.rs @@ -1,4 +1,4 @@ -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub struct _PyWeakReference { pub ob_base: crate::PyObject, pub wr_object: *mut crate::PyObject, diff --git a/pyo3-ffi/src/datetime.rs b/pyo3-ffi/src/datetime.rs index 7e5a250990f..a20b76aa91d 100644 --- a/pyo3-ffi/src/datetime.rs +++ b/pyo3-ffi/src/datetime.rs @@ -9,17 +9,16 @@ //! Support for `PyDateTime_CAPI` is limited as of PyPy 7.0.0. //! `DateTime_FromTimestamp` and `Date_FromTimestamp` are currently not supported. +#[cfg(GraalPy)] +use crate::{PyLong_AsLong, PyLong_Check, PyObject_GetAttrString, Py_DecRef}; use crate::{PyObject, PyObject_TypeCheck, PyTypeObject, Py_TYPE}; use std::cell::UnsafeCell; use std::os::raw::{c_char, c_int}; use std::ptr; #[cfg(not(PyPy))] -use { - crate::{PyCapsule_Import, Py_hash_t}, - std::ffi::CString, - std::os::raw::c_uchar, -}; - +use {crate::PyCapsule_Import, std::ffi::CString}; +#[cfg(not(any(PyPy, GraalPy)))] +use {crate::Py_hash_t, std::os::raw::c_uchar}; // Type struct wrappers const _PyDateTime_DATE_DATASIZE: usize = 4; const _PyDateTime_TIME_DATASIZE: usize = 6; @@ -30,26 +29,27 @@ const _PyDateTime_DATETIME_DATASIZE: usize = 10; /// Structure representing a `datetime.timedelta`. pub struct PyDateTime_Delta { pub ob_base: PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub hashcode: Py_hash_t, + #[cfg(not(GraalPy))] pub days: c_int, + #[cfg(not(GraalPy))] pub seconds: c_int, + #[cfg(not(GraalPy))] pub microseconds: c_int, } // skipped non-limited PyDateTime_TZInfo // skipped non-limited _PyDateTime_BaseTZInfo -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug, Copy, Clone)] /// Structure representing a `datetime.time` without a `tzinfo` member. pub struct _PyDateTime_BaseTime { pub ob_base: PyObject, - #[cfg(not(PyPy))] pub hashcode: Py_hash_t, pub hastzinfo: c_char, - #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_TIME_DATASIZE], } @@ -58,17 +58,19 @@ pub struct _PyDateTime_BaseTime { /// Structure representing a `datetime.time`. pub struct PyDateTime_Time { pub ob_base: PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub hashcode: Py_hash_t, + #[cfg(not(GraalPy))] pub hastzinfo: c_char, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub data: [c_uchar; _PyDateTime_TIME_DATASIZE], - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fold: c_uchar, /// # Safety /// /// Care should be taken when reading this field. If the time does not have a /// tzinfo then CPython may allocate as a `_PyDateTime_BaseTime` without this field. + #[cfg(not(GraalPy))] pub tzinfo: *mut PyObject, } @@ -77,24 +79,22 @@ pub struct PyDateTime_Time { /// Structure representing a `datetime.date` pub struct PyDateTime_Date { pub ob_base: PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub hashcode: Py_hash_t, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub hastzinfo: c_char, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub data: [c_uchar; _PyDateTime_DATE_DATASIZE], } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(C)] #[derive(Debug, Copy, Clone)] /// Structure representing a `datetime.datetime` without a `tzinfo` member. pub struct _PyDateTime_BaseDateTime { pub ob_base: PyObject, - #[cfg(not(PyPy))] pub hashcode: Py_hash_t, pub hastzinfo: c_char, - #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_DATETIME_DATASIZE], } @@ -103,17 +103,19 @@ pub struct _PyDateTime_BaseDateTime { /// Structure representing a `datetime.datetime`. pub struct PyDateTime_DateTime { pub ob_base: PyObject, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub hashcode: Py_hash_t, + #[cfg(not(GraalPy))] pub hastzinfo: c_char, - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub data: [c_uchar; _PyDateTime_DATETIME_DATASIZE], - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fold: c_uchar, /// # Safety /// /// Care should be taken when reading this field. If the time does not have a /// tzinfo then CPython may allocate as a `_PyDateTime_BaseDateTime` without this field. + #[cfg(not(GraalPy))] pub tzinfo: *mut PyObject, } @@ -121,7 +123,7 @@ pub struct PyDateTime_DateTime { // Accessor functions for PyDateTime_Date and PyDateTime_DateTime #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the year component of a `PyDateTime_Date` or `PyDateTime_DateTime`. /// Returns a signed integer greater than 0. pub unsafe fn PyDateTime_GET_YEAR(o: *mut PyObject) -> c_int { @@ -131,7 +133,7 @@ pub unsafe fn PyDateTime_GET_YEAR(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the month component of a `PyDateTime_Date` or `PyDateTime_DateTime`. /// Returns a signed integer in the range `[1, 12]`. pub unsafe fn PyDateTime_GET_MONTH(o: *mut PyObject) -> c_int { @@ -140,7 +142,7 @@ pub unsafe fn PyDateTime_GET_MONTH(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the day component of a `PyDateTime_Date` or `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[1, 31]`. pub unsafe fn PyDateTime_GET_DAY(o: *mut PyObject) -> c_int { @@ -149,28 +151,28 @@ pub unsafe fn PyDateTime_GET_DAY(o: *mut PyObject) -> c_int { } // Accessor macros for times -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_HOUR { ($o: expr, $offset:expr) => { c_int::from((*$o).data[$offset + 0]) }; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_MINUTE { ($o: expr, $offset:expr) => { c_int::from((*$o).data[$offset + 1]) }; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_SECOND { ($o: expr, $offset:expr) => { c_int::from((*$o).data[$offset + 2]) }; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_MICROSECOND { ($o: expr, $offset:expr) => { (c_int::from((*$o).data[$offset + 3]) << 16) @@ -179,14 +181,14 @@ macro_rules! _PyDateTime_GET_MICROSECOND { }; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_FOLD { ($o: expr) => { (*$o).fold }; } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _PyDateTime_GET_TZINFO { ($o: expr) => { if (*$o).hastzinfo != 0 { @@ -199,7 +201,7 @@ macro_rules! _PyDateTime_GET_TZINFO { // Accessor functions for DateTime #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the hour component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 23]` pub unsafe fn PyDateTime_DATE_GET_HOUR(o: *mut PyObject) -> c_int { @@ -207,7 +209,7 @@ pub unsafe fn PyDateTime_DATE_GET_HOUR(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the minute component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 59]` pub unsafe fn PyDateTime_DATE_GET_MINUTE(o: *mut PyObject) -> c_int { @@ -215,7 +217,7 @@ pub unsafe fn PyDateTime_DATE_GET_MINUTE(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the second component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 59]` pub unsafe fn PyDateTime_DATE_GET_SECOND(o: *mut PyObject) -> c_int { @@ -223,7 +225,7 @@ pub unsafe fn PyDateTime_DATE_GET_SECOND(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the microsecond component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 999999]` pub unsafe fn PyDateTime_DATE_GET_MICROSECOND(o: *mut PyObject) -> c_int { @@ -231,7 +233,7 @@ pub unsafe fn PyDateTime_DATE_GET_MICROSECOND(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the fold component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 1]` pub unsafe fn PyDateTime_DATE_GET_FOLD(o: *mut PyObject) -> c_uchar { @@ -239,7 +241,7 @@ pub unsafe fn PyDateTime_DATE_GET_FOLD(o: *mut PyObject) -> c_uchar { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the tzinfo component of a `PyDateTime_DateTime`. /// Returns a pointer to a `PyObject` that should be either NULL or an instance /// of a `datetime.tzinfo` subclass. @@ -249,7 +251,7 @@ pub unsafe fn PyDateTime_DATE_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { // Accessor functions for Time #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the hour component of a `PyDateTime_Time`. /// Returns a signed integer in the interval `[0, 23]` pub unsafe fn PyDateTime_TIME_GET_HOUR(o: *mut PyObject) -> c_int { @@ -257,7 +259,7 @@ pub unsafe fn PyDateTime_TIME_GET_HOUR(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the minute component of a `PyDateTime_Time`. /// Returns a signed integer in the interval `[0, 59]` pub unsafe fn PyDateTime_TIME_GET_MINUTE(o: *mut PyObject) -> c_int { @@ -265,7 +267,7 @@ pub unsafe fn PyDateTime_TIME_GET_MINUTE(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the second component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 59]` pub unsafe fn PyDateTime_TIME_GET_SECOND(o: *mut PyObject) -> c_int { @@ -273,14 +275,14 @@ pub unsafe fn PyDateTime_TIME_GET_SECOND(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the microsecond component of a `PyDateTime_DateTime`. /// Returns a signed integer in the interval `[0, 999999]` pub unsafe fn PyDateTime_TIME_GET_MICROSECOND(o: *mut PyObject) -> c_int { _PyDateTime_GET_MICROSECOND!((o as *mut PyDateTime_Time), 0) } -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[inline] /// Retrieve the fold component of a `PyDateTime_Time`. /// Returns a signed integer in the interval `[0, 1]` @@ -289,7 +291,7 @@ pub unsafe fn PyDateTime_TIME_GET_FOLD(o: *mut PyObject) -> c_uchar { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the tzinfo component of a `PyDateTime_Time`. /// Returns a pointer to a `PyObject` that should be either NULL or an instance /// of a `datetime.tzinfo` subclass. @@ -298,7 +300,7 @@ pub unsafe fn PyDateTime_TIME_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { } // Accessor functions -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _access_field { ($obj:expr, $type: ident, $field:ident) => { (*($obj as *mut $type)).$field @@ -306,7 +308,7 @@ macro_rules! _access_field { } // Accessor functions for PyDateTime_Delta -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] macro_rules! _access_delta_field { ($obj:expr, $field:ident) => { _access_field!($obj, PyDateTime_Delta, $field) @@ -314,7 +316,7 @@ macro_rules! _access_delta_field { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the days component of a `PyDateTime_Delta`. /// /// Returns a signed integer in the interval [-999999999, 999999999]. @@ -326,7 +328,7 @@ pub unsafe fn PyDateTime_DELTA_GET_DAYS(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the seconds component of a `PyDateTime_Delta`. /// /// Returns a signed integer in the interval [0, 86399]. @@ -338,7 +340,7 @@ pub unsafe fn PyDateTime_DELTA_GET_SECONDS(o: *mut PyObject) -> c_int { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] /// Retrieve the seconds component of a `PyDateTime_Delta`. /// /// Returns a signed integer in the interval [0, 999999]. @@ -349,6 +351,132 @@ pub unsafe fn PyDateTime_DELTA_GET_MICROSECONDS(o: *mut PyObject) -> c_int { _access_delta_field!(o, microseconds) } +// Accessor functions for GraalPy. The macros on GraalPy work differently, +// but copying them seems suboptimal +#[inline] +#[cfg(GraalPy)] +pub unsafe fn _get_attr(obj: *mut PyObject, field: &str) -> c_int { + let result = PyObject_GetAttrString(obj, field.as_ptr() as *const c_char); + Py_DecRef(result); // the original macros are borrowing + if PyLong_Check(result) == 1 { + PyLong_AsLong(result) as c_int + } else { + 0 + } +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_GET_YEAR(o: *mut PyObject) -> c_int { + _get_attr(o, "year\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_GET_MONTH(o: *mut PyObject) -> c_int { + _get_attr(o, "month\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_GET_DAY(o: *mut PyObject) -> c_int { + _get_attr(o, "day\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_HOUR(o: *mut PyObject) -> c_int { + _get_attr(o, "hour\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_MINUTE(o: *mut PyObject) -> c_int { + _get_attr(o, "minute\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_SECOND(o: *mut PyObject) -> c_int { + _get_attr(o, "second\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_MICROSECOND(o: *mut PyObject) -> c_int { + _get_attr(o, "microsecond\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_FOLD(o: *mut PyObject) -> c_int { + _get_attr(o, "fold\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DATE_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { + let res = PyObject_GetAttrString(o, "tzinfo\0".as_ptr() as *const c_char); + Py_DecRef(res); // the original macros are borrowing + res +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_HOUR(o: *mut PyObject) -> c_int { + _get_attr(o, "hour\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_MINUTE(o: *mut PyObject) -> c_int { + _get_attr(o, "minute\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_SECOND(o: *mut PyObject) -> c_int { + _get_attr(o, "second\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_MICROSECOND(o: *mut PyObject) -> c_int { + _get_attr(o, "microsecond\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_FOLD(o: *mut PyObject) -> c_int { + _get_attr(o, "fold\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_TIME_GET_TZINFO(o: *mut PyObject) -> *mut PyObject { + let res = PyObject_GetAttrString(o, "tzinfo\0".as_ptr() as *const c_char); + Py_DecRef(res); // the original macros are borrowing + res +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DELTA_GET_DAYS(o: *mut PyObject) -> c_int { + _get_attr(o, "days\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DELTA_GET_SECONDS(o: *mut PyObject) -> c_int { + _get_attr(o, "seconds\0") +} + +#[inline] +#[cfg(GraalPy)] +pub unsafe fn PyDateTime_DELTA_GET_MICROSECONDS(o: *mut PyObject) -> c_int { + _get_attr(o, "microseconds\0") +} + #[cfg(PyPy)] extern "C" { // skipped _PyDateTime_HAS_TZINFO (not in PyPy) diff --git a/pyo3-ffi/src/dictobject.rs b/pyo3-ffi/src/dictobject.rs index 8d522df97e2..99fc56b246b 100644 --- a/pyo3-ffi/src/dictobject.rs +++ b/pyo3-ffi/src/dictobject.rs @@ -109,6 +109,6 @@ extern "C" { pub static mut PyDictRevIterItem_Type: PyTypeObject; } -#[cfg(any(PyPy, Py_LIMITED_API))] +#[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) opaque_struct!(PyDictObject); diff --git a/pyo3-ffi/src/listobject.rs b/pyo3-ffi/src/listobject.rs index 32c6a2dc26a..a6f47eadf83 100644 --- a/pyo3-ffi/src/listobject.rs +++ b/pyo3-ffi/src/listobject.rs @@ -54,14 +54,16 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyList_AsTuple")] pub fn PyList_AsTuple(arg1: *mut PyObject) -> *mut PyObject; - // CPython macros exported as functions on PyPy - #[cfg(PyPy)] + // CPython macros exported as functions on PyPy or GraalPy + #[cfg(any(PyPy, GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyList_GET_ITEM")] + #[cfg_attr(GraalPy, link_name = "PyList_GetItem")] pub fn PyList_GET_ITEM(arg1: *mut PyObject, arg2: Py_ssize_t) -> *mut PyObject; #[cfg(PyPy)] #[cfg_attr(PyPy, link_name = "PyPyList_GET_SIZE")] pub fn PyList_GET_SIZE(arg1: *mut PyObject) -> Py_ssize_t; - #[cfg(PyPy)] + #[cfg(any(PyPy, GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyList_SET_ITEM")] + #[cfg_attr(GraalPy, link_name = "_PyList_SET_ITEM")] pub fn PyList_SET_ITEM(arg1: *mut PyObject, arg2: Py_ssize_t, arg3: *mut PyObject); } diff --git a/pyo3-ffi/src/methodobject.rs b/pyo3-ffi/src/methodobject.rs index e80d5668e78..3ed6b770e54 100644 --- a/pyo3-ffi/src/methodobject.rs +++ b/pyo3-ffi/src/methodobject.rs @@ -4,7 +4,7 @@ use crate::PyObject_TypeCheck; use std::os::raw::{c_char, c_int, c_void}; use std::{mem, ptr}; -#[cfg(all(Py_3_9, not(Py_LIMITED_API)))] +#[cfg(all(Py_3_9, not(Py_LIMITED_API), not(GraalPy)))] pub struct PyCFunctionObject { pub ob_base: PyObject, pub m_ml: *mut PyMethodDef, diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index 0e0243cd764..b33ee558a37 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -79,6 +79,7 @@ pub struct PyObject { #[derive(Debug, Copy, Clone)] pub struct PyVarObject { pub ob_base: PyObject, + #[cfg(not(GraalPy))] pub ob_size: Py_ssize_t, } @@ -98,12 +99,18 @@ pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { #[inline] #[cfg(not(Py_3_12))] pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { - (*ob).ob_refcnt + #[cfg(not(GraalPy))] + return (*ob).ob_refcnt; + #[cfg(GraalPy)] + return _Py_REFCNT(ob); } #[inline] pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { - (*ob).ob_type + #[cfg(not(GraalPy))] + return (*ob).ob_type; + #[cfg(GraalPy)] + return _Py_TYPE(ob); } // PyLong_Type defined in longobject.rs @@ -111,9 +118,14 @@ pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { #[inline] pub unsafe fn Py_SIZE(ob: *mut PyObject) -> Py_ssize_t { - debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyLong_Type)); - debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyBool_Type)); - (*ob.cast::()).ob_size + #[cfg(not(GraalPy))] + { + debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyLong_Type)); + debug_assert_ne!((*ob).ob_type, std::ptr::addr_of_mut!(crate::PyBool_Type)); + (*ob.cast::()).ob_size + } + #[cfg(GraalPy)] + _Py_SIZE(ob) } #[inline] @@ -464,8 +476,10 @@ extern "C" { pub fn _Py_Dealloc(arg1: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] + #[cfg_attr(GraalPy, link_name = "_Py_IncRef")] pub fn Py_IncRef(o: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] + #[cfg_attr(GraalPy, link_name = "_Py_DecRef")] pub fn Py_DecRef(o: *mut PyObject); #[cfg(Py_3_10)] @@ -474,11 +488,21 @@ extern "C" { #[cfg(Py_3_10)] #[cfg_attr(PyPy, link_name = "_PyPy_DecRef")] pub fn _Py_DecRef(o: *mut PyObject); + + #[cfg(GraalPy)] + pub fn _Py_REFCNT(arg1: *const PyObject) -> Py_ssize_t; + + #[cfg(GraalPy)] + pub fn _Py_TYPE(arg1: *const PyObject) -> *mut PyTypeObject; + + #[cfg(GraalPy)] + pub fn _Py_SIZE(arg1: *const PyObject) -> Py_ssize_t; } #[inline(always)] pub unsafe fn Py_INCREF(op: *mut PyObject) { #[cfg(any( + GraalPy, all(Py_LIMITED_API, Py_3_12), all( py_sys_config = "Py_REF_DEBUG", @@ -499,6 +523,7 @@ pub unsafe fn Py_INCREF(op: *mut PyObject) { all(Py_LIMITED_API, not(Py_3_12)), all( not(Py_LIMITED_API), + not(GraalPy), any( not(py_sys_config = "Py_REF_DEBUG"), all(py_sys_config = "Py_REF_DEBUG", Py_3_12), @@ -544,6 +569,7 @@ pub unsafe fn Py_INCREF(op: *mut PyObject) { )] pub unsafe fn Py_DECREF(op: *mut PyObject) { #[cfg(any( + GraalPy, all(Py_LIMITED_API, Py_3_12), all( py_sys_config = "Py_REF_DEBUG", @@ -564,6 +590,7 @@ pub unsafe fn Py_DECREF(op: *mut PyObject) { all(Py_LIMITED_API, not(Py_3_12)), all( not(Py_LIMITED_API), + not(GraalPy), any( not(py_sys_config = "Py_REF_DEBUG"), all(py_sys_config = "Py_REF_DEBUG", Py_3_12), @@ -669,13 +696,20 @@ pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "_PyPy_NoneStruct")] static mut _Py_NoneStruct: PyObject; + + #[cfg(GraalPy)] + static mut _Py_NoneStructReference: *mut PyObject; } #[inline] pub unsafe fn Py_None() -> *mut PyObject { - ptr::addr_of_mut!(_Py_NoneStruct) + #[cfg(not(GraalPy))] + return ptr::addr_of_mut!(_Py_NoneStruct); + #[cfg(GraalPy)] + return _Py_NoneStructReference; } #[inline] @@ -687,13 +721,20 @@ pub unsafe fn Py_IsNone(x: *mut PyObject) -> c_int { #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "_PyPy_NotImplementedStruct")] static mut _Py_NotImplementedStruct: PyObject; + + #[cfg(GraalPy)] + static mut _Py_NotImplementedStructReference: *mut PyObject; } #[inline] pub unsafe fn Py_NotImplemented() -> *mut PyObject { - ptr::addr_of_mut!(_Py_NotImplementedStruct) + #[cfg(not(GraalPy))] + return ptr::addr_of_mut!(_Py_NotImplementedStruct); + #[cfg(GraalPy)] + return _Py_NotImplementedStructReference; } // skipped Py_RETURN_NOTIMPLEMENTED diff --git a/pyo3-ffi/src/pyframe.rs b/pyo3-ffi/src/pyframe.rs index 43a9d1f6777..4dd3d2b31a5 100644 --- a/pyo3-ffi/src/pyframe.rs +++ b/pyo3-ffi/src/pyframe.rs @@ -1,3 +1,4 @@ +#[cfg(not(GraalPy))] #[cfg(any(Py_3_10, all(Py_3_9, not(Py_LIMITED_API))))] use crate::PyCodeObject; #[cfg(not(Py_LIMITED_API))] @@ -9,6 +10,7 @@ opaque_struct!(PyFrameObject); extern "C" { pub fn PyFrame_GetLineNumber(f: *mut PyFrameObject) -> c_int; + #[cfg(not(GraalPy))] #[cfg(any(Py_3_10, all(Py_3_9, not(Py_LIMITED_API))))] pub fn PyFrame_GetCode(f: *mut PyFrameObject) -> *mut PyCodeObject; } diff --git a/pyo3-ffi/src/pyhash.rs b/pyo3-ffi/src/pyhash.rs index f8072d85108..f42f9730f1b 100644 --- a/pyo3-ffi/src/pyhash.rs +++ b/pyo3-ffi/src/pyhash.rs @@ -1,6 +1,6 @@ -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] use crate::pyport::{Py_hash_t, Py_ssize_t}; -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] use std::os::raw::{c_char, c_void}; use std::os::raw::{c_int, c_ulong}; @@ -10,7 +10,7 @@ extern "C" { // skipped non-limited _Py_HashPointer // skipped non-limited _Py_HashPointerRaw - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] pub fn _Py_HashBytes(src: *const c_void, len: Py_ssize_t) -> Py_hash_t; } @@ -20,7 +20,7 @@ pub const _PyHASH_MULTIPLIER: c_ulong = 1000003; // skipped non-limited _Py_HashSecret_t -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyHash_FuncDef { @@ -30,7 +30,7 @@ pub struct PyHash_FuncDef { pub seed_bits: c_int, } -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] impl Default for PyHash_FuncDef { #[inline] fn default() -> Self { @@ -39,7 +39,7 @@ impl Default for PyHash_FuncDef { } extern "C" { - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] pub fn PyHash_GetFuncDef() -> *mut PyHash_FuncDef; } diff --git a/pyo3-ffi/src/pythonrun.rs b/pyo3-ffi/src/pythonrun.rs index e5f20de0058..10985b6068c 100644 --- a/pyo3-ffi/src/pythonrun.rs +++ b/pyo3-ffi/src/pythonrun.rs @@ -1,12 +1,12 @@ use crate::object::*; #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] use libc::FILE; -#[cfg(all(not(PyPy), any(Py_LIMITED_API, not(Py_3_10))))] +#[cfg(all(not(PyPy), any(Py_LIMITED_API, not(Py_3_10), GraalPy)))] use std::os::raw::c_char; use std::os::raw::c_int; extern "C" { - #[cfg(all(Py_LIMITED_API, not(PyPy)))] + #[cfg(any(all(Py_LIMITED_API, not(PyPy)), GraalPy))] pub fn Py_CompileString(string: *const c_char, p: *const c_char, s: c_int) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyErr_Print")] diff --git a/pyo3-ffi/src/setobject.rs b/pyo3-ffi/src/setobject.rs index 84a368a7f27..9d5351fc798 100644 --- a/pyo3-ffi/src/setobject.rs +++ b/pyo3-ffi/src/setobject.rs @@ -1,5 +1,5 @@ use crate::object::*; -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] use crate::pyport::Py_hash_t; use crate::pyport::Py_ssize_t; use std::os::raw::c_int; @@ -7,7 +7,7 @@ use std::ptr::addr_of_mut; pub const PySet_MINSIZE: usize = 8; -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct setentry { @@ -15,7 +15,7 @@ pub struct setentry { pub hash: Py_hash_t, } -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] #[repr(C)] #[derive(Debug)] pub struct PySetObject { @@ -32,7 +32,7 @@ pub struct PySetObject { // skipped #[inline] -#[cfg(all(not(PyPy), not(Py_LIMITED_API)))] +#[cfg(all(not(any(PyPy, GraalPy)), not(Py_LIMITED_API)))] pub unsafe fn PySet_GET_SIZE(so: *mut PyObject) -> Py_ssize_t { debug_assert_eq!(PyAnySet_Check(so), 1); let so = so.cast::(); @@ -92,7 +92,7 @@ extern "C" { } #[inline] -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyFrozenSet_CheckExact(ob: *mut PyObject) -> c_int { (Py_TYPE(ob) == addr_of_mut!(PyFrozenSet_Type)) as c_int } diff --git a/pyo3-ffi/src/sliceobject.rs b/pyo3-ffi/src/sliceobject.rs index 6f09906fcc4..a3ea153987c 100644 --- a/pyo3-ffi/src/sliceobject.rs +++ b/pyo3-ffi/src/sliceobject.rs @@ -5,21 +5,31 @@ use std::ptr::addr_of_mut; #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "_PyPy_EllipsisObject")] static mut _Py_EllipsisObject: PyObject; + + #[cfg(GraalPy)] + static mut _Py_EllipsisObjectReference: *mut PyObject; } #[inline] pub unsafe fn Py_Ellipsis() -> *mut PyObject { - addr_of_mut!(_Py_EllipsisObject) + #[cfg(not(GraalPy))] + return addr_of_mut!(_Py_EllipsisObject); + #[cfg(GraalPy)] + return _Py_EllipsisObjectReference; } #[cfg(not(Py_LIMITED_API))] #[repr(C)] pub struct PySliceObject { pub ob_base: PyObject, + #[cfg(not(GraalPy))] pub start: *mut PyObject, + #[cfg(not(GraalPy))] pub stop: *mut PyObject, + #[cfg(not(GraalPy))] pub step: *mut PyObject, } diff --git a/pyo3-ffi/src/structseq.rs b/pyo3-ffi/src/structseq.rs index 6dfc1daf158..f8566787b51 100644 --- a/pyo3-ffi/src/structseq.rs +++ b/pyo3-ffi/src/structseq.rs @@ -42,13 +42,13 @@ extern "C" { #[cfg(not(Py_LIMITED_API))] pub type PyStructSequence = crate::PyTupleObject; -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] #[inline] pub unsafe fn PyStructSequence_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) { crate::PyTuple_SET_ITEM(op, i, v) } -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] #[inline] pub unsafe fn PyStructSequence_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject { crate::PyTuple_GET_ITEM(op, i) diff --git a/pyo3-ffi/src/weakrefobject.rs b/pyo3-ffi/src/weakrefobject.rs index d065ae23e0f..7e11a9012e7 100644 --- a/pyo3-ffi/src/weakrefobject.rs +++ b/pyo3-ffi/src/weakrefobject.rs @@ -3,10 +3,10 @@ use std::os::raw::c_int; #[cfg(not(PyPy))] use std::ptr::addr_of_mut; -#[cfg(all(not(PyPy), Py_LIMITED_API))] +#[cfg(all(not(PyPy), Py_LIMITED_API, not(GraalPy)))] opaque_struct!(PyWeakReference); -#[cfg(all(not(PyPy), not(Py_LIMITED_API)))] +#[cfg(all(not(PyPy), not(Py_LIMITED_API), not(GraalPy)))] pub use crate::_PyWeakReference as PyWeakReference; #[cfg_attr(windows, link(name = "pythonXY"))] diff --git a/pytests/tests/test_awaitable.py b/pytests/tests/test_awaitable.py index 2bada317517..40f1e1cc01d 100644 --- a/pytests/tests/test_awaitable.py +++ b/pytests/tests/test_awaitable.py @@ -1,13 +1,22 @@ import pytest +import sys from pyo3_pytests.awaitable import IterAwaitable, FutureAwaitable +@pytest.mark.skipif( + sys.implementation.name == "graalpy", + reason="GraalPy's asyncio module has a bug with native classes, see oracle/graalpython#365", +) @pytest.mark.asyncio async def test_iter_awaitable(): assert await IterAwaitable(5) == 5 +@pytest.mark.skipif( + sys.implementation.name == "graalpy", + reason="GraalPy's asyncio module has a bug with native classes, see oracle/graalpython#365", +) @pytest.mark.asyncio async def test_future_awaitable(): assert await FutureAwaitable(5) == 5 diff --git a/pytests/tests/test_misc.py b/pytests/tests/test_misc.py index 6645f942f1a..de75f1c8a80 100644 --- a/pytests/tests/test_misc.py +++ b/pytests/tests/test_misc.py @@ -27,8 +27,8 @@ def test_multiple_imports_same_interpreter_ok(): reason="Cannot identify subinterpreters on Python older than 3.9", ) @pytest.mark.skipif( - platform.python_implementation() == "PyPy", - reason="PyPy does not support subinterpreters", + platform.python_implementation() in ("PyPy", "GraalVM"), + reason="PyPy and GraalPy do not support subinterpreters", ) def test_import_in_subinterpreter_forbidden(): import _xxsubinterpreters diff --git a/pytests/tests/test_objstore.py b/pytests/tests/test_objstore.py index bfd8bad84df..3f0d23fa97c 100644 --- a/pytests/tests/test_objstore.py +++ b/pytests/tests/test_objstore.py @@ -12,6 +12,11 @@ def test_objstore_doesnot_leak_memory(): # check refcount on PyPy getrefcount = getattr(sys, "getrefcount", lambda obj: 0) + if sys.implementation.name == "graalpy": + # GraalPy has an incomplete sys.getrefcount implementation + def getrefcount(obj): + return 0 + before = getrefcount(message) store = ObjStore() for _ in range(N): diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 027982461e9..44843141440 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -175,7 +175,7 @@ int_convert_u64_or_i64!( true ); -#[cfg(not(Py_LIMITED_API))] +#[cfg(all(not(Py_LIMITED_API), not(GraalPy)))] mod fast_128bit_int_conversion { use super::*; @@ -242,7 +242,7 @@ mod fast_128bit_int_conversion { } // For ABI3 we implement the conversion manually. -#[cfg(Py_LIMITED_API)] +#[cfg(any(Py_LIMITED_API, GraalPy))] mod slow_128bit_int_conversion { use super::*; const SHIFT: usize = 64; diff --git a/src/err/mod.rs b/src/err/mod.rs index de1e621fc13..8dd16b26b47 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -863,8 +863,17 @@ impl PyErr { /// associated with the exception, as accessible from Python through `__cause__`. pub fn cause(&self, py: Python<'_>) -> Option { use crate::ffi_ptr_ext::FfiPtrExt; - unsafe { ffi::PyException_GetCause(self.value_bound(py).as_ptr()).assume_owned_or_opt(py) } - .map(Self::from_value_bound) + let obj = unsafe { + ffi::PyException_GetCause(self.value_bound(py).as_ptr()).assume_owned_or_opt(py) + }; + // PyException_GetCause is documented as potentially returning PyNone, but only GraalPy seems to actually do that + #[cfg(GraalPy)] + if let Some(cause) = &obj { + if cause.is_none() { + return None; + } + } + obj.map(Self::from_value_bound) } /// Set the cause associated with the exception, pass `None` to clear it. diff --git a/src/exceptions.rs b/src/exceptions.rs index bd9c89c425f..add958257ab 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -409,14 +409,14 @@ impl_native_exception!( PyExc_FloatingPointError, native_doc!("FloatingPointError") ); -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] impl_native_exception!( PyOSError, PyExc_OSError, native_doc!("OSError"), ffi::PyOSErrorObject ); -#[cfg(PyPy)] +#[cfg(any(PyPy, GraalPy))] impl_native_exception!(PyOSError, PyExc_OSError, native_doc!("OSError")); impl_native_exception!(PyImportError, PyExc_ImportError, native_doc!("ImportError")); @@ -455,14 +455,14 @@ impl_native_exception!( PyExc_NotImplementedError, native_doc!("NotImplementedError") ); -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] impl_native_exception!( PySyntaxError, PyExc_SyntaxError, native_doc!("SyntaxError"), ffi::PySyntaxErrorObject ); -#[cfg(PyPy)] +#[cfg(any(PyPy, GraalPy))] impl_native_exception!(PySyntaxError, PyExc_SyntaxError, native_doc!("SyntaxError")); impl_native_exception!( PyReferenceError, @@ -470,14 +470,14 @@ impl_native_exception!( native_doc!("ReferenceError") ); impl_native_exception!(PySystemError, PyExc_SystemError, native_doc!("SystemError")); -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] impl_native_exception!( PySystemExit, PyExc_SystemExit, native_doc!("SystemExit"), ffi::PySystemExitObject ); -#[cfg(PyPy)] +#[cfg(any(PyPy, GraalPy))] impl_native_exception!(PySystemExit, PyExc_SystemExit, native_doc!("SystemExit")); impl_native_exception!(PyTypeError, PyExc_TypeError, native_doc!("TypeError")); impl_native_exception!( @@ -485,14 +485,14 @@ impl_native_exception!( PyExc_UnboundLocalError, native_doc!("UnboundLocalError") ); -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] impl_native_exception!( PyUnicodeError, PyExc_UnicodeError, native_doc!("UnicodeError"), ffi::PyUnicodeErrorObject ); -#[cfg(PyPy)] +#[cfg(any(PyPy, GraalPy))] impl_native_exception!( PyUnicodeError, PyExc_UnicodeError, diff --git a/src/gil.rs b/src/gil.rs index 5ca3167e66c..bb07489fa1f 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -78,7 +78,7 @@ fn gil_is_acquired() -> bool { /// Python::with_gil(|py| py.run_bound("print('Hello World')", None, None)) /// # } /// ``` -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub fn prepare_freethreaded_python() { // Protect against race conditions when Python is not yet initialized and multiple threads // concurrently call 'prepare_freethreaded_python()'. Note that we do not protect against @@ -125,7 +125,7 @@ pub fn prepare_freethreaded_python() { /// }); /// } /// ``` -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn with_embedded_python_interpreter(f: F) -> R where F: for<'p> FnOnce(Python<'p>) -> R, @@ -182,14 +182,14 @@ impl GILGuard { // auto-initialize so this avoids breaking existing builds. // - Otherwise, just check the GIL is initialized. cfg_if::cfg_if! { - if #[cfg(all(feature = "auto-initialize", not(PyPy)))] { + if #[cfg(all(feature = "auto-initialize", not(any(PyPy, GraalPy))))] { prepare_freethreaded_python(); } else { // This is a "hack" to make running `cargo test` for PyO3 convenient (i.e. no need // to specify `--features auto-initialize` manually. Tests within the crate itself // all depend on the auto-initialize feature for conciseness but Cargo does not // provide a mechanism to specify required features for tests. - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] if option_env!("CARGO_PRIMARY_PACKAGE").is_some() { prepare_freethreaded_python(); } diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index 9ca80e3918c..ba13f7a9841 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -2,10 +2,14 @@ use std::cell::UnsafeCell; -#[cfg(all(not(PyPy), Py_3_9, not(all(windows, Py_LIMITED_API, not(Py_3_10)))))] +#[cfg(all( + not(any(PyPy, GraalPy)), + Py_3_9, + not(all(windows, Py_LIMITED_API, not(Py_3_10))) +))] use portable_atomic::{AtomicI64, Ordering}; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] use crate::exceptions::PyImportError; use crate::{ffi, sync::GILOnceCell, types::PyModule, Bound, Py, PyResult, Python}; @@ -15,7 +19,11 @@ pub struct ModuleDef { ffi_def: UnsafeCell, initializer: ModuleInitializer, /// Interpreter ID where module was initialized (not applicable on PyPy). - #[cfg(all(not(PyPy), Py_3_9, not(all(windows, Py_LIMITED_API, not(Py_3_10)))))] + #[cfg(all( + not(any(PyPy, GraalPy)), + Py_3_9, + not(all(windows, Py_LIMITED_API, not(Py_3_10))) + ))] interpreter: AtomicI64, /// Initialized module object, cached to avoid reinitialization. module: GILOnceCell>, @@ -58,7 +66,11 @@ impl ModuleDef { ffi_def, initializer, // -1 is never expected to be a valid interpreter ID - #[cfg(all(not(PyPy), Py_3_9, not(all(windows, Py_LIMITED_API, not(Py_3_10)))))] + #[cfg(all( + not(any(PyPy, GraalPy)), + Py_3_9, + not(all(windows, Py_LIMITED_API, not(Py_3_10))) + ))] interpreter: AtomicI64::new(-1), module: GILOnceCell::new(), } @@ -85,7 +97,7 @@ impl ModuleDef { // that static data is not reused across interpreters. // // PyPy does not have subinterpreters, so no need to check interpreter ID. - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] { // PyInterpreterState_Get is only available on 3.9 and later, but is missing // from python3.dll for Windows stable API on 3.9 diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index 4d77f329125..db493817cba 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -24,12 +24,14 @@ pub unsafe fn module_init( } #[inline] +#[allow(clippy::used_underscore_binding)] pub unsafe fn noargs( slf: *mut ffi::PyObject, - args: *mut ffi::PyObject, + _args: *mut ffi::PyObject, f: for<'py> unsafe fn(Python<'py>, *mut ffi::PyObject) -> PyResult<*mut ffi::PyObject>, ) -> *mut ffi::PyObject { - debug_assert!(args.is_null()); + #[cfg(not(GraalPy))] // this is not specified and GraalPy does not pass null here + debug_assert!(_args.is_null()); trampoline(|py| f(py, slf)) } diff --git a/src/lib.rs b/src/lib.rs index fd5a520fdb5..e444912a63d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -321,7 +321,7 @@ pub use crate::err::{ }; #[allow(deprecated)] pub use crate::gil::GILPool; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter}; pub use crate::instance::{Borrowed, Bound, Py, PyNativeType, PyObject}; pub use crate::marker::Python; diff --git a/src/macros.rs b/src/macros.rs index 648bac180ff..09e2acfb45b 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -201,7 +201,7 @@ macro_rules! wrap_pymodule { /// /// Use it before [`prepare_freethreaded_python`](crate::prepare_freethreaded_python) and /// leave feature `auto-initialize` off -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[macro_export] macro_rules! append_to_inittab { ($module:ident) => { diff --git a/src/marker.rs b/src/marker.rs index 08b9042491b..67119b5e55e 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -400,7 +400,7 @@ impl Python<'_> { /// If the [`auto-initialize`] feature is enabled and the Python runtime is not already /// initialized, this function will initialize it. See #[cfg_attr( - not(PyPy), + not(any(PyPy, GraalPy)), doc = "[`prepare_freethreaded_python`](crate::prepare_freethreaded_python)" )] #[cfg_attr(PyPy, doc = "`prepare_freethreaded_python`")] diff --git a/src/types/any.rs b/src/types/any.rs index 19855abbb9a..ab4f5727623 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -6,7 +6,7 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::Bound; use crate::py_result_ext::PyResultExt; use crate::type_object::{HasPyGilRef, PyTypeCheck, PyTypeInfo}; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] use crate::types::PySuper; use crate::types::{PyDict, PyIterator, PyList, PyString, PyTuple, PyType}; use crate::{err, ffi, Py, PyNativeType, Python}; @@ -929,7 +929,7 @@ impl PyAny { /// Return a proxy object that delegates method calls to a parent or sibling class of type. /// /// This is equivalent to the Python expression `super()` - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn py_super(&self) -> PyResult<&PySuper> { self.as_borrowed().py_super().map(Bound::into_gil_ref) } @@ -1708,7 +1708,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// Return a proxy object that delegates method calls to a parent or sibling class of type. /// /// This is equivalent to the Python expression `super()` - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] fn py_super(&self) -> PyResult>; } @@ -1975,6 +1975,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { cfg_if::cfg_if! { if #[cfg(all( not(PyPy), + not(GraalPy), any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // PyObject_CallNoArgs was added to python in 3.9 but to limited API in 3.10 ))] { // Optimized path on python 3.9+ @@ -2010,7 +2011,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { N: IntoPy>, { cfg_if::cfg_if! { - if #[cfg(all(Py_3_9, not(any(Py_LIMITED_API, PyPy))))] { + if #[cfg(all(Py_3_9, not(any(Py_LIMITED_API, PyPy, GraalPy))))] { let py = self.py(); // Optimized path on python 3.9+ @@ -2265,7 +2266,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { inner(self, value.to_object(py).into_bound(py)) } - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] fn py_super(&self) -> PyResult> { PySuper::new_bound(&self.get_type(), self) } diff --git a/src/types/complex.rs b/src/types/complex.rs index 80ecffdc5cf..e711b054fe3 100644 --- a/src/types/complex.rs +++ b/src/types/complex.rs @@ -53,7 +53,7 @@ impl PyComplex { } } -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] mod not_limited_impls { use crate::ffi_ptr_ext::FfiPtrExt; use crate::Borrowed; @@ -264,10 +264,10 @@ pub trait PyComplexMethods<'py>: crate::sealed::Sealed { /// Returns the imaginary part of the complex number. fn imag(&self) -> c_double; /// Returns `|self|`. - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] fn abs(&self) -> c_double; /// Returns `self` raised to the power of `other`. - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] fn pow(&self, other: &Bound<'py, PyComplex>) -> Bound<'py, PyComplex>; } @@ -280,7 +280,7 @@ impl<'py> PyComplexMethods<'py> for Bound<'py, PyComplex> { unsafe { ffi::PyComplex_ImagAsDouble(self.as_ptr()) } } - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] fn abs(&self) -> c_double { unsafe { let val = (*self.as_ptr().cast::()).cval; @@ -288,7 +288,7 @@ impl<'py> PyComplexMethods<'py> for Bound<'py, PyComplex> { } } - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] fn pow(&self, other: &Bound<'py, PyComplex>) -> Bound<'py, PyComplex> { use crate::ffi_ptr_ext::FfiPtrExt; unsafe { diff --git a/src/types/datetime.rs b/src/types/datetime.rs index d46a2b77c33..e1620a6f647 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -11,6 +11,8 @@ use crate::ffi::{ PyDateTime_DATE_GET_FOLD, PyDateTime_DATE_GET_HOUR, PyDateTime_DATE_GET_MICROSECOND, PyDateTime_DATE_GET_MINUTE, PyDateTime_DATE_GET_SECOND, }; +#[cfg(GraalPy)] +use crate::ffi::{PyDateTime_DATE_GET_TZINFO, PyDateTime_TIME_GET_TZINFO, Py_IsNone}; use crate::ffi::{ PyDateTime_DELTA_GET_DAYS, PyDateTime_DELTA_GET_MICROSECONDS, PyDateTime_DELTA_GET_SECONDS, }; @@ -552,6 +554,7 @@ impl<'py> PyTzInfoAccess<'py> for &'py PyDateTime { impl<'py> PyTzInfoAccess<'py> for Bound<'py, PyDateTime> { fn get_tzinfo_bound(&self) -> Option> { let ptr = self.as_ptr() as *mut ffi::PyDateTime_DateTime; + #[cfg(not(GraalPy))] unsafe { if (*ptr).hastzinfo != 0 { Some( @@ -565,6 +568,20 @@ impl<'py> PyTzInfoAccess<'py> for Bound<'py, PyDateTime> { None } } + + #[cfg(GraalPy)] + unsafe { + let res = PyDateTime_DATE_GET_TZINFO(ptr as *mut ffi::PyObject); + if Py_IsNone(res) == 1 { + None + } else { + Some( + res.assume_borrowed(self.py()) + .to_owned() + .downcast_into_unchecked(), + ) + } + } } } @@ -740,6 +757,7 @@ impl<'py> PyTzInfoAccess<'py> for &'py PyTime { impl<'py> PyTzInfoAccess<'py> for Bound<'py, PyTime> { fn get_tzinfo_bound(&self) -> Option> { let ptr = self.as_ptr() as *mut ffi::PyDateTime_Time; + #[cfg(not(GraalPy))] unsafe { if (*ptr).hastzinfo != 0 { Some( @@ -753,6 +771,20 @@ impl<'py> PyTzInfoAccess<'py> for Bound<'py, PyTime> { None } } + + #[cfg(GraalPy)] + unsafe { + let res = PyDateTime_TIME_GET_TZINFO(ptr as *mut ffi::PyObject); + if Py_IsNone(res) == 1 { + None + } else { + Some( + res.assume_borrowed(self.py()) + .to_owned() + .downcast_into_unchecked(), + ) + } + } } } diff --git a/src/types/dict.rs b/src/types/dict.rs index 43bade5a80c..a4ba72a59c0 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -20,11 +20,11 @@ pyobject_native_type!( ); /// Represents a Python `dict_keys`. -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(transparent)] pub struct PyDictKeys(PyAny); -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pyobject_native_type_core!( PyDictKeys, pyobject_native_static_type_object!(ffi::PyDictKeys_Type), @@ -32,11 +32,11 @@ pyobject_native_type_core!( ); /// Represents a Python `dict_values`. -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(transparent)] pub struct PyDictValues(PyAny); -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pyobject_native_type_core!( PyDictValues, pyobject_native_static_type_object!(ffi::PyDictValues_Type), @@ -44,11 +44,11 @@ pyobject_native_type_core!( ); /// Represents a Python `dict_items`. -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] #[repr(transparent)] pub struct PyDictItems(PyAny); -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pyobject_native_type_core!( PyDictItems, pyobject_native_static_type_object!(ffi::PyDictItems_Type), @@ -76,14 +76,14 @@ impl PyDict { /// Deprecated form of [`from_sequence_bound`][PyDict::from_sequence_bound]. #[cfg_attr( - all(not(PyPy), not(feature = "gil-refs")), + all(not(any(PyPy, GraalPy)), not(feature = "gil-refs")), deprecated( since = "0.21.0", note = "`PyDict::from_sequence` will be replaced by `PyDict::from_sequence_bound` in a future PyO3 version" ) )] #[inline] - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn from_sequence(seq: &PyAny) -> PyResult<&PyDict> { Self::from_sequence_bound(&seq.as_borrowed()).map(Bound::into_gil_ref) } @@ -95,7 +95,7 @@ impl PyDict { /// /// Returns an error on invalid input. In the case of key collisions, /// this keeps the last entry seen. - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn from_sequence_bound<'py>(seq: &Bound<'py, PyAny>) -> PyResult> { let py = seq.py(); let dict = Self::new_bound(py); @@ -542,12 +542,12 @@ impl<'a, 'py> Borrowed<'a, 'py, PyDict> { } fn dict_len(dict: &Bound<'_, PyDict>) -> Py_ssize_t { - #[cfg(any(not(Py_3_8), PyPy, Py_LIMITED_API))] + #[cfg(any(not(Py_3_8), PyPy, GraalPy, Py_LIMITED_API))] unsafe { ffi::PyDict_Size(dict.as_ptr()) } - #[cfg(all(Py_3_8, not(PyPy), not(Py_LIMITED_API)))] + #[cfg(all(Py_3_8, not(PyPy), not(GraalPy), not(Py_LIMITED_API)))] unsafe { (*dict.as_ptr().cast::()).ma_used } @@ -824,7 +824,7 @@ where #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] use crate::exceptions; use crate::types::PyTuple; use std::collections::{BTreeMap, HashMap}; @@ -850,7 +850,7 @@ mod tests { } #[test] - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] fn test_from_sequence() { Python::with_gil(|py| { let items = PyList::new(py, &vec![("a", 1), ("b", 2)]); @@ -881,7 +881,7 @@ mod tests { } #[test] - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] fn test_from_sequence_err() { Python::with_gil(|py| { let items = PyList::new(py, &vec!["a", "b"]); @@ -955,7 +955,7 @@ mod tests { #[test] #[allow(deprecated)] - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] fn test_get_item_with_error() { Python::with_gil(|py| { let mut v = HashMap::new(); @@ -1388,7 +1388,7 @@ mod tests { }); } - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] fn abc_dict(py: Python<'_>) -> Bound<'_, PyDict> { let mut map = HashMap::<&'static str, i32>::new(); map.insert("a", 1); @@ -1398,7 +1398,7 @@ mod tests { } #[test] - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] fn dict_keys_view() { Python::with_gil(|py| { let dict = abc_dict(py); @@ -1410,7 +1410,7 @@ mod tests { } #[test] - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] fn dict_values_view() { Python::with_gil(|py| { let dict = abc_dict(py); @@ -1422,7 +1422,7 @@ mod tests { } #[test] - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] fn dict_items_view() { Python::with_gil(|py| { let dict = abc_dict(py); diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 8a60efb8caa..36d5578f701 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -60,7 +60,7 @@ impl<'py> PyFrozenSetBuilder<'py> { #[repr(transparent)] pub struct PyFrozenSet(PyAny); -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pyobject_native_type!( PyFrozenSet, ffi::PySetObject, @@ -68,7 +68,7 @@ pyobject_native_type!( #checkfunction=ffi::PyFrozenSet_Check ); -#[cfg(PyPy)] +#[cfg(any(PyPy, GraalPy))] pyobject_native_type_core!( PyFrozenSet, pyobject_native_static_type_object!(ffi::PyFrozenSet_Type), diff --git a/src/types/mod.rs b/src/types/mod.rs index cee45e8678d..5448999d4c7 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -5,7 +5,7 @@ pub use self::boolobject::{PyBool, PyBoolMethods}; pub use self::bytearray::{PyByteArray, PyByteArrayMethods}; pub use self::bytes::{PyBytes, PyBytesMethods}; pub use self::capsule::{PyCapsule, PyCapsuleMethods}; -#[cfg(all(not(Py_LIMITED_API), not(PyPy)))] +#[cfg(all(not(Py_LIMITED_API), not(PyPy), not(GraalPy)))] pub use self::code::PyCode; pub use self::complex::{PyComplex, PyComplexMethods}; #[allow(deprecated)] @@ -17,15 +17,15 @@ pub use self::datetime::{ PyTimeAccess, PyTzInfo, PyTzInfoAccess, }; pub use self::dict::{IntoPyDict, PyDict, PyDictMethods}; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub use self::dict::{PyDictItems, PyDictKeys, PyDictValues}; pub use self::ellipsis::PyEllipsis; pub use self::float::{PyFloat, PyFloatMethods}; -#[cfg(all(not(Py_LIMITED_API), not(PyPy)))] +#[cfg(all(not(Py_LIMITED_API), not(PyPy), not(GraalPy)))] pub use self::frame::PyFrame; pub use self::frozenset::{PyFrozenSet, PyFrozenSetBuilder, PyFrozenSetMethods}; pub use self::function::PyCFunction; -#[cfg(all(not(Py_LIMITED_API), not(PyPy)))] +#[cfg(all(not(Py_LIMITED_API), not(PyPy), not(GraalPy)))] pub use self::function::PyFunction; pub use self::iterator::PyIterator; pub use self::list::{PyList, PyListMethods}; @@ -36,7 +36,7 @@ pub use self::none::PyNone; pub use self::notimplemented::PyNotImplemented; pub use self::num::PyLong; pub use self::num::PyLong as PyInt; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pub use self::pysuper::PySuper; pub use self::sequence::{PySequence, PySequenceMethods}; pub use self::set::{PySet, PySetMethods}; @@ -328,7 +328,7 @@ pub(crate) mod boolobject; pub(crate) mod bytearray; pub(crate) mod bytes; pub(crate) mod capsule; -#[cfg(all(not(Py_LIMITED_API), not(PyPy)))] +#[cfg(all(not(Py_LIMITED_API), not(PyPy), not(GraalPy)))] mod code; pub(crate) mod complex; #[cfg(not(Py_LIMITED_API))] @@ -336,7 +336,7 @@ pub(crate) mod datetime; pub(crate) mod dict; mod ellipsis; pub(crate) mod float; -#[cfg(all(not(Py_LIMITED_API), not(PyPy)))] +#[cfg(all(not(Py_LIMITED_API), not(PyPy), not(GraalPy)))] mod frame; pub(crate) mod frozenset; mod function; @@ -348,7 +348,7 @@ pub(crate) mod module; mod none; mod notimplemented; mod num; -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] mod pysuper; pub(crate) mod sequence; pub(crate) mod set; diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 62abb66fa6e..2b37b6d14f0 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -134,7 +134,7 @@ impl PySequence { /// Returns the number of occurrences of `value` in self, that is, return the /// number of keys for which `self[key] == value`. #[inline] - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] pub fn count(&self, value: V) -> PyResult where V: ToPyObject, @@ -870,7 +870,7 @@ mod tests { } #[test] - #[cfg(not(PyPy))] + #[cfg(not(any(PyPy, GraalPy)))] fn test_seq_count() { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2, 3, 5, 8]; diff --git a/src/types/set.rs b/src/types/set.rs index c043fa5ba4f..4f1fcf8499f 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -14,7 +14,7 @@ use std::ptr; #[repr(transparent)] pub struct PySet(PyAny); -#[cfg(not(PyPy))] +#[cfg(not(any(PyPy, GraalPy)))] pyobject_native_type!( PySet, ffi::PySetObject, @@ -22,7 +22,7 @@ pyobject_native_type!( #checkfunction=ffi::PySet_Check ); -#[cfg(PyPy)] +#[cfg(any(PyPy, GraalPy))] pyobject_native_type_core!( PySet, pyobject_native_static_type_object!(ffi::PySet_Type), diff --git a/src/types/string.rs b/src/types/string.rs index 81c41adb545..4bbc6fb86b0 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -270,7 +270,7 @@ impl PyString { /// /// By using this API, you accept responsibility for testing that PyStringData behaves as /// expected on the targets where you plan to distribute your software. - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, GraalPy)))] pub unsafe fn data(&self) -> PyResult> { self.as_borrowed().data() } @@ -319,7 +319,7 @@ pub trait PyStringMethods<'py>: crate::sealed::Sealed { /// /// By using this API, you accept responsibility for testing that PyStringData behaves as /// expected on the targets where you plan to distribute your software. - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, GraalPy)))] unsafe fn data(&self) -> PyResult>; } @@ -345,7 +345,7 @@ impl<'py> PyStringMethods<'py> for Bound<'py, PyString> { } } - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, GraalPy)))] unsafe fn data(&self) -> PyResult> { self.as_borrowed().data() } @@ -408,7 +408,7 @@ impl<'a> Borrowed<'a, '_, PyString> { Cow::Owned(String::from_utf8_lossy(bytes.as_bytes()).into_owned()) } - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, GraalPy)))] unsafe fn data(self) -> PyResult> { let ptr = self.as_ptr(); diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 4dfe4436bf0..636a2f3e11f 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -34,9 +34,9 @@ fn new_from_iter<'py>( let mut counter: Py_ssize_t = 0; for obj in elements.take(len as usize) { - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] ffi::PyTuple_SET_ITEM(ptr, counter, obj.into_ptr()); - #[cfg(any(Py_LIMITED_API, PyPy))] + #[cfg(any(Py_LIMITED_API, PyPy, GraalPy))] ffi::PyTuple_SetItem(ptr, counter, obj.into_ptr()); counter += 1; } @@ -186,7 +186,7 @@ impl PyTuple { /// # Safety /// /// Caller must verify that the index is within the bounds of the tuple. - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] pub unsafe fn get_item_unchecked(&self, index: usize) -> &PyAny { self.as_borrowed() .get_borrowed_item_unchecked(index) @@ -194,7 +194,7 @@ impl PyTuple { } /// Returns `self` as a slice of objects. - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, GraalPy)))] pub fn as_slice(&self) -> &[&PyAny] { // This is safe because &PyAny has the same memory layout as *mut ffi::PyObject, // and because tuples are immutable. @@ -293,7 +293,7 @@ pub trait PyTupleMethods<'py>: crate::sealed::Sealed { /// # Safety /// /// Caller must verify that the index is within the bounds of the tuple. - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] unsafe fn get_item_unchecked(&self, index: usize) -> Bound<'py, PyAny>; /// Like [`get_item_unchecked`][PyTupleMethods::get_item_unchecked], but returns a borrowed object, @@ -302,11 +302,11 @@ pub trait PyTupleMethods<'py>: crate::sealed::Sealed { /// # Safety /// /// Caller must verify that the index is within the bounds of the tuple. - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] unsafe fn get_borrowed_item_unchecked<'a>(&'a self, index: usize) -> Borrowed<'a, 'py, PyAny>; /// Returns `self` as a slice of objects. - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, GraalPy)))] fn as_slice(&self) -> &[Bound<'py, PyAny>]; /// Determines if self contains `value`. @@ -339,9 +339,9 @@ pub trait PyTupleMethods<'py>: crate::sealed::Sealed { impl<'py> PyTupleMethods<'py> for Bound<'py, PyTuple> { fn len(&self) -> usize { unsafe { - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] let size = ffi::PyTuple_GET_SIZE(self.as_ptr()); - #[cfg(any(Py_LIMITED_API, PyPy))] + #[cfg(any(Py_LIMITED_API, PyPy, GraalPy))] let size = ffi::PyTuple_Size(self.as_ptr()); // non-negative Py_ssize_t should always fit into Rust uint size as usize @@ -376,17 +376,17 @@ impl<'py> PyTupleMethods<'py> for Bound<'py, PyTuple> { self.as_borrowed().get_borrowed_item(index) } - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] unsafe fn get_item_unchecked(&self, index: usize) -> Bound<'py, PyAny> { self.get_borrowed_item_unchecked(index).to_owned() } - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] unsafe fn get_borrowed_item_unchecked<'a>(&'a self, index: usize) -> Borrowed<'a, 'py, PyAny> { self.as_borrowed().get_borrowed_item_unchecked(index) } - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, GraalPy)))] fn as_slice(&self) -> &[Bound<'py, PyAny>] { // This is safe because Bound<'py, PyAny> has the same memory layout as *mut ffi::PyObject, // and because tuples are immutable. @@ -436,7 +436,7 @@ impl<'a, 'py> Borrowed<'a, 'py, PyTuple> { } } - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] unsafe fn get_borrowed_item_unchecked(self, index: usize) -> Borrowed<'a, 'py, PyAny> { ffi::PyTuple_GET_ITEM(self.as_ptr(), index as Py_ssize_t).assume_borrowed(self.py()) } @@ -591,9 +591,9 @@ impl<'a, 'py> BorrowedTupleIterator<'a, 'py> { tuple: Borrowed<'a, 'py, PyTuple>, index: usize, ) -> Borrowed<'a, 'py, PyAny> { - #[cfg(any(Py_LIMITED_API, PyPy))] + #[cfg(any(Py_LIMITED_API, PyPy, GraalPy))] let item = tuple.get_borrowed_item(index).expect("tuple.get failed"); - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] let item = tuple.get_borrowed_item_unchecked(index); item } @@ -696,10 +696,10 @@ fn type_output() -> TypeInfo { { let t = obj.downcast::()?; if t.len() == $length { - #[cfg(any(Py_LIMITED_API, PyPy))] + #[cfg(any(Py_LIMITED_API, PyPy, GraalPy))] return Ok(($(t.get_borrowed_item($n)?.extract::<$T>()?,)+)); - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] unsafe {return Ok(($(t.get_borrowed_item_unchecked($n).extract::<$T>()?,)+));} } else { Err(wrong_tuple_length(t, $length)) @@ -718,9 +718,9 @@ fn array_into_tuple(py: Python<'_>, array: [PyObject; N]) -> Py< let ptr = ffi::PyTuple_New(N.try_into().expect("0 < N <= 12")); let tup = Py::from_owned_ptr(py, ptr); for (index, obj) in array.into_iter().enumerate() { - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] ffi::PyTuple_SET_ITEM(ptr, index as ffi::Py_ssize_t, obj.into_ptr()); - #[cfg(any(Py_LIMITED_API, PyPy))] + #[cfg(any(Py_LIMITED_API, PyPy, GraalPy))] ffi::PyTuple_SetItem(ptr, index as ffi::Py_ssize_t, obj.into_ptr()); } tup @@ -1010,7 +1010,7 @@ mod tests { } #[test] - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, GraalPy)))] fn test_as_slice() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); @@ -1112,7 +1112,7 @@ mod tests { }); } - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] #[test] fn test_tuple_get_item_unchecked_sanity() { Python::with_gil(|py| { @@ -1457,7 +1457,7 @@ mod tests { .unwrap(), 2 ); - #[cfg(not(any(Py_LIMITED_API, PyPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] { assert_eq!( unsafe { tuple.get_item_unchecked(2) } From 1be2fad9bfa900dc2df412e32613641d9175d759 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 25 Mar 2024 23:36:08 +0000 Subject: [PATCH 240/349] release: 0.21.0 (#3983) --- CHANGELOG.md | 48 ++++++++++--------- Cargo.toml | 8 ++-- README.md | 4 +- examples/decorator/.template/pre-script.rhai | 2 +- .../maturin-starter/.template/pre-script.rhai | 2 +- examples/plugin/.template/pre-script.rhai | 2 +- .../.template/pre-script.rhai | 2 +- examples/word-count/.template/pre-script.rhai | 2 +- newsfragments/3247.added.md | 1 - newsfragments/3947.changed.md | 1 - newsfragments/3963.added.md | 1 - newsfragments/3971.added.md | 1 - newsfragments/3982.added.md | 1 - newsfragments/3991.added.md | 1 - pyo3-build-config/Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 4 +- pyo3-macros-backend/Cargo.toml | 4 +- pyo3-macros/Cargo.toml | 4 +- pyproject.toml | 2 +- 19 files changed, 45 insertions(+), 47 deletions(-) delete mode 100644 newsfragments/3247.added.md delete mode 100644 newsfragments/3947.changed.md delete mode 100644 newsfragments/3963.added.md delete mode 100644 newsfragments/3971.added.md delete mode 100644 newsfragments/3982.added.md delete mode 100644 newsfragments/3991.added.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3581d789ff3..2e1ccb8af4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,71 +10,74 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h -## [0.21.0-beta.0] - 2024-03-10 +## [0.21.0] - 2024-03-25 ### Added +- Add support for GraalPy (24.0 and up). [#3247](https://github.com/PyO3/pyo3/pull/3247) - Add `PyMemoryView` type. [#3514](https://github.com/PyO3/pyo3/pull/3514) -- Support `async fn` in macros with coroutine implementation [#3540](https://github.com/PyO3/pyo3/pull/3540) +- Allow `async fn` in for `#[pyfunction]` and `#[pymethods]`, with the `experimental-async` feature. [#3540](https://github.com/PyO3/pyo3/pull/3540) [#3588](https://github.com/PyO3/pyo3/pull/3588) [#3599](https://github.com/PyO3/pyo3/pull/3599) [#3931](https://github.com/PyO3/pyo3/pull/3931) - Implement `PyTypeInfo` for `PyEllipsis`, `PyNone` and `PyNotImplemented`. [#3577](https://github.com/PyO3/pyo3/pull/3577) - Support `#[pyclass]` on enums that have non-unit variants. [#3582](https://github.com/PyO3/pyo3/pull/3582) -- Add `__name__`/`__qualname__` attributes to `Coroutine` [#3588](https://github.com/PyO3/pyo3/pull/3588) -- Add `coroutine::CancelHandle` to catch coroutine cancellation [#3599](https://github.com/PyO3/pyo3/pull/3599) -- Add support for extracting Rust set types from `frozenset`. [#3632](https://github.com/PyO3/pyo3/pull/3632) +- Support `chrono` feature with `abi3` feature. [#3664](https://github.com/PyO3/pyo3/pull/3664) - `FromPyObject`, `IntoPy` and `ToPyObject` are implemented on `std::duration::Duration` [#3670](https://github.com/PyO3/pyo3/pull/3670) - Add `PyString::to_cow`. Add `Py::to_str`, `Py::to_cow`, and `Py::to_string_lossy`, as ways to access Python string data safely beyond the GIL lifetime. [#3677](https://github.com/PyO3/pyo3/pull/3677) - Add `Bound` and `Borrowed` smart pointers as a new API for accessing Python objects. [#3686](https://github.com/PyO3/pyo3/pull/3686) -- Add `PyNativeType::as_bound` to convert "GIL refs" to the new `Bound` smart pointer. [#3692](https://github.com/PyO3/pyo3/pull/3692) -- Add `FromPyObject::extract_bound` method, which can be implemented to avoid using the GIL Ref API in `FromPyObject` implementations. [#3706](https://github.com/PyO3/pyo3/pull/3706) +- Add `PyNativeType::as_borrowed` to convert "GIL refs" to the new `Bound` smart pointer. [#3692](https://github.com/PyO3/pyo3/pull/3692) +- Add `FromPyObject::extract_bound` method, to migrate `FromPyObject` implementations to the Bound API. [#3706](https://github.com/PyO3/pyo3/pull/3706) - Add `gil-refs` feature to allow continued use of the deprecated GIL Refs APIs. [#3707](https://github.com/PyO3/pyo3/pull/3707) -- Added methods to `PyAnyMethods` for binary operators (`add`, `sub`, etc.) [#3712](https://github.com/PyO3/pyo3/pull/3712) -- `chrono-tz` feature allowing conversion between `chrono_tz::Tz` and `zoneinfo.ZoneInfo` [#3730](https://github.com/PyO3/pyo3/pull/3730) -- Add definition for `PyType_GetModuleByDef` to `pyo3_ffi`. [#3734](https://github.com/PyO3/pyo3/pull/3734) +- Add methods to `PyAnyMethods` for binary operators (`add`, `sub`, etc.) [#3712](https://github.com/PyO3/pyo3/pull/3712) +- Add `chrono-tz` feature allowing conversion between `chrono_tz::Tz` and `zoneinfo.ZoneInfo` [#3730](https://github.com/PyO3/pyo3/pull/3730) +- Add FFI definition `PyType_GetModuleByDef`. [#3734](https://github.com/PyO3/pyo3/pull/3734) - Conversion between `std::time::SystemTime` and `datetime.datetime` [#3736](https://github.com/PyO3/pyo3/pull/3736) - Add `Py::as_any` and `Py::into_any`. [#3785](https://github.com/PyO3/pyo3/pull/3785) - Add `PyStringMethods::encode_utf8`. [#3801](https://github.com/PyO3/pyo3/pull/3801) -- Add `PyBackedStr` and `PyBackedBytes`, as alternatives to `&str` and `&bytes` where a Python object owns the data. [#3802](https://github.com/PyO3/pyo3/pull/3802) -- The ability to create Python modules with a Rust `mod` block - behind the `experimental-declarative-modules` feature. [#3815](https://github.com/PyO3/pyo3/pull/3815) +- Add `PyBackedStr` and `PyBackedBytes`, as alternatives to `&str` and `&bytes` where a Python object owns the data. [#3802](https://github.com/PyO3/pyo3/pull/3802) [#3991](https://github.com/PyO3/pyo3/pull/3991) +- Allow `#[pymodule]` macro on Rust `mod` blocks, with the `experimental-declarative-modules` feature. [#3815](https://github.com/PyO3/pyo3/pull/3815) - Implement `ExactSizeIterator` for `set` and `frozenset` iterators on `abi3` feature. [#3849](https://github.com/PyO3/pyo3/pull/3849) - Add `Py::drop_ref` to explicitly drop a `Py`` and immediately decrease the Python reference count if the GIL is already held. [#3871](https://github.com/PyO3/pyo3/pull/3871) +- Allow `#[pymodule]` macro on single argument functions that take `&Bound<'_, PyModule>`. [#3905](https://github.com/PyO3/pyo3/pull/3905) - Implement `FromPyObject` for `Cow`. [#3928](https://github.com/PyO3/pyo3/pull/3928) -- Add `experimental-async` feature. [#3931](https://github.com/PyO3/pyo3/pull/3931) +- Implement `Default` for `GILOnceCell`. [#3971](https://github.com/PyO3/pyo3/pull/3971) +- Add `PyDictMethods::into_mapping`, `PyListMethods::into_sequence` and `PyTupleMethods::into_sequence`. [#3982](https://github.com/PyO3/pyo3/pull/3982) ### Changed -- - `PyDict::from_sequence` now takes a single argument of type `&PyAny` (previously took two arguments `Python` and `PyObject`). [#3532](https://github.com/PyO3/pyo3/pull/3532) +- `PyDict::from_sequence` now takes a single argument of type `&PyAny` (previously took two arguments `Python` and `PyObject`). [#3532](https://github.com/PyO3/pyo3/pull/3532) - Deprecate `Py::is_ellipsis` and `PyAny::is_ellipsis` in favour of `any.is(py.Ellipsis())`. [#3577](https://github.com/PyO3/pyo3/pull/3577) - Split some `PyTypeInfo` functionality into new traits `HasPyGilRef` and `PyTypeCheck`. [#3600](https://github.com/PyO3/pyo3/pull/3600) - Deprecate `PyTryFrom` and `PyTryInto` traits in favor of `any.downcast()` via the `PyTypeCheck` and `PyTypeInfo` traits. [#3601](https://github.com/PyO3/pyo3/pull/3601) - Allow async methods to accept `&self`/`&mut self` [#3609](https://github.com/PyO3/pyo3/pull/3609) -- Values of type `bool` can now be extracted from NumPy's `bool_`. [#3638](https://github.com/PyO3/pyo3/pull/3638) -- Add `AsRefSource` to `PyNativeType`. [#3653](https://github.com/PyO3/pyo3/pull/3653) -- Changed `.is_true` to `.is_truthy` on `PyAny` and `Py` to clarify that the test is not based on identity with or equality to the True singleton. [#3657](https://github.com/PyO3/pyo3/pull/3657) +- `FromPyObject` for set types now also accept `frozenset` objects as input. [#3632](https://github.com/PyO3/pyo3/pull/3632) +- `FromPyObject` for `bool` now also accepts NumPy's `bool_` as input. [#3638](https://github.com/PyO3/pyo3/pull/3638) +- Add `AsRefSource` associated type to `PyNativeType`. [#3653](https://github.com/PyO3/pyo3/pull/3653) +- Rename `.is_true` to `.is_truthy` on `PyAny` and `Py` to clarify that the test is not based on identity with or equality to the True singleton. [#3657](https://github.com/PyO3/pyo3/pull/3657) - `PyType::name` is now `PyType::qualname` whereas `PyType::name` efficiently accesses the full name which includes the module name. [#3660](https://github.com/PyO3/pyo3/pull/3660) - The `Iter(A)NextOutput` types are now deprecated and `__(a)next__` can directly return anything which can be converted into Python objects, i.e. awaitables do not need to be wrapped into `IterANextOutput` or `Option` any more. `Option` can still be used as well and returning `None` will trigger the fast path for `__next__`, stopping iteration without having to raise a `StopIteration` exception. [#3661](https://github.com/PyO3/pyo3/pull/3661) -- Implements `FromPyObject` on `chrono::DateTime` for all `Tz` and not only `FixedOffset` and `Utc` [#3663](https://github.com/PyO3/pyo3/pull/3663) -- `chrono` conversions are compatible with `abi3` [#3664](https://github.com/PyO3/pyo3/pull/3664) +- Implement `FromPyObject` on `chrono::DateTime` for all `Tz`, not just `FixedOffset` and `Utc`. [#3663](https://github.com/PyO3/pyo3/pull/3663) - Add lifetime parameter to `PyTzInfoAccess` trait. For the deprecated gil-ref API, the trait is now implemented for `&'py PyTime` and `&'py PyDateTime` instead of `PyTime` and `PyDate`. [#3679](https://github.com/PyO3/pyo3/pull/3679) - Calls to `__traverse__` become no-ops for unsendable pyclasses if on the wrong thread, thereby avoiding hard aborts at the cost of potential leakage. [#3689](https://github.com/PyO3/pyo3/pull/3689) - Include `PyNativeType` in `pyo3::prelude`. [#3692](https://github.com/PyO3/pyo3/pull/3692) - Improve performance of `extract::` (and other integer types) by avoiding call to `__index__()` converting the value to an integer for 3.10+. Gives performance improvement of around 30% for successful extraction. [#3742](https://github.com/PyO3/pyo3/pull/3742) - Relax bound of `FromPyObject` for `Py` to just `T: PyTypeCheck`. [#3776](https://github.com/PyO3/pyo3/pull/3776) - `PySet` and `PyFrozenSet` iterators now always iterate the equivalent of `iter(set)`. (A "fast path" with no noticeable performance benefit was removed.) [#3849](https://github.com/PyO3/pyo3/pull/3849) -- The `#[pymodule]` macro now supports module functions that take a single argument as a `&Bound<'_, PyModule>`. [#3905](https://github.com/PyO3/pyo3/pull/3905) - Move implementations of `FromPyObject` for `&str`, `Cow`, `&[u8]` and `Cow<[u8]>` onto a temporary trait `FromPyObjectBound` when `gil-refs` feature is deactivated. [#3928](https://github.com/PyO3/pyo3/pull/3928) +- Deprecate `GILPool`, `Python::with_pool`, and `Python::new_pool`. [#3947](https://github.com/PyO3/pyo3/pull/3947) ### Removed - Remove all functionality deprecated in PyO3 0.19. [#3603](https://github.com/PyO3/pyo3/pull/3603) -- Remove `PyCode` and `PyCode_Type` on PyPy: `PyCode_Type` is not exposed by PyPy. [#3934](https://github.com/PyO3/pyo3/pull/3934) ### Fixed - Match PyPy 7.3.14 in removing PyPy-only symbol `Py_MAX_NDIMS` in favour of `PyBUF_MAX_NDIM`. [#3757](https://github.com/PyO3/pyo3/pull/3757) - Fix segmentation fault using `datetime` types when an invalid `datetime` module is on sys.path. [#3818](https://github.com/PyO3/pyo3/pull/3818) - Fix `non_local_definitions` lint warning triggered by many PyO3 macros. [#3901](https://github.com/PyO3/pyo3/pull/3901) +- Disable `PyCode` and `PyCode_Type` on PyPy: `PyCode_Type` is not exposed by PyPy. [#3934](https://github.com/PyO3/pyo3/pull/3934) + +## [0.21.0-beta.0] - 2024-03-10 +Prerelease of PyO3 0.21. See (the GitHub diff)[https://github.com/pyo3/pyo3/compare/v0.21.0-beta.0...v0.21.0] for what changed between 0.21.0-beta.0 and the final release. ## [0.20.3] - 2024-02-23 @@ -1706,6 +1709,7 @@ Yanked - Initial release +[0.21.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0 [0.21.0-beta.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0-beta.0 [0.20.3]: https://github.com/pyo3/pyo3/compare/v0.20.2...v0.20.3 [0.20.2]: https://github.com/pyo3/pyo3/compare/v0.20.1...v0.20.2 diff --git a/Cargo.toml b/Cargo.toml index ba5c3461180..dfbeb2d601d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.21.0-beta.0" +version = "0.21.0" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -22,10 +22,10 @@ memoffset = "0.9" portable-atomic = "1.0" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.0-beta.0" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.0" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.21.0-beta.0", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.21.0", optional = true } indoc = { version = "2.0.1", optional = true } unindent = { version = "0.2.1", optional = true } @@ -60,7 +60,7 @@ rayon = "1.6.1" futures = "0.3.28" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.0-beta.0", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.0", features = ["resolve-config"] } [features] default = ["macros"] diff --git a/README.md b/README.md index 461e56612ad..b28376e7f5e 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.20.3", features = ["extension-module"] } +pyo3 = { version = "0.21.0", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.20.3" +version = "0.21.0" features = ["auto-initialize"] ``` diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index f059903cc18..cebb06c0f61 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.3"); +variable::set("PYO3_VERSION", "0.21.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index f059903cc18..cebb06c0f61 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.3"); +variable::set("PYO3_VERSION", "0.21.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 90989a891ef..4b290f7f7e0 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.3"); +variable::set("PYO3_VERSION", "0.21.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index d5520ae1d28..fbbe7ed0e7c 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.3"); +variable::set("PYO3_VERSION", "0.21.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index f059903cc18..cebb06c0f61 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.20.3"); +variable::set("PYO3_VERSION", "0.21.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/newsfragments/3247.added.md b/newsfragments/3247.added.md deleted file mode 100644 index dd2641c3194..00000000000 --- a/newsfragments/3247.added.md +++ /dev/null @@ -1 +0,0 @@ -Added support for running PyO3 extensions on GraalPy. diff --git a/newsfragments/3947.changed.md b/newsfragments/3947.changed.md deleted file mode 100644 index 4618e9378fb..00000000000 --- a/newsfragments/3947.changed.md +++ /dev/null @@ -1 +0,0 @@ -Deprecate `GILPool`, `Python::with_pool`, and `Python::new_pool`. diff --git a/newsfragments/3963.added.md b/newsfragments/3963.added.md deleted file mode 100644 index 86da17acc89..00000000000 --- a/newsfragments/3963.added.md +++ /dev/null @@ -1 +0,0 @@ -added `Borrowed::to_owned` \ No newline at end of file diff --git a/newsfragments/3971.added.md b/newsfragments/3971.added.md deleted file mode 100644 index 12c0d2265bc..00000000000 --- a/newsfragments/3971.added.md +++ /dev/null @@ -1 +0,0 @@ -Implement `Default` for `GILOnceCell`. diff --git a/newsfragments/3982.added.md b/newsfragments/3982.added.md deleted file mode 100644 index abc89574d38..00000000000 --- a/newsfragments/3982.added.md +++ /dev/null @@ -1 +0,0 @@ -Added `Bound<'py, PyDict>::into_mapping`, `Bound<'py, PyList>::into_sequence` and `Bound<'py, PyTuple>::into_sequence`. diff --git a/newsfragments/3991.added.md b/newsfragments/3991.added.md deleted file mode 100644 index 1a59ecee79c..00000000000 --- a/newsfragments/3991.added.md +++ /dev/null @@ -1 +0,0 @@ -implemented `AsRef` and `AsRef<[u8]>` for `PyBackedStr`, `AsRef<[u8]>` for `PyBackedBytes`. diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index c00427eb18e..f19ded99697 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.21.0-beta.0" +version = "0.21.0" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 8dea584e304..091e5f898ef 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.21.0-beta.0" +version = "0.21.0" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,7 +38,7 @@ abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312"] generate-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0-beta.0", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0", features = ["resolve-config"] } [lints] workspace = true diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index abf71902e20..822dc372b47 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.21.0-beta.0" +version = "0.21.0" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -16,7 +16,7 @@ edition = "2021" [dependencies] heck = "0.4" proc-macro2 = { version = "1", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0-beta.0", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0", features = ["resolve-config"] } quote = { version = "1", default-features = false } [dependencies.syn] diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 008224e78ef..a70ccb0ae7b 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.21.0-beta.0" +version = "0.21.0" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -22,7 +22,7 @@ experimental-declarative-modules = [] proc-macro2 = { version = "1", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.0-beta.0" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.0" } [lints] workspace = true diff --git a/pyproject.toml b/pyproject.toml index 5d5ef42b1db..1a3cfa4aaad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.towncrier] filename = "CHANGELOG.md" -version = "0.21.0-beta.0" +version = "0.21.0" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" From 35faeff6f1db5bc4aa672355ae6174f8fe154fc4 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:53:11 +0100 Subject: [PATCH 241/349] handle `#[pyo3(from_py_with = "")]` in `#[setter]` methods (#3995) * handle `#[pyo3(from_py_with = "")]` in `#[setter]` methods * add newsfragment --- newsfragments/3995.fixed.md | 1 + pyo3-macros-backend/src/pymethod.rs | 32 +++++++++++++++- tests/test_getter_setter.rs | 12 ++++++ tests/ui/deprecations.rs | 6 +++ tests/ui/deprecations.stderr | 58 ++++++++++++++++------------- 5 files changed, 81 insertions(+), 28 deletions(-) create mode 100644 newsfragments/3995.fixed.md diff --git a/newsfragments/3995.fixed.md b/newsfragments/3995.fixed.md new file mode 100644 index 00000000000..e47a71b790d --- /dev/null +++ b/newsfragments/3995.fixed.md @@ -0,0 +1 @@ +handle `#[pyo3(from_py_with = "")]` in `#[setter]` methods \ No newline at end of file diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 4e6f46d96d5..1e476ca22f2 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -11,7 +11,7 @@ use crate::{ }; use crate::{quotes, utils}; use proc_macro2::{Span, TokenStream}; -use quote::{format_ident, quote, ToTokens}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::{ext::IdentExt, spanned::Spanned, Result}; /// Generated code for a single pymethod item. @@ -586,6 +586,34 @@ pub fn impl_py_setter_def( } }; + let extract = if let PropertyType::Function { spec, .. } = &property_type { + Some(spec) + } else { + None + } + .and_then(|spec| { + let (_, args) = split_off_python_arg(&spec.signature.arguments); + let value_arg = &args[0]; + let from_py_with = &value_arg.attrs.from_py_with.as_ref()?.value; + let name = value_arg.name.to_string(); + + Some(quote_spanned! { from_py_with.span() => + let e = #pyo3_path::impl_::deprecations::GilRefs::new(); + let from_py_with = #pyo3_path::impl_::deprecations::inspect_fn(#from_py_with, &e); + e.from_py_with_arg(); + let _val = #pyo3_path::impl_::extract_argument::from_py_with( + &_value.into(), + #name, + from_py_with as fn(_) -> _, + )?; + }) + }) + .unwrap_or_else(|| { + quote! { + let _val = #pyo3_path::FromPyObject::extract_bound(_value.into())?; + } + }); + let mut cfg_attrs = TokenStream::new(); if let PropertyType::Descriptor { field, .. } = &property_type { for attr in field @@ -611,7 +639,7 @@ pub fn impl_py_setter_def( .ok_or_else(|| { #pyo3_path::exceptions::PyAttributeError::new_err("can't delete attribute") })?; - let _val = #pyo3_path::FromPyObject::extract_bound(_value.into())?; + #extract #init_holders let result = #setter_impl; #check_gil_refs diff --git a/tests/test_getter_setter.rs b/tests/test_getter_setter.rs index 1009ce1bd91..5f886639cc0 100644 --- a/tests/test_getter_setter.rs +++ b/tests/test_getter_setter.rs @@ -41,12 +41,21 @@ impl ClassWithProperties { self.num = value; } + #[setter] + fn set_from_len(&mut self, #[pyo3(from_py_with = "extract_len")] value: i32) { + self.num = value; + } + #[getter] fn get_data_list<'py>(&self, py: Python<'py>) -> Bound<'py, PyList> { PyList::new_bound(py, [self.num]) } } +fn extract_len(any: &Bound<'_, PyAny>) -> PyResult { + any.len().map(|len| len as i32) +} + #[test] fn class_with_properties() { Python::with_gil(|py| { @@ -64,6 +73,9 @@ fn class_with_properties() { py_run!(py, inst, "assert inst.get_num() == inst.unwrapped == 42"); py_run!(py, inst, "assert inst.data_list == [42]"); + py_run!(py, inst, "inst.from_len = [0, 0, 0]"); + py_run!(py, inst, "assert inst.get_num() == 3"); + let d = [("C", py.get_type_bound::())].into_py_dict_bound(py); py_assert!(py, *d, "C.DATA.__doc__ == 'a getter for data'"); }); diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index 6f3a1feda50..0ed4d09eebb 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -26,6 +26,12 @@ impl MyClass { #[staticmethod] fn static_method_gil_ref(_any: &PyAny) {} + + #[setter] + fn set_foo_gil_ref(&self, #[pyo3(from_py_with = "extract_gil_ref")] _value: i32) {} + + #[setter] + fn set_foo_bound(&self, #[pyo3(from_py_with = "extract_bound")] _value: i32) {} } fn main() {} diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index 1d87f31c3f8..e2d9cf36ebb 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -34,70 +34,76 @@ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg` 28 | fn static_method_gil_ref(_any: &PyAny) {} | ^ +error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor + --> tests/ui/deprecations.rs:31:53 + | +31 | fn set_foo_gil_ref(&self, #[pyo3(from_py_with = "extract_gil_ref")] _value: i32) {} + | ^^^^^^^^^^^^^^^^^ + error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:41:43 + --> tests/ui/deprecations.rs:47:43 | -41 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { +47 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:51:19 + --> tests/ui/deprecations.rs:57:19 | -51 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { +57 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:57:57 + --> tests/ui/deprecations.rs:63:57 | -57 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +63 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:90:27 + --> tests/ui/deprecations.rs:96:27 | -90 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, +96 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:96:29 - | -96 | fn pyfunction_gil_ref(_any: &PyAny) {} - | ^ + --> tests/ui/deprecations.rs:102:29 + | +102 | fn pyfunction_gil_ref(_any: &PyAny) {} + | ^ error: use of deprecated method `pyo3::deprecations::OptionGilRefs::>::function_arg`: use `Option<&Bound<'_, T>>` instead for this function argument - --> tests/ui/deprecations.rs:99:36 - | -99 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} - | ^^^^^^ + --> tests/ui/deprecations.rs:105:36 + | +105 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} + | ^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:106:27 + --> tests/ui/deprecations.rs:112:27 | -106 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] +112 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:116:27 + --> tests/ui/deprecations.rs:122:27 | -116 | #[pyo3(from_py_with = "PyAny::len")] usize, +122 | #[pyo3(from_py_with = "PyAny::len")] usize, | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:122:31 + --> tests/ui/deprecations.rs:128:31 | -122 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), +128 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:129:27 + --> tests/ui/deprecations.rs:135:27 | -129 | #[pyo3(from_py_with = "extract_gil_ref")] +135 | #[pyo3(from_py_with = "extract_gil_ref")] | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::>::is_python`: use `wrap_pyfunction_bound!` instead - --> tests/ui/deprecations.rs:142:13 + --> tests/ui/deprecations.rs:148:13 | -142 | let _ = wrap_pyfunction!(double, py); +148 | let _ = wrap_pyfunction!(double, py); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From 7c03d65cda997c28422e9a41c42d79f19e89aba9 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 27 Mar 2024 19:16:22 +0800 Subject: [PATCH 242/349] chore: add Python scripting in database example (#3999) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b28376e7f5e..7d79f028dcd 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,7 @@ about this topic. - [fastuuid](https://github.com/thedrow/fastuuid/) _Python bindings to Rust's UUID library._ - [feos](https://github.com/feos-org/feos) _Lightning fast thermodynamic modeling in Rust with fully developed Python interface._ - [forust](https://github.com/jinlow/forust) _A lightweight gradient boosted decision tree library written in Rust._ +- [greptimedb](https://github.com/GreptimeTeam/greptimedb/tree/main/src/script) _Support [Python scripting](https://docs.greptime.com/user-guide/python-scripts/overview) in the database_ - [haem](https://github.com/BooleanCat/haem) _A Python library for working on Bioinformatics problems._ - [html-py-ever](https://github.com/PyO3/setuptools-rust/tree/main/examples/html-py-ever) _Using [html5ever](https://github.com/servo/html5ever) through [kuchiki](https://github.com/kuchiki-rs/kuchiki) to speed up html parsing and css-selecting._ - [hyperjson](https://github.com/mre/hyperjson) _A hyper-fast Python module for reading/writing JSON data using Rust's serde-json._ From bcfc848703bed30b4ffc225aeb440de3e79cbead Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 27 Mar 2024 11:48:56 +0000 Subject: [PATCH 243/349] docs: fix link in CHANGELOG (#4001) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e1ccb8af4a..047a4b21e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,7 +77,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h ## [0.21.0-beta.0] - 2024-03-10 -Prerelease of PyO3 0.21. See (the GitHub diff)[https://github.com/pyo3/pyo3/compare/v0.21.0-beta.0...v0.21.0] for what changed between 0.21.0-beta.0 and the final release. +Prerelease of PyO3 0.21. See [the GitHub diff](https://github.com/pyo3/pyo3/compare/v0.21.0-beta.0...v0.21.0) for what changed between 0.21.0-beta.0 and the final release. ## [0.20.3] - 2024-02-23 From dd1710256d00ac2538f2fb8068fe63254b9f469c Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:41:04 +0100 Subject: [PATCH 244/349] use `extract_argument` for `#[setter]` extraction (#3998) * use `extract_argument` for `#[setter]` extraction * add newsfragment --- newsfragments/3998.changed.md | 1 + newsfragments/3998.fixed.md | 1 + pyo3-macros-backend/src/pymethod.rs | 19 ++++++++-- tests/test_getter_setter.rs | 9 +++++ tests/ui/deprecations.rs | 6 ++++ tests/ui/deprecations.stderr | 54 ++++++++++++++++------------- 6 files changed, 64 insertions(+), 26 deletions(-) create mode 100644 newsfragments/3998.changed.md create mode 100644 newsfragments/3998.fixed.md diff --git a/newsfragments/3998.changed.md b/newsfragments/3998.changed.md new file mode 100644 index 00000000000..c02c6546c95 --- /dev/null +++ b/newsfragments/3998.changed.md @@ -0,0 +1 @@ +Warn on uses of GIL Refs for `#[setter]` function arguments. \ No newline at end of file diff --git a/newsfragments/3998.fixed.md b/newsfragments/3998.fixed.md new file mode 100644 index 00000000000..11f5d006ec0 --- /dev/null +++ b/newsfragments/3998.fixed.md @@ -0,0 +1 @@ +Allow extraction of `&Bound` in `#[setter]` methods. \ No newline at end of file diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 1e476ca22f2..22802a01177 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -586,6 +586,8 @@ pub fn impl_py_setter_def( } }; + // TODO: rework this to make use of `impl_::params::impl_arg_param` which + // handles all these cases already. let extract = if let PropertyType::Function { spec, .. } = &property_type { Some(spec) } else { @@ -609,8 +611,21 @@ pub fn impl_py_setter_def( }) }) .unwrap_or_else(|| { + let (span, name) = match &property_type { + PropertyType::Descriptor { field, .. } => (field.ty.span(), field.ident.as_ref().map(|i|i.to_string()).unwrap_or_default()), + PropertyType::Function { spec, .. } => { + let (_, args) = split_off_python_arg(&spec.signature.arguments); + (args[0].ty.span(), args[0].name.to_string()) + } + }; + + let holder = holders.push_holder(span); + let gil_refs_checker = holders.push_gil_refs_checker(span); quote! { - let _val = #pyo3_path::FromPyObject::extract_bound(_value.into())?; + let _val = #pyo3_path::impl_::deprecations::inspect_type( + #pyo3_path::impl_::extract_argument::extract_argument(_value.into(), &mut #holder, #name)?, + &#gil_refs_checker + ); } }); @@ -639,8 +654,8 @@ pub fn impl_py_setter_def( .ok_or_else(|| { #pyo3_path::exceptions::PyAttributeError::new_err("can't delete attribute") })?; - #extract #init_holders + #extract let result = #setter_impl; #check_gil_refs #pyo3_path::callback::convert(py, result) diff --git a/tests/test_getter_setter.rs b/tests/test_getter_setter.rs index 5f886639cc0..b0b15d78cd0 100644 --- a/tests/test_getter_setter.rs +++ b/tests/test_getter_setter.rs @@ -46,6 +46,12 @@ impl ClassWithProperties { self.num = value; } + #[setter] + fn set_from_any(&mut self, value: &Bound<'_, PyAny>) -> PyResult<()> { + self.num = value.extract()?; + Ok(()) + } + #[getter] fn get_data_list<'py>(&self, py: Python<'py>) -> Bound<'py, PyList> { PyList::new_bound(py, [self.num]) @@ -76,6 +82,9 @@ fn class_with_properties() { py_run!(py, inst, "inst.from_len = [0, 0, 0]"); py_run!(py, inst, "assert inst.get_num() == 3"); + py_run!(py, inst, "inst.from_any = 15"); + py_run!(py, inst, "assert inst.get_num() == 15"); + let d = [("C", py.get_type_bound::())].into_py_dict_bound(py); py_assert!(py, *d, "C.DATA.__doc__ == 'a getter for data'"); }); diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index 0ed4d09eebb..dcc9b7b1d84 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -32,6 +32,12 @@ impl MyClass { #[setter] fn set_foo_bound(&self, #[pyo3(from_py_with = "extract_bound")] _value: i32) {} + + #[setter] + fn set_bar_gil_ref(&self, _value: &PyAny) {} + + #[setter] + fn set_bar_bound(&self, _value: &Bound<'_, PyAny>) {} } fn main() {} diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index e2d9cf36ebb..e692702f23e 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -41,69 +41,75 @@ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_ | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:47:43 + --> tests/ui/deprecations.rs:37:39 | -47 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { +37 | fn set_bar_gil_ref(&self, _value: &PyAny) {} + | ^ + +error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument + --> tests/ui/deprecations.rs:53:43 + | +53 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:57:19 + --> tests/ui/deprecations.rs:63:19 | -57 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { +63 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:63:57 + --> tests/ui/deprecations.rs:69:57 | -63 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +69 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:96:27 - | -96 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, - | ^^^^^^^^^^^^^^^^^ + --> tests/ui/deprecations.rs:102:27 + | +102 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, + | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:102:29 + --> tests/ui/deprecations.rs:108:29 | -102 | fn pyfunction_gil_ref(_any: &PyAny) {} +108 | fn pyfunction_gil_ref(_any: &PyAny) {} | ^ error: use of deprecated method `pyo3::deprecations::OptionGilRefs::>::function_arg`: use `Option<&Bound<'_, T>>` instead for this function argument - --> tests/ui/deprecations.rs:105:36 + --> tests/ui/deprecations.rs:111:36 | -105 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} +111 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} | ^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:112:27 + --> tests/ui/deprecations.rs:118:27 | -112 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] +118 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:122:27 + --> tests/ui/deprecations.rs:128:27 | -122 | #[pyo3(from_py_with = "PyAny::len")] usize, +128 | #[pyo3(from_py_with = "PyAny::len")] usize, | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:128:31 + --> tests/ui/deprecations.rs:134:31 | -128 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), +134 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:135:27 + --> tests/ui/deprecations.rs:141:27 | -135 | #[pyo3(from_py_with = "extract_gil_ref")] +141 | #[pyo3(from_py_with = "extract_gil_ref")] | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::>::is_python`: use `wrap_pyfunction_bound!` instead - --> tests/ui/deprecations.rs:148:13 + --> tests/ui/deprecations.rs:154:13 | -148 | let _ = wrap_pyfunction!(double, py); +154 | let _ = wrap_pyfunction!(double, py); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From b053e83c083b3fca057cf877490739a4799895ef Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 29 Mar 2024 11:28:42 +0000 Subject: [PATCH 245/349] implement `Send` and `Sync` for `PyBackedStr` and `PyBackedBytes` (#4007) --- newsfragments/4007.fixed.md | 1 + src/pybacked.rs | 34 ++++++++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 newsfragments/4007.fixed.md diff --git a/newsfragments/4007.fixed.md b/newsfragments/4007.fixed.md new file mode 100644 index 00000000000..ff905fb4e94 --- /dev/null +++ b/newsfragments/4007.fixed.md @@ -0,0 +1 @@ +Implement `Send` and `Sync` for `PyBackedStr` and `PyBackedBytes`. diff --git a/src/pybacked.rs b/src/pybacked.rs index dbb0865285e..01b9c2a6f00 100644 --- a/src/pybacked.rs +++ b/src/pybacked.rs @@ -16,14 +16,14 @@ use crate::{ pub struct PyBackedStr { #[allow(dead_code)] // only held so that the storage is not dropped storage: Py, - data: NonNull<[u8]>, + data: NonNull, } impl Deref for PyBackedStr { type Target = str; fn deref(&self) -> &str { - // Safety: `data` is known to be immutable utf8 string and owned by self - unsafe { std::str::from_utf8_unchecked(self.data.as_ref()) } + // Safety: `data` is known to be immutable and owned by self + unsafe { self.data.as_ref() } } } @@ -39,13 +39,18 @@ impl AsRef<[u8]> for PyBackedStr { } } +// Safety: the underlying Python str (or bytes) is immutable and +// safe to share between threads +unsafe impl Send for PyBackedStr {} +unsafe impl Sync for PyBackedStr {} + impl TryFrom> for PyBackedStr { type Error = PyErr; fn try_from(py_string: Bound<'_, PyString>) -> Result { #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] { let s = py_string.to_str()?; - let data = NonNull::from(s.as_bytes()); + let data = NonNull::from(s); Ok(Self { storage: py_string.as_any().to_owned().unbind(), data, @@ -54,8 +59,8 @@ impl TryFrom> for PyBackedStr { #[cfg(not(any(Py_3_10, not(Py_LIMITED_API))))] { let bytes = py_string.encode_utf8()?; - let b = bytes.as_bytes(); - let data = NonNull::from(b); + let s = unsafe { std::str::from_utf8_unchecked(bytes.as_bytes()) }; + let data = NonNull::from(s); Ok(Self { storage: bytes.into_any().unbind(), data, @@ -100,6 +105,11 @@ impl AsRef<[u8]> for PyBackedBytes { } } +// Safety: the underlying Python bytes or Rust bytes is immutable and +// safe to share between threads +unsafe impl Send for PyBackedBytes {} +unsafe impl Sync for PyBackedBytes {} + impl From> for PyBackedBytes { fn from(py_bytes: Bound<'_, PyBytes>) -> Self { let b = py_bytes.as_bytes(); @@ -201,4 +211,16 @@ mod test { assert_eq!(&*py_backed_bytes, b"abcde"); }); } + + #[test] + fn test_backed_types_send_sync() { + fn is_send() {} + fn is_sync() {} + + is_send::(); + is_sync::(); + + is_send::(); + is_sync::(); + } } From cf74624de9c88e51914c7d2a2dbae5c4d4a6e63d Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 29 Mar 2024 11:54:33 +0000 Subject: [PATCH 246/349] refactor to remove add_to_module functions from generated code (#4009) * refactor to remove add_to_module functions from generated code * mrsv, newsfragment * clippy --- newsfragments/4009.fixed.md | 1 + pyo3-macros-backend/src/module.rs | 14 +++---- pyo3-macros-backend/src/pyclass.rs | 8 ++-- pyo3-macros-backend/src/pyfunction.rs | 10 +---- src/impl_/pymodule.rs | 57 +++++++++++++++++++++++++-- src/macros.rs | 10 ++--- src/types/mod.rs | 13 ++---- tests/test_declarative_module.rs | 2 +- 8 files changed, 75 insertions(+), 40 deletions(-) create mode 100644 newsfragments/4009.fixed.md diff --git a/newsfragments/4009.fixed.md b/newsfragments/4009.fixed.md new file mode 100644 index 00000000000..a5d378e12ad --- /dev/null +++ b/newsfragments/4009.fixed.md @@ -0,0 +1 @@ +Fix some uncovered code blocks emitted by `#[pymodule]`, `#[pyfunction]` and `#[pyclass]` macros. diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index dc2b3bfc86b..080a279a88c 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -267,7 +267,7 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { use #pyo3_path::impl_::pymodule::PyAddToModule; #( #(#module_items_cfg_attrs)* - #module_items::add_to_module(module)?; + #module_items::_PYO3_DEF.add_to_module(module)?; )* #pymodule_init Ok(()) @@ -358,21 +358,19 @@ fn module_initialization(options: PyModuleOptions, ident: &syn::Ident) -> TokenS let pyinit_symbol = format!("PyInit_{}", name); quote! { + #[doc(hidden)] pub const __PYO3_NAME: &'static str = concat!(stringify!(#name), "\0"); pub(super) struct MakeDef; - pub static DEF: #pyo3_path::impl_::pymodule::ModuleDef = MakeDef::make_def(); - - pub fn add_to_module(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { - use #pyo3_path::prelude::PyModuleMethods; - module.add_submodule(DEF.make_module(module.py())?.bind(module.py())) - } + #[doc(hidden)] + pub static _PYO3_DEF: #pyo3_path::impl_::pymodule::ModuleDef = MakeDef::make_def(); /// This autogenerated function is called by the python interpreter when importing /// the module. + #[doc(hidden)] #[export_name = #pyinit_symbol] pub unsafe extern "C" fn __pyo3_init() -> *mut #pyo3_path::ffi::PyObject { - #pyo3_path::impl_::trampoline::module_init(|py| DEF.make_module(py)) + #pyo3_path::impl_::trampoline::module_init(|py| _PYO3_DEF.make_module(py)) } } } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 5122d205640..88cf9149e4d 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1620,11 +1620,9 @@ impl<'a> PyClassImplsBuilder<'a> { let Ctx { pyo3_path } = ctx; let cls = self.cls; quote! { - impl #pyo3_path::impl_::pymodule::PyAddToModule for #cls { - fn add_to_module(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { - use #pyo3_path::types::PyModuleMethods; - module.add_class::() - } + impl #cls { + #[doc(hidden)] + const _PYO3_DEF: #pyo3_path::impl_::pymodule::AddClassToModule = #pyo3_path::impl_::pymodule::AddClassToModule::new(); } } } diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index cce9f74824c..25d2536ecb5 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -269,13 +269,7 @@ pub fn impl_wrap_pyfunction( #[doc(hidden)] #vis mod #name { pub(crate) struct MakeDef; - pub const DEF: #pyo3_path::impl_::pymethods::PyMethodDef = MakeDef::DEF; - - pub fn add_to_module(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { - use #pyo3_path::prelude::PyModuleMethods; - use ::std::convert::Into; - module.add_function(#pyo3_path::types::PyCFunction::internal_new(module.py(), &DEF, module.into())?) - } + pub const _PYO3_DEF: #pyo3_path::impl_::pymethods::PyMethodDef = MakeDef::_PYO3_DEF; } // Generate the definition inside an anonymous function in the same scope as the original function - @@ -283,7 +277,7 @@ pub fn impl_wrap_pyfunction( // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pyfunction] is // inside a function body) impl #name::MakeDef { - const DEF: #pyo3_path::impl_::pymethods::PyMethodDef = #methoddef; + const _PYO3_DEF: #pyo3_path::impl_::pymethods::PyMethodDef = #methoddef; } #[allow(non_snake_case)] diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index ba13f7a9841..20560aeb8d5 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -1,6 +1,6 @@ //! Implementation details of `#[pymodule]` which need to be accessible from proc-macro generated code. -use std::cell::UnsafeCell; +use std::{cell::UnsafeCell, marker::PhantomData}; #[cfg(all( not(any(PyPy, GraalPy)), @@ -11,7 +11,12 @@ use portable_atomic::{AtomicI64, Ordering}; #[cfg(not(any(PyPy, GraalPy)))] use crate::exceptions::PyImportError; -use crate::{ffi, sync::GILOnceCell, types::PyModule, Bound, Py, PyResult, Python}; +use crate::{ + ffi, + sync::GILOnceCell, + types::{PyCFunction, PyModule, PyModuleMethods}, + Bound, Py, PyClass, PyMethodDef, PyResult, PyTypeInfo, Python, +}; /// `Sync` wrapper of `ffi::PyModuleDef`. pub struct ModuleDef { @@ -149,7 +154,53 @@ impl ModuleDef { /// /// Currently only implemented for classes. pub trait PyAddToModule { - fn add_to_module(module: &Bound<'_, PyModule>) -> PyResult<()>; + fn add_to_module(&'static self, module: &Bound<'_, PyModule>) -> PyResult<()>; +} + +/// For adding native types (non-pyclass) to a module. +pub struct AddTypeToModule(PhantomData); + +impl AddTypeToModule { + #[allow(clippy::new_without_default)] + pub const fn new() -> Self { + AddTypeToModule(PhantomData) + } +} + +impl PyAddToModule for AddTypeToModule { + fn add_to_module(&'static self, module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add(T::NAME, T::type_object_bound(module.py())) + } +} + +/// For adding a class to a module. +pub struct AddClassToModule(PhantomData); + +impl AddClassToModule { + #[allow(clippy::new_without_default)] + pub const fn new() -> Self { + AddClassToModule(PhantomData) + } +} + +impl PyAddToModule for AddClassToModule { + fn add_to_module(&'static self, module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() + } +} + +/// For adding a function to a module. +impl PyAddToModule for PyMethodDef { + fn add_to_module(&'static self, module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(PyCFunction::internal_new(module.py(), self, Some(module))?) + } +} + +/// For adding a module to a module. +impl PyAddToModule for ModuleDef { + fn add_to_module(&'static self, module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_submodule(self.make_module(module.py())?.bind(module.py())) + } } #[cfg(test)] diff --git a/src/macros.rs b/src/macros.rs index 09e2acfb45b..0267c2663eb 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -138,7 +138,7 @@ macro_rules! wrap_pyfunction { use $function as wrapped_pyfunction; $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( py_or_module, - &wrapped_pyfunction::DEF, + &wrapped_pyfunction::_PYO3_DEF, ) } }; @@ -150,7 +150,7 @@ macro_rules! wrap_pyfunction { check_gil_refs.is_python(); $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( py_or_module, - &wrapped_pyfunction::DEF, + &wrapped_pyfunction::_PYO3_DEF, ) }}; } @@ -166,7 +166,7 @@ macro_rules! wrap_pyfunction_bound { use $function as wrapped_pyfunction; $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( $crate::impl_::pyfunction::OnlyBound(py_or_module), - &wrapped_pyfunction::DEF, + &wrapped_pyfunction::_PYO3_DEF, ) } }; @@ -174,7 +174,7 @@ macro_rules! wrap_pyfunction_bound { use $function as wrapped_pyfunction; $crate::impl_::pyfunction::WrapPyFunctionArg::wrap_pyfunction( $crate::impl_::pyfunction::OnlyBound($py_or_module), - &wrapped_pyfunction::DEF, + &wrapped_pyfunction::_PYO3_DEF, ) }}; } @@ -189,7 +189,7 @@ macro_rules! wrap_pymodule { ($module:path) => { &|py| { use $module as wrapped_pymodule; - wrapped_pymodule::DEF + wrapped_pymodule::_PYO3_DEF .make_module(py) .expect("failed to wrap pymodule") } diff --git a/src/types/mod.rs b/src/types/mod.rs index 5448999d4c7..a03d01b301a 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -250,16 +250,9 @@ macro_rules! pyobject_native_type_info( )? } - impl<$($generics,)*> $crate::impl_::pymodule::PyAddToModule for $name { - fn add_to_module( - module: &$crate::Bound<'_, $crate::types::PyModule>, - ) -> $crate::PyResult<()> { - use $crate::types::PyModuleMethods; - module.add( - ::NAME, - ::type_object_bound(module.py()), - ) - } + impl $name { + #[doc(hidden)] + const _PYO3_DEF: $crate::impl_::pymodule::AddTypeToModule = $crate::impl_::pymodule::AddTypeToModule::new(); } }; ); diff --git a/tests/test_declarative_module.rs b/tests/test_declarative_module.rs index 9eea8e2df07..5f8e2c9fa55 100644 --- a/tests/test_declarative_module.rs +++ b/tests/test_declarative_module.rs @@ -145,7 +145,7 @@ mod class_initialization_module { #[cfg(not(Py_LIMITED_API))] fn test_class_initialization_fails() { Python::with_gil(|py| { - let err = class_initialization_module::DEF + let err = class_initialization_module::_PYO3_DEF .make_module(py) .unwrap_err(); assert_eq!( From 60e3f44dcf0396d43ebd3cea60aba483db72cc81 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 29 Mar 2024 10:16:45 -0400 Subject: [PATCH 247/349] Ensure all arguments to _run are strings (#4013) Otherwise you get errors like https://github.com/PyO3/pyo3/actions/runs/8480723060/job/23236947935?pr=4012 --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 3ee76da5f08..3d82c8d3746 100644 --- a/noxfile.py +++ b/noxfile.py @@ -406,7 +406,7 @@ def check_guide(session: nox.Session): session, "lychee", "--include-fragments", - PYO3_GUIDE_SRC, + str(PYO3_GUIDE_SRC), *remap_args, *session.posargs, ) From ae3ac7d87289da5c4456953aa1edf355a0b5b65e Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 29 Mar 2024 11:49:50 -0400 Subject: [PATCH 248/349] Fixed error string when failing to import an exception (#4012) --- newsfragments/4012.fixed.md | 1 + src/exceptions.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/4012.fixed.md diff --git a/newsfragments/4012.fixed.md b/newsfragments/4012.fixed.md new file mode 100644 index 00000000000..352ec928487 --- /dev/null +++ b/newsfragments/4012.fixed.md @@ -0,0 +1 @@ +Fixed the error message when a class referenced in `pyo3::import_exception!` does not exist diff --git a/src/exceptions.rs b/src/exceptions.rs index add958257ab..66b5b57dc0e 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -123,7 +123,7 @@ macro_rules! import_exception { ::std::panic!("Can not import module {}: {}\n{}", stringify!($module), err, traceback); }); let cls = imp.getattr(stringify!($name)).expect(concat!( - "Can not load exception class: {}.{}", + "Can not load exception class: ", stringify!($module), ".", stringify!($name) From c6f25e42a285c22a29c82b8da1ff8d365f9db632 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 29 Mar 2024 15:51:38 +0000 Subject: [PATCH 249/349] ci: relax check in pyobject_drop test (#4014) --- src/gil.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/gil.rs b/src/gil.rs index bb07489fa1f..e2f36037755 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -556,11 +556,11 @@ mod tests { } #[cfg(not(target_arch = "wasm32"))] - fn pool_dirty_with( - inc_refs: Vec>, - dec_refs: Vec>, - ) -> bool { - *POOL.pointer_ops.lock() == (inc_refs, dec_refs) + fn pool_dec_refs_contains(obj: &PyObject) -> bool { + POOL.pointer_ops + .lock() + .1 + .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } #[test] @@ -633,11 +633,13 @@ mod tests { assert_eq!(obj.get_refcnt(py), 2); assert!(pool_inc_refs_does_not_contain(&obj)); + assert!(pool_dec_refs_does_not_contain(&obj)); - // With the GIL held, reference cound will be decreased immediately. + // With the GIL held, reference count will be decreased immediately. drop(reference); assert_eq!(obj.get_refcnt(py), 1); + assert!(pool_inc_refs_does_not_contain(&obj)); assert!(pool_dec_refs_does_not_contain(&obj)); }); } @@ -652,6 +654,7 @@ mod tests { assert_eq!(obj.get_refcnt(py), 2); assert!(pool_inc_refs_does_not_contain(&obj)); + assert!(pool_dec_refs_does_not_contain(&obj)); // Drop reference in a separate thread which doesn't have the GIL. std::thread::spawn(move || drop(reference)).join().unwrap(); @@ -659,10 +662,8 @@ mod tests { // The reference count should not have changed (the GIL has always // been held by this thread), it is remembered to release later. assert_eq!(obj.get_refcnt(py), 2); - assert!(pool_dirty_with( - vec![], - vec![NonNull::new(obj.as_ptr()).unwrap()] - )); + assert!(pool_inc_refs_does_not_contain(&obj)); + assert!(pool_dec_refs_contains(&obj)); obj }); From de03ca270adcabfe0881595879da09b7762527d6 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 29 Mar 2024 17:28:08 -0400 Subject: [PATCH 250/349] Remove dead code in macros backend (#4011) --- pyo3-macros-backend/src/method.rs | 10 ---------- pyo3-macros-backend/src/pyclass.rs | 12 ++---------- pyo3-macros-backend/src/pyfunction.rs | 3 --- 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 1a46d333c3b..31dfb075958 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -269,7 +269,6 @@ pub struct FnSpec<'a> { // r# can be removed by syn::ext::IdentExt::unraw() pub python_name: syn::Ident, pub signature: FunctionSignature<'a>, - pub output: syn::Type, pub convention: CallingConvention, pub text_signature: Option, pub asyncness: Option, @@ -277,13 +276,6 @@ pub struct FnSpec<'a> { pub deprecations: Deprecations<'a>, } -pub fn get_return_info(output: &syn::ReturnType) -> syn::Type { - match output { - syn::ReturnType::Default => syn::Type::Infer(syn::parse_quote! {_}), - syn::ReturnType::Type(_, ty) => *ty.clone(), - } -} - pub fn parse_method_receiver(arg: &syn::FnArg) -> Result { match arg { syn::FnArg::Receiver( @@ -329,7 +321,6 @@ impl<'a> FnSpec<'a> { ensure_signatures_on_valid_method(&fn_type, signature.as_ref(), text_signature.as_ref())?; let name = &sig.ident; - let ty = get_return_info(&sig.output); let python_name = python_name.as_ref().unwrap_or(name).unraw(); let arguments: Vec<_> = sig @@ -361,7 +352,6 @@ impl<'a> FnSpec<'a> { convention, python_name, signature, - output: ty, text_signature, asyncness: sig.asyncness, unsafety: sig.unsafety, diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 88cf9149e4d..cf8d9aae801 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -975,13 +975,8 @@ fn impl_complex_enum_struct_variant_cls( let field_type = field.ty; let field_with_type = quote! { #field_name: #field_type }; - let field_getter = complex_enum_variant_field_getter( - &variant_cls_type, - field_name, - field_type, - field.span, - ctx, - )?; + let field_getter = + complex_enum_variant_field_getter(&variant_cls_type, field_name, field.span, ctx)?; let field_getter_impl = quote! { fn #field_name(slf: #pyo3_path::PyRef) -> #pyo3_path::PyResult<#field_type> { @@ -1188,7 +1183,6 @@ fn complex_enum_struct_variant_new<'a>( name: &format_ident!("__pymethod_constructor__"), python_name: format_ident!("__new__"), signature, - output: variant_cls_type.clone(), convention: crate::method::CallingConvention::TpNew, text_signature: None, asyncness: None, @@ -1202,7 +1196,6 @@ fn complex_enum_struct_variant_new<'a>( fn complex_enum_variant_field_getter<'a>( variant_cls_type: &'a syn::Type, field_name: &'a syn::Ident, - field_type: &'a syn::Type, field_span: Span, ctx: &Ctx, ) -> Result { @@ -1215,7 +1208,6 @@ fn complex_enum_variant_field_getter<'a>( name: field_name, python_name: field_name.clone(), signature, - output: field_type.clone(), convention: crate::method::CallingConvention::Noargs, text_signature: None, asyncness: None, diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 25d2536ecb5..9f9557a3664 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -240,15 +240,12 @@ pub fn impl_wrap_pyfunction( FunctionSignature::from_arguments(arguments)? }; - let ty = method::get_return_info(&func.sig.output); - let spec = method::FnSpec { tp, name: &func.sig.ident, convention: CallingConvention::from_signature(&signature), python_name, signature, - output: ty, text_signature, asyncness: func.sig.asyncness, unsafety: func.sig.unsafety, From 0e093a5911f0dd4e6e919c306359e41f9b98ccaf Mon Sep 17 00:00:00 2001 From: geo7 Date: Fri, 29 Mar 2024 22:45:47 +0000 Subject: [PATCH 251/349] Update getting-started.md (#4004) Missing word --- guide/src/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/getting-started.md b/guide/src/getting-started.md index ef9ca5254b8..59cf5ba6ded 100644 --- a/guide/src/getting-started.md +++ b/guide/src/getting-started.md @@ -33,7 +33,7 @@ You can read more about `pyenv`'s configuration options [here](https://github.co ### Building -There are a number of build and Python package management systems such as [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](./building-and-distribution.md#manual-builds). We recommend the use of `maturin`, which you can install [here](https://maturin.rs/installation.html). It is developed to work with PyO3 and provides the most "batteries included" experience, especially if you are aiming to publish to PyPI. `maturin` is just a Python package, so you can add it in the same you already install Python packages. +There are a number of build and Python package management systems such as [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](./building-and-distribution.md#manual-builds). We recommend the use of `maturin`, which you can install [here](https://maturin.rs/installation.html). It is developed to work with PyO3 and provides the most "batteries included" experience, especially if you are aiming to publish to PyPI. `maturin` is just a Python package, so you can add it in the same way you already install Python packages. System Python: ```bash From 4d033c4497029c12a8788c28f9ffba6fb18234f4 Mon Sep 17 00:00:00 2001 From: Rikus Honey Date: Sat, 30 Mar 2024 09:39:00 +0200 Subject: [PATCH 252/349] Fix broken hyperlink to types.md (#4018) --- guide/src/types.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/guide/src/types.md b/guide/src/types.md index 0f1fa3d0af1..a402e52090a 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -268,8 +268,8 @@ assert_eq!((x, y, z), (1, 2, 3)); # Python::with_gil(example).unwrap() ``` -To avoid copying data, [`#[pyclass]`][pyclass] types can directly reference Rust data stored within the Python objects without needing to `.extract()`. See the [corresponding documentation in the class section of the guide](./class. -md#bound-and-interior-mutability) for more detail. +To avoid copying data, [`#[pyclass]`][pyclass] types can directly reference Rust data stored within the Python objects without needing to `.extract()`. See the [corresponding documentation in the class section of the guide](./class.md#bound-and-interior-mutability) +for more detail. ## The GIL Refs API From 74d9d23ba0d6d0866650bc105e9611bc8ca6d03b Mon Sep 17 00:00:00 2001 From: Weijie Guo Date: Sat, 30 Mar 2024 16:48:19 +0800 Subject: [PATCH 253/349] async method should allow args not only receiver (#4015) * async method should allow args not only receiver * add changelog md --- newsfragments/4015.fixed.md | 1 + pyo3-macros-backend/src/method.rs | 22 ++++++++++++++++++--- tests/test_coroutine.rs | 33 +++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 newsfragments/4015.fixed.md diff --git a/newsfragments/4015.fixed.md b/newsfragments/4015.fixed.md new file mode 100644 index 00000000000..a8f4f636c76 --- /dev/null +++ b/newsfragments/4015.fixed.md @@ -0,0 +1 @@ +Fix the bug that an async `#[pymethod]` with receiver can't have any other args. \ No newline at end of file diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 31dfb075958..f4fdb19377a 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -1,7 +1,7 @@ use std::fmt::Display; use proc_macro2::{Span, TokenStream}; -use quote::{quote, quote_spanned, ToTokens}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::{ext::IdentExt, spanned::Spanned, Ident, Result}; use crate::utils::Ctx; @@ -518,17 +518,33 @@ impl<'a> FnSpec<'a> { Some(cls) => quote!(Some(<#cls as #pyo3_path::PyTypeInfo>::NAME)), None => quote!(None), }; + let evaluate_args = || -> (Vec, TokenStream) { + let mut arg_names = Vec::with_capacity(args.len()); + let mut evaluate_arg = quote! {}; + for arg in &args { + let arg_name = format_ident!("arg_{}", arg_names.len()); + arg_names.push(arg_name.clone()); + evaluate_arg.extend(quote! { + let #arg_name = #arg + }); + } + (arg_names, evaluate_arg) + }; let future = match self.tp { FnType::Fn(SelfType::Receiver { mutable: false, .. }) => { + let (arg_name, evaluate_arg) = evaluate_args(); quote! {{ + #evaluate_arg; let __guard = #pyo3_path::impl_::coroutine::RefGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; - async move { function(&__guard, #(#args),*).await } + async move { function(&__guard, #(#arg_name),*).await } }} } FnType::Fn(SelfType::Receiver { mutable: true, .. }) => { + let (arg_name, evaluate_arg) = evaluate_args(); quote! {{ + #evaluate_arg; let mut __guard = #pyo3_path::impl_::coroutine::RefMutGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; - async move { function(&mut __guard, #(#args),*).await } + async move { function(&mut __guard, #(#arg_name),*).await } }} } _ => { diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index 23f6a6722d4..0e698deb9af 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -245,6 +245,39 @@ fn coroutine_panic() { }) } +#[test] +fn test_async_method_receiver_with_other_args() { + #[pyclass] + struct Value(i32); + #[pymethods] + impl Value { + #[new] + fn new() -> Self { + Self(0) + } + async fn get_value_plus_with(&self, v: i32) -> i32 { + self.0 + v + } + async fn set_value(&mut self, new_value: i32) -> i32 { + self.0 = new_value; + self.0 + } + } + + Python::with_gil(|gil| { + let test = r#" + import asyncio + + v = Value() + assert asyncio.run(v.get_value_plus_with(3)) == 3 + assert asyncio.run(v.set_value(10)) == 10 + assert asyncio.run(v.get_value_plus_with(1)) == 11 + "#; + let locals = [("Value", gil.get_type_bound::())].into_py_dict_bound(gil); + py_run!(gil, *locals, test); + }); +} + #[test] fn test_async_method_receiver() { #[pyclass] From 22e8dd10e1667c46c258c56a7aeb8ecf2e4fad30 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 30 Mar 2024 20:29:39 +0000 Subject: [PATCH 254/349] add benchmark for class / method calls (#4016) --- pytests/src/pyclasses.rs | 6 ++++++ pytests/tests/test_pyclasses.py | 26 +++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index 8e957c77955..ac817627cfe 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -11,6 +11,12 @@ impl EmptyClass { fn new() -> Self { EmptyClass {} } + + fn method(&self) {} + + fn __len__(&self) -> usize { + 0 + } } /// This is for demonstrating how to return a value from __next__ diff --git a/pytests/tests/test_pyclasses.py b/pytests/tests/test_pyclasses.py index 9a9b44b52e8..74f883e3808 100644 --- a/pytests/tests/test_pyclasses.py +++ b/pytests/tests/test_pyclasses.py @@ -8,14 +8,38 @@ def test_empty_class_init(benchmark): benchmark(pyclasses.EmptyClass) +def test_method_call(benchmark): + obj = pyclasses.EmptyClass() + assert benchmark(obj.method) is None + + +def test_proto_call(benchmark): + obj = pyclasses.EmptyClass() + assert benchmark(len, obj) == 0 + + class EmptyClassPy: - pass + def method(self): + pass + + def __len__(self) -> int: + return 0 def test_empty_class_init_py(benchmark): benchmark(EmptyClassPy) +def test_method_call_py(benchmark): + obj = EmptyClassPy() + assert benchmark(obj.method) == pyclasses.EmptyClass().method() + + +def test_proto_call_py(benchmark): + obj = EmptyClassPy() + assert benchmark(len, obj) == len(pyclasses.EmptyClass()) + + def test_iter(): i = pyclasses.PyClassIter() assert next(i) == 1 From 9d932c1061c06000c9424c6542f7c3f266d32836 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 30 Mar 2024 22:10:21 +0000 Subject: [PATCH 255/349] add `#[inline]` hints on many `Bound` and `Borrowed` methods (#4024) --- newsfragments/4024.changed.md | 1 + src/instance.rs | 25 +++++++++++++++++++++++++ src/pycell.rs | 1 + 3 files changed, 27 insertions(+) create mode 100644 newsfragments/4024.changed.md diff --git a/newsfragments/4024.changed.md b/newsfragments/4024.changed.md new file mode 100644 index 00000000000..65afd8f0d0d --- /dev/null +++ b/newsfragments/4024.changed.md @@ -0,0 +1 @@ +Add `#[inline]` hints on many `Bound` and `Borrowed` methods. diff --git a/src/instance.rs b/src/instance.rs index 5f054d0d2d4..88a550ffd30 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -103,6 +103,7 @@ impl<'py> Bound<'py, PyAny> { /// /// - `ptr` must be a valid pointer to a Python object /// - `ptr` must be an owned Python reference, as the `Bound<'py, PyAny>` will assume ownership + #[inline] pub unsafe fn from_owned_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self(py, ManuallyDrop::new(Py::from_owned_ptr(py, ptr))) } @@ -113,6 +114,7 @@ impl<'py> Bound<'py, PyAny> { /// /// - `ptr` must be a valid pointer to a Python object, or null /// - `ptr` must be an owned Python reference, as the `Bound<'py, PyAny>` will assume ownership + #[inline] pub unsafe fn from_owned_ptr_or_opt(py: Python<'py>, ptr: *mut ffi::PyObject) -> Option { Py::from_owned_ptr_or_opt(py, ptr).map(|obj| Self(py, ManuallyDrop::new(obj))) } @@ -124,6 +126,7 @@ impl<'py> Bound<'py, PyAny> { /// /// - `ptr` must be a valid pointer to a Python object, or null /// - `ptr` must be an owned Python reference, as the `Bound<'py, PyAny>` will assume ownership + #[inline] pub unsafe fn from_owned_ptr_or_err( py: Python<'py>, ptr: *mut ffi::PyObject, @@ -137,6 +140,7 @@ impl<'py> Bound<'py, PyAny> { /// # Safety /// /// - `ptr` must be a valid pointer to a Python object + #[inline] pub unsafe fn from_borrowed_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self(py, ManuallyDrop::new(Py::from_borrowed_ptr(py, ptr))) } @@ -147,6 +151,7 @@ impl<'py> Bound<'py, PyAny> { /// # Safety /// /// - `ptr` must be a valid pointer to a Python object, or null + #[inline] pub unsafe fn from_borrowed_ptr_or_opt( py: Python<'py>, ptr: *mut ffi::PyObject, @@ -160,6 +165,7 @@ impl<'py> Bound<'py, PyAny> { /// # Safety /// /// - `ptr` must be a valid pointer to a Python object, or null + #[inline] pub unsafe fn from_borrowed_ptr_or_err( py: Python<'py>, ptr: *mut ffi::PyObject, @@ -235,6 +241,7 @@ where /// /// Panics if the value is currently mutably borrowed. For a non-panicking variant, use /// [`try_borrow`](#method.try_borrow). + #[inline] pub fn borrow(&self) -> PyRef<'py, T> { PyRef::borrow(self) } @@ -268,6 +275,7 @@ where /// # Panics /// Panics if the value is currently borrowed. For a non-panicking variant, use /// [`try_borrow_mut`](#method.try_borrow_mut). + #[inline] pub fn borrow_mut(&self) -> PyRefMut<'py, T> where T: PyClass, @@ -282,6 +290,7 @@ where /// This is the non-panicking variant of [`borrow`](#method.borrow). /// /// For frozen classes, the simpler [`get`][Self::get] is available. + #[inline] pub fn try_borrow(&self) -> Result, PyBorrowError> { PyRef::try_borrow(self) } @@ -291,6 +300,7 @@ where /// The borrow lasts while the returned [`PyRefMut`] exists. /// /// This is the non-panicking variant of [`borrow_mut`](#method.borrow_mut). + #[inline] pub fn try_borrow_mut(&self) -> Result, PyBorrowMutError> where T: PyClass, @@ -321,6 +331,7 @@ where /// py_counter.get().value.fetch_add(1, Ordering::Relaxed); /// }); /// ``` + #[inline] pub fn get(&self) -> &T where T: PyClass + Sync, @@ -328,6 +339,7 @@ where self.1.get() } + #[inline] pub(crate) fn get_class_object(&self) -> &PyClassObject { self.1.get_class_object() } @@ -503,6 +515,7 @@ impl<'py, T> Bound<'py, T> { } unsafe impl AsPyPointer for Bound<'_, T> { + #[inline] fn as_ptr(&self) -> *mut ffi::PyObject { self.1.as_ptr() } @@ -559,6 +572,7 @@ impl<'a, 'py> Borrowed<'a, 'py, PyAny> { /// - similar to `std::slice::from_raw_parts`, the lifetime `'a` is completely defined by /// the caller and it is the caller's responsibility to ensure that the reference this is /// derived from is valid for the lifetime `'a`. + #[inline] pub unsafe fn from_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self( NonNull::new(ptr).unwrap_or_else(|| crate::err::panic_after_error(py)), @@ -578,6 +592,7 @@ impl<'a, 'py> Borrowed<'a, 'py, PyAny> { /// - similar to `std::slice::from_raw_parts`, the lifetime `'a` is completely defined by /// the caller and it is the caller's responsibility to ensure that the reference this is /// derived from is valid for the lifetime `'a`. + #[inline] pub unsafe fn from_ptr_or_opt(py: Python<'py>, ptr: *mut ffi::PyObject) -> Option { NonNull::new(ptr).map(|ptr| Self(ptr, PhantomData, py)) } @@ -594,6 +609,7 @@ impl<'a, 'py> Borrowed<'a, 'py, PyAny> { /// - similar to `std::slice::from_raw_parts`, the lifetime `'a` is completely defined by /// the caller and it is the caller's responsibility to ensure that the reference this is /// derived from is valid for the lifetime `'a`. + #[inline] pub unsafe fn from_ptr_or_err(py: Python<'py>, ptr: *mut ffi::PyObject) -> PyResult { NonNull::new(ptr).map_or_else( || Err(PyErr::fetch(py)), @@ -605,6 +621,7 @@ impl<'a, 'py> Borrowed<'a, 'py, PyAny> { /// This is similar to `std::slice::from_raw_parts`, the lifetime `'a` is completely defined by /// the caller and it's the caller's responsibility to ensure that the reference this is /// derived from is valid for the lifetime `'a`. + #[inline] pub(crate) unsafe fn from_ptr_unchecked(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self(NonNull::new_unchecked(ptr), PhantomData, py) } @@ -627,6 +644,7 @@ impl<'a, 'py> Borrowed<'a, 'py, PyAny> { /// /// # Safety /// Callers must ensure that the type is valid or risk type confusion. + #[inline] pub(crate) unsafe fn downcast_unchecked(self) -> Borrowed<'a, 'py, T> { Borrowed(self.0, PhantomData, self.2) } @@ -634,6 +652,7 @@ impl<'a, 'py> Borrowed<'a, 'py, PyAny> { impl<'a, 'py, T> From<&'a Bound<'py, T>> for Borrowed<'a, 'py, T> { /// Create borrow on a Bound + #[inline] fn from(instance: &'a Bound<'py, T>) -> Self { instance.as_borrowed() } @@ -1118,6 +1137,7 @@ where /// /// Panics if the value is currently mutably borrowed. For a non-panicking variant, use /// [`try_borrow`](#method.try_borrow). + #[inline] pub fn borrow<'py>(&'py self, py: Python<'py>) -> PyRef<'py, T> { self.bind(py).borrow() } @@ -1154,6 +1174,7 @@ where /// # Panics /// Panics if the value is currently borrowed. For a non-panicking variant, use /// [`try_borrow_mut`](#method.try_borrow_mut). + #[inline] pub fn borrow_mut<'py>(&'py self, py: Python<'py>) -> PyRefMut<'py, T> where T: PyClass, @@ -1171,6 +1192,7 @@ where /// /// Equivalent to `self.as_ref(py).borrow_mut()` - /// see [`PyCell::try_borrow`](crate::pycell::PyCell::try_borrow). + #[inline] pub fn try_borrow<'py>(&'py self, py: Python<'py>) -> Result, PyBorrowError> { self.bind(py).try_borrow() } @@ -1183,6 +1205,7 @@ where /// /// Equivalent to `self.as_ref(py).try_borrow_mut()` - /// see [`PyCell::try_borrow_mut`](crate::pycell::PyCell::try_borrow_mut). + #[inline] pub fn try_borrow_mut<'py>( &'py self, py: Python<'py>, @@ -1216,6 +1239,7 @@ where /// /// cell.get().value.fetch_add(1, Ordering::Relaxed); /// ``` + #[inline] pub fn get(&self) -> &T where T: PyClass + Sync, @@ -1225,6 +1249,7 @@ where } /// Get a view on the underlying `PyClass` contents. + #[inline] pub(crate) fn get_class_object(&self) -> &PyClassObject { let class_object = self.as_ptr().cast::>(); // Safety: Bound is known to contain an object which is laid out in memory as a diff --git a/src/pycell.rs b/src/pycell.rs index ccc55a756d6..a649ad02412 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -847,6 +847,7 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { self.inner.clone().into_ptr() } + #[inline] pub(crate) fn borrow(obj: &Bound<'py, T>) -> Self { Self::try_borrow(obj).expect("Already borrowed") } From cff4aa3cc81aa14cfee3fa45c0faea191f38be7e Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 30 Mar 2024 23:26:34 +0000 Subject: [PATCH 256/349] ci: defer test-debug to the merge queue (#4023) --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63f6c31a599..12db5867f90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -439,6 +439,7 @@ jobs: run: nox -s test-emscripten test-debug: + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] runs-on: ubuntu-latest steps: From 3af9a1f4e06216ad7b14a401d3cfcdc66eed12d7 Mon Sep 17 00:00:00 2001 From: Weijie Guo Date: Sun, 31 Mar 2024 16:03:38 +0800 Subject: [PATCH 257/349] docs: fix example in types.md (#4028) --- guide/src/types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/types.md b/guide/src/types.md index a402e52090a..cd5e1052920 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -78,7 +78,7 @@ Or, without the type annotations: use pyo3::prelude::*; use pyo3::types::PyList; -# fn example(py: Python<'_>) -> PyResult<()> { +fn example(py: Python<'_>) -> PyResult<()> { let x = PyList::empty_bound(py); x.append(1)?; let y = x.clone(); From 336b1c982b081aff45468ac64eac374cd79f3683 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 31 Mar 2024 20:55:31 +0100 Subject: [PATCH 258/349] add `import_exception_bound!` macro (#4027) * add `import_exception_bound!` macro * newsfragment and tidy up --- newsfragments/4027.added.md | 1 + src/exceptions.rs | 113 +++++++++++++++++++++++------------- src/impl_.rs | 1 + src/impl_/exceptions.rs | 28 +++++++++ src/sync.rs | 8 ++- 5 files changed, 108 insertions(+), 43 deletions(-) create mode 100644 newsfragments/4027.added.md create mode 100644 src/impl_/exceptions.rs diff --git a/newsfragments/4027.added.md b/newsfragments/4027.added.md new file mode 100644 index 00000000000..ccf32952da2 --- /dev/null +++ b/newsfragments/4027.added.md @@ -0,0 +1 @@ +Add `import_exception_bound!` macro to import exception types without generating GIL Ref functionality for them. diff --git a/src/exceptions.rs b/src/exceptions.rs index 66b5b57dc0e..c650d7af079 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -29,18 +29,7 @@ macro_rules! impl_exception_boilerplate { } } - impl $name { - /// Creates a new [`PyErr`] of this type. - /// - /// [`PyErr`]: https://docs.rs/pyo3/latest/pyo3/struct.PyErr.html "PyErr in pyo3" - #[inline] - pub fn new_err(args: A) -> $crate::PyErr - where - A: $crate::PyErrArguments + ::std::marker::Send + ::std::marker::Sync + 'static, - { - $crate::PyErr::new::<$name, A>(args) - } - } + $crate::impl_exception_boilerplate_bound!($name); impl ::std::error::Error for $name { fn source(&self) -> ::std::option::Option<&(dyn ::std::error::Error + 'static)> { @@ -59,6 +48,25 @@ macro_rules! impl_exception_boilerplate { }; } +#[doc(hidden)] +#[macro_export] +macro_rules! impl_exception_boilerplate_bound { + ($name: ident) => { + impl $name { + /// Creates a new [`PyErr`] of this type. + /// + /// [`PyErr`]: https://docs.rs/pyo3/latest/pyo3/struct.PyErr.html "PyErr in pyo3" + #[inline] + pub fn new_err(args: A) -> $crate::PyErr + where + A: $crate::PyErrArguments + ::std::marker::Send + ::std::marker::Sync + 'static, + { + $crate::PyErr::new::<$name, A>(args) + } + } + }; +} + /// Defines a Rust type for an exception defined in Python code. /// /// # Syntax @@ -105,34 +113,57 @@ macro_rules! import_exception { impl $name { fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { - use $crate::sync::GILOnceCell; - use $crate::prelude::PyTracebackMethods; - use $crate::prelude::PyAnyMethods; - static TYPE_OBJECT: GILOnceCell<$crate::Py<$crate::types::PyType>> = - GILOnceCell::new(); + use $crate::types::PyTypeMethods; + static TYPE_OBJECT: $crate::impl_::exceptions::ImportedExceptionTypeObject = + $crate::impl_::exceptions::ImportedExceptionTypeObject::new(stringify!($module), stringify!($name)); + TYPE_OBJECT.get(py).as_type_ptr() + } + } + }; +} - TYPE_OBJECT - .get_or_init(py, || { - let imp = py - .import_bound(stringify!($module)) - .unwrap_or_else(|err| { - let traceback = err - .traceback_bound(py) - .map(|tb| tb.format().expect("raised exception will have a traceback")) - .unwrap_or_default(); - ::std::panic!("Can not import module {}: {}\n{}", stringify!($module), err, traceback); - }); - let cls = imp.getattr(stringify!($name)).expect(concat!( - "Can not load exception class: ", - stringify!($module), - ".", - stringify!($name) - )); - - cls.extract() - .expect("Imported exception should be a type object") - }) - .as_ptr() as *mut _ +/// Variant of [`import_exception`](crate::import_exception) that does not emit code needed to +/// use the imported exception type as a GIL Ref. +/// +/// This is useful only during migration as a way to avoid generating needless code. +#[macro_export] +macro_rules! import_exception_bound { + ($module: expr, $name: ident) => { + /// A Rust type representing an exception defined in Python code. + /// + /// This type was created by the [`pyo3::import_exception_bound!`] macro - see its documentation + /// for more information. + /// + /// [`pyo3::import_exception_bound!`]: https://docs.rs/pyo3/latest/pyo3/macro.import_exception.html "import_exception in pyo3" + #[repr(transparent)] + #[allow(non_camel_case_types)] // E.g. `socket.herror` + pub struct $name($crate::PyAny); + + $crate::impl_exception_boilerplate_bound!($name); + + // FIXME remove this: was necessary while `PyTypeInfo` requires `HasPyGilRef`, + // should change in 0.22. + unsafe impl $crate::type_object::HasPyGilRef for $name { + type AsRefTarget = $crate::PyAny; + } + + $crate::pyobject_native_type_info!( + $name, + $name::type_object_raw, + ::std::option::Option::Some(stringify!($module)) + ); + + impl $crate::types::DerefToPyAny for $name {} + + impl $name { + fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { + use $crate::types::PyTypeMethods; + static TYPE_OBJECT: $crate::impl_::exceptions::ImportedExceptionTypeObject = + $crate::impl_::exceptions::ImportedExceptionTypeObject::new( + stringify!($module), + stringify!($name), + ); + TYPE_OBJECT.get(py).as_type_ptr() } } }; @@ -849,8 +880,8 @@ mod tests { use crate::types::{IntoPyDict, PyDict}; use crate::{PyErr, PyNativeType}; - import_exception!(socket, gaierror); - import_exception!(email.errors, MessageError); + import_exception_bound!(socket, gaierror); + import_exception_bound!(email.errors, MessageError); #[test] fn test_check_exception() { diff --git a/src/impl_.rs b/src/impl_.rs index ea71b257c0e..71ba397cb94 100644 --- a/src/impl_.rs +++ b/src/impl_.rs @@ -9,6 +9,7 @@ #[cfg(feature = "experimental-async")] pub mod coroutine; pub mod deprecations; +pub mod exceptions; pub mod extract_argument; pub mod freelist; pub mod frompyobject; diff --git a/src/impl_/exceptions.rs b/src/impl_/exceptions.rs new file mode 100644 index 00000000000..eafac1edfa2 --- /dev/null +++ b/src/impl_/exceptions.rs @@ -0,0 +1,28 @@ +use crate::{sync::GILOnceCell, types::PyType, Bound, Py, Python}; + +pub struct ImportedExceptionTypeObject { + imported_value: GILOnceCell>, + module: &'static str, + name: &'static str, +} + +impl ImportedExceptionTypeObject { + pub const fn new(module: &'static str, name: &'static str) -> Self { + Self { + imported_value: GILOnceCell::new(), + module, + name, + } + } + + pub fn get<'py>(&self, py: Python<'py>) -> &Bound<'py, PyType> { + self.imported_value + .get_or_try_init_type_ref(py, self.module, self.name) + .unwrap_or_else(|e| { + panic!( + "failed to import exception {}.{}: {}", + self.module, self.name, e + ) + }) + } +} diff --git a/src/sync.rs b/src/sync.rs index 5af4940461d..856ba84d1e3 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -201,13 +201,17 @@ impl GILOnceCell> { /// /// This is a shorthand method for `get_or_init` which imports the type from Python on init. pub(crate) fn get_or_try_init_type_ref<'py>( - &'py self, + &self, py: Python<'py>, module_name: &str, attr_name: &str, ) -> PyResult<&Bound<'py, PyType>> { self.get_or_try_init(py, || { - py.import_bound(module_name)?.getattr(attr_name)?.extract() + let type_object = py + .import_bound(module_name)? + .getattr(attr_name)? + .downcast_into()?; + Ok(type_object.unbind()) }) .map(|ty| ty.bind(py)) } From 63ba371db021985e01818694cc0987ee213fd4b9 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:10:40 +0200 Subject: [PATCH 259/349] added various std traits for `PyBackedStr` and `PyBackedBytes` (#4020) * added various std traits for `PyBackedStr` and `PyBackedBytes` * add newsfragment * add tests --- newsfragments/4020.added.md | 1 + src/pybacked.rs | 263 +++++++++++++++++++++++++++++++++++- 2 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 newsfragments/4020.added.md diff --git a/newsfragments/4020.added.md b/newsfragments/4020.added.md new file mode 100644 index 00000000000..53d0d6b8fd3 --- /dev/null +++ b/newsfragments/4020.added.md @@ -0,0 +1 @@ +Adds `Clone`, `Debug`, `PartialEq`, `Eq`, `PartialOrd`, `Ord` and `Hash` implementation for `PyBackedBytes` and `PyBackedStr`, and `Display` for `PyBackedStr`. diff --git a/src/pybacked.rs b/src/pybacked.rs index 01b9c2a6f00..e0bacb86144 100644 --- a/src/pybacked.rs +++ b/src/pybacked.rs @@ -1,6 +1,6 @@ //! Contains types for working with Python objects that own the underlying data. -use std::{ops::Deref, ptr::NonNull}; +use std::{ops::Deref, ptr::NonNull, sync::Arc}; use crate::{ types::{ @@ -13,6 +13,7 @@ use crate::{ /// A wrapper around `str` where the storage is owned by a Python `bytes` or `str` object. /// /// This type gives access to the underlying data via a `Deref` implementation. +#[derive(Clone)] pub struct PyBackedStr { #[allow(dead_code)] // only held so that the storage is not dropped storage: Py, @@ -44,6 +45,14 @@ impl AsRef<[u8]> for PyBackedStr { unsafe impl Send for PyBackedStr {} unsafe impl Sync for PyBackedStr {} +impl std::fmt::Display for PyBackedStr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.deref().fmt(f) + } +} + +impl_traits!(PyBackedStr, str); + impl TryFrom> for PyBackedStr { type Error = PyErr; fn try_from(py_string: Bound<'_, PyString>) -> Result { @@ -79,6 +88,7 @@ impl FromPyObject<'_> for PyBackedStr { /// A wrapper around `[u8]` where the storage is either owned by a Python `bytes` object, or a Rust `Box<[u8]>`. /// /// This type gives access to the underlying data via a `Deref` implementation. +#[derive(Clone)] pub struct PyBackedBytes { #[allow(dead_code)] // only held so that the storage is not dropped storage: PyBackedBytesStorage, @@ -86,9 +96,10 @@ pub struct PyBackedBytes { } #[allow(dead_code)] +#[derive(Clone)] enum PyBackedBytesStorage { Python(Py), - Rust(Box<[u8]>), + Rust(Arc<[u8]>), } impl Deref for PyBackedBytes { @@ -110,6 +121,32 @@ impl AsRef<[u8]> for PyBackedBytes { unsafe impl Send for PyBackedBytes {} unsafe impl Sync for PyBackedBytes {} +impl PartialEq<[u8; N]> for PyBackedBytes { + fn eq(&self, other: &[u8; N]) -> bool { + self.deref() == other + } +} + +impl PartialEq for [u8; N] { + fn eq(&self, other: &PyBackedBytes) -> bool { + self == other.deref() + } +} + +impl PartialEq<&[u8; N]> for PyBackedBytes { + fn eq(&self, other: &&[u8; N]) -> bool { + self.deref() == *other + } +} + +impl PartialEq for &[u8; N] { + fn eq(&self, other: &PyBackedBytes) -> bool { + self == &other.deref() + } +} + +impl_traits!(PyBackedBytes, [u8]); + impl From> for PyBackedBytes { fn from(py_bytes: Bound<'_, PyBytes>) -> Self { let b = py_bytes.as_bytes(); @@ -123,7 +160,7 @@ impl From> for PyBackedBytes { impl From> for PyBackedBytes { fn from(py_bytearray: Bound<'_, PyByteArray>) -> Self { - let s = py_bytearray.to_vec().into_boxed_slice(); + let s = Arc::<[u8]>::from(py_bytearray.to_vec()); let data = NonNull::from(s.as_ref()); Self { storage: PyBackedBytesStorage::Rust(s), @@ -144,10 +181,85 @@ impl FromPyObject<'_> for PyBackedBytes { } } +macro_rules! impl_traits { + ($slf:ty, $equiv:ty) => { + impl std::fmt::Debug for $slf { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.deref().fmt(f) + } + } + + impl PartialEq for $slf { + fn eq(&self, other: &Self) -> bool { + self.deref() == other.deref() + } + } + + impl PartialEq<$equiv> for $slf { + fn eq(&self, other: &$equiv) -> bool { + self.deref() == other + } + } + + impl PartialEq<&$equiv> for $slf { + fn eq(&self, other: &&$equiv) -> bool { + self.deref() == *other + } + } + + impl PartialEq<$slf> for $equiv { + fn eq(&self, other: &$slf) -> bool { + self == other.deref() + } + } + + impl PartialEq<$slf> for &$equiv { + fn eq(&self, other: &$slf) -> bool { + self == &other.deref() + } + } + + impl Eq for $slf {} + + impl PartialOrd for $slf { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl PartialOrd<$equiv> for $slf { + fn partial_cmp(&self, other: &$equiv) -> Option { + self.deref().partial_cmp(other) + } + } + + impl PartialOrd<$slf> for $equiv { + fn partial_cmp(&self, other: &$slf) -> Option { + self.partial_cmp(other.deref()) + } + } + + impl Ord for $slf { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.deref().cmp(other.deref()) + } + } + + impl std::hash::Hash for $slf { + fn hash(&self, state: &mut H) { + self.deref().hash(state) + } + } + }; +} +use impl_traits; + #[cfg(test)] mod test { use super::*; use crate::Python; + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; #[test] fn py_backed_str_empty() { @@ -223,4 +335,149 @@ mod test { is_send::(); is_sync::(); } + + #[test] + fn test_backed_str_clone() { + Python::with_gil(|py| { + let s1: PyBackedStr = PyString::new_bound(py, "hello").try_into().unwrap(); + let s2 = s1.clone(); + assert_eq!(s1, s2); + + drop(s1); + assert_eq!(s2, "hello"); + }); + } + + #[test] + fn test_backed_str_eq() { + Python::with_gil(|py| { + let s1: PyBackedStr = PyString::new_bound(py, "hello").try_into().unwrap(); + let s2: PyBackedStr = PyString::new_bound(py, "hello").try_into().unwrap(); + assert_eq!(s1, "hello"); + assert_eq!(s1, s2); + + let s3: PyBackedStr = PyString::new_bound(py, "abcde").try_into().unwrap(); + assert_eq!("abcde", s3); + assert_ne!(s1, s3); + }); + } + + #[test] + fn test_backed_str_hash() { + Python::with_gil(|py| { + let h = { + let mut hasher = DefaultHasher::new(); + "abcde".hash(&mut hasher); + hasher.finish() + }; + + let s1: PyBackedStr = PyString::new_bound(py, "abcde").try_into().unwrap(); + let h1 = { + let mut hasher = DefaultHasher::new(); + s1.hash(&mut hasher); + hasher.finish() + }; + + assert_eq!(h, h1); + }); + } + + #[test] + fn test_backed_str_ord() { + Python::with_gil(|py| { + let mut a = vec!["a", "c", "d", "b", "f", "g", "e"]; + let mut b = a + .iter() + .map(|s| PyString::new_bound(py, s).try_into().unwrap()) + .collect::>(); + + a.sort(); + b.sort(); + + assert_eq!(a, b); + }) + } + + #[test] + fn test_backed_bytes_from_bytes_clone() { + Python::with_gil(|py| { + let b1: PyBackedBytes = PyBytes::new_bound(py, b"abcde").into(); + let b2 = b1.clone(); + assert_eq!(b1, b2); + + drop(b1); + assert_eq!(b2, b"abcde"); + }); + } + + #[test] + fn test_backed_bytes_from_bytearray_clone() { + Python::with_gil(|py| { + let b1: PyBackedBytes = PyByteArray::new_bound(py, b"abcde").into(); + let b2 = b1.clone(); + assert_eq!(b1, b2); + + drop(b1); + assert_eq!(b2, b"abcde"); + }); + } + + #[test] + fn test_backed_bytes_eq() { + Python::with_gil(|py| { + let b1: PyBackedBytes = PyBytes::new_bound(py, b"abcde").into(); + let b2: PyBackedBytes = PyByteArray::new_bound(py, b"abcde").into(); + + assert_eq!(b1, b"abcde"); + assert_eq!(b1, b2); + + let b3: PyBackedBytes = PyBytes::new_bound(py, b"hello").into(); + assert_eq!(b"hello", b3); + assert_ne!(b1, b3); + }); + } + + #[test] + fn test_backed_bytes_hash() { + Python::with_gil(|py| { + let h = { + let mut hasher = DefaultHasher::new(); + b"abcde".hash(&mut hasher); + hasher.finish() + }; + + let b1: PyBackedBytes = PyBytes::new_bound(py, b"abcde").into(); + let h1 = { + let mut hasher = DefaultHasher::new(); + b1.hash(&mut hasher); + hasher.finish() + }; + + let b2: PyBackedBytes = PyByteArray::new_bound(py, b"abcde").into(); + let h2 = { + let mut hasher = DefaultHasher::new(); + b2.hash(&mut hasher); + hasher.finish() + }; + + assert_eq!(h, h1); + assert_eq!(h, h2); + }); + } + + #[test] + fn test_backed_bytes_ord() { + Python::with_gil(|py| { + let mut a = vec![b"a", b"c", b"d", b"b", b"f", b"g", b"e"]; + let mut b = a + .iter() + .map(|&b| PyBytes::new_bound(py, b).into()) + .collect::>(); + + a.sort(); + b.sort(); + + assert_eq!(a, b); + }) + } } From 8f87b8636df73ef7d1690bd848389382280aa335 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:10:18 +0200 Subject: [PATCH 260/349] refactor `#[setter]` argument extraction (#4002) --- pyo3-macros-backend/src/method.rs | 4 ++ pyo3-macros-backend/src/params.rs | 39 +++++++----- pyo3-macros-backend/src/pymethod.rs | 97 +++++++++++++++++------------ src/impl_/extract_argument.rs | 4 +- tests/ui/static_ref.stderr | 23 +++---- 5 files changed, 97 insertions(+), 70 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index f4fdb19377a..155c554025d 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -63,6 +63,10 @@ impl<'a> FnArg<'a> { } } } + + pub fn is_regular(&self) -> bool { + !self.py && !self.is_cancel_handle && !self.is_kwargs && !self.is_varargs + } } fn handle_argument_error(pat: &syn::Pat) -> syn::Error { diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index cab28698b71..fa50d260986 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -73,7 +73,7 @@ pub fn is_forwarded_args(signature: &FunctionSignature<'_>) -> bool { ) } -fn check_arg_for_gil_refs( +pub(crate) fn check_arg_for_gil_refs( tokens: TokenStream, gil_refs_checker: syn::Ident, ctx: &Ctx, @@ -120,7 +120,11 @@ pub fn impl_arg_params( .iter() .enumerate() .map(|(i, arg)| { - impl_arg_param(arg, i, &mut 0, &args_array, holders, ctx).map(|tokens| { + let from_py_with = + syn::Ident::new(&format!("from_py_with_{}", i), Span::call_site()); + let arg_value = quote!(#args_array[0].as_deref()); + + impl_arg_param(arg, from_py_with, arg_value, holders, ctx).map(|tokens| { check_arg_for_gil_refs( tokens, holders.push_gil_refs_checker(arg.ty.span()), @@ -161,14 +165,20 @@ pub fn impl_arg_params( let num_params = positional_parameter_names.len() + keyword_only_parameters.len(); - let mut option_pos = 0; + let mut option_pos = 0usize; let param_conversion = spec .signature .arguments .iter() .enumerate() .map(|(i, arg)| { - impl_arg_param(arg, i, &mut option_pos, &args_array, holders, ctx).map(|tokens| { + let from_py_with = syn::Ident::new(&format!("from_py_with_{}", i), Span::call_site()); + let arg_value = quote!(#args_array[#option_pos].as_deref()); + if arg.is_regular() { + option_pos += 1; + } + + impl_arg_param(arg, from_py_with, arg_value, holders, ctx).map(|tokens| { check_arg_for_gil_refs(tokens, holders.push_gil_refs_checker(arg.ty.span()), ctx) }) }) @@ -234,11 +244,10 @@ pub fn impl_arg_params( /// Re option_pos: The option slice doesn't contain the py: Python argument, so the argument /// index and the index in option diverge when using py: Python -fn impl_arg_param( +pub(crate) fn impl_arg_param( arg: &FnArg<'_>, - pos: usize, - option_pos: &mut usize, - args_array: &syn::Ident, + from_py_with: syn::Ident, + arg_value: TokenStream, // expected type: Option<&'a Bound<'py, PyAny>> holders: &mut Holders, ctx: &Ctx, ) -> Result { @@ -291,9 +300,6 @@ fn impl_arg_param( }); } - let arg_value = quote_arg_span!(#args_array[#option_pos]); - *option_pos += 1; - let mut default = arg.default.as_ref().map(|expr| quote!(#expr)); // Option arguments have special treatment: the default should be specified _without_ the @@ -312,11 +318,10 @@ fn impl_arg_param( .map(|attr| &attr.value) .is_some() { - let from_py_with = syn::Ident::new(&format!("from_py_with_{}", pos), Span::call_site()); if let Some(default) = default { quote_arg_span! { #pyo3_path::impl_::extract_argument::from_py_with_with_default( - #arg_value.as_deref(), + #arg_value, #name_str, #from_py_with as fn(_) -> _, #[allow(clippy::redundant_closure)] @@ -328,7 +333,7 @@ fn impl_arg_param( } else { quote_arg_span! { #pyo3_path::impl_::extract_argument::from_py_with( - &#pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value), + #pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value), #name_str, #from_py_with as fn(_) -> _, )? @@ -338,7 +343,7 @@ fn impl_arg_param( let holder = holders.push_holder(arg.name.span()); quote_arg_span! { #pyo3_path::impl_::extract_argument::extract_optional_argument( - #arg_value.as_deref(), + #arg_value, &mut #holder, #name_str, #[allow(clippy::redundant_closure)] @@ -351,7 +356,7 @@ fn impl_arg_param( let holder = holders.push_holder(arg.name.span()); quote_arg_span! { #pyo3_path::impl_::extract_argument::extract_argument_with_default( - #arg_value.as_deref(), + #arg_value, &mut #holder, #name_str, #[allow(clippy::redundant_closure)] @@ -364,7 +369,7 @@ fn impl_arg_param( let holder = holders.push_holder(arg.name.span()); quote_arg_span! { #pyo3_path::impl_::extract_argument::extract_argument( - &#pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value), + #pyo3_path::impl_::extract_argument::unwrap_required_argument(#arg_value), &mut #holder, #name_str )? diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 22802a01177..ee7d3d7aaee 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use crate::attributes::{NameAttribute, RenamingRule}; use crate::method::{CallingConvention, ExtractErrorMode}; -use crate::params::Holders; +use crate::params::{check_arg_for_gil_refs, impl_arg_param, Holders}; use crate::utils::Ctx; use crate::utils::PythonDoc; use crate::{ @@ -586,48 +586,63 @@ pub fn impl_py_setter_def( } }; - // TODO: rework this to make use of `impl_::params::impl_arg_param` which - // handles all these cases already. - let extract = if let PropertyType::Function { spec, .. } = &property_type { - Some(spec) - } else { - None - } - .and_then(|spec| { - let (_, args) = split_off_python_arg(&spec.signature.arguments); - let value_arg = &args[0]; - let from_py_with = &value_arg.attrs.from_py_with.as_ref()?.value; - let name = value_arg.name.to_string(); - - Some(quote_spanned! { from_py_with.span() => - let e = #pyo3_path::impl_::deprecations::GilRefs::new(); - let from_py_with = #pyo3_path::impl_::deprecations::inspect_fn(#from_py_with, &e); - e.from_py_with_arg(); - let _val = #pyo3_path::impl_::extract_argument::from_py_with( - &_value.into(), - #name, - from_py_with as fn(_) -> _, - )?; - }) - }) - .unwrap_or_else(|| { - let (span, name) = match &property_type { - PropertyType::Descriptor { field, .. } => (field.ty.span(), field.ident.as_ref().map(|i|i.to_string()).unwrap_or_default()), - PropertyType::Function { spec, .. } => { - let (_, args) = split_off_python_arg(&spec.signature.arguments); - (args[0].ty.span(), args[0].name.to_string()) - } - }; + let extract = match &property_type { + PropertyType::Function { spec, .. } => { + let (_, args) = split_off_python_arg(&spec.signature.arguments); + let value_arg = &args[0]; + let (from_py_with, ident) = if let Some(from_py_with) = + &value_arg.attrs.from_py_with.as_ref().map(|f| &f.value) + { + let ident = syn::Ident::new("from_py_with", from_py_with.span()); + ( + quote_spanned! { from_py_with.span() => + let e = #pyo3_path::impl_::deprecations::GilRefs::new(); + let #ident = #pyo3_path::impl_::deprecations::inspect_fn(#from_py_with, &e); + e.from_py_with_arg(); + }, + ident, + ) + } else { + (quote!(), syn::Ident::new("dummy", Span::call_site())) + }; - let holder = holders.push_holder(span); - let gil_refs_checker = holders.push_gil_refs_checker(span); - quote! { - let _val = #pyo3_path::impl_::deprecations::inspect_type( - #pyo3_path::impl_::extract_argument::extract_argument(_value.into(), &mut #holder, #name)?, - &#gil_refs_checker - ); + let extract = impl_arg_param( + &args[0], + ident, + quote!(::std::option::Option::Some(_value.into())), + &mut holders, + ctx, + ) + .map(|tokens| { + check_arg_for_gil_refs( + tokens, + holders.push_gil_refs_checker(value_arg.ty.span()), + ctx, + ) + })?; + quote! { + #from_py_with + let _val = #extract; + } + } + PropertyType::Descriptor { field, .. } => { + let span = field.ty.span(); + let name = field + .ident + .as_ref() + .map(|i| i.to_string()) + .unwrap_or_default(); + + let holder = holders.push_holder(span); + let gil_refs_checker = holders.push_gil_refs_checker(span); + quote! { + let _val = #pyo3_path::impl_::deprecations::inspect_type( + #pyo3_path::impl_::extract_argument::extract_argument(_value.into(), &mut #holder, #name)?, + &#gil_refs_checker + ); + } } - }); + }; let mut cfg_attrs = TokenStream::new(); if let PropertyType::Descriptor { field, .. } = &property_type { diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index 4dcef02c5e6..485b8645086 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -223,7 +223,9 @@ pub fn argument_extraction_error(py: Python<'_>, arg_name: &str, error: PyErr) - /// `argument` must not be `None` #[doc(hidden)] #[inline] -pub unsafe fn unwrap_required_argument(argument: Option>) -> PyArg<'_> { +pub unsafe fn unwrap_required_argument<'a, 'py>( + argument: Option<&'a Bound<'py, PyAny>>, +) -> &'a Bound<'py, PyAny> { match argument { Some(value) => value, #[cfg(debug_assertions)] diff --git a/tests/ui/static_ref.stderr b/tests/ui/static_ref.stderr index 50b054f6245..6004c4037e5 100644 --- a/tests/ui/static_ref.stderr +++ b/tests/ui/static_ref.stderr @@ -9,6 +9,18 @@ error: lifetime may not live long enough | = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) +error[E0597]: `output[_]` does not live long enough + --> tests/ui/static_ref.rs:4:1 + | +4 | #[pyfunction] + | ^^^^^^^^^^^^- + | | | + | | `output[_]` dropped here while still borrowed + | borrowed value does not live long enough + | argument requires that `output[_]` is borrowed for `'static` + | + = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) + error[E0597]: `holder_0` does not live long enough --> tests/ui/static_ref.rs:5:15 | @@ -21,17 +33,6 @@ error[E0597]: `holder_0` does not live long enough 5 | fn static_ref(list: &'static Bound<'_, PyList>) -> usize { | ^^^^^^^ borrowed value does not live long enough -error[E0716]: temporary value dropped while borrowed - --> tests/ui/static_ref.rs:5:21 - | -4 | #[pyfunction] - | ------------- - | | | - | | temporary value is freed at the end of this statement - | argument requires that borrow lasts for `'static` -5 | fn static_ref(list: &'static Bound<'_, PyList>) -> usize { - | ^ creates a temporary value which is freed while still in use - error: lifetime may not live long enough --> tests/ui/static_ref.rs:9:1 | From 8cabd2619c349ffede8e141c09e540c0c83adb79 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 1 Apr 2024 16:18:57 +0100 Subject: [PATCH 261/349] docs: updates to guide for PyO3 0.21 feedback (#4031) * docs: add notes on smart pointer conversions * guide: add more notes on `.extract::<&str>()` to migration guide --- guide/src/migration.md | 16 +++++++++++++++- guide/src/types.md | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index 69cf8922255..d855f69d396 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -246,14 +246,26 @@ Because the new `Bound` API brings ownership out of the PyO3 framework and in - `Bound::iter_borrowed` is slightly more efficient than `Bound::iter`. The default iteration of `Bound` cannot return borrowed references because Rust does not (yet) have "lending iterators". Similarly `Bound::get_borrowed_item` is more efficient than `Bound::get_item` for the same reason. - `&Bound` does not implement `FromPyObject` (although it might be possible to do this in the future once the GIL Refs API is completely removed). Use `bound_any.downcast::()` instead of `bound_any.extract::<&Bound>()`. - `Bound::to_str` now borrows from the `Bound` rather than from the `'py` lifetime, so code will need to store the smart pointer as a value in some cases where previously `&PyString` was just used as a temporary. (There are some more details relating to this in [the section below](#deactivating-the-gil-refs-feature).) +- `.extract::<&str>()` now borrows from the source Python object. The simplest way to update is to change to `.extract::()`, which retains ownership of the Python reference. See more information [in the section on deactivating the `gil-refs` feature](#deactivating-the-gil-refs-feature). -To convert between `&PyAny` and `&Bound` you can use the `as_borrowed()` method: +To convert between `&PyAny` and `&Bound` use the `as_borrowed()` method: ```rust,ignore let gil_ref: &PyAny = ...; let bound: &Bound = &gil_ref.as_borrowed(); ``` +To convert between `Py` and `Bound` use the `bind()` / `into_bound()` methods, and `as_unbound()` / `unbind()` to go back from `Bound` to `Py`. + +```rust,ignore +let obj: Py = ...; +let bound: &Bound<'py, PyList> = obj.bind(py); +let bound: Bound<'py, PyList> = obj.into_bound(py); + +let obj: &Py = bound.as_unbound(); +let obj: Py = bound.unbind(); +``` +
⚠️ Warning: dangling pointer trap 💣 @@ -325,6 +337,8 @@ There is just one case of code that changes upon disabling these features: `From To make PyO3's core functionality continue to work while the GIL Refs API is in the process of being removed, disabling the `gil-refs` feature moves the implementations of `FromPyObject` for `&str`, `Cow<'_, str>`, `&[u8]`, `Cow<'_, u8>` to a new temporary trait `FromPyObjectBound`. This trait is the expected future form of `FromPyObject` and has an additional lifetime `'a` to enable these types to borrow data from Python objects. +PyO3 0.21 has introduced the [`PyBackedStr`]({{#PYO3_DOCS_URL}}/pyo3/pybacked/struct.PyBackedStr.html) and [`PyBackedBytes`]({{#PYO3_DOCS_URL}}/pyo3/pybacked/struct.PyBackedBytes.html) types to help with this case. The easiest way to avoid lifetime challenges from extracting `&str` is to use these. For more complex types like `Vec<&str>`, is now impossible to extract directly from a Python object and `Vec` is the recommended upgrade path. + A key thing to note here is because extracting to these types now ties them to the input lifetime, some extremely common patterns may need to be split into multiple Rust lines. For example, the following snippet of calling `.extract::<&str>()` directly on the result of `.getattr()` needs to be adjusted when deactivating the `gil-refs-migration` feature. Before: diff --git a/guide/src/types.md b/guide/src/types.md index cd5e1052920..4c63d175991 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -160,6 +160,45 @@ for i in 0..=2 { # Python::with_gil(example).unwrap(); ``` +### Casting between smart pointer types + +To convert between `Py` and `Bound<'py, T>` use the `bind()` / `into_bound()` methods. Use the `as_unbound()` / `unbind()` methods to go back from `Bound<'py, T>` to `Py`. + +```rust,ignore +let obj: Py = ...; +let bound: &Bound<'py, PyAny> = obj.bind(py); +let bound: Bound<'py, PyAny> = obj.into_bound(py); + +let obj: &Py = bound.as_unbound(); +let obj: Py = bound.unbind(); +``` + +To convert between `Bound<'py, T>` and `Borrowed<'a, 'py, T>` use the `as_borrowed()` method. `Borrowed<'a, 'py, T>` has a deref coercion to `Bound<'py, T>`. Use the `to_owned()` method to increment the Python reference count and to create a new `Bound<'py, T>` from the `Borrowed<'a, 'py, T>`. + +```rust,ignore +let bound: Bound<'py, PyAny> = ...; +let borrowed: Borrowed<'_, 'py, PyAny> = bound.as_borrowed(); + +// deref coercion +let bound: &Bound<'py, PyAny> = &borrowed; + +// create a new Bound by increase the Python reference count +let bound: Bound<'py, PyAny> = borrowed.to_owned(); +``` + +To convert between `Py` and `Borrowed<'a, 'py, T>` use the `bind_borrowed()` method. Use either `as_unbound()` or `.to_owned().unbind()` to go back to `Py` from `Borrowed<'a, 'py, T>`, via `Bound<'py, T>`. + +```rust,ignore +let obj: Py = ...; +let borrowed: Borrowed<'_, 'py, PyAny> = bound.as_borrowed(); + +// via deref coercion to Bound and then using Bound::as_unbound +let obj: &Py = borrowed.as_unbound(); + +// via a new Bound by increasing the Python reference count, and unbind it +let obj: Py = borrowed.to_owned().unbind(). +``` + ## Concrete Python types In all of `Py`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`, the type parameter `T` denotes the type of the Python object referred to by the smart pointer. From c1f11fb4bddc836f820e0db3db514b4742b8a3e7 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Mon, 1 Apr 2024 19:51:58 +0100 Subject: [PATCH 262/349] release: 0.21.1 (#4032) --- CHANGELOG.md | 24 +++++++++++++++++++ Cargo.toml | 8 +++---- README.md | 4 ++-- examples/decorator/.template/pre-script.rhai | 2 +- .../maturin-starter/.template/pre-script.rhai | 2 +- examples/plugin/.template/pre-script.rhai | 2 +- .../.template/pre-script.rhai | 2 +- examples/word-count/.template/pre-script.rhai | 2 +- newsfragments/3995.fixed.md | 1 - newsfragments/3998.changed.md | 1 - newsfragments/3998.fixed.md | 1 - newsfragments/4007.fixed.md | 1 - newsfragments/4009.fixed.md | 1 - newsfragments/4012.fixed.md | 1 - newsfragments/4015.fixed.md | 1 - newsfragments/4020.added.md | 1 - newsfragments/4024.changed.md | 1 - newsfragments/4027.added.md | 1 - pyo3-build-config/Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 4 ++-- pyo3-macros-backend/Cargo.toml | 4 ++-- pyo3-macros/Cargo.toml | 4 ++-- pyproject.toml | 2 +- 23 files changed, 43 insertions(+), 29 deletions(-) delete mode 100644 newsfragments/3995.fixed.md delete mode 100644 newsfragments/3998.changed.md delete mode 100644 newsfragments/3998.fixed.md delete mode 100644 newsfragments/4007.fixed.md delete mode 100644 newsfragments/4009.fixed.md delete mode 100644 newsfragments/4012.fixed.md delete mode 100644 newsfragments/4015.fixed.md delete mode 100644 newsfragments/4020.added.md delete mode 100644 newsfragments/4024.changed.md delete mode 100644 newsfragments/4027.added.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 047a4b21e1a..6f4ce218021 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,28 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.21.1] - 2024-04-01 + +### Added + +- Implement `Send` and `Sync` for `PyBackedStr` and `PyBackedBytes`. [#4007](https://github.com/PyO3/pyo3/pull/4007) +- Implement `Clone`, `Debug`, `PartialEq`, `Eq`, `PartialOrd`, `Ord` and `Hash` implementation for `PyBackedBytes` and `PyBackedStr`, and `Display` for `PyBackedStr`. [#4020](https://github.com/PyO3/pyo3/pull/4020) +- Add `import_exception_bound!` macro to import exception types without generating GIL Ref functionality for them. [#4027](https://github.com/PyO3/pyo3/pull/4027) + +### Changed + +- Emit deprecation warning for uses of GIL Refs as `#[setter]` function arguments. [#3998](https://github.com/PyO3/pyo3/pull/3998) +- Add `#[inline]` hints on many `Bound` and `Borrowed` methods. [#4024](https://github.com/PyO3/pyo3/pull/4024) + +### Fixed + +- Handle `#[pyo3(from_py_with = "")]` in `#[setter]` methods [#3995](https://github.com/PyO3/pyo3/pull/3995) +- Allow extraction of `&Bound` in `#[setter]` methods. [#3998](https://github.com/PyO3/pyo3/pull/3998) +- Fix some uncovered code blocks emitted by `#[pymodule]`, `#[pyfunction]` and `#[pyclass]` macros. [#4009](https://github.com/PyO3/pyo3/pull/4009) +- Fix typo in the panic message when a class referenced in `pyo3::import_exception!` does not exist. [#4012](https://github.com/PyO3/pyo3/pull/4012) +- Fix compile error when using an async `#[pymethod]` with a receiver and additional arguments. [#4015](https://github.com/PyO3/pyo3/pull/4015) + + ## [0.21.0] - 2024-03-25 ### Added @@ -1709,6 +1731,8 @@ Yanked - Initial release +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.21.1...HEAD +[0.21.1]: https://github.com/pyo3/pyo3/compare/v0.21.0...v0.21.1 [0.21.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0 [0.21.0-beta.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0-beta.0 [0.20.3]: https://github.com/pyo3/pyo3/compare/v0.20.2...v0.20.3 diff --git a/Cargo.toml b/Cargo.toml index dfbeb2d601d..fdd1fa9d29a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.21.0" +version = "0.21.1" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -22,10 +22,10 @@ memoffset = "0.9" portable-atomic = "1.0" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.0" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.1" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.21.0", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.21.1", optional = true } indoc = { version = "2.0.1", optional = true } unindent = { version = "0.2.1", optional = true } @@ -60,7 +60,7 @@ rayon = "1.6.1" futures = "0.3.28" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.0", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.1", features = ["resolve-config"] } [features] default = ["macros"] diff --git a/README.md b/README.md index 7d79f028dcd..8e7e2f75bca 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.21.0", features = ["extension-module"] } +pyo3 = { version = "0.21.1", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.21.0" +version = "0.21.1" features = ["auto-initialize"] ``` diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index cebb06c0f61..1dc689ef6d4 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.0"); +variable::set("PYO3_VERSION", "0.21.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index cebb06c0f61..1dc689ef6d4 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.0"); +variable::set("PYO3_VERSION", "0.21.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 4b290f7f7e0..78adb883f43 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.0"); +variable::set("PYO3_VERSION", "0.21.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index fbbe7ed0e7c..212f62f76fe 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.0"); +variable::set("PYO3_VERSION", "0.21.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index cebb06c0f61..1dc689ef6d4 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.0"); +variable::set("PYO3_VERSION", "0.21.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/newsfragments/3995.fixed.md b/newsfragments/3995.fixed.md deleted file mode 100644 index e47a71b790d..00000000000 --- a/newsfragments/3995.fixed.md +++ /dev/null @@ -1 +0,0 @@ -handle `#[pyo3(from_py_with = "")]` in `#[setter]` methods \ No newline at end of file diff --git a/newsfragments/3998.changed.md b/newsfragments/3998.changed.md deleted file mode 100644 index c02c6546c95..00000000000 --- a/newsfragments/3998.changed.md +++ /dev/null @@ -1 +0,0 @@ -Warn on uses of GIL Refs for `#[setter]` function arguments. \ No newline at end of file diff --git a/newsfragments/3998.fixed.md b/newsfragments/3998.fixed.md deleted file mode 100644 index 11f5d006ec0..00000000000 --- a/newsfragments/3998.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Allow extraction of `&Bound` in `#[setter]` methods. \ No newline at end of file diff --git a/newsfragments/4007.fixed.md b/newsfragments/4007.fixed.md deleted file mode 100644 index ff905fb4e94..00000000000 --- a/newsfragments/4007.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Implement `Send` and `Sync` for `PyBackedStr` and `PyBackedBytes`. diff --git a/newsfragments/4009.fixed.md b/newsfragments/4009.fixed.md deleted file mode 100644 index a5d378e12ad..00000000000 --- a/newsfragments/4009.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix some uncovered code blocks emitted by `#[pymodule]`, `#[pyfunction]` and `#[pyclass]` macros. diff --git a/newsfragments/4012.fixed.md b/newsfragments/4012.fixed.md deleted file mode 100644 index 352ec928487..00000000000 --- a/newsfragments/4012.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixed the error message when a class referenced in `pyo3::import_exception!` does not exist diff --git a/newsfragments/4015.fixed.md b/newsfragments/4015.fixed.md deleted file mode 100644 index a8f4f636c76..00000000000 --- a/newsfragments/4015.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix the bug that an async `#[pymethod]` with receiver can't have any other args. \ No newline at end of file diff --git a/newsfragments/4020.added.md b/newsfragments/4020.added.md deleted file mode 100644 index 53d0d6b8fd3..00000000000 --- a/newsfragments/4020.added.md +++ /dev/null @@ -1 +0,0 @@ -Adds `Clone`, `Debug`, `PartialEq`, `Eq`, `PartialOrd`, `Ord` and `Hash` implementation for `PyBackedBytes` and `PyBackedStr`, and `Display` for `PyBackedStr`. diff --git a/newsfragments/4024.changed.md b/newsfragments/4024.changed.md deleted file mode 100644 index 65afd8f0d0d..00000000000 --- a/newsfragments/4024.changed.md +++ /dev/null @@ -1 +0,0 @@ -Add `#[inline]` hints on many `Bound` and `Borrowed` methods. diff --git a/newsfragments/4027.added.md b/newsfragments/4027.added.md deleted file mode 100644 index ccf32952da2..00000000000 --- a/newsfragments/4027.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `import_exception_bound!` macro to import exception types without generating GIL Ref functionality for them. diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index f19ded99697..1eb269c2132 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.21.0" +version = "0.21.1" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 091e5f898ef..64753976fab 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.21.0" +version = "0.21.1" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,7 +38,7 @@ abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312"] generate-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.1", features = ["resolve-config"] } [lints] workspace = true diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 822dc372b47..6ca4eeade8c 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.21.0" +version = "0.21.1" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -16,7 +16,7 @@ edition = "2021" [dependencies] heck = "0.4" proc-macro2 = { version = "1", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.0", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.1", features = ["resolve-config"] } quote = { version = "1", default-features = false } [dependencies.syn] diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index a70ccb0ae7b..39a4b9198c6 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.21.0" +version = "0.21.1" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -22,7 +22,7 @@ experimental-declarative-modules = [] proc-macro2 = { version = "1", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.0" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.1" } [lints] workspace = true diff --git a/pyproject.toml b/pyproject.toml index 1a3cfa4aaad..d474753ccd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.towncrier] filename = "CHANGELOG.md" -version = "0.21.0" +version = "0.21.1" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" From a4aea230ed6087a5fccc415d6456c536645e0df1 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 2 Apr 2024 18:43:51 +0100 Subject: [PATCH 263/349] fix compile error for multiple async method arguments (#4035) --- newsfragments/4035.fixed.md | 1 + pyo3-macros-backend/src/method.rs | 25 ++++-------- tests/test_coroutine.rs | 66 +++++++++++++++---------------- 3 files changed, 41 insertions(+), 51 deletions(-) create mode 100644 newsfragments/4035.fixed.md diff --git a/newsfragments/4035.fixed.md b/newsfragments/4035.fixed.md new file mode 100644 index 00000000000..5425c5cbaf7 --- /dev/null +++ b/newsfragments/4035.fixed.md @@ -0,0 +1 @@ +Fix compile error for `async fn` in `#[pymethods]` with a `&self` receiver and more than one additional argument. diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 155c554025d..6af0ec97d04 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -522,33 +522,22 @@ impl<'a> FnSpec<'a> { Some(cls) => quote!(Some(<#cls as #pyo3_path::PyTypeInfo>::NAME)), None => quote!(None), }; - let evaluate_args = || -> (Vec, TokenStream) { - let mut arg_names = Vec::with_capacity(args.len()); - let mut evaluate_arg = quote! {}; - for arg in &args { - let arg_name = format_ident!("arg_{}", arg_names.len()); - arg_names.push(arg_name.clone()); - evaluate_arg.extend(quote! { - let #arg_name = #arg - }); - } - (arg_names, evaluate_arg) - }; + let arg_names = (0..args.len()) + .map(|i| format_ident!("arg_{}", i)) + .collect::>(); let future = match self.tp { FnType::Fn(SelfType::Receiver { mutable: false, .. }) => { - let (arg_name, evaluate_arg) = evaluate_args(); quote! {{ - #evaluate_arg; + #(let #arg_names = #args;)* let __guard = #pyo3_path::impl_::coroutine::RefGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; - async move { function(&__guard, #(#arg_name),*).await } + async move { function(&__guard, #(#arg_names),*).await } }} } FnType::Fn(SelfType::Receiver { mutable: true, .. }) => { - let (arg_name, evaluate_arg) = evaluate_args(); quote! {{ - #evaluate_arg; + #(let #arg_names = #args;)* let mut __guard = #pyo3_path::impl_::coroutine::RefMutGuard::<#cls>::new(&#pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_slf))?; - async move { function(&mut __guard, #(#arg_name),*).await } + async move { function(&mut __guard, #(#arg_names),*).await } }} } _ => { diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index 0e698deb9af..4abba9f36b4 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -245,39 +245,6 @@ fn coroutine_panic() { }) } -#[test] -fn test_async_method_receiver_with_other_args() { - #[pyclass] - struct Value(i32); - #[pymethods] - impl Value { - #[new] - fn new() -> Self { - Self(0) - } - async fn get_value_plus_with(&self, v: i32) -> i32 { - self.0 + v - } - async fn set_value(&mut self, new_value: i32) -> i32 { - self.0 = new_value; - self.0 - } - } - - Python::with_gil(|gil| { - let test = r#" - import asyncio - - v = Value() - assert asyncio.run(v.get_value_plus_with(3)) == 3 - assert asyncio.run(v.set_value(10)) == 10 - assert asyncio.run(v.get_value_plus_with(1)) == 11 - "#; - let locals = [("Value", gil.get_type_bound::())].into_py_dict_bound(gil); - py_run!(gil, *locals, test); - }); -} - #[test] fn test_async_method_receiver() { #[pyclass] @@ -341,3 +308,36 @@ fn test_async_method_receiver() { assert!(IS_DROPPED.load(Ordering::SeqCst)); } + +#[test] +fn test_async_method_receiver_with_other_args() { + #[pyclass] + struct Value(i32); + #[pymethods] + impl Value { + #[new] + fn new() -> Self { + Self(0) + } + async fn get_value_plus_with(&self, v1: i32, v2: i32) -> i32 { + self.0 + v1 + v2 + } + async fn set_value(&mut self, new_value: i32) -> i32 { + self.0 = new_value; + self.0 + } + } + + Python::with_gil(|gil| { + let test = r#" + import asyncio + + v = Value() + assert asyncio.run(v.get_value_plus_with(3, 0)) == 3 + assert asyncio.run(v.set_value(10)) == 10 + assert asyncio.run(v.get_value_plus_with(1, 1)) == 12 + "#; + let locals = [("Value", gil.get_type_bound::())].into_py_dict_bound(gil); + py_run!(gil, *locals, test); + }); +} From 7a00b4d357c0250c2212e856782143f283a7c2bb Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Thu, 4 Apr 2024 21:08:51 +0200 Subject: [PATCH 264/349] add descriptive error msg for `__traverse__` receivers other than `&self` (#4045) * add descriptive error msg for `__traverse__` receivers other than `self` * add newsfragment * improve error message --- newsfragments/4045.fixed.md | 1 + pyo3-macros-backend/src/pymethod.rs | 21 ++++++++++-- tests/ui/traverse.rs | 50 ++++++++++++++++++++++++++--- tests/ui/traverse.stderr | 42 +++++++++++++----------- 4 files changed, 88 insertions(+), 26 deletions(-) create mode 100644 newsfragments/4045.fixed.md diff --git a/newsfragments/4045.fixed.md b/newsfragments/4045.fixed.md new file mode 100644 index 00000000000..6b2bbcfa01d --- /dev/null +++ b/newsfragments/4045.fixed.md @@ -0,0 +1 @@ +Add better error message on wrong receiver extraction in `__traverse__`. diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index ee7d3d7aaee..abe8c7ac8a3 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -434,9 +434,24 @@ fn impl_traverse_slot( let Ctx { pyo3_path } = ctx; if let (Some(py_arg), _) = split_off_python_arg(&spec.signature.arguments) { return Err(syn::Error::new_spanned(py_arg.ty, "__traverse__ may not take `Python`. \ - Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. \ - Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, \ - i.e. `Python::with_gil` will panic.")); + Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` \ + should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited \ + inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic.")); + } + + // check that the receiver does not try to smuggle an (implicit) `Python` token into here + if let FnType::Fn(SelfType::TryFromBoundRef(span)) + | FnType::Fn(SelfType::Receiver { + mutable: true, + span, + }) = spec.tp + { + bail_spanned! { span => + "__traverse__ may not take a receiver other than `&self`. Usually, an implementation of \ + `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` \ + should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited \ + inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic." + } } let rust_fn_ident = spec.name; diff --git a/tests/ui/traverse.rs b/tests/ui/traverse.rs index 034224951c9..0cf7170db21 100644 --- a/tests/ui/traverse.rs +++ b/tests/ui/traverse.rs @@ -1,13 +1,55 @@ use pyo3::prelude::*; -use pyo3::PyVisit; use pyo3::PyTraverseError; +use pyo3::PyVisit; #[pyclass] struct TraverseTriesToTakePyRef {} #[pymethods] impl TraverseTriesToTakePyRef { - fn __traverse__(slf: PyRef, visit: PyVisit) {} + fn __traverse__(slf: PyRef, visit: PyVisit) -> Result<(), PyTraverseError> { + Ok(()) + } +} + +#[pyclass] +struct TraverseTriesToTakePyRefMut {} + +#[pymethods] +impl TraverseTriesToTakePyRefMut { + fn __traverse__(slf: PyRefMut, visit: PyVisit) -> Result<(), PyTraverseError> { + Ok(()) + } +} + +#[pyclass] +struct TraverseTriesToTakeBound {} + +#[pymethods] +impl TraverseTriesToTakeBound { + fn __traverse__(slf: Bound<'_, Self>, visit: PyVisit) -> Result<(), PyTraverseError> { + Ok(()) + } +} + +#[pyclass] +struct TraverseTriesToTakeMutSelf {} + +#[pymethods] +impl TraverseTriesToTakeMutSelf { + fn __traverse__(&mut self, visit: PyVisit) -> Result<(), PyTraverseError> { + Ok(()) + } +} + +#[pyclass] +struct TraverseTriesToTakeSelf {} + +#[pymethods] +impl TraverseTriesToTakeSelf { + fn __traverse__(&self, visit: PyVisit) -> Result<(), PyTraverseError> { + Ok(()) + } } #[pyclass] @@ -19,9 +61,7 @@ impl Class { Ok(()) } - fn __clear__(&mut self) { - } + fn __clear__(&mut self) {} } - fn main() {} diff --git a/tests/ui/traverse.stderr b/tests/ui/traverse.stderr index 4448e67e13a..5b1d1b6b2ec 100644 --- a/tests/ui/traverse.stderr +++ b/tests/ui/traverse.stderr @@ -1,23 +1,29 @@ -error: __traverse__ may not take `Python`. Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. - --> tests/ui/traverse.rs:18:32 +error: __traverse__ may not take a receiver other than `&self`. Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. + --> tests/ui/traverse.rs:10:26 | -18 | fn __traverse__(&self, py: Python<'_>, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { - | ^^^^^^^^^^ +10 | fn __traverse__(slf: PyRef, visit: PyVisit) -> Result<(), PyTraverseError> { + | ^^^^^ + +error: __traverse__ may not take a receiver other than `&self`. Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. + --> tests/ui/traverse.rs:20:26 + | +20 | fn __traverse__(slf: PyRefMut, visit: PyVisit) -> Result<(), PyTraverseError> { + | ^^^^^^^^ -error[E0308]: mismatched types - --> tests/ui/traverse.rs:9:6 +error: __traverse__ may not take a receiver other than `&self`. Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. + --> tests/ui/traverse.rs:30:26 | -8 | #[pymethods] - | ------------ arguments to this function are incorrect -9 | impl TraverseTriesToTakePyRef { - | ______^ -10 | | fn __traverse__(slf: PyRef, visit: PyVisit) {} - | |___________________^ expected fn pointer, found fn item +30 | fn __traverse__(slf: Bound<'_, Self>, visit: PyVisit) -> Result<(), PyTraverseError> { + | ^^^^^ + +error: __traverse__ may not take a receiver other than `&self`. Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. + --> tests/ui/traverse.rs:40:21 | - = note: expected fn pointer `for<'a, 'b> fn(&'a TraverseTriesToTakePyRef, PyVisit<'b>) -> Result<(), PyTraverseError>` - found fn item `for<'a, 'b> fn(pyo3::PyRef<'a, TraverseTriesToTakePyRef, >, PyVisit<'b>) {TraverseTriesToTakePyRef::__traverse__}` -note: function defined here - --> src/impl_/pymethods.rs +40 | fn __traverse__(&mut self, visit: PyVisit) -> Result<(), PyTraverseError> { + | ^ + +error: __traverse__ may not take `Python`. Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. + --> tests/ui/traverse.rs:60:32 | - | pub unsafe fn _call_traverse( - | ^^^^^^^^^^^^^^ +60 | fn __traverse__(&self, py: Python<'_>, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + | ^^^^^^^^^^ From 2f0869a6d643e790e71dd5a676a30a7e539e5138 Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Sun, 7 Apr 2024 00:36:52 +0200 Subject: [PATCH 265/349] fix: allow impl of Ungil for deprecated PyCell in nightly (#4053) --- src/marker.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/marker.rs b/src/marker.rs index 67119b5e55e..b1f2d399209 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -280,6 +280,7 @@ mod nightly { impl !Ungil for crate::PyAny {} // All the borrowing wrappers + #[allow(deprecated)] impl !Ungil for crate::PyCell {} impl !Ungil for crate::PyRef<'_, T> {} impl !Ungil for crate::PyRefMut<'_, T> {} From 47f9ef41747ff1e7d89f584797ea9fc0d88dd1d5 Mon Sep 17 00:00:00 2001 From: "Jeong, Heon" Date: Thu, 11 Apr 2024 14:07:56 -0700 Subject: [PATCH 266/349] Fix typo (#4052) Fix incorrect closing brackets --- guide/src/ecosystem/async-await.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index 0319fa05063..9c0ad19bdef 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -120,7 +120,7 @@ Export an async function that makes use of `async-std`: use pyo3::{prelude::*, wrap_pyfunction}; #[pyfunction] -fn rust_sleep(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>>> { +fn rust_sleep(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { pyo3_asyncio::async_std::future_into_py(py, async { async_std::task::sleep(std::time::Duration::from_secs(1)).await; Ok(Python::with_gil(|py| py.None())) From 631c25f2f9ef2d2f73ea1510cc6aba0581b0af16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 21:08:44 +0000 Subject: [PATCH 267/349] build(deps): bump peaceiris/actions-gh-pages from 3 to 4 (#4062) Bumps [peaceiris/actions-gh-pages](https://github.com/peaceiris/actions-gh-pages) from 3 to 4. - [Release notes](https://github.com/peaceiris/actions-gh-pages/releases) - [Changelog](https://github.com/peaceiris/actions-gh-pages/blob/main/CHANGELOG.md) - [Commits](https://github.com/peaceiris/actions-gh-pages/compare/v3...v4) --- updated-dependencies: - dependency-name: peaceiris/actions-gh-pages dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index a9a7669054a..a101eb6ad25 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -46,7 +46,7 @@ jobs: - name: Deploy docs and the guide if: ${{ github.event_name == 'release' }} - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./target/guide/ From 9a6b1962d39c0acd6ddc42b186151611018cbd69 Mon Sep 17 00:00:00 2001 From: Liam Date: Thu, 11 Apr 2024 22:11:51 +0100 Subject: [PATCH 268/349] Minor: Fix a typo in Contributing.md (#4066) --- Contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Contributing.md b/Contributing.md index 29d1bb758a7..054099b431a 100644 --- a/Contributing.md +++ b/Contributing.md @@ -190,7 +190,7 @@ Second, there is a Python-based benchmark contained in the `pytests` subdirector You can view what code is and isn't covered by PyO3's tests. We aim to have 100% coverage - please check coverage and add tests if you notice a lack of coverage! -- First, ensure the llmv-cov cargo plugin is installed. You may need to run the plugin through cargo once before using it with `nox`. +- First, ensure the llvm-cov cargo plugin is installed. You may need to run the plugin through cargo once before using it with `nox`. ```shell cargo install cargo-llvm-cov cargo llvm-cov From c8b59d71176a371f9feaa8abfa884f9fd451fbb5 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 12 Apr 2024 08:34:08 +0200 Subject: [PATCH 269/349] add `#[doc(hidden)]` to the Rust module created by `#[pymodule]` (#4067) * add `#[doc(hidden)]` to the Rust module created by `#[pymodule]` * add newsfragment --- newsfragments/4067.fixed.md | 1 + pyo3-macros-backend/src/module.rs | 1 + tests/test_compile_error.rs | 1 + tests/ui/pymodule_missing_docs.rs | 12 ++++++++++++ 4 files changed, 15 insertions(+) create mode 100644 newsfragments/4067.fixed.md create mode 100644 tests/ui/pymodule_missing_docs.rs diff --git a/newsfragments/4067.fixed.md b/newsfragments/4067.fixed.md new file mode 100644 index 00000000000..869b6addf15 --- /dev/null +++ b/newsfragments/4067.fixed.md @@ -0,0 +1 @@ +fixes `missing_docs` lint to trigger on documented `#[pymodule]` functions \ No newline at end of file diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 080a279a88c..8ff720cc616 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -324,6 +324,7 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result Ok(quote! { #function + #[doc(hidden)] #vis mod #ident { #initialization } diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 38b5c4727c6..44049620598 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -53,4 +53,5 @@ fn test_compile_errors() { #[cfg(feature = "experimental-async")] #[cfg(any(not(Py_LIMITED_API), Py_3_10))] // to avoid PyFunctionArgument for &str t.compile_fail("tests/ui/invalid_cancel_handle.rs"); + t.pass("tests/ui/pymodule_missing_docs.rs"); } diff --git a/tests/ui/pymodule_missing_docs.rs b/tests/ui/pymodule_missing_docs.rs new file mode 100644 index 00000000000..1b196fa65e0 --- /dev/null +++ b/tests/ui/pymodule_missing_docs.rs @@ -0,0 +1,12 @@ +#![deny(missing_docs)] +//! Some crate docs + +use pyo3::prelude::*; + +/// Some module documentation +#[pymodule] +pub fn python_module(_m: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} + +fn main() {} From ee5216f406889a9b993bfbe2e9dd977027e57075 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 12 Apr 2024 08:34:27 +0200 Subject: [PATCH 270/349] fix declarative-modules compile error (#4054) * fix declarative-modules compile error * add newsfragment --- newsfragments/4054.fixed.md | 1 + pyo3-macros-backend/src/pyclass.rs | 2 +- tests/test_declarative_module.rs | 11 +++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 newsfragments/4054.fixed.md diff --git a/newsfragments/4054.fixed.md b/newsfragments/4054.fixed.md new file mode 100644 index 00000000000..4b8da92ca4d --- /dev/null +++ b/newsfragments/4054.fixed.md @@ -0,0 +1 @@ +Fixes a compile error when exporting a `#[pyclass]` living in a different Rust module using the declarative-module feature. diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index cf8d9aae801..aff23b879f5 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1614,7 +1614,7 @@ impl<'a> PyClassImplsBuilder<'a> { quote! { impl #cls { #[doc(hidden)] - const _PYO3_DEF: #pyo3_path::impl_::pymodule::AddClassToModule = #pyo3_path::impl_::pymodule::AddClassToModule::new(); + pub const _PYO3_DEF: #pyo3_path::impl_::pymodule::AddClassToModule = #pyo3_path::impl_::pymodule::AddClassToModule::new(); } } } diff --git a/tests/test_declarative_module.rs b/tests/test_declarative_module.rs index 5f8e2c9fa55..8e432c3ae58 100644 --- a/tests/test_declarative_module.rs +++ b/tests/test_declarative_module.rs @@ -9,6 +9,13 @@ use pyo3::types::PyBool; #[path = "../src/tests/common.rs"] mod common; +mod some_module { + use pyo3::prelude::*; + + #[pyclass] + pub struct SomePyClass; +} + #[pyclass] struct ValueClass { value: usize, @@ -50,6 +57,10 @@ mod declarative_module { #[pymodule_export] use super::{declarative_module2, double, MyError, ValueClass as Value}; + // test for #4036 + #[pymodule_export] + use super::some_module::SomePyClass; + #[pymodule] mod inner { use super::*; From 30348b4d3fc433508cdc9b71ae7a973ec5e40235 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 13 Apr 2024 15:43:06 +0800 Subject: [PATCH 271/349] Link libpython for AIX target (#4073) --- newsfragments/4073.fixed.md | 1 + pyo3-build-config/src/impl_.rs | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 newsfragments/4073.fixed.md diff --git a/newsfragments/4073.fixed.md b/newsfragments/4073.fixed.md new file mode 100644 index 00000000000..0f77647e42d --- /dev/null +++ b/newsfragments/4073.fixed.md @@ -0,0 +1 @@ +fixes undefined symbol errors when building extension module on AIX by linking `libpython` diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index d5373db9655..2ee68503faa 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -775,6 +775,8 @@ pub fn is_linking_libpython() -> bool { /// Must be called from a PyO3 crate build script. fn is_linking_libpython_for_target(target: &Triple) -> bool { target.operating_system == OperatingSystem::Windows + // See https://github.com/PyO3/pyo3/issues/4068#issuecomment-2051159852 + || target.operating_system == OperatingSystem::Aix || target.environment == Environment::Android || target.environment == Environment::Androideabi || !is_extension_module() From 4e5167db4241e9c003c922b148fefd870eb0edad Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sat, 13 Apr 2024 09:57:13 +0200 Subject: [PATCH 272/349] Extend guide on interaction between method receivers and lifetime elision. (#4069) --- guide/src/class.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/guide/src/class.md b/guide/src/class.md index f353cc4787e..b5ef95cb2f7 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -998,6 +998,44 @@ impl MyClass { Note that `text_signature` on `#[new]` is not compatible with compilation in `abi3` mode until Python 3.10 or greater. +### Method receivers and lifetime elision + +PyO3 supports writing instance methods using the normal method receivers for shared `&self` and unique `&mut self` references. This interacts with [lifetime elision][lifetime-elision] insofar as the lifetime of a such a receiver is assigned to all elided output lifetime parameters. + +This is a good default for general Rust code where return values are more likely to borrow from the receiver than from the other arguments, if they contain any lifetimes at all. However, when returning bound references `Bound<'py, T>` in PyO3-based code, the GIL lifetime `'py` should usually be derived from a GIL token `py: Python<'py>` passed as an argument instead of the receiver. + +Specifically, signatures like + +```rust,ignore +fn frobnicate(&self, py: Python) -> Bound; +``` + +will not work as they are inferred as + +```rust,ignore +fn frobnicate<'a, 'py>(&'a self, py: Python<'py>) -> Bound<'a, Foo>; +``` + +instead of the intended + +```rust,ignore +fn frobnicate<'a, 'py>(&'a self, py: Python<'py>) -> Bound<'py, Foo>; +``` + +and should usually be written as + +```rust,ignore +fn frobnicate<'py>(&self, py: Python<'py>) -> Bound<'py, Foo>; +``` + +The same problem does not exist for `#[pyfunction]`s as the special case for receiver lifetimes does not apply and indeed a signature like + +```rust,ignore +fn frobnicate(bar: &Bar, py: Python) -> Bound; +``` + +will yield compiler error [E0106 "missing lifetime specifier"][compiler-error-e0106]. + ## `#[pyclass]` enums Enum support in PyO3 comes in two flavors, depending on what kind of variants the enum has: simple and complex. @@ -1329,3 +1367,6 @@ impl pyo3::impl_::pyclass::PyClassImpl for MyClass { [classattr]: https://docs.python.org/3/tutorial/classes.html#class-and-instance-variables [`multiple-pymethods`]: features.md#multiple-pymethods + +[lifetime-elision]: https://doc.rust-lang.org/reference/lifetime-elision.html +[compiler-error-e0106]: https://doc.rust-lang.org/error_codes/E0106.html From 721100a9e981391de784022bf564285f772314ea Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sat, 13 Apr 2024 14:59:44 +0200 Subject: [PATCH 273/349] Suppress non_local_definitions lint as we often want the non-local effects in macro code (#4074) * Resolve references to legacy numerical constants and use the associated constants instead * Suppress non_local_definitions lint as we often want the non-local effects in macro code --- pyo3-ffi/src/pyport.rs | 4 ++-- pyo3-macros-backend/src/module.rs | 2 ++ pyo3-macros-backend/src/pyfunction.rs | 1 + pyo3-macros-backend/src/pymethod.rs | 1 + src/callback.rs | 7 +------ src/conversions/std/num.rs | 28 +++++++++++++-------------- src/impl_/pyclass.rs | 1 + src/pycell/impl_.rs | 2 +- src/types/any.rs | 1 + tests/test_proto_methods.rs | 2 +- 10 files changed, 25 insertions(+), 24 deletions(-) diff --git a/pyo3-ffi/src/pyport.rs b/pyo3-ffi/src/pyport.rs index 741b0db7bf8..a144c67fb1b 100644 --- a/pyo3-ffi/src/pyport.rs +++ b/pyo3-ffi/src/pyport.rs @@ -11,8 +11,8 @@ pub type Py_ssize_t = ::libc::ssize_t; pub type Py_hash_t = Py_ssize_t; pub type Py_uhash_t = ::libc::size_t; -pub const PY_SSIZE_T_MIN: Py_ssize_t = std::isize::MIN as Py_ssize_t; -pub const PY_SSIZE_T_MAX: Py_ssize_t = std::isize::MAX as Py_ssize_t; +pub const PY_SSIZE_T_MIN: Py_ssize_t = isize::MIN as Py_ssize_t; +pub const PY_SSIZE_T_MAX: Py_ssize_t = isize::MAX as Py_ssize_t; #[cfg(target_endian = "big")] pub const PY_BIG_ENDIAN: usize = 1; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 8ff720cc616..3153279a2b8 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -249,6 +249,7 @@ pub fn pymodule_module_impl(mut module: syn::ItemMod) -> Result { #initialization + #[allow(unknown_lints, non_local_definitions)] impl MakeDef { const fn make_def() -> #pyo3_path::impl_::pymodule::ModuleDef { use #pyo3_path::impl_::pymodule as impl_; @@ -333,6 +334,7 @@ pub fn pymodule_function_impl(mut function: syn::ItemFn) -> Result // this avoids complications around the fact that the generated module has a different scope // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pymodule] is // inside a function body) + #[allow(unknown_lints, non_local_definitions)] impl #ident::MakeDef { const fn make_def() -> #pyo3_path::impl_::pymodule::ModuleDef { fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 9f9557a3664..7c355533b83 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -273,6 +273,7 @@ pub fn impl_wrap_pyfunction( // this avoids complications around the fact that the generated module has a different scope // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pyfunction] is // inside a function body) + #[allow(unknown_lints, non_local_definitions)] impl #name::MakeDef { const _PYO3_DEF: #pyo3_path::impl_::pymethods::PyMethodDef = #methoddef; } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index abe8c7ac8a3..7a4c54db9da 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -366,6 +366,7 @@ pub fn impl_py_method_def_new( #deprecations use #pyo3_path::impl_::pyclass::*; + #[allow(unknown_lints, non_local_definitions)] impl PyClassNewTextSignature<#cls> for PyClassImplCollector<#cls> { #[inline] fn new_text_signature(self) -> ::std::option::Option<&'static str> { diff --git a/src/callback.rs b/src/callback.rs index a56b268aa1e..1e446039904 100644 --- a/src/callback.rs +++ b/src/callback.rs @@ -4,7 +4,6 @@ use crate::err::{PyErr, PyResult}; use crate::exceptions::PyOverflowError; use crate::ffi::{self, Py_hash_t}; use crate::{IntoPy, PyObject, Python}; -use std::isize; use std::os::raw::c_int; /// A type which can be the return type of a python C-API callback @@ -85,11 +84,7 @@ impl IntoPyCallbackOutput<()> for () { impl IntoPyCallbackOutput for usize { #[inline] fn convert(self, _py: Python<'_>) -> PyResult { - if self <= (isize::MAX as usize) { - Ok(self as isize) - } else { - Err(PyOverflowError::new_err(())) - } + self.try_into().map_err(|_err| PyOverflowError::new_err(())) } } diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 44843141440..e2072d210e0 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -445,7 +445,7 @@ mod test_128bit_integers { #[test] fn test_i128_max() { Python::with_gil(|py| { - let v = std::i128::MAX; + let v = i128::MAX; let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert_eq!(v as u128, obj.extract::(py).unwrap()); @@ -456,7 +456,7 @@ mod test_128bit_integers { #[test] fn test_i128_min() { Python::with_gil(|py| { - let v = std::i128::MIN; + let v = i128::MIN; let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert!(obj.extract::(py).is_err()); @@ -467,7 +467,7 @@ mod test_128bit_integers { #[test] fn test_u128_max() { Python::with_gil(|py| { - let v = std::u128::MAX; + let v = u128::MAX; let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert!(obj.extract::(py).is_err()); @@ -495,7 +495,7 @@ mod test_128bit_integers { #[test] fn test_nonzero_i128_max() { Python::with_gil(|py| { - let v = NonZeroI128::new(std::i128::MAX).unwrap(); + let v = NonZeroI128::new(i128::MAX).unwrap(); let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert_eq!( @@ -509,7 +509,7 @@ mod test_128bit_integers { #[test] fn test_nonzero_i128_min() { Python::with_gil(|py| { - let v = NonZeroI128::new(std::i128::MIN).unwrap(); + let v = NonZeroI128::new(i128::MIN).unwrap(); let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert!(obj.extract::(py).is_err()); @@ -520,7 +520,7 @@ mod test_128bit_integers { #[test] fn test_nonzero_u128_max() { Python::with_gil(|py| { - let v = NonZeroU128::new(std::u128::MAX).unwrap(); + let v = NonZeroU128::new(u128::MAX).unwrap(); let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert!(obj.extract::(py).is_err()); @@ -573,7 +573,7 @@ mod tests { #[test] fn test_u32_max() { Python::with_gil(|py| { - let v = std::u32::MAX; + let v = u32::MAX; let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert_eq!(u64::from(v), obj.extract::(py).unwrap()); @@ -584,7 +584,7 @@ mod tests { #[test] fn test_i64_max() { Python::with_gil(|py| { - let v = std::i64::MAX; + let v = i64::MAX; let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert_eq!(v as u64, obj.extract::(py).unwrap()); @@ -595,7 +595,7 @@ mod tests { #[test] fn test_i64_min() { Python::with_gil(|py| { - let v = std::i64::MIN; + let v = i64::MIN; let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert!(obj.extract::(py).is_err()); @@ -606,7 +606,7 @@ mod tests { #[test] fn test_u64_max() { Python::with_gil(|py| { - let v = std::u64::MAX; + let v = u64::MAX; let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert!(obj.extract::(py).is_err()); @@ -664,7 +664,7 @@ mod tests { #[test] fn test_nonzero_u32_max() { Python::with_gil(|py| { - let v = NonZeroU32::new(std::u32::MAX).unwrap(); + let v = NonZeroU32::new(u32::MAX).unwrap(); let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert_eq!(NonZeroU64::from(v), obj.extract::(py).unwrap()); @@ -675,7 +675,7 @@ mod tests { #[test] fn test_nonzero_i64_max() { Python::with_gil(|py| { - let v = NonZeroI64::new(std::i64::MAX).unwrap(); + let v = NonZeroI64::new(i64::MAX).unwrap(); let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert_eq!( @@ -689,7 +689,7 @@ mod tests { #[test] fn test_nonzero_i64_min() { Python::with_gil(|py| { - let v = NonZeroI64::new(std::i64::MIN).unwrap(); + let v = NonZeroI64::new(i64::MIN).unwrap(); let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert!(obj.extract::(py).is_err()); @@ -700,7 +700,7 @@ mod tests { #[test] fn test_nonzero_u64_max() { Python::with_gil(|py| { - let v = NonZeroU64::new(std::u64::MAX).unwrap(); + let v = NonZeroU64::new(u64::MAX).unwrap(); let obj = v.to_object(py); assert_eq!(v, obj.extract::(py).unwrap()); assert!(obj.extract::(py).is_err()); diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 1a144f736e0..1302834ca4b 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -852,6 +852,7 @@ slot_fragment_trait! { #[macro_export] macro_rules! generate_pyclass_richcompare_slot { ($cls:ty) => {{ + #[allow(unknown_lints, non_local_definitions)] impl $cls { #[allow(non_snake_case)] unsafe extern "C" fn __pymethod___richcmp____( diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 378bec04993..1bdd44b9e9c 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -55,7 +55,7 @@ struct BorrowFlag(usize); impl BorrowFlag { pub(crate) const UNUSED: BorrowFlag = BorrowFlag(0); - const HAS_MUTABLE_BORROW: BorrowFlag = BorrowFlag(usize::max_value()); + const HAS_MUTABLE_BORROW: BorrowFlag = BorrowFlag(usize::MAX); const fn increment(self) -> Self { Self(self.0 + 1) } diff --git a/src/types/any.rs b/src/types/any.rs index ab4f5727623..a5ea4c80d4c 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -2491,6 +2491,7 @@ class SimpleClass: #[cfg(feature = "macros")] #[test] + #[allow(unknown_lints, non_local_definitions)] fn test_hasattr_error() { use crate::exceptions::PyValueError; use crate::prelude::*; diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index dd707990c0b..c5d7306086d 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -3,7 +3,7 @@ use pyo3::exceptions::{PyAttributeError, PyIndexError, PyValueError}; use pyo3::types::{PyDict, PyList, PyMapping, PySequence, PySlice, PyType}; use pyo3::{prelude::*, py_run}; -use std::{isize, iter}; +use std::iter; #[path = "../src/tests/common.rs"] mod common; From cc7e16f4d6c1c4e9a19d0b514a4ef8ece6d14776 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sun, 14 Apr 2024 16:19:57 +0200 Subject: [PATCH 274/349] Refactoring of `FnArg` (#4033) * refactor `FnArg` * add UI tests * use enum variant types * add comment * remove dead code * remove last FIXME * review feedback davidhewitt --- pyo3-macros-backend/src/method.rs | 179 ++++++++++++++---- pyo3-macros-backend/src/params.rs | 172 +++++++---------- pyo3-macros-backend/src/pyclass.rs | 30 +-- .../src/pyfunction/signature.rs | 110 +++++++---- pyo3-macros-backend/src/pymethod.rs | 53 +++--- tests/ui/invalid_cancel_handle.rs | 6 + tests/ui/invalid_cancel_handle.stderr | 6 + tests/ui/invalid_pyfunctions.rs | 10 +- tests/ui/invalid_pyfunctions.stderr | 20 +- 9 files changed, 348 insertions(+), 238 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 6af0ec97d04..982cf62946e 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -6,7 +6,7 @@ use syn::{ext::IdentExt, spanned::Spanned, Ident, Result}; use crate::utils::Ctx; use crate::{ - attributes::{TextSignatureAttribute, TextSignatureAttributeValue}, + attributes::{FromPyWithAttribute, TextSignatureAttribute, TextSignatureAttributeValue}, deprecations::{Deprecation, Deprecations}, params::{impl_arg_params, Holders}, pyfunction::{ @@ -17,19 +17,109 @@ use crate::{ }; #[derive(Clone, Debug)] -pub struct FnArg<'a> { +pub struct RegularArg<'a> { pub name: &'a syn::Ident, pub ty: &'a syn::Type, - pub optional: Option<&'a syn::Type>, - pub default: Option, - pub py: bool, - pub attrs: PyFunctionArgPyO3Attributes, - pub is_varargs: bool, - pub is_kwargs: bool, - pub is_cancel_handle: bool, + pub from_py_with: Option, + pub default_value: Option, + pub option_wrapped_type: Option<&'a syn::Type>, +} + +/// Pythons *args argument +#[derive(Clone, Debug)] +pub struct VarargsArg<'a> { + pub name: &'a syn::Ident, + pub ty: &'a syn::Type, +} + +/// Pythons **kwarg argument +#[derive(Clone, Debug)] +pub struct KwargsArg<'a> { + pub name: &'a syn::Ident, + pub ty: &'a syn::Type, +} + +#[derive(Clone, Debug)] +pub struct CancelHandleArg<'a> { + pub name: &'a syn::Ident, + pub ty: &'a syn::Type, +} + +#[derive(Clone, Debug)] +pub struct PyArg<'a> { + pub name: &'a syn::Ident, + pub ty: &'a syn::Type, +} + +#[derive(Clone, Debug)] +pub enum FnArg<'a> { + Regular(RegularArg<'a>), + VarArgs(VarargsArg<'a>), + KwArgs(KwargsArg<'a>), + Py(PyArg<'a>), + CancelHandle(CancelHandleArg<'a>), } impl<'a> FnArg<'a> { + pub fn name(&self) -> &'a syn::Ident { + match self { + FnArg::Regular(RegularArg { name, .. }) => name, + FnArg::VarArgs(VarargsArg { name, .. }) => name, + FnArg::KwArgs(KwargsArg { name, .. }) => name, + FnArg::Py(PyArg { name, .. }) => name, + FnArg::CancelHandle(CancelHandleArg { name, .. }) => name, + } + } + + pub fn ty(&self) -> &'a syn::Type { + match self { + FnArg::Regular(RegularArg { ty, .. }) => ty, + FnArg::VarArgs(VarargsArg { ty, .. }) => ty, + FnArg::KwArgs(KwargsArg { ty, .. }) => ty, + FnArg::Py(PyArg { ty, .. }) => ty, + FnArg::CancelHandle(CancelHandleArg { ty, .. }) => ty, + } + } + + #[allow(clippy::wrong_self_convention)] + pub fn from_py_with(&self) -> Option<&FromPyWithAttribute> { + if let FnArg::Regular(RegularArg { from_py_with, .. }) = self { + from_py_with.as_ref() + } else { + None + } + } + + pub fn to_varargs_mut(&mut self) -> Result<&mut Self> { + if let Self::Regular(RegularArg { + name, + ty, + option_wrapped_type: None, + .. + }) = self + { + *self = Self::VarArgs(VarargsArg { name, ty }); + Ok(self) + } else { + bail_spanned!(self.name().span() => "args cannot be optional") + } + } + + pub fn to_kwargs_mut(&mut self) -> Result<&mut Self> { + if let Self::Regular(RegularArg { + name, + ty, + option_wrapped_type: Some(..), + .. + }) = self + { + *self = Self::KwArgs(KwargsArg { name, ty }); + Ok(self) + } else { + bail_spanned!(self.name().span() => "kwargs must be Option<_>") + } + } + /// Transforms a rust fn arg parsed with syn into a method::FnArg pub fn parse(arg: &'a mut syn::FnArg) -> Result { match arg { @@ -41,32 +131,43 @@ impl<'a> FnArg<'a> { bail_spanned!(cap.ty.span() => IMPL_TRAIT_ERR); } - let arg_attrs = PyFunctionArgPyO3Attributes::from_attrs(&mut cap.attrs)?; + let PyFunctionArgPyO3Attributes { + from_py_with, + cancel_handle, + } = PyFunctionArgPyO3Attributes::from_attrs(&mut cap.attrs)?; let ident = match &*cap.pat { syn::Pat::Ident(syn::PatIdent { ident, .. }) => ident, other => return Err(handle_argument_error(other)), }; - let is_cancel_handle = arg_attrs.cancel_handle.is_some(); + if utils::is_python(&cap.ty) { + return Ok(Self::Py(PyArg { + name: ident, + ty: &cap.ty, + })); + } - Ok(FnArg { + if cancel_handle.is_some() { + // `PyFunctionArgPyO3Attributes::from_attrs` validates that + // only compatible attributes are specified, either + // `cancel_handle` or `from_py_with`, dublicates and any + // combination of the two are already rejected. + return Ok(Self::CancelHandle(CancelHandleArg { + name: ident, + ty: &cap.ty, + })); + } + + Ok(Self::Regular(RegularArg { name: ident, ty: &cap.ty, - optional: utils::option_type_argument(&cap.ty), - default: None, - py: utils::is_python(&cap.ty), - attrs: arg_attrs, - is_varargs: false, - is_kwargs: false, - is_cancel_handle, - }) + from_py_with, + default_value: None, + option_wrapped_type: utils::option_type_argument(&cap.ty), + })) } } } - - pub fn is_regular(&self) -> bool { - !self.py && !self.is_cancel_handle && !self.is_kwargs && !self.is_varargs - } } fn handle_argument_error(pat: &syn::Pat) -> syn::Error { @@ -492,12 +593,14 @@ impl<'a> FnSpec<'a> { .signature .arguments .iter() - .filter(|arg| arg.is_cancel_handle); + .filter(|arg| matches!(arg, FnArg::CancelHandle(..))); let cancel_handle = cancel_handle_iter.next(); - if let Some(arg) = cancel_handle { - ensure_spanned!(self.asyncness.is_some(), arg.name.span() => "`cancel_handle` attribute can only be used with `async fn`"); - if let Some(arg2) = cancel_handle_iter.next() { - bail_spanned!(arg2.name.span() => "`cancel_handle` may only be specified once"); + if let Some(FnArg::CancelHandle(CancelHandleArg { name, .. })) = cancel_handle { + ensure_spanned!(self.asyncness.is_some(), name.span() => "`cancel_handle` attribute can only be used with `async fn`"); + if let Some(FnArg::CancelHandle(CancelHandleArg { name, .. })) = + cancel_handle_iter.next() + { + bail_spanned!(name.span() => "`cancel_handle` may only be specified once"); } } @@ -605,14 +708,10 @@ impl<'a> FnSpec<'a> { .signature .arguments .iter() - .map(|arg| { - if arg.py { - quote!(py) - } else if arg.is_cancel_handle { - quote!(__cancel_handle) - } else { - unreachable!() - } + .map(|arg| match arg { + FnArg::Py(..) => quote!(py), + FnArg::CancelHandle(..) => quote!(__cancel_handle), + _ => unreachable!("`CallingConvention::Noargs` should not contain any arguments (reaching Python) except for `self`, which is handled below."), }) .collect(); let call = rust_call(args, &mut holders); @@ -635,7 +734,7 @@ impl<'a> FnSpec<'a> { } CallingConvention::Fastcall => { let mut holders = Holders::new(); - let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders, ctx)?; + let (arg_convert, args) = impl_arg_params(self, cls, true, &mut holders, ctx); let call = rust_call(args, &mut holders); let init_holders = holders.init_holders(ctx); let check_gil_refs = holders.check_gil_refs(); @@ -660,7 +759,7 @@ impl<'a> FnSpec<'a> { } CallingConvention::Varargs => { let mut holders = Holders::new(); - let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx)?; + let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx); let call = rust_call(args, &mut holders); let init_holders = holders.init_holders(ctx); let check_gil_refs = holders.check_gil_refs(); @@ -684,7 +783,7 @@ impl<'a> FnSpec<'a> { } CallingConvention::TpNew => { let mut holders = Holders::new(); - let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx)?; + let (arg_convert, args) = impl_arg_params(self, cls, false, &mut holders, ctx); let self_arg = self .tp .self_arg(cls, ExtractErrorMode::Raise, &mut holders, ctx); diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index fa50d260986..d9f77fa07bc 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -1,13 +1,12 @@ use crate::utils::Ctx; use crate::{ - method::{FnArg, FnSpec}, + method::{FnArg, FnSpec, RegularArg}, pyfunction::FunctionSignature, quotes::some_wrap, }; use proc_macro2::{Span, TokenStream}; -use quote::{quote, quote_spanned}; +use quote::{format_ident, quote, quote_spanned}; use syn::spanned::Spanned; -use syn::Result; pub struct Holders { holders: Vec, @@ -60,16 +59,7 @@ impl Holders { pub fn is_forwarded_args(signature: &FunctionSignature<'_>) -> bool { matches!( signature.arguments.as_slice(), - [ - FnArg { - is_varargs: true, - .. - }, - FnArg { - is_kwargs: true, - .. - }, - ] + [FnArg::VarArgs(..), FnArg::KwArgs(..),] ) } @@ -90,7 +80,7 @@ pub fn impl_arg_params( fastcall: bool, holders: &mut Holders, ctx: &Ctx, -) -> Result<(TokenStream, Vec)> { +) -> (TokenStream, Vec) { let args_array = syn::Ident::new("output", Span::call_site()); let Ctx { pyo3_path } = ctx; @@ -100,9 +90,8 @@ pub fn impl_arg_params( .iter() .enumerate() .filter_map(|(i, arg)| { - let from_py_with = &arg.attrs.from_py_with.as_ref()?.value; - let from_py_with_holder = - syn::Ident::new(&format!("from_py_with_{}", i), Span::call_site()); + let from_py_with = &arg.from_py_with()?.value; + let from_py_with_holder = format_ident!("from_py_with_{}", i); Some(quote_spanned! { from_py_with.span() => let e = #pyo3_path::impl_::deprecations::GilRefs::new(); let #from_py_with_holder = #pyo3_path::impl_::deprecations::inspect_fn(#from_py_with, &e); @@ -119,28 +108,16 @@ pub fn impl_arg_params( .arguments .iter() .enumerate() - .map(|(i, arg)| { - let from_py_with = - syn::Ident::new(&format!("from_py_with_{}", i), Span::call_site()); - let arg_value = quote!(#args_array[0].as_deref()); - - impl_arg_param(arg, from_py_with, arg_value, holders, ctx).map(|tokens| { - check_arg_for_gil_refs( - tokens, - holders.push_gil_refs_checker(arg.ty.span()), - ctx, - ) - }) - }) - .collect::>()?; - return Ok(( + .map(|(i, arg)| impl_arg_param(arg, i, &mut 0, holders, ctx)) + .collect(); + return ( quote! { let _args = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &_args); let _kwargs = #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr_or_opt(py, &_kwargs); #from_py_with }, arg_convert, - )); + ); }; let positional_parameter_names = &spec.signature.python_signature.positional_parameters; @@ -171,18 +148,8 @@ pub fn impl_arg_params( .arguments .iter() .enumerate() - .map(|(i, arg)| { - let from_py_with = syn::Ident::new(&format!("from_py_with_{}", i), Span::call_site()); - let arg_value = quote!(#args_array[#option_pos].as_deref()); - if arg.is_regular() { - option_pos += 1; - } - - impl_arg_param(arg, from_py_with, arg_value, holders, ctx).map(|tokens| { - check_arg_for_gil_refs(tokens, holders.push_gil_refs_checker(arg.ty.span()), ctx) - }) - }) - .collect::>()?; + .map(|(i, arg)| impl_arg_param(arg, i, &mut option_pos, holders, ctx)) + .collect(); let args_handler = if spec.signature.python_signature.varargs.is_some() { quote! { #pyo3_path::impl_::extract_argument::TupleVarargs } @@ -224,7 +191,7 @@ pub fn impl_arg_params( }; // create array of arguments, and then parse - Ok(( + ( quote! { const DESCRIPTION: #pyo3_path::impl_::extract_argument::FunctionDescription = #pyo3_path::impl_::extract_argument::FunctionDescription { cls_name: #cls_name, @@ -239,18 +206,64 @@ pub fn impl_arg_params( #from_py_with }, param_conversion, - )) + ) +} + +fn impl_arg_param( + arg: &FnArg<'_>, + pos: usize, + option_pos: &mut usize, + holders: &mut Holders, + ctx: &Ctx, +) -> TokenStream { + let Ctx { pyo3_path } = ctx; + let args_array = syn::Ident::new("output", Span::call_site()); + + match arg { + FnArg::Regular(arg) => { + let from_py_with = format_ident!("from_py_with_{}", pos); + let arg_value = quote!(#args_array[#option_pos].as_deref()); + *option_pos += 1; + let tokens = impl_regular_arg_param(arg, from_py_with, arg_value, holders, ctx); + check_arg_for_gil_refs(tokens, holders.push_gil_refs_checker(arg.ty.span()), ctx) + } + FnArg::VarArgs(arg) => { + let holder = holders.push_holder(arg.name.span()); + let name_str = arg.name.to_string(); + quote_spanned! { arg.name.span() => + #pyo3_path::impl_::extract_argument::extract_argument( + &_args, + &mut #holder, + #name_str + )? + } + } + FnArg::KwArgs(arg) => { + let holder = holders.push_holder(arg.name.span()); + let name_str = arg.name.to_string(); + quote_spanned! { arg.name.span() => + #pyo3_path::impl_::extract_argument::extract_optional_argument( + _kwargs.as_deref(), + &mut #holder, + #name_str, + || ::std::option::Option::None + )? + } + } + FnArg::Py(..) => quote! { py }, + FnArg::CancelHandle(..) => quote! { __cancel_handle }, + } } /// Re option_pos: The option slice doesn't contain the py: Python argument, so the argument /// index and the index in option diverge when using py: Python -pub(crate) fn impl_arg_param( - arg: &FnArg<'_>, +pub(crate) fn impl_regular_arg_param( + arg: &RegularArg<'_>, from_py_with: syn::Ident, arg_value: TokenStream, // expected type: Option<&'a Bound<'py, PyAny>> holders: &mut Holders, ctx: &Ctx, -) -> Result { +) -> TokenStream { let Ctx { pyo3_path } = ctx; let pyo3_path = pyo3_path.to_tokens_spanned(arg.ty.span()); @@ -260,64 +273,19 @@ pub(crate) fn impl_arg_param( ($($tokens:tt)*) => { quote_spanned!(arg.ty.span() => $($tokens)*) } } - if arg.py { - return Ok(quote! { py }); - } - - if arg.is_cancel_handle { - return Ok(quote! { __cancel_handle }); - } - - let name = arg.name; - let name_str = name.to_string(); - - if arg.is_varargs { - ensure_spanned!( - arg.optional.is_none(), - arg.name.span() => "args cannot be optional" - ); - let holder = holders.push_holder(arg.ty.span()); - return Ok(quote_arg_span! { - #pyo3_path::impl_::extract_argument::extract_argument( - &_args, - &mut #holder, - #name_str - )? - }); - } else if arg.is_kwargs { - ensure_spanned!( - arg.optional.is_some(), - arg.name.span() => "kwargs must be Option<_>" - ); - let holder = holders.push_holder(arg.name.span()); - return Ok(quote_arg_span! { - #pyo3_path::impl_::extract_argument::extract_optional_argument( - _kwargs.as_deref(), - &mut #holder, - #name_str, - || ::std::option::Option::None - )? - }); - } - - let mut default = arg.default.as_ref().map(|expr| quote!(#expr)); + let name_str = arg.name.to_string(); + let mut default = arg.default_value.as_ref().map(|expr| quote!(#expr)); // Option arguments have special treatment: the default should be specified _without_ the // Some() wrapper. Maybe this should be changed in future?! - if arg.optional.is_some() { + if arg.option_wrapped_type.is_some() { default = Some(default.map_or_else( || quote!(::std::option::Option::None), |tokens| some_wrap(tokens, ctx), )); } - let tokens = if arg - .attrs - .from_py_with - .as_ref() - .map(|attr| &attr.value) - .is_some() - { + if arg.from_py_with.is_some() { if let Some(default) = default { quote_arg_span! { #pyo3_path::impl_::extract_argument::from_py_with_with_default( @@ -339,7 +307,7 @@ pub(crate) fn impl_arg_param( )? } } - } else if arg.optional.is_some() { + } else if arg.option_wrapped_type.is_some() { let holder = holders.push_holder(arg.name.span()); quote_arg_span! { #pyo3_path::impl_::extract_argument::extract_optional_argument( @@ -374,7 +342,5 @@ pub(crate) fn impl_arg_param( #name_str )? } - }; - - Ok(tokens) + } } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index aff23b879f5..d9c84655b42 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -7,7 +7,7 @@ use crate::attributes::{ }; use crate::deprecations::Deprecations; use crate::konst::{ConstAttributes, ConstSpec}; -use crate::method::{FnArg, FnSpec}; +use crate::method::{FnArg, FnSpec, PyArg, RegularArg}; use crate::pyimpl::{gen_py_const, PyClassMethodsType}; use crate::pymethod::{ impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, MethodAndSlotDef, PropertyType, @@ -1143,36 +1143,22 @@ fn complex_enum_struct_variant_new<'a>( let arg_py_type: syn::Type = parse_quote!(#pyo3_path::Python<'_>); let args = { - let mut no_pyo3_attrs = vec![]; - let attrs = crate::pyfunction::PyFunctionArgPyO3Attributes::from_attrs(&mut no_pyo3_attrs)?; - let mut args = vec![ // py: Python<'_> - FnArg { + FnArg::Py(PyArg { name: &arg_py_ident, ty: &arg_py_type, - optional: None, - default: None, - py: true, - attrs: attrs.clone(), - is_varargs: false, - is_kwargs: false, - is_cancel_handle: false, - }, + }), ]; for field in &variant.fields { - args.push(FnArg { + args.push(FnArg::Regular(RegularArg { name: field.ident, ty: field.ty, - optional: None, - default: None, - py: false, - attrs: attrs.clone(), - is_varargs: false, - is_kwargs: false, - is_cancel_handle: false, - }); + from_py_with: None, + default_value: None, + option_wrapped_type: None, + })); } args }; diff --git a/pyo3-macros-backend/src/pyfunction/signature.rs b/pyo3-macros-backend/src/pyfunction/signature.rs index baf01285658..3daa79c89f5 100644 --- a/pyo3-macros-backend/src/pyfunction/signature.rs +++ b/pyo3-macros-backend/src/pyfunction/signature.rs @@ -10,7 +10,7 @@ use syn::{ use crate::{ attributes::{kw, KeywordAttribute}, - method::FnArg, + method::{FnArg, RegularArg}, }; pub struct Signature { @@ -351,36 +351,39 @@ impl<'a> FunctionSignature<'a> { let mut next_non_py_argument_checked = |name: &syn::Ident| { for fn_arg in args_iter.by_ref() { - if fn_arg.py { - // If the user incorrectly tried to include py: Python in the - // signature, give a useful error as a hint. - ensure_spanned!( - name != fn_arg.name, - name.span() => "arguments of type `Python` must not be part of the signature" - ); - // Otherwise try next argument. - continue; - } - if fn_arg.is_cancel_handle { - // If the user incorrectly tried to include cancel: CoroutineCancel in the - // signature, give a useful error as a hint. - ensure_spanned!( - name != fn_arg.name, - name.span() => "`cancel_handle` argument must not be part of the signature" - ); - // Otherwise try next argument. - continue; + match fn_arg { + crate::method::FnArg::Py(..) => { + // If the user incorrectly tried to include py: Python in the + // signature, give a useful error as a hint. + ensure_spanned!( + name != fn_arg.name(), + name.span() => "arguments of type `Python` must not be part of the signature" + ); + // Otherwise try next argument. + continue; + } + crate::method::FnArg::CancelHandle(..) => { + // If the user incorrectly tried to include cancel: CoroutineCancel in the + // signature, give a useful error as a hint. + ensure_spanned!( + name != fn_arg.name(), + name.span() => "`cancel_handle` argument must not be part of the signature" + ); + // Otherwise try next argument. + continue; + } + _ => { + ensure_spanned!( + name == fn_arg.name(), + name.span() => format!( + "expected argument from function definition `{}` but got argument `{}`", + fn_arg.name().unraw(), + name.unraw(), + ) + ); + return Ok(fn_arg); + } } - - ensure_spanned!( - name == fn_arg.name, - name.span() => format!( - "expected argument from function definition `{}` but got argument `{}`", - fn_arg.name.unraw(), - name.unraw(), - ) - ); - return Ok(fn_arg); } bail_spanned!( name.span() => "signature entry does not have a corresponding function argument" @@ -398,7 +401,15 @@ impl<'a> FunctionSignature<'a> { arg.span(), )?; if let Some((_, default)) = &arg.eq_and_default { - fn_arg.default = Some(default.clone()); + if let FnArg::Regular(arg) = fn_arg { + arg.default_value = Some(default.clone()); + } else { + unreachable!( + "`Python` and `CancelHandle` are already handled above and `*args`/`**kwargs` are \ + parsed and transformed below. Because the have to come last and are only allowed \ + once, this has to be a regular argument." + ); + } } } SignatureItem::VarargsSep(sep) => { @@ -406,12 +417,12 @@ impl<'a> FunctionSignature<'a> { } SignatureItem::Varargs(varargs) => { let fn_arg = next_non_py_argument_checked(&varargs.ident)?; - fn_arg.is_varargs = true; + fn_arg.to_varargs_mut()?; parse_state.add_varargs(&mut python_signature, varargs)?; } SignatureItem::Kwargs(kwargs) => { let fn_arg = next_non_py_argument_checked(&kwargs.ident)?; - fn_arg.is_kwargs = true; + fn_arg.to_kwargs_mut()?; parse_state.add_kwargs(&mut python_signature, kwargs)?; } SignatureItem::PosargsSep(sep) => { @@ -421,9 +432,11 @@ impl<'a> FunctionSignature<'a> { } // Ensure no non-py arguments remain - if let Some(arg) = args_iter.find(|arg| !arg.py && !arg.is_cancel_handle) { + if let Some(arg) = + args_iter.find(|arg| !matches!(arg, FnArg::Py(..) | FnArg::CancelHandle(..))) + { bail_spanned!( - attribute.kw.span() => format!("missing signature entry for argument `{}`", arg.name) + attribute.kw.span() => format!("missing signature entry for argument `{}`", arg.name()) ); } @@ -439,15 +452,20 @@ impl<'a> FunctionSignature<'a> { let mut python_signature = PythonSignature::default(); for arg in &arguments { // Python<'_> arguments don't show in Python signature - if arg.py || arg.is_cancel_handle { + if matches!(arg, FnArg::Py(..) | FnArg::CancelHandle(..)) { continue; } - if arg.optional.is_none() { + if let FnArg::Regular(RegularArg { + ty, + option_wrapped_type: None, + .. + }) = arg + { // This argument is required, all previous arguments must also have been required ensure_spanned!( python_signature.required_positional_parameters == python_signature.positional_parameters.len(), - arg.ty.span() => "required arguments after an `Option<_>` argument are ambiguous\n\ + ty.span() => "required arguments after an `Option<_>` argument are ambiguous\n\ = help: add a `#[pyo3(signature)]` annotation on this function to unambiguously specify the default values for all optional parameters" ); @@ -457,7 +475,7 @@ impl<'a> FunctionSignature<'a> { python_signature .positional_parameters - .push(arg.name.unraw().to_string()); + .push(arg.name().unraw().to_string()); } Ok(Self { @@ -469,8 +487,12 @@ impl<'a> FunctionSignature<'a> { fn default_value_for_parameter(&self, parameter: &str) -> String { let mut default = "...".to_string(); - if let Some(fn_arg) = self.arguments.iter().find(|arg| arg.name == parameter) { - if let Some(arg_default) = fn_arg.default.as_ref() { + if let Some(fn_arg) = self.arguments.iter().find(|arg| arg.name() == parameter) { + if let FnArg::Regular(RegularArg { + default_value: Some(arg_default), + .. + }) = fn_arg + { match arg_default { // literal values syn::Expr::Lit(syn::ExprLit { lit, .. }) => match lit { @@ -496,7 +518,11 @@ impl<'a> FunctionSignature<'a> { // others, unsupported yet so defaults to `...` _ => {} } - } else if fn_arg.optional.is_some() { + } else if let FnArg::Regular(RegularArg { + option_wrapped_type: Some(..), + .. + }) = fn_arg + { // functions without a `#[pyo3(signature = (...))]` option // will treat trailing `Option` arguments as having a default of `None` default = "None".to_string(); diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 7a4c54db9da..aac804316f8 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -1,8 +1,8 @@ use std::borrow::Cow; use crate::attributes::{NameAttribute, RenamingRule}; -use crate::method::{CallingConvention, ExtractErrorMode}; -use crate::params::{check_arg_for_gil_refs, impl_arg_param, Holders}; +use crate::method::{CallingConvention, ExtractErrorMode, PyArg}; +use crate::params::{check_arg_for_gil_refs, impl_regular_arg_param, Holders}; use crate::utils::Ctx; use crate::utils::PythonDoc; use crate::{ @@ -487,7 +487,7 @@ fn impl_py_class_attribute( let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); ensure_spanned!( args.is_empty(), - args[0].ty.span() => "#[classattr] can only have one argument (of type pyo3::Python)" + args[0].ty().span() => "#[classattr] can only have one argument (of type pyo3::Python)" ); let name = &spec.name; @@ -537,7 +537,7 @@ fn impl_call_setter( bail_spanned!(spec.name.span() => "setter function expected to have one argument"); } else if args.len() > 1 { bail_spanned!( - args[1].ty.span() => + args[1].ty().span() => "setter function can have at most two arguments ([pyo3::Python,] and value)" ); } @@ -607,7 +607,7 @@ pub fn impl_py_setter_def( let (_, args) = split_off_python_arg(&spec.signature.arguments); let value_arg = &args[0]; let (from_py_with, ident) = if let Some(from_py_with) = - &value_arg.attrs.from_py_with.as_ref().map(|f| &f.value) + &value_arg.from_py_with().as_ref().map(|f| &f.value) { let ident = syn::Ident::new("from_py_with", from_py_with.span()); ( @@ -622,20 +622,21 @@ pub fn impl_py_setter_def( (quote!(), syn::Ident::new("dummy", Span::call_site())) }; - let extract = impl_arg_param( - &args[0], + let arg = if let FnArg::Regular(arg) = &value_arg { + arg + } else { + bail_spanned!(value_arg.name().span() => "The #[setter] value argument can't be *args, **kwargs or `cancel_handle`."); + }; + + let tokens = impl_regular_arg_param( + arg, ident, quote!(::std::option::Option::Some(_value.into())), &mut holders, ctx, - ) - .map(|tokens| { - check_arg_for_gil_refs( - tokens, - holders.push_gil_refs_checker(value_arg.ty.span()), - ctx, - ) - })?; + ); + let extract = + check_arg_for_gil_refs(tokens, holders.push_gil_refs_checker(arg.ty.span()), ctx); quote! { #from_py_with let _val = #extract; @@ -721,7 +722,7 @@ fn impl_call_getter( let slf = self_type.receiver(cls, ExtractErrorMode::Raise, holders, ctx); ensure_spanned!( args.is_empty(), - args[0].ty.span() => "getter function can only have one argument (of type pyo3::Python)" + args[0].ty().span() => "getter function can only have one argument (of type pyo3::Python)" ); let name = &spec.name; @@ -843,9 +844,9 @@ pub fn impl_py_getter_def( } /// Split an argument of pyo3::Python from the front of the arg list, if present -fn split_off_python_arg<'a>(args: &'a [FnArg<'a>]) -> (Option<&FnArg<'_>>, &[FnArg<'_>]) { +fn split_off_python_arg<'a>(args: &'a [FnArg<'a>]) -> (Option<&PyArg<'_>>, &[FnArg<'_>]) { match args { - [py, args @ ..] if utils::is_python(py.ty) => (Some(py), args), + [FnArg::Py(py), args @ ..] => (Some(py), args), args => (None, args), } } @@ -1052,14 +1053,14 @@ impl Ty { ctx: &Ctx, ) -> TokenStream { let Ctx { pyo3_path } = ctx; - let name_str = arg.name.unraw().to_string(); + let name_str = arg.name().unraw().to_string(); match self { Ty::Object => extract_object( extract_error_mode, holders, &name_str, quote! { #ident }, - arg.ty.span(), + arg.ty().span(), ctx ), Ty::MaybeNullObject => extract_object( @@ -1073,7 +1074,7 @@ impl Ty { #ident } }, - arg.ty.span(), + arg.ty().span(), ctx ), Ty::NonNullObject => extract_object( @@ -1081,7 +1082,7 @@ impl Ty { holders, &name_str, quote! { #ident.as_ptr() }, - arg.ty.span(), + arg.ty().span(), ctx ), Ty::IPowModulo => extract_object( @@ -1089,7 +1090,7 @@ impl Ty { holders, &name_str, quote! { #ident.as_ptr() }, - arg.ty.span(), + arg.ty().span(), ctx ), Ty::CompareOp => extract_error_mode.handle_error( @@ -1100,7 +1101,7 @@ impl Ty { ctx ), Ty::PySsizeT => { - let ty = arg.ty; + let ty = arg.ty(); extract_error_mode.handle_error( quote! { ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| #pyo3_path::exceptions::PyValueError::new_err(e.to_string())) @@ -1523,12 +1524,12 @@ fn extract_proto_arguments( let mut non_python_args = 0; for arg in &spec.signature.arguments { - if arg.py { + if let FnArg::Py(..) = arg { args.push(quote! { py }); } else { let ident = syn::Ident::new(&format!("arg{}", non_python_args), Span::call_site()); let conversions = proto_args.get(non_python_args) - .ok_or_else(|| err_spanned!(arg.ty.span() => format!("Expected at most {} non-python arguments", proto_args.len())))? + .ok_or_else(|| err_spanned!(arg.ty().span() => format!("Expected at most {} non-python arguments", proto_args.len())))? .extract(&ident, arg, extract_error_mode, holders, ctx); non_python_args += 1; args.push(conversions); diff --git a/tests/ui/invalid_cancel_handle.rs b/tests/ui/invalid_cancel_handle.rs index 59076b14418..cff6c5dcbad 100644 --- a/tests/ui/invalid_cancel_handle.rs +++ b/tests/ui/invalid_cancel_handle.rs @@ -19,4 +19,10 @@ async fn cancel_handle_wrong_type(#[pyo3(cancel_handle)] _param: String) {} #[pyfunction] async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} +#[pyfunction] +async fn cancel_handle_and_from_py_with( + #[pyo3(cancel_handle, from_py_with = "cancel_handle")] _param: pyo3::coroutine::CancelHandle, +) { +} + fn main() {} diff --git a/tests/ui/invalid_cancel_handle.stderr b/tests/ui/invalid_cancel_handle.stderr index ffd0b3fd0da..41a2c0854b7 100644 --- a/tests/ui/invalid_cancel_handle.stderr +++ b/tests/ui/invalid_cancel_handle.stderr @@ -16,6 +16,12 @@ error: `cancel_handle` attribute can only be used with `async fn` 14 | fn cancel_handle_synchronous(#[pyo3(cancel_handle)] _param: String) {} | ^^^^^^ +error: `from_py_with` and `cancel_handle` cannot be specified together + --> tests/ui/invalid_cancel_handle.rs:24:12 + | +24 | #[pyo3(cancel_handle, from_py_with = "cancel_handle")] _param: pyo3::coroutine::CancelHandle, + | ^^^^^^^^^^^^^ + error[E0308]: mismatched types --> tests/ui/invalid_cancel_handle.rs:16:1 | diff --git a/tests/ui/invalid_pyfunctions.rs b/tests/ui/invalid_pyfunctions.rs index eaa241c074b..1a95c9e4a34 100644 --- a/tests/ui/invalid_pyfunctions.rs +++ b/tests/ui/invalid_pyfunctions.rs @@ -1,5 +1,5 @@ use pyo3::prelude::*; -use pyo3::types::PyString; +use pyo3::types::{PyDict, PyString, PyTuple}; #[pyfunction] fn generic_function(value: T) {} @@ -16,6 +16,14 @@ fn destructured_argument((a, b): (i32, i32)) {} #[pyfunction] fn function_with_required_after_option(_opt: Option, _x: i32) {} +#[pyfunction] +#[pyo3(signature=(*args))] +fn function_with_optional_args(args: Option>) {} + +#[pyfunction] +#[pyo3(signature=(**kwargs))] +fn function_with_required_kwargs(kwargs: Bound<'_, PyDict>) {} + #[pyfunction(pass_module)] fn pass_module_but_no_arguments<'py>() {} diff --git a/tests/ui/invalid_pyfunctions.stderr b/tests/ui/invalid_pyfunctions.stderr index 299b20687cb..893d7cbec2c 100644 --- a/tests/ui/invalid_pyfunctions.stderr +++ b/tests/ui/invalid_pyfunctions.stderr @@ -29,16 +29,28 @@ error: required arguments after an `Option<_>` argument are ambiguous 17 | fn function_with_required_after_option(_opt: Option, _x: i32) {} | ^^^ +error: args cannot be optional + --> tests/ui/invalid_pyfunctions.rs:21:32 + | +21 | fn function_with_optional_args(args: Option>) {} + | ^^^^ + +error: kwargs must be Option<_> + --> tests/ui/invalid_pyfunctions.rs:25:34 + | +25 | fn function_with_required_kwargs(kwargs: Bound<'_, PyDict>) {} + | ^^^^^^ + error: expected `&PyModule` or `Py` as first argument with `pass_module` - --> tests/ui/invalid_pyfunctions.rs:20:37 + --> tests/ui/invalid_pyfunctions.rs:28:37 | -20 | fn pass_module_but_no_arguments<'py>() {} +28 | fn pass_module_but_no_arguments<'py>() {} | ^^ error[E0277]: the trait bound `&str: From>` is not satisfied - --> tests/ui/invalid_pyfunctions.rs:24:13 + --> tests/ui/invalid_pyfunctions.rs:32:13 | -24 | string: &str, +32 | string: &str, | ^ the trait `From>` is not implemented for `&str`, which is required by `BoundRef<'_, '_, pyo3::prelude::PyModule>: Into<_>` | = help: the following other types implement trait `From`: From 8ed5c17b9311c53d94c8fd21f2763fc602979779 Mon Sep 17 00:00:00 2001 From: David Matos Date: Sun, 14 Apr 2024 22:07:17 +0200 Subject: [PATCH 275/349] Allow use of scientific notation on rust decimal (#4079) * Allow use of scientific notation on rust decimal * Add newsfragment * Pull out var * Fix clippy warning * Modify let bindings location --- newsfragments/4079.added.md | 1 + src/conversions/rust_decimal.rs | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 newsfragments/4079.added.md diff --git a/newsfragments/4079.added.md b/newsfragments/4079.added.md new file mode 100644 index 00000000000..afe26728f9a --- /dev/null +++ b/newsfragments/4079.added.md @@ -0,0 +1 @@ +Added support for scientific notation in `Decimal` conversion diff --git a/src/conversions/rust_decimal.rs b/src/conversions/rust_decimal.rs index b75cd875128..782ca2e80f0 100644 --- a/src/conversions/rust_decimal.rs +++ b/src/conversions/rust_decimal.rs @@ -64,8 +64,11 @@ impl FromPyObject<'_> for Decimal { if let Ok(val) = obj.extract() { Ok(Decimal::new(val, 0)) } else { - Decimal::from_str(&obj.str()?.to_cow()?) - .map_err(|e| PyValueError::new_err(e.to_string())) + let py_str = &obj.str()?; + let rs_str = &py_str.to_cow()?; + Decimal::from_str(rs_str).or_else(|_| { + Decimal::from_scientific(rs_str).map_err(|e| PyValueError::new_err(e.to_string())) + }) } } } @@ -194,6 +197,23 @@ mod test_rust_decimal { }) } + #[test] + fn test_scientific_notation() { + Python::with_gil(|py| { + let locals = PyDict::new_bound(py); + py.run_bound( + "import decimal\npy_dec = decimal.Decimal(\"1e3\")", + None, + Some(&locals), + ) + .unwrap(); + let py_dec = locals.get_item("py_dec").unwrap().unwrap(); + let roundtripped: Decimal = py_dec.extract().unwrap(); + let rs_dec = Decimal::from_scientific("1e3").unwrap(); + assert_eq!(rs_dec, roundtripped); + }) + } + #[test] fn test_infinity() { Python::with_gil(|py| { From a5201c04afc73605ac3ef467a42d3052af5feb14 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 14 Apr 2024 22:38:40 +0100 Subject: [PATCH 276/349] Deprecate the `PySet::empty` gil-ref constructor (#4082) * Deprecate the `PySet::empty` gil-ref constructor * add newsfragment --- newsfragments/4082.changed.md | 1 + src/types/set.rs | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 newsfragments/4082.changed.md diff --git a/newsfragments/4082.changed.md b/newsfragments/4082.changed.md new file mode 100644 index 00000000000..231ea0ae576 --- /dev/null +++ b/newsfragments/4082.changed.md @@ -0,0 +1 @@ +Deprecate the `PySet::empty()` gil-ref constructor. diff --git a/src/types/set.rs b/src/types/set.rs index 4f1fcf8499f..f648bc2be1f 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -58,7 +58,14 @@ impl PySet { } /// Deprecated form of [`PySet::empty_bound`]. - pub fn empty(py: Python<'_>) -> PyResult<&'_ PySet> { + #[cfg_attr( + not(feature = "gil-refs"), + deprecated( + since = "0.21.2", + note = "`PySet::empty` will be replaced by `PySet::empty_bound` in a future PyO3 version" + ) + )] + pub fn empty(py: Python<'_>) -> PyResult<&PySet> { Self::empty_bound(py).map(Bound::into_gil_ref) } From 2ad2a3f208a92d3a6e11909b621edd8d87b356e4 Mon Sep 17 00:00:00 2001 From: David Matos Date: Tue, 16 Apr 2024 10:17:41 +0200 Subject: [PATCH 277/349] docs: Make contributing.md slightly more clear for newer contributors (#4080) * docs: Make contributing.md slightly more clear for newer contributors * Remove accidental backticks Rearrange overview of commands * Placed on wrong line * Add extra overview command --- .github/pull_request_template.md | 2 +- Contributing.md | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b344525cabe..11375e966b3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,6 +8,6 @@ Please consider adding the following to your pull request: - docs to all new functions and / or detail in the guide - tests for all new or changed functions -PyO3's CI pipeline will check your pull request. To run its tests +PyO3's CI pipeline will check your pull request, thus make sure you have checked the `Contributing.md` guidelines. To run most of its tests locally, you can run ```nox```. See ```nox --list-sessions``` for a list of supported actions. diff --git a/Contributing.md b/Contributing.md index 054099b431a..111e814ac8f 100644 --- a/Contributing.md +++ b/Contributing.md @@ -92,9 +92,7 @@ Here are a few things to note when you are writing PRs. ### Continuous Integration -The PyO3 repo uses GitHub Actions. PRs are blocked from merging if CI is not successful. - -Formatting, linting and tests are checked for all Rust and Python code. In addition, all warnings in Rust code are disallowed (using `RUSTFLAGS="-D warnings"`). +The PyO3 repo uses GitHub Actions. PRs are blocked from merging if CI is not successful. Formatting, linting and tests are checked for all Rust and Python code. In addition, all warnings in Rust code are disallowed (using `RUSTFLAGS="-D warnings"`). Tests run with all supported Python versions with the latest stable Rust compiler, as well as for Python 3.9 with the minimum supported Rust version. @@ -103,6 +101,24 @@ If you are adding a new feature, you should add it to the `full` feature in our You can run these tests yourself with `nox`. Use `nox -l` to list the full set of subcommands you can run. +#### Linting Python code +`nox -s ruff` + +#### Linting Rust code +`nox -s rustfmt` + +#### Semver checks +`cargo semver-checks check-release` + +#### Clippy +`nox -s clippy-all` + +#### Tests +`cargo test --features full` + +#### Check all conditional compilation +`nox -s check-feature-powerset` + #### UI Tests PyO3 uses [`trybuild`][trybuild] to develop UI tests to capture error messages from the Rust compiler for some of the macro functionality. From 03f59eaf45c5786e7a9c591b8f81b1243a50b5ef Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 16 Apr 2024 22:12:18 +0200 Subject: [PATCH 278/349] fix declarative module compile error with `create_exception!` (#4086) * fix declarative module compile error with `create_exception!` * add newsfragment --- newsfragments/4086.fixed.md | 1 + src/types/mod.rs | 2 +- tests/test_declarative_module.rs | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 newsfragments/4086.fixed.md diff --git a/newsfragments/4086.fixed.md b/newsfragments/4086.fixed.md new file mode 100644 index 00000000000..e9cae7733f9 --- /dev/null +++ b/newsfragments/4086.fixed.md @@ -0,0 +1 @@ +Fixes a compile error when exporting an exception created with `create_exception!` living in a different Rust module using the `declarative-module` feature. \ No newline at end of file diff --git a/src/types/mod.rs b/src/types/mod.rs index a03d01b301a..e7aead7fbe5 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -252,7 +252,7 @@ macro_rules! pyobject_native_type_info( impl $name { #[doc(hidden)] - const _PYO3_DEF: $crate::impl_::pymodule::AddTypeToModule = $crate::impl_::pymodule::AddTypeToModule::new(); + pub const _PYO3_DEF: $crate::impl_::pymodule::AddTypeToModule = $crate::impl_::pymodule::AddTypeToModule::new(); } }; ); diff --git a/tests/test_declarative_module.rs b/tests/test_declarative_module.rs index 8e432c3ae58..2e46f4a64d1 100644 --- a/tests/test_declarative_module.rs +++ b/tests/test_declarative_module.rs @@ -10,10 +10,14 @@ use pyo3::types::PyBool; mod common; mod some_module { + use pyo3::create_exception; + use pyo3::exceptions::PyException; use pyo3::prelude::*; #[pyclass] pub struct SomePyClass; + + create_exception!(some_module, SomeException, PyException); } #[pyclass] @@ -61,6 +65,10 @@ mod declarative_module { #[pymodule_export] use super::some_module::SomePyClass; + // test for #4036 + #[pymodule_export] + use super::some_module::SomeException; + #[pymodule] mod inner { use super::*; From 9761abf3a543b3e3abfa0c41613331103ad79e13 Mon Sep 17 00:00:00 2001 From: Bruno Kolenbrander <59372212+mejrs@users.noreply.github.com> Date: Wed, 17 Apr 2024 03:07:11 +0200 Subject: [PATCH 279/349] Specify higher target-lexicon version (#4087) --- pyo3-build-config/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 1eb269c2132..a0942831340 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -13,11 +13,11 @@ edition = "2021" [dependencies] once_cell = "1" python3-dll-a = { version = "0.2.6", optional = true } -target-lexicon = "0.12" +target-lexicon = "0.12.14" [build-dependencies] python3-dll-a = { version = "0.2.6", optional = true } -target-lexicon = "0.12" +target-lexicon = "0.12.14" [features] default = [] From 03c50a18392cdb183aa059d65d74dbca9fdc3ec3 Mon Sep 17 00:00:00 2001 From: Jacob Zhong Date: Thu, 18 Apr 2024 15:33:07 +0800 Subject: [PATCH 280/349] Change the types of `PySliceIndices` and `PySlice::indices (#3761) * Change the type of `PySliceIndices::slicelength` and `PySlice::indices()` * Fix example * Fix fmt --- examples/getitem/src/lib.rs | 5 ++--- newsfragments/3761.changed.md | 1 + src/types/slice.rs | 23 +++++++++++++---------- 3 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 newsfragments/3761.changed.md diff --git a/examples/getitem/src/lib.rs b/examples/getitem/src/lib.rs index c3c662ab92f..ce162a70bf9 100644 --- a/examples/getitem/src/lib.rs +++ b/examples/getitem/src/lib.rs @@ -2,7 +2,6 @@ use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; use pyo3::types::PySlice; -use std::os::raw::c_long; #[derive(FromPyObject)] enum IntOrSlice<'py> { @@ -29,7 +28,7 @@ impl ExampleContainer { } else if let Ok(slice) = key.downcast::() { // METHOD 1 - the use PySliceIndices to help with bounds checking and for cases when only start or end are provided // in this case the start/stop/step all filled in to give valid values based on the max_length given - let index = slice.indices(self.max_length as c_long).unwrap(); + let index = slice.indices(self.max_length as isize).unwrap(); let _delta = index.stop - index.start; // METHOD 2 - Do the getattr manually really only needed if you have some special cases for stop/_step not being present @@ -62,7 +61,7 @@ impl ExampleContainer { fn __setitem__(&self, idx: IntOrSlice, value: u32) -> PyResult<()> { match idx { IntOrSlice::Slice(slice) => { - let index = slice.indices(self.max_length as c_long).unwrap(); + let index = slice.indices(self.max_length as isize).unwrap(); println!( "Got a slice! {}-{}, step: {}, value: {}", index.start, index.stop, index.step, value diff --git a/newsfragments/3761.changed.md b/newsfragments/3761.changed.md new file mode 100644 index 00000000000..fd0847211d8 --- /dev/null +++ b/newsfragments/3761.changed.md @@ -0,0 +1 @@ +Change the type of `PySliceIndices::slicelength` and the `length` parameter of `PySlice::indices()`. diff --git a/src/types/slice.rs b/src/types/slice.rs index 8e7545208dc..b6895d09e10 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -1,13 +1,12 @@ use crate::err::{PyErr, PyResult}; -use crate::ffi::{self, Py_ssize_t}; +use crate::ffi; use crate::ffi_ptr_ext::FfiPtrExt; use crate::types::any::PyAnyMethods; use crate::{Bound, PyAny, PyNativeType, PyObject, Python, ToPyObject}; -use std::os::raw::c_long; /// Represents a Python `slice`. /// -/// Only `c_long` indices supported at the moment by the `PySlice` object. +/// Only `isize` indices supported at the moment by the `PySlice` object. #[repr(transparent)] pub struct PySlice(PyAny); @@ -22,13 +21,17 @@ pyobject_native_type!( #[derive(Debug, Eq, PartialEq)] pub struct PySliceIndices { /// Start of the slice + /// + /// It can be -1 when the step is negative, otherwise it's non-negative. pub start: isize, /// End of the slice + /// + /// It can be -1 when the step is negative, otherwise it's non-negative. pub stop: isize, /// Increment to use when iterating the slice from `start` to `stop`. pub step: isize, /// The length of the slice calculated from the original input sequence. - pub slicelength: isize, + pub slicelength: usize, } impl PySliceIndices { @@ -94,7 +97,7 @@ impl PySlice { /// assuming a sequence of length `length`, and stores the length of the /// slice in its `slicelength` member. #[inline] - pub fn indices(&self, length: c_long) -> PyResult { + pub fn indices(&self, length: isize) -> PyResult { self.as_borrowed().indices(length) } } @@ -109,12 +112,11 @@ pub trait PySliceMethods<'py>: crate::sealed::Sealed { /// Retrieves the start, stop, and step indices from the slice object, /// assuming a sequence of length `length`, and stores the length of the /// slice in its `slicelength` member. - fn indices(&self, length: c_long) -> PyResult; + fn indices(&self, length: isize) -> PyResult; } impl<'py> PySliceMethods<'py> for Bound<'py, PySlice> { - fn indices(&self, length: c_long) -> PyResult { - // non-negative Py_ssize_t should always fit into Rust usize + fn indices(&self, length: isize) -> PyResult { unsafe { let mut slicelength: isize = 0; let mut start: isize = 0; @@ -122,7 +124,7 @@ impl<'py> PySliceMethods<'py> for Bound<'py, PySlice> { let mut step: isize = 0; let r = ffi::PySlice_GetIndicesEx( self.as_ptr(), - length as Py_ssize_t, + length, &mut start, &mut stop, &mut step, @@ -133,7 +135,8 @@ impl<'py> PySliceMethods<'py> for Bound<'py, PySlice> { start, stop, step, - slicelength, + // non-negative isize should always fit into usize + slicelength: slicelength as _, }) } else { Err(PyErr::fetch(self.py())) From e64eb7290338352e6344545bfc836d186f315c6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 08:58:51 +0000 Subject: [PATCH 281/349] build(deps): update chrono-tz requirement from >= 0.6, < 0.9 to >= 0.6, < 0.10 (#4061) * build(deps): update chrono-tz requirement from >= 0.6, < 0.9 to >= 0.6, < 0.10 Updates the requirements on [chrono-tz](https://github.com/chronotope/chrono-tz) to permit the latest version. - [Release notes](https://github.com/chronotope/chrono-tz/releases) - [Changelog](https://github.com/chronotope/chrono-tz/blob/main/CHANGELOG.md) - [Commits](https://github.com/chronotope/chrono-tz/compare/v0.6.0...v0.9.0) --- updated-dependencies: - dependency-name: chrono-tz dependency-type: direct:production ... Signed-off-by: dependabot[bot] * newsfragment --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Hewitt --- Cargo.toml | 4 ++-- newsfragments/4061.packaging.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 newsfragments/4061.packaging.md diff --git a/Cargo.toml b/Cargo.toml index fdd1fa9d29a..789d52c6356 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ inventory = { version = "0.3.0", optional = true } # crate integrations that can be added using the eponymous features anyhow = { version = "1.0", optional = true } chrono = { version = "0.4.25", default-features = false, optional = true } -chrono-tz = { version = ">= 0.6, < 0.9", default-features = false, optional = true } +chrono-tz = { version = ">= 0.6, < 0.10", default-features = false, optional = true } either = { version = "1.9", optional = true } eyre = { version = ">= 0.4, < 0.7", optional = true } hashbrown = { version = ">= 0.9, < 0.15", optional = true } @@ -49,7 +49,7 @@ smallvec = { version = "1.0", optional = true } [dev-dependencies] assert_approx_eq = "1.1.0" chrono = "0.4.25" -chrono-tz = ">= 0.6, < 0.9" +chrono-tz = ">= 0.6, < 0.10" # Required for "and $N others" normalization trybuild = ">=1.0.70" proptest = { version = "1.0", default-features = false, features = ["std"] } diff --git a/newsfragments/4061.packaging.md b/newsfragments/4061.packaging.md new file mode 100644 index 00000000000..5e51f50290d --- /dev/null +++ b/newsfragments/4061.packaging.md @@ -0,0 +1 @@ +Extend range of supported versions of `chrono-tz` optional dependency to include version 0.10. From 2c205d4586bef8f839eb6a55eb91b905074ddce9 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 18 Apr 2024 09:59:02 +0100 Subject: [PATCH 282/349] release notes for 0.21.2 (#4091) --- CHANGELOG.md | 17 ++++++++++++++++- Cargo.toml | 8 ++++---- README.md | 4 ++-- examples/decorator/.template/pre-script.rhai | 2 +- .../maturin-starter/.template/pre-script.rhai | 2 +- examples/plugin/.template/pre-script.rhai | 2 +- .../.template/pre-script.rhai | 2 +- examples/word-count/.template/pre-script.rhai | 2 +- newsfragments/4035.fixed.md | 1 - newsfragments/4045.fixed.md | 1 - newsfragments/4054.fixed.md | 1 - newsfragments/4067.fixed.md | 1 - newsfragments/4073.fixed.md | 1 - newsfragments/4082.changed.md | 1 - pyo3-build-config/Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 4 ++-- pyo3-macros-backend/Cargo.toml | 4 ++-- pyo3-macros/Cargo.toml | 4 ++-- pyproject.toml | 2 +- 19 files changed, 35 insertions(+), 26 deletions(-) delete mode 100644 newsfragments/4035.fixed.md delete mode 100644 newsfragments/4045.fixed.md delete mode 100644 newsfragments/4054.fixed.md delete mode 100644 newsfragments/4067.fixed.md delete mode 100644 newsfragments/4073.fixed.md delete mode 100644 newsfragments/4082.changed.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f4ce218021..86055a9a80c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,20 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.21.2] - 2024-04-16 + +### Changed + +- Deprecate the `PySet::empty()` gil-ref constructor. [#4082](https://github.com/PyO3/pyo3/pull/4082) + +### Fixed + +- Fix compile error for `async fn` in `#[pymethods]` with a `&self` receiver and more than one additional argument. [#4035](https://github.com/PyO3/pyo3/pull/4035) +- Improve error message for wrong receiver type in `__traverse__`. [#4045](https://github.com/PyO3/pyo3/pull/4045) +- Fix compile error when exporting a `#[pyclass]` living in a different Rust module using the `experimental-declarative-modules` feature. [#4054](https://github.com/PyO3/pyo3/pull/4054) +- Fix `missing_docs` lint triggering on documented `#[pymodule]` functions. [#4067](https://github.com/PyO3/pyo3/pull/4067) +- Fix undefined symbol errors for extension modules on AIX (by linking `libpython`). [#4073](https://github.com/PyO3/pyo3/pull/4073) + ## [0.21.1] - 2024-04-01 ### Added @@ -1731,7 +1745,8 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.21.1...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.21.2...HEAD +[0.21.2]: https://github.com/pyo3/pyo3/compare/v0.21.1...v0.21.2 [0.21.1]: https://github.com/pyo3/pyo3/compare/v0.21.0...v0.21.1 [0.21.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0 [0.21.0-beta.0]: https://github.com/pyo3/pyo3/compare/v0.20.3...v0.21.0-beta.0 diff --git a/Cargo.toml b/Cargo.toml index 789d52c6356..90bcc03c80c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.21.1" +version = "0.21.2" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -22,10 +22,10 @@ memoffset = "0.9" portable-atomic = "1.0" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.1" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.2" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.21.1", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.21.2", optional = true } indoc = { version = "2.0.1", optional = true } unindent = { version = "0.2.1", optional = true } @@ -60,7 +60,7 @@ rayon = "1.6.1" futures = "0.3.28" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.1", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.2", features = ["resolve-config"] } [features] default = ["macros"] diff --git a/README.md b/README.md index 8e7e2f75bca..4da2447e8c4 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.21.1", features = ["extension-module"] } +pyo3 = { version = "0.21.2", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.21.1" +version = "0.21.2" features = ["auto-initialize"] ``` diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 1dc689ef6d4..37372854cd8 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.1"); +variable::set("PYO3_VERSION", "0.21.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 1dc689ef6d4..37372854cd8 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.1"); +variable::set("PYO3_VERSION", "0.21.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 78adb883f43..7c2f375fbfb 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.1"); +variable::set("PYO3_VERSION", "0.21.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 212f62f76fe..dd2950665eb 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.1"); +variable::set("PYO3_VERSION", "0.21.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 1dc689ef6d4..37372854cd8 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.21.1"); +variable::set("PYO3_VERSION", "0.21.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/newsfragments/4035.fixed.md b/newsfragments/4035.fixed.md deleted file mode 100644 index 5425c5cbaf7..00000000000 --- a/newsfragments/4035.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix compile error for `async fn` in `#[pymethods]` with a `&self` receiver and more than one additional argument. diff --git a/newsfragments/4045.fixed.md b/newsfragments/4045.fixed.md deleted file mode 100644 index 6b2bbcfa01d..00000000000 --- a/newsfragments/4045.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Add better error message on wrong receiver extraction in `__traverse__`. diff --git a/newsfragments/4054.fixed.md b/newsfragments/4054.fixed.md deleted file mode 100644 index 4b8da92ca4d..00000000000 --- a/newsfragments/4054.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixes a compile error when exporting a `#[pyclass]` living in a different Rust module using the declarative-module feature. diff --git a/newsfragments/4067.fixed.md b/newsfragments/4067.fixed.md deleted file mode 100644 index 869b6addf15..00000000000 --- a/newsfragments/4067.fixed.md +++ /dev/null @@ -1 +0,0 @@ -fixes `missing_docs` lint to trigger on documented `#[pymodule]` functions \ No newline at end of file diff --git a/newsfragments/4073.fixed.md b/newsfragments/4073.fixed.md deleted file mode 100644 index 0f77647e42d..00000000000 --- a/newsfragments/4073.fixed.md +++ /dev/null @@ -1 +0,0 @@ -fixes undefined symbol errors when building extension module on AIX by linking `libpython` diff --git a/newsfragments/4082.changed.md b/newsfragments/4082.changed.md deleted file mode 100644 index 231ea0ae576..00000000000 --- a/newsfragments/4082.changed.md +++ /dev/null @@ -1 +0,0 @@ -Deprecate the `PySet::empty()` gil-ref constructor. diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index a0942831340..60bf9a13ef9 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.21.1" +version = "0.21.2" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 64753976fab..8f7767254f1 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.21.1" +version = "0.21.2" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,7 +38,7 @@ abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312"] generate-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.1", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.2", features = ["resolve-config"] } [lints] workspace = true diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 6ca4eeade8c..f2675f2b75e 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.21.1" +version = "0.21.2" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -16,7 +16,7 @@ edition = "2021" [dependencies] heck = "0.4" proc-macro2 = { version = "1", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.1", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.2", features = ["resolve-config"] } quote = { version = "1", default-features = false } [dependencies.syn] diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 39a4b9198c6..690924c76a5 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.21.1" +version = "0.21.2" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -22,7 +22,7 @@ experimental-declarative-modules = [] proc-macro2 = { version = "1", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.1" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.2" } [lints] workspace = true diff --git a/pyproject.toml b/pyproject.toml index d474753ccd1..9a70116f301 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.towncrier] filename = "CHANGELOG.md" -version = "0.21.1" +version = "0.21.2" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" From b11174e96d85de439141670aa1b65cc0f5f6bcd1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:22:34 +0100 Subject: [PATCH 283/349] Update heck requirement from 0.4 to 0.5 (#3966) * Update heck requirement from 0.4 to 0.5 --- updated-dependencies: - dependency-name: heck dependency-type: direct:production ... Signed-off-by: dependabot[bot] * newsfragment --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Hewitt --- newsfragments/3966.packaging.md | 1 + pyo3-macros-backend/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3966.packaging.md diff --git a/newsfragments/3966.packaging.md b/newsfragments/3966.packaging.md new file mode 100644 index 00000000000..81220bc4e85 --- /dev/null +++ b/newsfragments/3966.packaging.md @@ -0,0 +1 @@ +Update `heck` dependency to 0.5. diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index f2675f2b75e..c2ffd53b0fc 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -14,7 +14,7 @@ edition = "2021" # not to depend on proc-macro itself. # See https://github.com/PyO3/pyo3/pull/810 for more. [dependencies] -heck = "0.4" +heck = "0.5" proc-macro2 = { version = "1", default-features = false } pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.2", features = ["resolve-config"] } quote = { version = "1", default-features = false } From d42c00d21dbff0fdf688ba49ed3ab3a41aad2bd7 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 19 Apr 2024 09:24:26 +0200 Subject: [PATCH 284/349] feature gate deprecated APIs for `PySet` (#4096) --- src/types/set.rs | 60 +++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/src/types/set.rs b/src/types/set.rs index f648bc2be1f..83938f3bf42 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -31,12 +31,10 @@ pyobject_native_type_core!( impl PySet { /// Deprecated form of [`PySet::new_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PySet::new` will be replaced by `PySet::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PySet::new` will be replaced by `PySet::new_bound` in a future PyO3 version" )] #[inline] pub fn new<'a, 'p, T: ToPyObject + 'a>( @@ -58,12 +56,10 @@ impl PySet { } /// Deprecated form of [`PySet::empty_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.2", - note = "`PySet::empty` will be replaced by `PySet::empty_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.2", + note = "`PySet::empty` will be replaced by `PySet::empty_bound` in a future PyO3 version" )] pub fn empty(py: Python<'_>) -> PyResult<&PySet> { Self::empty_bound(py).map(Bound::into_gil_ref) @@ -396,27 +392,29 @@ pub(crate) fn new_from_iter( } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::PySet; - use crate::{Python, ToPyObject}; + use crate::{ + types::{PyAnyMethods, PySetMethods}, + Python, ToPyObject, + }; use std::collections::HashSet; #[test] fn test_set_new() { Python::with_gil(|py| { - let set = PySet::new(py, &[1]).unwrap(); + let set = PySet::new_bound(py, &[1]).unwrap(); assert_eq!(1, set.len()); let v = vec![1]; - assert!(PySet::new(py, &[v]).is_err()); + assert!(PySet::new_bound(py, &[v]).is_err()); }); } #[test] fn test_set_empty() { Python::with_gil(|py| { - let set = PySet::empty(py).unwrap(); + let set = PySet::empty_bound(py).unwrap(); assert_eq!(0, set.len()); assert!(set.is_empty()); }); @@ -427,11 +425,11 @@ mod tests { Python::with_gil(|py| { let mut v = HashSet::new(); let ob = v.to_object(py); - let set: &PySet = ob.downcast(py).unwrap(); + let set = ob.downcast_bound::(py).unwrap(); assert_eq!(0, set.len()); v.insert(7); let ob = v.to_object(py); - let set2: &PySet = ob.downcast(py).unwrap(); + let set2 = ob.downcast_bound::(py).unwrap(); assert_eq!(1, set2.len()); }); } @@ -439,7 +437,7 @@ mod tests { #[test] fn test_set_clear() { Python::with_gil(|py| { - let set = PySet::new(py, &[1]).unwrap(); + let set = PySet::new_bound(py, &[1]).unwrap(); assert_eq!(1, set.len()); set.clear(); assert_eq!(0, set.len()); @@ -449,7 +447,7 @@ mod tests { #[test] fn test_set_contains() { Python::with_gil(|py| { - let set = PySet::new(py, &[1]).unwrap(); + let set = PySet::new_bound(py, &[1]).unwrap(); assert!(set.contains(1).unwrap()); }); } @@ -457,7 +455,7 @@ mod tests { #[test] fn test_set_discard() { Python::with_gil(|py| { - let set = PySet::new(py, &[1]).unwrap(); + let set = PySet::new_bound(py, &[1]).unwrap(); assert!(!set.discard(2).unwrap()); assert_eq!(1, set.len()); @@ -472,7 +470,7 @@ mod tests { #[test] fn test_set_add() { Python::with_gil(|py| { - let set = PySet::new(py, &[1, 2]).unwrap(); + let set = PySet::new_bound(py, &[1, 2]).unwrap(); set.add(1).unwrap(); // Add a dupliated element assert!(set.contains(1).unwrap()); }); @@ -481,13 +479,13 @@ mod tests { #[test] fn test_set_pop() { Python::with_gil(|py| { - let set = PySet::new(py, &[1]).unwrap(); + let set = PySet::new_bound(py, &[1]).unwrap(); let val = set.pop(); assert!(val.is_some()); let val2 = set.pop(); assert!(val2.is_none()); assert!(py - .eval("print('Exception state should not be set.')", None, None) + .eval_bound("print('Exception state should not be set.')", None, None) .is_ok()); }); } @@ -495,7 +493,7 @@ mod tests { #[test] fn test_set_iter() { Python::with_gil(|py| { - let set = PySet::new(py, &[1]).unwrap(); + let set = PySet::new_bound(py, &[1]).unwrap(); for el in set { assert_eq!(1i32, el.extract::<'_, i32>().unwrap()); @@ -520,9 +518,9 @@ mod tests { #[should_panic] fn test_set_iter_mutation() { Python::with_gil(|py| { - let set = PySet::new(py, &[1, 2, 3, 4, 5]).unwrap(); + let set = PySet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); - for _ in set { + for _ in &set { let _ = set.add(42); } }); @@ -532,9 +530,9 @@ mod tests { #[should_panic] fn test_set_iter_mutation_same_len() { Python::with_gil(|py| { - let set = PySet::new(py, &[1, 2, 3, 4, 5]).unwrap(); + let set = PySet::new_bound(py, &[1, 2, 3, 4, 5]).unwrap(); - for item in set { + for item in &set { let item: i32 = item.extract().unwrap(); let _ = set.del_item(item); let _ = set.add(item + 10); @@ -545,7 +543,7 @@ mod tests { #[test] fn test_set_iter_size_hint() { Python::with_gil(|py| { - let set = PySet::new(py, &[1]).unwrap(); + let set = PySet::new_bound(py, &[1]).unwrap(); let mut iter = set.iter(); // Exact size From cd28e1408e9f4d2eee4832e0123b8771d11b4731 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 19 Apr 2024 12:44:36 +0100 Subject: [PATCH 285/349] add `#[track_caller]` to all `Py`/`Bound`/`Borrowed` methods which panic (#4098) --- newsfragments/4098.changed.md | 1 + src/err/mod.rs | 1 + src/ffi_ptr_ext.rs | 2 ++ src/instance.rs | 9 +++++++++ src/pycell.rs | 2 ++ 5 files changed, 15 insertions(+) create mode 100644 newsfragments/4098.changed.md diff --git a/newsfragments/4098.changed.md b/newsfragments/4098.changed.md new file mode 100644 index 00000000000..5df526a52e3 --- /dev/null +++ b/newsfragments/4098.changed.md @@ -0,0 +1 @@ +Add `#[track_caller]` to all `Py`, `Bound<'py, T>` and `Borrowed<'a, 'py, T>` methods which can panic. diff --git a/src/err/mod.rs b/src/err/mod.rs index 8dd16b26b47..a61c8c62d31 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -1091,6 +1091,7 @@ fn display_downcast_error( ) } +#[track_caller] pub fn panic_after_error(_py: Python<'_>) -> ! { unsafe { ffi::PyErr_Print(); diff --git a/src/ffi_ptr_ext.rs b/src/ffi_ptr_ext.rs index 3ca8671f1f6..183b0e3734e 100644 --- a/src/ffi_ptr_ext.rs +++ b/src/ffi_ptr_ext.rs @@ -39,6 +39,7 @@ impl FfiPtrExt for *mut ffi::PyObject { } #[inline] + #[track_caller] unsafe fn assume_owned(self, py: Python<'_>) -> Bound<'_, PyAny> { Bound::from_owned_ptr(py, self) } @@ -57,6 +58,7 @@ impl FfiPtrExt for *mut ffi::PyObject { } #[inline] + #[track_caller] unsafe fn assume_borrowed<'a>(self, py: Python<'_>) -> Borrowed<'a, '_, PyAny> { Borrowed::from_ptr(py, self) } diff --git a/src/instance.rs b/src/instance.rs index 88a550ffd30..b2715abe2b9 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -104,6 +104,7 @@ impl<'py> Bound<'py, PyAny> { /// - `ptr` must be a valid pointer to a Python object /// - `ptr` must be an owned Python reference, as the `Bound<'py, PyAny>` will assume ownership #[inline] + #[track_caller] pub unsafe fn from_owned_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self(py, ManuallyDrop::new(Py::from_owned_ptr(py, ptr))) } @@ -141,6 +142,7 @@ impl<'py> Bound<'py, PyAny> { /// /// - `ptr` must be a valid pointer to a Python object #[inline] + #[track_caller] pub unsafe fn from_borrowed_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self(py, ManuallyDrop::new(Py::from_borrowed_ptr(py, ptr))) } @@ -242,6 +244,7 @@ where /// Panics if the value is currently mutably borrowed. For a non-panicking variant, use /// [`try_borrow`](#method.try_borrow). #[inline] + #[track_caller] pub fn borrow(&self) -> PyRef<'py, T> { PyRef::borrow(self) } @@ -276,6 +279,7 @@ where /// Panics if the value is currently borrowed. For a non-panicking variant, use /// [`try_borrow_mut`](#method.try_borrow_mut). #[inline] + #[track_caller] pub fn borrow_mut(&self) -> PyRefMut<'py, T> where T: PyClass, @@ -573,6 +577,7 @@ impl<'a, 'py> Borrowed<'a, 'py, PyAny> { /// the caller and it is the caller's responsibility to ensure that the reference this is /// derived from is valid for the lifetime `'a`. #[inline] + #[track_caller] pub unsafe fn from_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self { Self( NonNull::new(ptr).unwrap_or_else(|| crate::err::panic_after_error(py)), @@ -1138,6 +1143,7 @@ where /// Panics if the value is currently mutably borrowed. For a non-panicking variant, use /// [`try_borrow`](#method.try_borrow). #[inline] + #[track_caller] pub fn borrow<'py>(&'py self, py: Python<'py>) -> PyRef<'py, T> { self.bind(py).borrow() } @@ -1175,6 +1181,7 @@ where /// Panics if the value is currently borrowed. For a non-panicking variant, use /// [`try_borrow_mut`](#method.try_borrow_mut). #[inline] + #[track_caller] pub fn borrow_mut<'py>(&'py self, py: Python<'py>) -> PyRefMut<'py, T> where T: PyClass, @@ -1585,6 +1592,7 @@ impl Py { /// # Panics /// Panics if `ptr` is null. #[inline] + #[track_caller] pub unsafe fn from_owned_ptr(py: Python<'_>, ptr: *mut ffi::PyObject) -> Py { match NonNull::new(ptr) { Some(nonnull_ptr) => Py(nonnull_ptr, PhantomData), @@ -1628,6 +1636,7 @@ impl Py { /// # Panics /// Panics if `ptr` is null. #[inline] + #[track_caller] pub unsafe fn from_borrowed_ptr(py: Python<'_>, ptr: *mut ffi::PyObject) -> Py { match Self::from_borrowed_ptr_or_opt(py, ptr) { Some(slf) => slf, diff --git a/src/pycell.rs b/src/pycell.rs index a649ad02412..80ccff0a030 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -652,6 +652,7 @@ impl<'py, T: PyClass> PyRef<'py, T> { self.inner.clone().into_ptr() } + #[track_caller] pub(crate) fn borrow(obj: &Bound<'py, T>) -> Self { Self::try_borrow(obj).expect("Already mutably borrowed") } @@ -848,6 +849,7 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { } #[inline] + #[track_caller] pub(crate) fn borrow(obj: &Bound<'py, T>) -> Self { Self::try_borrow(obj).expect("Already borrowed") } From 947b372dcc776f717e5a9862c87c85609a9e8d0d Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 19 Apr 2024 20:35:52 +0100 Subject: [PATCH 286/349] change `PyAnyMethods::dir` to be fallible (#4100) --- newsfragments/4100.changed.md | 1 + src/types/any.rs | 34 +++++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 newsfragments/4100.changed.md diff --git a/newsfragments/4100.changed.md b/newsfragments/4100.changed.md new file mode 100644 index 00000000000..13fd2e02aa8 --- /dev/null +++ b/newsfragments/4100.changed.md @@ -0,0 +1 @@ +Change `PyAnyMethods::dir` to be fallible and return `PyResult>` (and similar for `PyAny::dir`). diff --git a/src/types/any.rs b/src/types/any.rs index a5ea4c80d4c..6ba34c86bc6 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -845,8 +845,8 @@ impl PyAny { /// Returns the list of attributes of this object. /// /// This is equivalent to the Python expression `dir(self)`. - pub fn dir(&self) -> &PyList { - self.as_borrowed().dir().into_gil_ref() + pub fn dir(&self) -> PyResult<&PyList> { + self.as_borrowed().dir().map(Bound::into_gil_ref) } /// Checks whether this object is an instance of type `ty`. @@ -1674,7 +1674,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// Returns the list of attributes of this object. /// /// This is equivalent to the Python expression `dir(self)`. - fn dir(&self) -> Bound<'py, PyList>; + fn dir(&self) -> PyResult>; /// Checks whether this object is an instance of type `ty`. /// @@ -2220,10 +2220,10 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { Ok(v as usize) } - fn dir(&self) -> Bound<'py, PyList> { + fn dir(&self) -> PyResult> { unsafe { ffi::PyObject_Dir(self.as_ptr()) - .assume_owned(self.py()) + .assume_owned_or_err(self.py()) .downcast_into_unchecked() } } @@ -2471,6 +2471,7 @@ class SimpleClass: .unwrap(); let a = obj .dir() + .unwrap() .into_iter() .map(|x| x.extract::().unwrap()); let b = dir.into_iter().map(|x| x.extract::().unwrap()); @@ -2745,4 +2746,27 @@ class SimpleClass: assert!(not_container.is_empty().is_err()); }); } + + #[cfg(feature = "macros")] + #[test] + #[allow(unknown_lints, non_local_definitions)] + fn test_fallible_dir() { + use crate::exceptions::PyValueError; + use crate::prelude::*; + + #[pyclass(crate = "crate")] + struct DirFail; + + #[pymethods(crate = "crate")] + impl DirFail { + fn __dir__(&self) -> PyResult { + Err(PyValueError::new_err("uh-oh!")) + } + } + + Python::with_gil(|py| { + let obj = Bound::new(py, DirFail).unwrap(); + assert!(obj.dir().unwrap_err().is_instance_of::(py)); + }) + } } From b0ad1e10aaf15a446635fd9bb8488079dc250624 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Mon, 22 Apr 2024 09:19:01 +0200 Subject: [PATCH 287/349] feature gate deprecated APIs for `PyTuple` (#4107) --- src/types/tuple.rs | 85 +++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 636a2f3e11f..563a81983fa 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -59,12 +59,10 @@ pyobject_native_type_core!(PyTuple, pyobject_native_static_type_object!(ffi::PyT impl PyTuple { /// Deprecated form of `PyTuple::new_bound`. #[track_caller] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyTuple::new` will be replaced by `PyTuple::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyTuple::new` will be replaced by `PyTuple::new_bound` in a future PyO3 version" )] pub fn new( py: Python<'_>, @@ -117,12 +115,10 @@ impl PyTuple { } /// Deprecated form of `PyTuple::empty_bound`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyTuple::empty` will be replaced by `PyTuple::empty_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyTuple::empty` will be replaced by `PyTuple::empty_bound` in a future PyO3 version" )] pub fn empty(py: Python<'_>) -> &PyTuple { Self::empty_bound(py).into_gil_ref() @@ -832,24 +828,23 @@ tuple_conversion!( ); #[cfg(test)] -#[allow(deprecated)] // TODO: remove allow when GIL Pool is removed mod tests { - use crate::types::{any::PyAnyMethods, tuple::PyTupleMethods, PyAny, PyList, PyTuple}; + use crate::types::{any::PyAnyMethods, tuple::PyTupleMethods, PyList, PyTuple}; use crate::{Python, ToPyObject}; use std::collections::HashSet; #[test] fn test_new() { Python::with_gil(|py| { - let ob = PyTuple::new(py, [1, 2, 3]); + let ob = PyTuple::new_bound(py, [1, 2, 3]); assert_eq!(3, ob.len()); - let ob: &PyAny = ob.into(); + let ob = ob.as_any(); assert_eq!((1, 2, 3), ob.extract().unwrap()); let mut map = HashSet::new(); map.insert(1); map.insert(2); - PyTuple::new(py, map); + PyTuple::new_bound(py, map); }); } @@ -857,10 +852,10 @@ mod tests { fn test_len() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); - let tuple: &PyTuple = ob.downcast(py).unwrap(); + let tuple = ob.downcast_bound::(py).unwrap(); assert_eq!(3, tuple.len()); assert!(!tuple.is_empty()); - let ob: &PyAny = tuple.into(); + let ob = tuple.as_any(); assert_eq!((1, 2, 3), ob.extract().unwrap()); }); } @@ -868,7 +863,7 @@ mod tests { #[test] fn test_empty() { Python::with_gil(|py| { - let tuple = PyTuple::empty(py); + let tuple = PyTuple::empty_bound(py); assert!(tuple.is_empty()); assert_eq!(0, tuple.len()); }); @@ -877,7 +872,7 @@ mod tests { #[test] fn test_slice() { Python::with_gil(|py| { - let tup = PyTuple::new(py, [2, 3, 5, 7]); + let tup = PyTuple::new_bound(py, [2, 3, 5, 7]); let slice = tup.get_slice(1, 3); assert_eq!(2, slice.len()); let slice = tup.get_slice(1, 7); @@ -889,7 +884,7 @@ mod tests { fn test_iter() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); - let tuple: &PyTuple = ob.downcast(py).unwrap(); + let tuple = ob.downcast_bound::(py).unwrap(); assert_eq!(3, tuple.len()); let mut iter = tuple.iter(); @@ -913,7 +908,7 @@ mod tests { fn test_iter_rev() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); - let tuple: &PyTuple = ob.downcast(py).unwrap(); + let tuple = ob.downcast_bound::(py).unwrap(); assert_eq!(3, tuple.len()); let mut iter = tuple.iter().rev(); @@ -983,7 +978,7 @@ mod tests { fn test_into_iter() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); - let tuple: &PyTuple = ob.downcast(py).unwrap(); + let tuple = ob.downcast_bound::(py).unwrap(); assert_eq!(3, tuple.len()); for (i, item) in tuple.iter().enumerate() { @@ -1014,7 +1009,7 @@ mod tests { fn test_as_slice() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); - let tuple: &PyTuple = ob.downcast(py).unwrap(); + let tuple = ob.downcast_bound::(py).unwrap(); let slice = tuple.as_slice(); assert_eq!(3, slice.len()); @@ -1092,7 +1087,7 @@ mod tests { fn test_tuple_get_item_invalid_index() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); - let tuple: &PyTuple = ob.downcast(py).unwrap(); + let tuple = ob.downcast_bound::(py).unwrap(); let obj = tuple.get_item(5); assert!(obj.is_err()); assert_eq!( @@ -1106,7 +1101,7 @@ mod tests { fn test_tuple_get_item_sanity() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); - let tuple: &PyTuple = ob.downcast(py).unwrap(); + let tuple = ob.downcast_bound::(py).unwrap(); let obj = tuple.get_item(0); assert_eq!(obj.unwrap().extract::().unwrap(), 1); }); @@ -1117,13 +1112,15 @@ mod tests { fn test_tuple_get_item_unchecked_sanity() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); - let tuple: &PyTuple = ob.downcast(py).unwrap(); + let tuple = ob.downcast_bound::(py).unwrap(); let obj = unsafe { tuple.get_item_unchecked(0) }; assert_eq!(obj.extract::().unwrap(), 1); }); } #[test] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_tuple_index_trait() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); @@ -1136,6 +1133,8 @@ mod tests { #[test] #[should_panic] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_tuple_index_trait_panic() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); @@ -1145,6 +1144,8 @@ mod tests { } #[test] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_tuple_index_trait_ranges() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); @@ -1165,6 +1166,8 @@ mod tests { #[test] #[should_panic = "range start index 5 out of range for tuple of length 3"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_tuple_index_trait_range_panic_start() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); @@ -1175,6 +1178,8 @@ mod tests { #[test] #[should_panic = "range end index 10 out of range for tuple of length 3"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_tuple_index_trait_range_panic_end() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); @@ -1185,6 +1190,8 @@ mod tests { #[test] #[should_panic = "slice index starts at 2 but ends at 1"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_tuple_index_trait_range_panic_wrong_order() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); @@ -1196,6 +1203,8 @@ mod tests { #[test] #[should_panic = "range start index 8 out of range for tuple of length 3"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_tuple_index_trait_range_from_panic() { Python::with_gil(|py| { let ob = (1, 2, 3).to_object(py); @@ -1208,7 +1217,7 @@ mod tests { fn test_tuple_contains() { Python::with_gil(|py| { let ob = (1, 1, 2, 3, 5, 8).to_object(py); - let tuple: &PyTuple = ob.downcast(py).unwrap(); + let tuple = ob.downcast_bound::(py).unwrap(); assert_eq!(6, tuple.len()); let bad_needle = 7i32.to_object(py); @@ -1226,7 +1235,7 @@ mod tests { fn test_tuple_index() { Python::with_gil(|py| { let ob = (1, 1, 2, 3, 5, 8).to_object(py); - let tuple: &PyTuple = ob.downcast(py).unwrap(); + let tuple = ob.downcast_bound::(py).unwrap(); assert_eq!(0, tuple.index(1i32).unwrap()); assert_eq!(2, tuple.index(2i32).unwrap()); assert_eq!(3, tuple.index(3i32).unwrap()); @@ -1263,7 +1272,7 @@ mod tests { fn too_long_iterator() { Python::with_gil(|py| { let iter = FaultyIter(0..usize::MAX, 73); - let _tuple = PyTuple::new(py, iter); + let _tuple = PyTuple::new_bound(py, iter); }) } @@ -1274,7 +1283,7 @@ mod tests { fn too_short_iterator() { Python::with_gil(|py| { let iter = FaultyIter(0..35, 73); - let _tuple = PyTuple::new(py, iter); + let _tuple = PyTuple::new_bound(py, iter); }) } @@ -1286,14 +1295,14 @@ mod tests { Python::with_gil(|py| { let iter = FaultyIter(0..0, usize::MAX); - let _tuple = PyTuple::new(py, iter); + let _tuple = PyTuple::new_bound(py, iter); }) } #[cfg(feature = "macros")] #[test] fn bad_clone_mem_leaks() { - use crate::{IntoPy, Py}; + use crate::{IntoPy, Py, PyAny}; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; static NEEDS_DESTRUCTING_COUNT: AtomicUsize = AtomicUsize::new(0); @@ -1346,7 +1355,7 @@ mod tests { Python::with_gil(|py| { std::panic::catch_unwind(|| { let iter = FaultyIter(0..50, 50); - let _tuple = PyTuple::new(py, iter); + let _tuple = PyTuple::new_bound(py, iter); }) .unwrap_err(); }); @@ -1361,7 +1370,7 @@ mod tests { #[cfg(feature = "macros")] #[test] fn bad_clone_mem_leaks_2() { - use crate::{IntoPy, Py}; + use crate::{IntoPy, Py, PyAny}; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; static NEEDS_DESTRUCTING_COUNT: AtomicUsize = AtomicUsize::new(0); @@ -1412,9 +1421,9 @@ mod tests { #[test] fn test_tuple_to_list() { Python::with_gil(|py| { - let tuple = PyTuple::new(py, vec![1, 2, 3]); + let tuple = PyTuple::new_bound(py, vec![1, 2, 3]); let list = tuple.to_list(); - let list_expected = PyList::new(py, vec![1, 2, 3]); + let list_expected = PyList::new_bound(py, vec![1, 2, 3]); assert!(list.eq(list_expected).unwrap()); }) } From da2d1aa8b46636b62ea8fd07d7a3a412a3a28280 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Mon, 22 Apr 2024 09:20:32 +0200 Subject: [PATCH 288/349] feature gate deprecated APIs for `PyString` (#4101) --- src/types/string.rs | 122 +++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 59 deletions(-) diff --git a/src/types/string.rs b/src/types/string.rs index 4bbc6fb86b0..4aa73341ae9 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -136,12 +136,10 @@ pyobject_native_type_core!(PyString, pyobject_native_static_type_object!(ffi::Py impl PyString { /// Deprecated form of [`PyString::new_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyString::new` will be replaced by `PyString::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyString::new` will be replaced by `PyString::new_bound` in a future PyO3 version" )] pub fn new<'py>(py: Python<'py>, s: &str) -> &'py Self { Self::new_bound(py, s).into_gil_ref() @@ -161,12 +159,10 @@ impl PyString { } /// Deprecated form of [`PyString::intern_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyString::intern` will be replaced by `PyString::intern_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyString::intern` will be replaced by `PyString::intern_bound` in a future PyO3 version" )] pub fn intern<'py>(py: Python<'py>, s: &str) -> &'py Self { Self::intern_bound(py, s).into_gil_ref() @@ -176,8 +172,8 @@ impl PyString { /// /// This will return a reference to the same Python string object if called repeatedly with the same string. /// - /// Note that while this is more memory efficient than [`PyString::new`], it unconditionally allocates a - /// temporary Python string object and is thereby slower than [`PyString::new`]. + /// Note that while this is more memory efficient than [`PyString::new_bound`], it unconditionally allocates a + /// temporary Python string object and is thereby slower than [`PyString::new_bound`]. /// /// Panics if out of memory. pub fn intern_bound<'py>(py: Python<'py>, s: &str) -> Bound<'py, PyString> { @@ -193,12 +189,10 @@ impl PyString { } /// Deprecated form of [`PyString::from_object_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyString::from_object` will be replaced by `PyString::from_object_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyString::from_object` will be replaced by `PyString::from_object_bound` in a future PyO3 version" )] pub fn from_object<'py>(src: &'py PyAny, encoding: &str, errors: &str) -> PyResult<&'py Self> { Self::from_object_bound(&src.as_borrowed(), encoding, errors).map(Bound::into_gil_ref) @@ -502,37 +496,37 @@ impl IntoPy> for &'_ Py { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; use crate::{PyObject, ToPyObject}; #[test] - fn test_to_str_utf8() { + fn test_to_cow_utf8() { Python::with_gil(|py| { let s = "ascii 🐈"; - let obj: PyObject = PyString::new(py, s).into(); - let py_string: &PyString = obj.downcast(py).unwrap(); - assert_eq!(s, py_string.to_str().unwrap()); + let py_string = PyString::new_bound(py, s); + assert_eq!(s, py_string.to_cow().unwrap()); }) } #[test] - fn test_to_str_surrogate() { + fn test_to_cow_surrogate() { Python::with_gil(|py| { - let obj: PyObject = py.eval(r"'\ud800'", None, None).unwrap().into(); - let py_string: &PyString = obj.downcast(py).unwrap(); - assert!(py_string.to_str().is_err()); + let py_string = py + .eval_bound(r"'\ud800'", None, None) + .unwrap() + .downcast_into::() + .unwrap(); + assert!(py_string.to_cow().is_err()); }) } #[test] - fn test_to_str_unicode() { + fn test_to_cow_unicode() { Python::with_gil(|py| { let s = "哈哈🐈"; - let obj: PyObject = PyString::new(py, s).into(); - let py_string: &PyString = obj.downcast(py).unwrap(); - assert_eq!(s, py_string.to_str().unwrap()); + let py_string = PyString::new_bound(py, s); + assert_eq!(s, py_string.to_cow().unwrap()); }) } @@ -548,7 +542,7 @@ mod tests { #[test] fn test_encode_utf8_surrogate() { Python::with_gil(|py| { - let obj: PyObject = py.eval(r"'\ud800'", None, None).unwrap().into(); + let obj: PyObject = py.eval_bound(r"'\ud800'", None, None).unwrap().into(); assert!(obj .bind(py) .downcast::() @@ -561,11 +555,12 @@ mod tests { #[test] fn test_to_string_lossy() { Python::with_gil(|py| { - let obj: PyObject = py - .eval(r"'🐈 Hello \ud800World'", None, None) + let py_string = py + .eval_bound(r"'🐈 Hello \ud800World'", None, None) .unwrap() - .into(); - let py_string: &PyString = obj.downcast(py).unwrap(); + .downcast_into::() + .unwrap(); + assert_eq!(py_string.to_string_lossy(), "🐈 Hello ���World"); }) } @@ -574,7 +569,7 @@ mod tests { fn test_debug_string() { Python::with_gil(|py| { let v = "Hello\n".to_object(py); - let s: &PyString = v.downcast(py).unwrap(); + let s = v.downcast_bound::(py).unwrap(); assert_eq!(format!("{:?}", s), "'Hello\\n'"); }) } @@ -583,7 +578,7 @@ mod tests { fn test_display_string() { Python::with_gil(|py| { let v = "Hello\n".to_object(py); - let s: &PyString = v.downcast(py).unwrap(); + let s = v.downcast_bound::(py).unwrap(); assert_eq!(format!("{}", s), "Hello\n"); }) } @@ -592,7 +587,7 @@ mod tests { #[cfg(not(Py_LIMITED_API))] fn test_string_data_ucs1() { Python::with_gil(|py| { - let s = PyString::new(py, "hello, world"); + let s = PyString::new_bound(py, "hello, world"); let data = unsafe { s.data().unwrap() }; assert_eq!(data, PyStringData::Ucs1(b"hello, world")); @@ -615,11 +610,13 @@ mod tests { ) }; assert!(!ptr.is_null()); - let s: &PyString = unsafe { py.from_owned_ptr(ptr) }; + let s = unsafe { ptr.assume_owned(py).downcast_into_unchecked::() }; let data = unsafe { s.data().unwrap() }; assert_eq!(data, PyStringData::Ucs1(b"f\xfe")); let err = data.to_string(py).unwrap_err(); - assert!(err.get_type(py).is(py.get_type::())); + assert!(err + .get_type_bound(py) + .is(&py.get_type_bound::())); assert!(err .to_string() .contains("'utf-8' codec can't decode byte 0xfe in position 1")); @@ -631,7 +628,7 @@ mod tests { #[cfg(not(Py_LIMITED_API))] fn test_string_data_ucs2() { Python::with_gil(|py| { - let s = py.eval("'foo\\ud800'", None, None).unwrap(); + let s = py.eval_bound("'foo\\ud800'", None, None).unwrap(); let py_string = s.downcast::().unwrap(); let data = unsafe { py_string.data().unwrap() }; @@ -657,11 +654,13 @@ mod tests { ) }; assert!(!ptr.is_null()); - let s: &PyString = unsafe { py.from_owned_ptr(ptr) }; + let s = unsafe { ptr.assume_owned(py).downcast_into_unchecked::() }; let data = unsafe { s.data().unwrap() }; assert_eq!(data, PyStringData::Ucs2(&[0xff22, 0xd800])); let err = data.to_string(py).unwrap_err(); - assert!(err.get_type(py).is(py.get_type::())); + assert!(err + .get_type_bound(py) + .is(&py.get_type_bound::())); assert!(err .to_string() .contains("'utf-16' codec can't decode bytes in position 0-3")); @@ -674,7 +673,7 @@ mod tests { fn test_string_data_ucs4() { Python::with_gil(|py| { let s = "哈哈🐈"; - let py_string = PyString::new(py, s); + let py_string = PyString::new_bound(py, s); let data = unsafe { py_string.data().unwrap() }; assert_eq!(data, PyStringData::Ucs4(&[21704, 21704, 128008])); @@ -696,11 +695,13 @@ mod tests { ) }; assert!(!ptr.is_null()); - let s: &PyString = unsafe { py.from_owned_ptr(ptr) }; + let s = unsafe { ptr.assume_owned(py).downcast_into_unchecked::() }; let data = unsafe { s.data().unwrap() }; assert_eq!(data, PyStringData::Ucs4(&[0x20000, 0xd800])); let err = data.to_string(py).unwrap_err(); - assert!(err.get_type(py).is(py.get_type::())); + assert!(err + .get_type_bound(py) + .is(&py.get_type_bound::())); assert!(err .to_string() .contains("'utf-32' codec can't decode bytes in position 0-7")); @@ -711,16 +712,16 @@ mod tests { #[test] fn test_intern_string() { Python::with_gil(|py| { - let py_string1 = PyString::intern(py, "foo"); - assert_eq!(py_string1.to_str().unwrap(), "foo"); + let py_string1 = PyString::intern_bound(py, "foo"); + assert_eq!(py_string1.to_cow().unwrap(), "foo"); - let py_string2 = PyString::intern(py, "foo"); - assert_eq!(py_string2.to_str().unwrap(), "foo"); + let py_string2 = PyString::intern_bound(py, "foo"); + assert_eq!(py_string2.to_cow().unwrap(), "foo"); assert_eq!(py_string1.as_ptr(), py_string2.as_ptr()); - let py_string3 = PyString::intern(py, "bar"); - assert_eq!(py_string3.to_str().unwrap(), "bar"); + let py_string3 = PyString::intern_bound(py, "bar"); + assert_eq!(py_string3.to_cow().unwrap(), "bar"); assert_ne!(py_string1.as_ptr(), py_string3.as_ptr()); }); @@ -730,7 +731,7 @@ mod tests { fn test_py_to_str_utf8() { Python::with_gil(|py| { let s = "ascii 🐈"; - let py_string: Py = PyString::new(py, s).into_py(py); + let py_string: Py = PyString::new_bound(py, s).into_py(py); #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] assert_eq!(s, py_string.to_str(py).unwrap()); @@ -742,8 +743,11 @@ mod tests { #[test] fn test_py_to_str_surrogate() { Python::with_gil(|py| { - let py_string: Py = - py.eval(r"'\ud800'", None, None).unwrap().extract().unwrap(); + let py_string: Py = py + .eval_bound(r"'\ud800'", None, None) + .unwrap() + .extract() + .unwrap(); #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] assert!(py_string.to_str(py).is_err()); @@ -756,7 +760,7 @@ mod tests { fn test_py_to_string_lossy() { Python::with_gil(|py| { let py_string: Py = py - .eval(r"'🐈 Hello \ud800World'", None, None) + .eval_bound(r"'🐈 Hello \ud800World'", None, None) .unwrap() .extract() .unwrap(); From c2ac9a98e2316e42e59cf8b07f0e24ff244afbb7 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 22 Apr 2024 10:56:39 +0200 Subject: [PATCH 289/349] fix vectorcall argument handling (#4104) Fixes #4093 - Make PY_VECTORCALL_ARGUMENTS_OFFSET a size_t like on CPython - Remove the assert in PyVectorcall_NARGS and use checked cast --- newsfragments/4104.fixed.md | 1 + pyo3-ffi/src/cpython/abstract_.rs | 16 +++++++--------- 2 files changed, 8 insertions(+), 9 deletions(-) create mode 100644 newsfragments/4104.fixed.md diff --git a/newsfragments/4104.fixed.md b/newsfragments/4104.fixed.md new file mode 100644 index 00000000000..3eff1654b4f --- /dev/null +++ b/newsfragments/4104.fixed.md @@ -0,0 +1 @@ +Changes definitions of `PY_VECTORCALL_ARGUMENTS_OFFSET` and `PyVectorcall_NARGS` to fix a false-positive assertion. diff --git a/pyo3-ffi/src/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs index cf95f6711d4..ad28216cbff 100644 --- a/pyo3-ffi/src/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -4,8 +4,6 @@ use std::os::raw::{c_char, c_int}; #[cfg(not(Py_3_11))] use crate::Py_buffer; -#[cfg(Py_3_8)] -use crate::pyport::PY_SSIZE_T_MAX; #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] use crate::{ vectorcallfunc, PyCallable_Check, PyThreadState, PyThreadState_GET, PyTuple_Check, @@ -42,14 +40,14 @@ extern "C" { } #[cfg(Py_3_8)] -const PY_VECTORCALL_ARGUMENTS_OFFSET: Py_ssize_t = - 1 << (8 * std::mem::size_of::() as Py_ssize_t - 1); +const PY_VECTORCALL_ARGUMENTS_OFFSET: size_t = + 1 << (8 * std::mem::size_of::() as size_t - 1); #[cfg(Py_3_8)] #[inline(always)] pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t { - assert!(n <= (PY_SSIZE_T_MAX as size_t)); - (n as Py_ssize_t) & !PY_VECTORCALL_ARGUMENTS_OFFSET + let n = n & !PY_VECTORCALL_ARGUMENTS_OFFSET; + n.try_into().expect("cannot fail due to mask") } #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] @@ -184,7 +182,7 @@ pub unsafe fn PyObject_CallOneArg(func: *mut PyObject, arg: *mut PyObject) -> *m let args = args_array.as_ptr().offset(1); // For PY_VECTORCALL_ARGUMENTS_OFFSET let tstate = PyThreadState_GET(); let nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - _PyObject_VectorcallTstate(tstate, func, args, nargsf as size_t, std::ptr::null_mut()) + _PyObject_VectorcallTstate(tstate, func, args, nargsf, std::ptr::null_mut()) } extern "C" { @@ -206,7 +204,7 @@ pub unsafe fn PyObject_CallMethodNoArgs( PyObject_VectorcallMethod( name, &self_, - 1 | PY_VECTORCALL_ARGUMENTS_OFFSET as size_t, + 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, std::ptr::null_mut(), ) } @@ -223,7 +221,7 @@ pub unsafe fn PyObject_CallMethodOneArg( PyObject_VectorcallMethod( name, args.as_ptr(), - 2 | PY_VECTORCALL_ARGUMENTS_OFFSET as size_t, + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, std::ptr::null_mut(), ) } From 5d2f5b5702319150d41258de77f589119134ee74 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 23 Apr 2024 07:48:27 +0200 Subject: [PATCH 290/349] feature gate deprecated APIs for `PyDict` (#4108) --- src/conversion.rs | 1 + src/instance.rs | 6 +-- src/types/dict.rs | 86 ++++++++++++++++++++----------------------- src/types/iterator.rs | 9 +++-- src/types/mapping.rs | 1 + 5 files changed, 49 insertions(+), 54 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index dfa53eac83e..ced209abade 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -674,6 +674,7 @@ mod test_no_clone {} #[cfg(test)] mod tests { + #[cfg(feature = "gil-refs")] #[allow(deprecated)] mod deprecated { use super::super::PyTryFrom; diff --git a/src/instance.rs b/src/instance.rs index b2715abe2b9..02b3fc76241 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -2034,7 +2034,7 @@ mod tests { #[test] fn test_call_for_non_existing_method() { Python::with_gil(|py| { - let obj: PyObject = PyDict::new(py).into(); + let obj: PyObject = PyDict::new_bound(py).into(); assert!(obj.call_method0(py, "asdf").is_err()); assert!(obj .call_method(py, "nonexistent_method", (1,), None) @@ -2047,7 +2047,7 @@ mod tests { #[test] fn py_from_dict() { let dict: Py = Python::with_gil(|py| { - let native = PyDict::new(py); + let native = PyDict::new_bound(py); Py::from(native) }); @@ -2057,7 +2057,7 @@ mod tests { #[test] fn pyobject_from_py() { Python::with_gil(|py| { - let dict: Py = PyDict::new(py).into(); + let dict: Py = PyDict::new_bound(py).unbind(); let cnt = dict.get_refcnt(py); let p: PyObject = dict.into(); assert_eq!(p.get_refcnt(py), cnt); diff --git a/src/types/dict.rs b/src/types/dict.rs index a4ba72a59c0..64b3adcad44 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -57,12 +57,10 @@ pyobject_native_type_core!( impl PyDict { /// Deprecated form of [`new_bound`][PyDict::new_bound]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyDict::new` will be replaced by `PyDict::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyDict::new` will be replaced by `PyDict::new_bound` in a future PyO3 version" )] #[inline] pub fn new(py: Python<'_>) -> &PyDict { @@ -75,12 +73,10 @@ impl PyDict { } /// Deprecated form of [`from_sequence_bound`][PyDict::from_sequence_bound]. - #[cfg_attr( - all(not(any(PyPy, GraalPy)), not(feature = "gil-refs")), - deprecated( - since = "0.21.0", - note = "`PyDict::from_sequence` will be replaced by `PyDict::from_sequence_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyDict::from_sequence` will be replaced by `PyDict::from_sequence_bound` in a future PyO3 version" )] #[inline] #[cfg(not(any(PyPy, GraalPy)))] @@ -751,12 +747,10 @@ pub(crate) use borrowed_iter::BorrowedDictIter; pub trait IntoPyDict: Sized { /// Converts self into a `PyDict` object pointer. Whether pointer owned or borrowed /// depends on implementation. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`IntoPyDict::into_py_dict` will be replaced by `IntoPyDict::into_py_dict_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`IntoPyDict::into_py_dict` will be replaced by `IntoPyDict::into_py_dict_bound` in a future PyO3 version" )] fn into_py_dict(self, py: Python<'_>) -> &PyDict { Self::into_py_dict_bound(self, py).into_gil_ref() @@ -821,7 +815,6 @@ where } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; #[cfg(not(any(PyPy, GraalPy)))] @@ -853,8 +846,8 @@ mod tests { #[cfg(not(any(PyPy, GraalPy)))] fn test_from_sequence() { Python::with_gil(|py| { - let items = PyList::new(py, &vec![("a", 1), ("b", 2)]); - let dict = PyDict::from_sequence(items).unwrap(); + let items = PyList::new_bound(py, &vec![("a", 1), ("b", 2)]); + let dict = PyDict::from_sequence_bound(&items).unwrap(); assert_eq!( 1, dict.get_item("a") @@ -884,8 +877,8 @@ mod tests { #[cfg(not(any(PyPy, GraalPy)))] fn test_from_sequence_err() { Python::with_gil(|py| { - let items = PyList::new(py, &vec!["a", "b"]); - assert!(PyDict::from_sequence(items).is_err()); + let items = PyList::new_bound(py, &vec!["a", "b"]); + assert!(PyDict::from_sequence_bound(&items).is_err()); }); } @@ -913,11 +906,11 @@ mod tests { Python::with_gil(|py| { let mut v = HashMap::new(); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); assert_eq!(0, dict.len()); v.insert(7, 32); let ob = v.to_object(py); - let dict2: &PyDict = ob.downcast(py).unwrap(); + let dict2 = ob.downcast_bound::(py).unwrap(); assert_eq!(1, dict2.len()); }); } @@ -928,7 +921,7 @@ mod tests { let mut v = HashMap::new(); v.insert(7, 32); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); assert!(dict.contains(7i32).unwrap()); assert!(!dict.contains(8i32).unwrap()); }); @@ -940,7 +933,7 @@ mod tests { let mut v = HashMap::new(); v.insert(7, 32); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); assert_eq!( 32, dict.get_item(7i32) @@ -961,7 +954,7 @@ mod tests { let mut v = HashMap::new(); v.insert(7, 32); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast::(py).unwrap(); assert_eq!( 32, dict.get_item_with_error(7i32) @@ -984,7 +977,7 @@ mod tests { let mut v = HashMap::new(); v.insert(7, 32); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); assert!(dict.set_item(7i32, 42i32).is_ok()); // change assert!(dict.set_item(8i32, 123i32).is_ok()); // insert assert_eq!( @@ -1010,11 +1003,10 @@ mod tests { fn test_set_item_refcnt() { Python::with_gil(|py| { let cnt; - let obj = py.eval("object()", None, None).unwrap(); + let obj = py.eval_bound("object()", None, None).unwrap(); { - let _pool = unsafe { crate::GILPool::new() }; cnt = obj.get_refcnt(); - let _dict = [(10, obj)].into_py_dict_bound(py); + let _dict = [(10, &obj)].into_py_dict_bound(py); } { assert_eq!(cnt, obj.get_refcnt()); @@ -1028,7 +1020,7 @@ mod tests { let mut v = HashMap::new(); v.insert(7, 32); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); assert!(dict.set_item(7i32, 42i32).is_ok()); // change assert!(dict.set_item(8i32, 123i32).is_ok()); // insert assert_eq!(32i32, v[&7i32]); // not updated! @@ -1042,7 +1034,7 @@ mod tests { let mut v = HashMap::new(); v.insert(7, 32); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); assert!(dict.del_item(7i32).is_ok()); assert_eq!(0, dict.len()); assert!(dict.get_item(7i32).unwrap().is_none()); @@ -1055,7 +1047,7 @@ mod tests { let mut v = HashMap::new(); v.insert(7, 32); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); assert!(dict.del_item(7i32).is_ok()); // change assert_eq!(32i32, *v.get(&7i32).unwrap()); // not updated! }); @@ -1069,7 +1061,7 @@ mod tests { v.insert(8, 42); v.insert(9, 123); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut key_sum = 0; let mut value_sum = 0; @@ -1091,7 +1083,7 @@ mod tests { v.insert(8, 42); v.insert(9, 123); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut key_sum = 0; for el in dict.keys() { @@ -1109,7 +1101,7 @@ mod tests { v.insert(8, 42); v.insert(9, 123); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut values_sum = 0; for el in dict.values() { @@ -1127,7 +1119,7 @@ mod tests { v.insert(8, 42); v.insert(9, 123); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); let mut key_sum = 0; let mut value_sum = 0; for (key, value) in dict { @@ -1168,7 +1160,7 @@ mod tests { v.insert(9, 123); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); for (key, value) in dict { dict.set_item(key, value.extract::().unwrap() + 7) @@ -1186,7 +1178,7 @@ mod tests { v.insert(i * 2, i * 2); } let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); for (i, (key, value)) in dict.iter().enumerate() { let key = key.extract::().unwrap(); @@ -1211,7 +1203,7 @@ mod tests { v.insert(i * 2, i * 2); } let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); for (i, (key, value)) in dict.iter().enumerate() { let key = key.extract::().unwrap(); @@ -1235,7 +1227,7 @@ mod tests { v.insert(8, 42); v.insert(9, 123); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); let mut iter = dict.iter(); assert_eq!(iter.size_hint(), (v.len(), Some(v.len()))); @@ -1261,7 +1253,7 @@ mod tests { v.insert(8, 42); v.insert(9, 123); let ob = v.to_object(py); - let dict: &PyDict = ob.downcast(py).unwrap(); + let dict = ob.downcast_bound::(py).unwrap(); let mut key_sum = 0; let mut value_sum = 0; for (key, value) in dict { @@ -1404,7 +1396,7 @@ mod tests { let dict = abc_dict(py); let keys = dict.call_method0("keys").unwrap(); assert!(keys - .is_instance(&py.get_type::().as_borrowed()) + .is_instance(&py.get_type_bound::().as_borrowed()) .unwrap()); }) } @@ -1416,7 +1408,7 @@ mod tests { let dict = abc_dict(py); let values = dict.call_method0("values").unwrap(); assert!(values - .is_instance(&py.get_type::().as_borrowed()) + .is_instance(&py.get_type_bound::().as_borrowed()) .unwrap()); }) } @@ -1428,7 +1420,7 @@ mod tests { let dict = abc_dict(py); let items = dict.call_method0("items").unwrap(); assert!(items - .is_instance(&py.get_type::().as_borrowed()) + .is_instance(&py.get_type_bound::().as_borrowed()) .unwrap()); }) } diff --git a/src/types/iterator.rs b/src/types/iterator.rs index bd01fe81a78..60f36f22de2 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -158,7 +158,7 @@ mod tests { use super::PyIterator; use crate::exceptions::PyTypeError; use crate::gil::GILPool; - use crate::types::{PyDict, PyList}; + use crate::types::{PyAnyMethods, PyDict, PyList}; use crate::{Py, PyAny, Python, ToPyObject}; #[test] @@ -244,10 +244,11 @@ def fibonacci(target): "#; Python::with_gil(|py| { - let context = PyDict::new(py); - py.run(fibonacci_generator, None, Some(context)).unwrap(); + let context = PyDict::new_bound(py); + py.run_bound(fibonacci_generator, None, Some(&context)) + .unwrap(); - let generator = py.eval("fibonacci(5)", None, Some(context)).unwrap(); + let generator = py.eval_bound("fibonacci(5)", None, Some(&context)).unwrap(); for (actual, expected) in generator.iter().unwrap().zip(&[1, 1, 2, 3, 5]) { let actual = actual.unwrap().extract::().unwrap(); assert_eq!(actual, *expected) diff --git a/src/types/mapping.rs b/src/types/mapping.rs index a5a93163bbd..a91dad3679f 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -435,6 +435,7 @@ mod tests { } #[test] + #[cfg(feature = "gil-refs")] #[allow(deprecated)] fn test_mapping_try_from() { use crate::PyTryFrom; From f5fee94afcaf11edceceed45192aabd71aeb9415 Mon Sep 17 00:00:00 2001 From: Bruno Kolenbrander <59372212+mejrs@users.noreply.github.com> Date: Tue, 23 Apr 2024 20:01:41 +0200 Subject: [PATCH 291/349] Scope macro imports more tightly (#4088) --- pyo3-macros-backend/src/module.rs | 11 ++++++----- pytests/src/enums.rs | 4 +++- tests/test_no_imports.rs | 5 ++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 3153279a2b8..626cde121e6 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -382,10 +382,7 @@ fn module_initialization(options: PyModuleOptions, ident: &syn::Ident) -> TokenS fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn) -> Result<()> { let ctx = &Ctx::new(&options.krate); let Ctx { pyo3_path } = ctx; - let mut stmts: Vec = vec![syn::parse_quote!( - #[allow(unknown_lints, unused_imports, redundant_imports)] - use #pyo3_path::{PyNativeType, types::PyModuleMethods}; - )]; + let mut stmts: Vec = Vec::new(); for mut stmt in func.block.stmts.drain(..) { if let syn::Stmt::Item(Item::Fn(func)) = &mut stmt { @@ -395,7 +392,11 @@ fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn let name = &func.sig.ident; let statements: Vec = syn::parse_quote! { #wrapped_function - #module_name.as_borrowed().add_function(#pyo3_path::wrap_pyfunction!(#name, #module_name.as_borrowed())?)?; + { + #[allow(unknown_lints, unused_imports, redundant_imports)] + use #pyo3_path::{PyNativeType, types::PyModuleMethods}; + #module_name.as_borrowed().add_function(#pyo3_path::wrap_pyfunction!(#name, #module_name.as_borrowed())?)?; + } }; stmts.extend(statements); } diff --git a/pytests/src/enums.rs b/pytests/src/enums.rs index 4bb269fbbd2..0a1bc49bb63 100644 --- a/pytests/src/enums.rs +++ b/pytests/src/enums.rs @@ -1,5 +1,7 @@ use pyo3::{ - pyclass, pyfunction, pymodule, types::PyModule, wrap_pyfunction_bound, Bound, PyResult, + pyclass, pyfunction, pymodule, + types::{PyModule, PyModuleMethods}, + wrap_pyfunction_bound, Bound, PyResult, }; #[pymodule] diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index 35c978b0da5..022d61e084d 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -30,7 +30,10 @@ fn basic_module_bound(m: &pyo3::Bound<'_, pyo3::types::PyModule>) -> pyo3::PyRes 42 } - m.add_function(pyo3::wrap_pyfunction_bound!(basic_function, m)?)?; + pyo3::types::PyModuleMethods::add_function( + m, + pyo3::wrap_pyfunction_bound!(basic_function, m)?, + )?; Ok(()) } From 013a4476ead64247257c1292a61201e8ee18adfc Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 24 Apr 2024 19:33:53 +0200 Subject: [PATCH 292/349] feature gate deprecated APIs for `datetime` types (#4111) --- src/types/datetime.rs | 145 ++++++++++++++++++------------------------ src/types/mod.rs | 2 +- 2 files changed, 64 insertions(+), 83 deletions(-) diff --git a/src/types/datetime.rs b/src/types/datetime.rs index e1620a6f647..12db67ab961 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -173,12 +173,10 @@ pub trait PyTimeAccess { /// Trait for accessing the components of a struct containing a tzinfo. pub trait PyTzInfoAccess<'py> { /// Deprecated form of `get_tzinfo_bound`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`get_tzinfo` will be replaced by `get_tzinfo_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`get_tzinfo` will be replaced by `get_tzinfo_bound` in a future PyO3 version" )] fn get_tzinfo(&self) -> Option<&'py PyTzInfo> { self.get_tzinfo_bound().map(Bound::into_gil_ref) @@ -205,12 +203,10 @@ pyobject_native_type!( impl PyDate { /// Deprecated form of [`PyDate::new_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyDate::new` will be replaced by `PyDate::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyDate::new` will be replaced by `PyDate::new_bound` in a future PyO3 version" )] pub fn new(py: Python<'_>, year: i32, month: u8, day: u8) -> PyResult<&PyDate> { Self::new_bound(py, year, month, day).map(Bound::into_gil_ref) @@ -227,12 +223,10 @@ impl PyDate { } /// Deprecated form of [`PyDate::from_timestamp_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyDate::from_timestamp` will be replaced by `PyDate::from_timestamp_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyDate::from_timestamp` will be replaced by `PyDate::from_timestamp_bound` in a future PyO3 version" )] pub fn from_timestamp(py: Python<'_>, timestamp: i64) -> PyResult<&PyDate> { Self::from_timestamp_bound(py, timestamp).map(Bound::into_gil_ref) @@ -296,12 +290,10 @@ pyobject_native_type!( impl PyDateTime { /// Deprecated form of [`PyDateTime::new_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyDateTime::new` will be replaced by `PyDateTime::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyDateTime::new` will be replaced by `PyDateTime::new_bound` in a future PyO3 version" )] #[allow(clippy::too_many_arguments)] pub fn new<'py>( @@ -361,12 +353,10 @@ impl PyDateTime { } /// Deprecated form of [`PyDateTime::new_bound_with_fold`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyDateTime::new_with_fold` will be replaced by `PyDateTime::new_bound_with_fold` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyDateTime::new_with_fold` will be replaced by `PyDateTime::new_bound_with_fold` in a future PyO3 version" )] #[allow(clippy::too_many_arguments)] pub fn new_with_fold<'py>( @@ -436,12 +426,10 @@ impl PyDateTime { } /// Deprecated form of [`PyDateTime::from_timestamp_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyDateTime::from_timestamp` will be replaced by `PyDateTime::from_timestamp_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyDateTime::from_timestamp` will be replaced by `PyDateTime::from_timestamp_bound` in a future PyO3 version" )] pub fn from_timestamp<'py>( py: Python<'py>, @@ -598,12 +586,10 @@ pyobject_native_type!( impl PyTime { /// Deprecated form of [`PyTime::new_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyTime::new` will be replaced by `PyTime::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyTime::new` will be replaced by `PyTime::new_bound` in a future PyO3 version" )] pub fn new<'py>( py: Python<'py>, @@ -649,12 +635,10 @@ impl PyTime { } /// Deprecated form of [`PyTime::new_bound_with_fold`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyTime::new_with_fold` will be replaced by `PyTime::new_bound_with_fold` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyTime::new_with_fold` will be replaced by `PyTime::new_bound_with_fold` in a future PyO3 version" )] pub fn new_with_fold<'py>( py: Python<'py>, @@ -677,7 +661,7 @@ impl PyTime { .map(Bound::into_gil_ref) } - /// Alternate constructor that takes a `fold` argument. See [`PyDateTime::new_with_fold`]. + /// Alternate constructor that takes a `fold` argument. See [`PyDateTime::new_bound_with_fold`]. pub fn new_bound_with_fold<'py>( py: Python<'py>, hour: u8, @@ -791,7 +775,7 @@ impl<'py> PyTzInfoAccess<'py> for Bound<'py, PyTime> { /// Bindings for `datetime.tzinfo`. /// /// This is an abstract base class and cannot be constructed directly. -/// For concrete time zone implementations, see [`timezone_utc`] and +/// For concrete time zone implementations, see [`timezone_utc_bound`] and /// the [`zoneinfo` module](https://docs.python.org/3/library/zoneinfo.html). #[repr(transparent)] pub struct PyTzInfo(PyAny); @@ -804,12 +788,10 @@ pyobject_native_type!( ); /// Deprecated form of [`timezone_utc_bound`]. -#[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`timezone_utc` will be replaced by `timezone_utc_bound` in a future PyO3 version" - ) +#[cfg(feature = "gil-refs")] +#[deprecated( + since = "0.21.0", + note = "`timezone_utc` will be replaced by `timezone_utc_bound` in a future PyO3 version" )] pub fn timezone_utc(py: Python<'_>) -> &PyTzInfo { timezone_utc_bound(py).into_gil_ref() @@ -858,12 +840,10 @@ pyobject_native_type!( impl PyDelta { /// Deprecated form of [`PyDelta::new_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyDelta::new` will be replaced by `PyDelta::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyDelta::new` will be replaced by `PyDelta::new_bound` in a future PyO3 version" )] pub fn new( py: Python<'_>, @@ -936,7 +916,6 @@ fn opt_to_pyobj(opt: Option<&Bound<'_, PyTzInfo>>) -> *mut ffi::PyObject { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; #[cfg(feature = "macros")] @@ -947,14 +926,15 @@ mod tests { #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_datetime_fromtimestamp() { Python::with_gil(|py| { - let dt = PyDateTime::from_timestamp(py, 100.0, None).unwrap(); + let dt = PyDateTime::from_timestamp_bound(py, 100.0, None).unwrap(); py_run!( py, dt, "import datetime; assert dt == datetime.datetime.fromtimestamp(100)" ); - let dt = PyDateTime::from_timestamp(py, 100.0, Some(timezone_utc(py))).unwrap(); + let dt = + PyDateTime::from_timestamp_bound(py, 100.0, Some(&timezone_utc_bound(py))).unwrap(); py_run!( py, dt, @@ -968,7 +948,7 @@ mod tests { #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_date_fromtimestamp() { Python::with_gil(|py| { - let dt = PyDate::from_timestamp(py, 100).unwrap(); + let dt = PyDate::from_timestamp_bound(py, 100).unwrap(); py_run!( py, dt, @@ -981,8 +961,10 @@ mod tests { #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_new_with_fold() { Python::with_gil(|py| { - let a = PyDateTime::new_with_fold(py, 2021, 1, 23, 20, 32, 40, 341516, None, false); - let b = PyDateTime::new_with_fold(py, 2021, 1, 23, 20, 32, 40, 341516, None, true); + let a = + PyDateTime::new_bound_with_fold(py, 2021, 1, 23, 20, 32, 40, 341516, None, false); + let b = + PyDateTime::new_bound_with_fold(py, 2021, 1, 23, 20, 32, 40, 341516, None, true); assert!(!a.unwrap().get_fold()); assert!(b.unwrap().get_fold()); @@ -991,26 +973,25 @@ mod tests { #[test] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons - #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] fn test_get_tzinfo() { crate::Python::with_gil(|py| { - let utc = timezone_utc(py); + let utc = timezone_utc_bound(py); - let dt = PyDateTime::new(py, 2018, 1, 1, 0, 0, 0, 0, Some(utc)).unwrap(); + let dt = PyDateTime::new_bound(py, 2018, 1, 1, 0, 0, 0, 0, Some(&utc)).unwrap(); - assert!(dt.get_tzinfo().unwrap().eq(utc).unwrap()); + assert!(dt.get_tzinfo_bound().unwrap().eq(&utc).unwrap()); - let dt = PyDateTime::new(py, 2018, 1, 1, 0, 0, 0, 0, None).unwrap(); + let dt = PyDateTime::new_bound(py, 2018, 1, 1, 0, 0, 0, 0, None).unwrap(); - assert!(dt.get_tzinfo().is_none()); + assert!(dt.get_tzinfo_bound().is_none()); - let t = PyTime::new(py, 0, 0, 0, 0, Some(utc)).unwrap(); + let t = PyTime::new_bound(py, 0, 0, 0, 0, Some(&utc)).unwrap(); - assert!(t.get_tzinfo().unwrap().eq(utc).unwrap()); + assert!(t.get_tzinfo_bound().unwrap().eq(utc).unwrap()); - let t = PyTime::new(py, 0, 0, 0, 0, None).unwrap(); + let t = PyTime::new_bound(py, 0, 0, 0, 0, None).unwrap(); - assert!(t.get_tzinfo().is_none()); + assert!(t.get_tzinfo_bound().is_none()); }); } @@ -1024,9 +1005,9 @@ mod tests { .unwrap() .call_method1("utcoffset", ((),)) .unwrap() - .extract::<&PyDelta>() + .downcast_into::() .unwrap() - .eq(PyDelta::new(py, 0, -3600, 0, true).unwrap()) + .eq(PyDelta::new_bound(py, 0, -3600, 0, true).unwrap()) .unwrap() ); @@ -1035,9 +1016,9 @@ mod tests { .unwrap() .call_method1("utcoffset", ((),)) .unwrap() - .extract::<&PyDelta>() + .downcast_into::() .unwrap() - .eq(PyDelta::new(py, 0, 3600, 0, true).unwrap()) + .eq(PyDelta::new_bound(py, 0, 3600, 0, true).unwrap()) .unwrap() ); diff --git a/src/types/mod.rs b/src/types/mod.rs index e7aead7fbe5..d127cbbca20 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -9,7 +9,7 @@ pub use self::capsule::{PyCapsule, PyCapsuleMethods}; pub use self::code::PyCode; pub use self::complex::{PyComplex, PyComplexMethods}; #[allow(deprecated)] -#[cfg(not(Py_LIMITED_API))] +#[cfg(all(not(Py_LIMITED_API), feature = "gil-refs"))] pub use self::datetime::timezone_utc; #[cfg(not(Py_LIMITED_API))] pub use self::datetime::{ From c951bf86de4c32c1e4d026687d6b54f975aea802 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 24 Apr 2024 19:34:19 +0200 Subject: [PATCH 293/349] feature gate deprecated APIs for `PyCapsule` (#4112) --- src/types/capsule.rs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/types/capsule.rs b/src/types/capsule.rs index fe5a47e1796..c2b73a0d0f7 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -47,12 +47,10 @@ pyobject_native_type_core!(PyCapsule, pyobject_native_static_type_object!(ffi::P impl PyCapsule { /// Deprecated form of [`PyCapsule::new_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyCapsule::new` will be replaced by `PyCapsule::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyCapsule::new` will be replaced by `PyCapsule::new_bound` in a future PyO3 version" )] pub fn new( py: Python<'_>, @@ -102,12 +100,10 @@ impl PyCapsule { } /// Deprecated form of [`PyCapsule::new_bound_with_destructor`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyCapsule::new_with_destructor` will be replaced by `PyCapsule::new_bound_with_destructor` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyCapsule::new_with_destructor` will be replaced by `PyCapsule::new_bound_with_destructor` in a future PyO3 version" )] pub fn new_with_destructor< T: 'static + Send + AssertNotZeroSized, @@ -441,7 +437,6 @@ fn name_ptr_ignore_error(slf: &Bound<'_, PyCapsule>) -> *const c_char { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use libc::c_void; From 6c2e6f8bcceb2d954ca8c645777bce3d6e77de88 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 24 Apr 2024 23:13:27 +0200 Subject: [PATCH 294/349] feature gate deprecated APIs for `PyFrozenSet` (#4118) --- src/types/frozenset.rs | 43 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 36d5578f701..1fbbba44615 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -39,12 +39,10 @@ impl<'py> PyFrozenSetBuilder<'py> { } /// Deprecated form of [`PyFrozenSetBuilder::finalize_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyFrozenSetBuilder::finalize` will be replaced by `PyFrozenSetBuilder::finalize_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyFrozenSetBuilder::finalize` will be replaced by `PyFrozenSetBuilder::finalize_bound` in a future PyO3 version" )] pub fn finalize(self) -> &'py PyFrozenSet { self.finalize_bound().into_gil_ref() @@ -78,12 +76,10 @@ pyobject_native_type_core!( impl PyFrozenSet { /// Deprecated form of [`PyFrozenSet::new_bound`]. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyFrozenSet::new` will be replaced by `PyFrozenSet::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyFrozenSet::new` will be replaced by `PyFrozenSet::new_bound` in a future PyO3 version" )] pub fn new<'a, 'p, T: ToPyObject + 'a>( py: Python<'p>, @@ -104,12 +100,10 @@ impl PyFrozenSet { } /// Deprecated form of [`PyFrozenSet::empty_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyFrozenSet::empty` will be replaced by `PyFrozenSet::empty_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyFrozenSet::empty` will be replaced by `PyFrozenSet::empty_bound` in a future PyO3 version" )] pub fn empty(py: Python<'_>) -> PyResult<&'_ PyFrozenSet> { Self::empty_bound(py).map(Bound::into_gil_ref) @@ -324,25 +318,24 @@ pub(crate) fn new_from_iter( } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::*; #[test] fn test_frozenset_new_and_len() { Python::with_gil(|py| { - let set = PyFrozenSet::new(py, &[1]).unwrap(); + let set = PyFrozenSet::new_bound(py, &[1]).unwrap(); assert_eq!(1, set.len()); let v = vec![1]; - assert!(PyFrozenSet::new(py, &[v]).is_err()); + assert!(PyFrozenSet::new_bound(py, &[v]).is_err()); }); } #[test] fn test_frozenset_empty() { Python::with_gil(|py| { - let set = PyFrozenSet::empty(py).unwrap(); + let set = PyFrozenSet::empty_bound(py).unwrap(); assert_eq!(0, set.len()); assert!(set.is_empty()); }); @@ -351,7 +344,7 @@ mod tests { #[test] fn test_frozenset_contains() { Python::with_gil(|py| { - let set = PyFrozenSet::new(py, &[1]).unwrap(); + let set = PyFrozenSet::new_bound(py, &[1]).unwrap(); assert!(set.contains(1).unwrap()); }); } @@ -359,7 +352,7 @@ mod tests { #[test] fn test_frozenset_iter() { Python::with_gil(|py| { - let set = PyFrozenSet::new(py, &[1]).unwrap(); + let set = PyFrozenSet::new_bound(py, &[1]).unwrap(); for el in set { assert_eq!(1i32, el.extract::().unwrap()); @@ -381,7 +374,7 @@ mod tests { #[test] fn test_frozenset_iter_size_hint() { Python::with_gil(|py| { - let set = PyFrozenSet::new(py, &[1]).unwrap(); + let set = PyFrozenSet::new_bound(py, &[1]).unwrap(); let mut iter = set.iter(); // Exact size From 3cb286e0d2749a67f1861d0d62295ffb32ff2fac Mon Sep 17 00:00:00 2001 From: Alexander Clausen Date: Thu, 25 Apr 2024 17:36:29 +0200 Subject: [PATCH 295/349] docs: fix typo in trait-bounds.md (#4124) --- guide/src/trait-bounds.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/trait-bounds.md b/guide/src/trait-bounds.md index 0644e679190..eb67bc42413 100644 --- a/guide/src/trait-bounds.md +++ b/guide/src/trait-bounds.md @@ -1,6 +1,6 @@ # Using in Python a Rust function with trait bounds -PyO3 allows for easy conversion from Rust to Python for certain functions and classes (see the [conversion table](conversions/tables.md). +PyO3 allows for easy conversion from Rust to Python for certain functions and classes (see the [conversion table](conversions/tables.md)). However, it is not always straightforward to convert Rust code that requires a given trait implementation as an argument. This tutorial explains how to convert a Rust function that takes a trait as argument for use in Python with classes implementing the same methods as the trait. From 8734b76f60058475ddc3b80c86b8106796870fac Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Thu, 25 Apr 2024 19:44:42 +0200 Subject: [PATCH 296/349] feature gate deprecated APIs for `PyIterator` (#4119) --- src/types/iterator.rs | 59 ++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/src/types/iterator.rs b/src/types/iterator.rs index 60f36f22de2..53330705869 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -1,9 +1,9 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::Borrowed; use crate::py_result_ext::PyResultExt; -use crate::{ - ffi, AsPyPointer, Bound, PyAny, PyDowncastError, PyErr, PyNativeType, PyResult, PyTypeCheck, -}; +#[cfg(feature = "gil-refs")] +use crate::PyDowncastError; +use crate::{ffi, AsPyPointer, Bound, PyAny, PyErr, PyNativeType, PyResult, PyTypeCheck}; /// A Python iterator object. /// @@ -32,12 +32,10 @@ pyobject_native_type_extract!(PyIterator); impl PyIterator { /// Deprecated form of `PyIterator::from_bound_object`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyIterator::from_object` will be replaced by `PyIterator::from_bound_object` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyIterator::from_object` will be replaced by `PyIterator::from_bound_object` in a future PyO3 version" )] pub fn from_object(obj: &PyAny) -> PyResult<&PyIterator> { Self::from_bound_object(&obj.as_borrowed()).map(Bound::into_gil_ref) @@ -128,6 +126,7 @@ impl PyTypeCheck for PyIterator { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl<'v> crate::PyTryFrom<'v> for PyIterator { fn try_from>(value: V) -> Result<&'v PyIterator, PyDowncastError<'v>> { @@ -153,19 +152,17 @@ impl<'v> crate::PyTryFrom<'v> for PyIterator { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::PyIterator; use crate::exceptions::PyTypeError; - use crate::gil::GILPool; - use crate::types::{PyAnyMethods, PyDict, PyList}; - use crate::{Py, PyAny, Python, ToPyObject}; + use crate::types::{PyAnyMethods, PyDict, PyList, PyListMethods}; + use crate::{Python, ToPyObject}; #[test] fn vec_iter() { Python::with_gil(|py| { let obj = vec![10, 20].to_object(py); - let inst = obj.as_ref(py); + let inst = obj.bind(py); let mut it = inst.iter().unwrap(); assert_eq!( 10_i32, @@ -188,7 +185,7 @@ mod tests { }); Python::with_gil(|py| { - let inst = obj.as_ref(py); + let inst = obj.bind(py); let mut it = inst.iter().unwrap(); assert_eq!( @@ -206,26 +203,24 @@ mod tests { fn iter_item_refcnt() { Python::with_gil(|py| { let count; - let obj = py.eval("object()", None, None).unwrap(); + let obj = py.eval_bound("object()", None, None).unwrap(); let list = { - let _pool = unsafe { GILPool::new() }; - let list = PyList::empty(py); + let list = PyList::empty_bound(py); list.append(10).unwrap(); - list.append(obj).unwrap(); + list.append(&obj).unwrap(); count = obj.get_refcnt(); list.to_object(py) }; { - let _pool = unsafe { GILPool::new() }; - let inst = list.as_ref(py); + let inst = list.bind(py); let mut it = inst.iter().unwrap(); assert_eq!( 10_i32, it.next().unwrap().unwrap().extract::<'_, i32>().unwrap() ); - assert!(it.next().unwrap().unwrap().is(obj)); + assert!(it.next().unwrap().unwrap().is(&obj)); assert!(it.next().is_none()); } assert_eq!(count, obj.get_refcnt()); @@ -293,18 +288,20 @@ def fibonacci(target): fn int_not_iterable() { Python::with_gil(|py| { let x = 5.to_object(py); - let err = PyIterator::from_object(x.as_ref(py)).unwrap_err(); + let err = PyIterator::from_bound_object(x.bind(py)).unwrap_err(); assert!(err.is_instance_of::(py)); }); } #[test] - + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn iterator_try_from() { Python::with_gil(|py| { - let obj: Py = vec![10, 20].to_object(py).as_ref(py).iter().unwrap().into(); - let iter: &PyIterator = obj.downcast(py).unwrap(); + let obj: crate::Py = + vec![10, 20].to_object(py).as_ref(py).iter().unwrap().into(); + let iter = ::try_from(obj.as_ref(py)).unwrap(); assert!(obj.is(iter)); }); } @@ -321,14 +318,14 @@ def fibonacci(target): #[crate::pymethods(crate = "crate")] impl Downcaster { - fn downcast_iterator(&mut self, obj: &PyAny) { + fn downcast_iterator(&mut self, obj: &crate::Bound<'_, crate::PyAny>) { self.failed = Some(obj.downcast::().unwrap_err().into()); } } // Regression test for 2913 Python::with_gil(|py| { - let downcaster = Py::new(py, Downcaster { failed: None }).unwrap(); + let downcaster = crate::Py::new(py, Downcaster { failed: None }).unwrap(); crate::py_run!( py, downcaster, @@ -360,13 +357,13 @@ def fibonacci(target): #[cfg(feature = "macros")] fn python_class_iterator() { #[crate::pyfunction(crate = "crate")] - fn assert_iterator(obj: &PyAny) { + fn assert_iterator(obj: &crate::Bound<'_, crate::PyAny>) { assert!(obj.downcast::().is_ok()) } // Regression test for 2913 Python::with_gil(|py| { - let assert_iterator = crate::wrap_pyfunction!(assert_iterator, py).unwrap(); + let assert_iterator = crate::wrap_pyfunction_bound!(assert_iterator, py).unwrap(); crate::py_run!( py, assert_iterator, @@ -385,7 +382,7 @@ def fibonacci(target): #[cfg(not(Py_LIMITED_API))] fn length_hint_becomes_size_hint_lower_bound() { Python::with_gil(|py| { - let list = py.eval("[1, 2, 3]", None, None).unwrap(); + let list = py.eval_bound("[1, 2, 3]", None, None).unwrap(); let iter = list.iter().unwrap(); let hint = iter.size_hint(); assert_eq!(hint, (3, None)); From c66ed292ec1849e031592b961c47ec21b37407ca Mon Sep 17 00:00:00 2001 From: David Matos Date: Fri, 26 Apr 2024 08:00:13 +0200 Subject: [PATCH 297/349] Disable PyUnicode_DATA on PyPy (#4116) * Disable PYUNICODE_DATA on PyPy * Add newsfragment * Adjust import on PyString --- newsfragments/4116.fixed.md | 1 + pyo3-ffi/src/cpython/unicodeobject.rs | 10 +++++----- src/ffi/tests.rs | 15 +++++++++------ src/types/string.rs | 20 ++++++++++---------- 4 files changed, 25 insertions(+), 21 deletions(-) create mode 100644 newsfragments/4116.fixed.md diff --git a/newsfragments/4116.fixed.md b/newsfragments/4116.fixed.md new file mode 100644 index 00000000000..63531aceb39 --- /dev/null +++ b/newsfragments/4116.fixed.md @@ -0,0 +1 @@ +Disable `PyUnicode_DATA` on PyPy: Not exposed by PyPy. diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index 9ab523a2d7f..feb78cf0c82 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -449,19 +449,19 @@ pub const PyUnicode_1BYTE_KIND: c_uint = 1; pub const PyUnicode_2BYTE_KIND: c_uint = 2; pub const PyUnicode_4BYTE_KIND: c_uint = 4; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_1BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS1 { PyUnicode_DATA(op) as *mut Py_UCS1 } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_2BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS2 { PyUnicode_DATA(op) as *mut Py_UCS2 } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { PyUnicode_DATA(op) as *mut Py_UCS4 @@ -487,7 +487,7 @@ pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void { } } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { debug_assert!(!(*(op as *mut PyUnicodeObject)).data.any.is_null()); @@ -495,7 +495,7 @@ pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { (*(op as *mut PyUnicodeObject)).data.any } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, PyPy)))] #[inline] pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { debug_assert!(crate::PyUnicode_Check(op) != 0); diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index 3532172c933..5aee1618472 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -2,11 +2,14 @@ use crate::ffi::*; use crate::types::any::PyAnyMethods; use crate::Python; +#[cfg(all(PyPy, feature = "macros"))] +use crate::types::PyString; + +#[cfg(not(any(Py_LIMITED_API, PyPy)))] +use crate::types::PyString; + #[cfg(not(Py_LIMITED_API))] -use crate::{ - types::{PyDict, PyString}, - Bound, IntoPy, Py, PyAny, -}; +use crate::{types::PyDict, Bound, IntoPy, Py, PyAny}; #[cfg(not(any(Py_3_12, Py_LIMITED_API)))] use libc::wchar_t; @@ -160,7 +163,7 @@ fn ascii_object_bitfield() { } #[test] -#[cfg(not(Py_LIMITED_API))] +#[cfg(not(any(Py_LIMITED_API, PyPy)))] #[cfg_attr(Py_3_10, allow(deprecated))] fn ascii() { Python::with_gil(|py| { @@ -202,7 +205,7 @@ fn ascii() { } #[test] -#[cfg(not(Py_LIMITED_API))] +#[cfg(not(any(Py_LIMITED_API, PyPy)))] #[cfg_attr(Py_3_10, allow(deprecated))] fn ucs4() { Python::with_gil(|py| { diff --git a/src/types/string.rs b/src/types/string.rs index 4aa73341ae9..09c5903547c 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -264,7 +264,7 @@ impl PyString { /// /// By using this API, you accept responsibility for testing that PyStringData behaves as /// expected on the targets where you plan to distribute your software. - #[cfg(not(any(Py_LIMITED_API, GraalPy)))] + #[cfg(not(any(Py_LIMITED_API, GraalPy, PyPy)))] pub unsafe fn data(&self) -> PyResult> { self.as_borrowed().data() } @@ -313,7 +313,7 @@ pub trait PyStringMethods<'py>: crate::sealed::Sealed { /// /// By using this API, you accept responsibility for testing that PyStringData behaves as /// expected on the targets where you plan to distribute your software. - #[cfg(not(any(Py_LIMITED_API, GraalPy)))] + #[cfg(not(any(Py_LIMITED_API, GraalPy, PyPy)))] unsafe fn data(&self) -> PyResult>; } @@ -339,7 +339,7 @@ impl<'py> PyStringMethods<'py> for Bound<'py, PyString> { } } - #[cfg(not(any(Py_LIMITED_API, GraalPy)))] + #[cfg(not(any(Py_LIMITED_API, GraalPy, PyPy)))] unsafe fn data(&self) -> PyResult> { self.as_borrowed().data() } @@ -402,7 +402,7 @@ impl<'a> Borrowed<'a, '_, PyString> { Cow::Owned(String::from_utf8_lossy(bytes.as_bytes()).into_owned()) } - #[cfg(not(any(Py_LIMITED_API, GraalPy)))] + #[cfg(not(any(Py_LIMITED_API, GraalPy, PyPy)))] unsafe fn data(self) -> PyResult> { let ptr = self.as_ptr(); @@ -584,7 +584,7 @@ mod tests { } #[test] - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, PyPy)))] fn test_string_data_ucs1() { Python::with_gil(|py| { let s = PyString::new_bound(py, "hello, world"); @@ -597,7 +597,7 @@ mod tests { } #[test] - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, PyPy)))] fn test_string_data_ucs1_invalid() { Python::with_gil(|py| { // 0xfe is not allowed in UTF-8. @@ -625,7 +625,7 @@ mod tests { } #[test] - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, PyPy)))] fn test_string_data_ucs2() { Python::with_gil(|py| { let s = py.eval_bound("'foo\\ud800'", None, None).unwrap(); @@ -641,7 +641,7 @@ mod tests { } #[test] - #[cfg(all(not(Py_LIMITED_API), target_endian = "little"))] + #[cfg(all(not(any(Py_LIMITED_API, PyPy)), target_endian = "little"))] fn test_string_data_ucs2_invalid() { Python::with_gil(|py| { // U+FF22 (valid) & U+d800 (never valid) @@ -669,7 +669,7 @@ mod tests { } #[test] - #[cfg(not(Py_LIMITED_API))] + #[cfg(not(any(Py_LIMITED_API, PyPy)))] fn test_string_data_ucs4() { Python::with_gil(|py| { let s = "哈哈🐈"; @@ -682,7 +682,7 @@ mod tests { } #[test] - #[cfg(all(not(Py_LIMITED_API), target_endian = "little"))] + #[cfg(all(not(any(Py_LIMITED_API, PyPy)), target_endian = "little"))] fn test_string_data_ucs4_invalid() { Python::with_gil(|py| { // U+20000 (valid) & U+d800 (never valid) From 8ff5e5b0ab6002922d8d9b523b1b50fb5cf80b9a Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sat, 27 Apr 2024 01:12:11 -0400 Subject: [PATCH 298/349] Fix lychee for guide (#4130) * Fix lychee for guide * Update nightly in netlify --- .netlify/build.sh | 1 + guide/src/parallelism.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.netlify/build.sh b/.netlify/build.sh index e1d86788ca1..a61180be59a 100755 --- a/.netlify/build.sh +++ b/.netlify/build.sh @@ -2,6 +2,7 @@ set -uex +rustup update nightly rustup default nightly PYO3_VERSION=$(cargo search pyo3 --limit 1 | head -1 | tr -s ' ' | cut -d ' ' -f 3 | tr -d '"') diff --git a/guide/src/parallelism.md b/guide/src/parallelism.md index 792e0ed8de4..a288b14be19 100644 --- a/guide/src/parallelism.md +++ b/guide/src/parallelism.md @@ -1,6 +1,6 @@ # Parallelism -CPython has the infamous [Global Interpreter Lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock), which prevents several threads from executing Python bytecode in parallel. This makes threading in Python a bad fit for [CPU-bound](https://stackoverflow.com/questions/868568/) tasks and often forces developers to accept the overhead of multiprocessing. +CPython has the infamous [Global Interpreter Lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock), which prevents several threads from executing Python bytecode in parallel. This makes threading in Python a bad fit for [CPU-bound](https://en.wikipedia.org/wiki/CPU-bound) tasks and often forces developers to accept the overhead of multiprocessing. In PyO3 parallelism can be easily achieved in Rust-only code. Let's take a look at our [word-count](https://github.com/PyO3/pyo3/blob/main/examples/word-count/src/lib.rs) example, where we have a `search` function that utilizes the [rayon](https://github.com/rayon-rs/rayon) crate to count words in parallel. ```rust,no_run From 059c8b386201a2aaa7a76b06392308ddcdfaa6e3 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 27 Apr 2024 13:34:27 +0200 Subject: [PATCH 299/349] feature gate deprecated APIs for `PyBytes` and `PyPyByteArray` (#4131) --- src/types/bytearray.rs | 30 ++++++++++++------------------ src/types/bytes.rs | 30 ++++++++++++------------------ 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 7227519975b..8bc4bf55d5b 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -15,12 +15,10 @@ pyobject_native_type_core!(PyByteArray, pyobject_native_static_type_object!(ffi: impl PyByteArray { /// Deprecated form of [`PyByteArray::new_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyByteArray::new` will be replaced by `PyByteArray::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyByteArray::new` will be replaced by `PyByteArray::new_bound` in a future PyO3 version" )] pub fn new<'py>(py: Python<'py>, src: &[u8]) -> &'py PyByteArray { Self::new_bound(py, src).into_gil_ref() @@ -40,12 +38,10 @@ impl PyByteArray { } /// Deprecated form of [`PyByteArray::new_bound_with`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyByteArray::new_with` will be replaced by `PyByteArray::new_bound_with` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyByteArray::new_with` will be replaced by `PyByteArray::new_bound_with` in a future PyO3 version" )] pub fn new_with(py: Python<'_>, len: usize, init: F) -> PyResult<&PyByteArray> where @@ -104,12 +100,10 @@ impl PyByteArray { } /// Deprecated form of [`PyByteArray::from_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyByteArray::from` will be replaced by `PyByteArray::from_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyByteArray::from` will be replaced by `PyByteArray::from_bound` in a future PyO3 version" )] pub fn from(src: &PyAny) -> PyResult<&PyByteArray> { PyByteArray::from_bound(&src.as_borrowed()).map(Bound::into_gil_ref) diff --git a/src/types/bytes.rs b/src/types/bytes.rs index 44c87560514..f3da65c7c97 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -17,12 +17,10 @@ pyobject_native_type_core!(PyBytes, pyobject_native_static_type_object!(ffi::PyB impl PyBytes { /// Deprecated form of [`PyBytes::new_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyBytes::new` will be replaced by `PyBytes::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyBytes::new` will be replaced by `PyBytes::new_bound` in a future PyO3 version" )] pub fn new<'p>(py: Python<'p>, s: &[u8]) -> &'p PyBytes { Self::new_bound(py, s).into_gil_ref() @@ -43,12 +41,10 @@ impl PyBytes { } /// Deprecated form of [`PyBytes::new_bound_with`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyBytes::new_with` will be replaced by `PyBytes::new_bound_with` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyBytes::new_with` will be replaced by `PyBytes::new_bound_with` in a future PyO3 version" )] pub fn new_with(py: Python<'_>, len: usize, init: F) -> PyResult<&PyBytes> where @@ -103,12 +99,10 @@ impl PyBytes { /// /// # Safety /// See [`PyBytes::bound_from_ptr`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyBytes::from_ptr` will be replaced by `PyBytes::bound_from_ptr` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyBytes::from_ptr` will be replaced by `PyBytes::bound_from_ptr` in a future PyO3 version" )] pub unsafe fn from_ptr(py: Python<'_>, ptr: *const u8, len: usize) -> &PyBytes { Self::bound_from_ptr(py, ptr, len).into_gil_ref() From 6fb972b2324ba115517ba886943e98f1b1dc84b2 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 27 Apr 2024 17:34:13 +0200 Subject: [PATCH 300/349] feature gate deprecated APIs for `PyEllipsis`, `PyNone` and `PyNotImplemented` (#4132) --- src/types/ellipsis.rs | 10 ++++------ src/types/none.rs | 10 ++++------ src/types/notimplemented.rs | 10 ++++------ 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/types/ellipsis.rs b/src/types/ellipsis.rs index 4d091c6f9d0..cbeaf489c17 100644 --- a/src/types/ellipsis.rs +++ b/src/types/ellipsis.rs @@ -12,12 +12,10 @@ pyobject_native_type_extract!(PyEllipsis); impl PyEllipsis { /// Returns the `Ellipsis` object. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyEllipsis::get` will be replaced by `PyEllipsis::get_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyEllipsis::get` will be replaced by `PyEllipsis::get_bound` in a future PyO3 version" )] #[inline] pub fn get(py: Python<'_>) -> &PyEllipsis { diff --git a/src/types/none.rs b/src/types/none.rs index a14af044808..0ab1570b92d 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -14,12 +14,10 @@ pyobject_native_type_extract!(PyNone); impl PyNone { /// Returns the `None` object. /// Deprecated form of [`PyNone::get_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyNone::get` will be replaced by `PyNone::get_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyNone::get` will be replaced by `PyNone::get_bound` in a future PyO3 version" )] #[inline] pub fn get(py: Python<'_>) -> &PyNone { diff --git a/src/types/notimplemented.rs b/src/types/notimplemented.rs index 9d31e670e2b..7fad1220b26 100644 --- a/src/types/notimplemented.rs +++ b/src/types/notimplemented.rs @@ -12,12 +12,10 @@ pyobject_native_type_extract!(PyNotImplemented); impl PyNotImplemented { /// Returns the `NotImplemented` object. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyNotImplemented::get` will be replaced by `PyNotImplemented::get_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyNotImplemented::get` will be replaced by `PyNotImplemented::get_bound` in a future PyO3 version" )] #[inline] pub fn get(py: Python<'_>) -> &PyNotImplemented { From 9e1960ea348d6beab648c5c05fb8a0bad306d9e1 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 28 Apr 2024 12:11:28 -0400 Subject: [PATCH 301/349] Update MSRV to 1.63 (#4129) * Bump MSRV to 1.63 * Drop parking_lot in favor of std::sync * Make portable-atomic dep conditional * Remove no longer required cfg --- .github/workflows/build.yml | 2 +- .github/workflows/ci.yml | 4 +- Cargo.toml | 7 +-- README.md | 4 +- build.rs | 2 +- guide/src/building-and-distribution.md | 2 +- guide/src/getting-started.md | 2 +- newsfragments/4129.changed.md | 1 + noxfile.py | 21 ++------ pyo3-build-config/src/lib.rs | 5 -- pyo3-ffi/README.md | 2 +- pyo3-ffi/build.rs | 2 +- pyo3-ffi/src/lib.rs | 2 +- src/coroutine/cancel.rs | 9 ++-- src/gil.rs | 67 +++++++++++--------------- src/impl_/pymodule.rs | 10 +++- src/lib.rs | 2 +- tests/test_coroutine.rs | 3 ++ 18 files changed, 67 insertions(+), 80 deletions(-) create mode 100644 newsfragments/4129.changed.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d2f74401dd6..40d4a0012fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,7 +54,7 @@ jobs: name: Prepare LD_LIBRARY_PATH (Ubuntu only) run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV - - if: inputs.rust == '1.56.0' + - if: inputs.rust == '1.63.0' name: Prepare minimal package versions (MSRV only) run: nox -s set-minimal-package-versions diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12db5867f90..0c245f33483 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: - toolchain: 1.56.0 + toolchain: 1.63.0 targets: x86_64-unknown-linux-gnu components: rust-src - uses: actions/setup-python@v5 @@ -255,7 +255,7 @@ jobs: ] include: # Test minimal supported Rust version - - rust: 1.56.0 + - rust: 1.63.0 python-version: "3.12" platform: { diff --git a/Cargo.toml b/Cargo.toml index 90bcc03c80c..9202c69ef92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,14 +12,12 @@ categories = ["api-bindings", "development-tools::ffi"] license = "MIT OR Apache-2.0" exclude = ["/.gitignore", ".cargo/config", "/codecov.yml", "/Makefile", "/pyproject.toml", "/noxfile.py", "/.github", "/tests/test_compile_error.rs", "/tests/ui"] edition = "2021" -rust-version = "1.56" +rust-version = "1.63" [dependencies] cfg-if = "1.0" libc = "0.2.62" -parking_lot = ">= 0.11, < 0.13" memoffset = "0.9" -portable-atomic = "1.0" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.2" } @@ -46,6 +44,9 @@ rust_decimal = { version = "1.0.0", default-features = false, optional = true } serde = { version = "1.0", optional = true } smallvec = { version = "1.0", optional = true } +[target.'cfg(not(target_has_atomic = "64"))'.dependencies] +portable-atomic = "1.0" + [dev-dependencies] assert_approx_eq = "1.1.0" chrono = "0.4.25" diff --git a/README.md b/README.md index 4da2447e8c4..72653e8fc77 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![benchmark](https://img.shields.io/badge/benchmark-✓-Green?logo=github)](https://pyo3.rs/dev/bench/) [![codecov](https://img.shields.io/codecov/c/gh/PyO3/pyo3?logo=codecov)](https://codecov.io/gh/PyO3/pyo3) [![crates.io](https://img.shields.io/crates/v/pyo3?logo=rust)](https://crates.io/crates/pyo3) -[![minimum rustc 1.56](https://img.shields.io/badge/rustc-1.56+-blue?logo=rust)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) +[![minimum rustc 1.63](https://img.shields.io/badge/rustc-1.63+-blue?logo=rust)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) [![discord server](https://img.shields.io/discord/1209263839632424990?logo=discord)](https://discord.gg/33kcChzH7f) [![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green?logo=github)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) @@ -18,7 +18,7 @@ PyO3 supports the following software versions: - Python 3.7 and up (CPython, PyPy, and GraalPy) - - Rust 1.56 and up + - Rust 1.63 and up You can use PyO3 to write a native Python module in Rust, or to embed Python in a Rust binary. The following sections explain each of these in turn. diff --git a/build.rs b/build.rs index 7f0ae6e31c8..28592004df2 100644 --- a/build.rs +++ b/build.rs @@ -39,7 +39,7 @@ fn configure_pyo3() -> Result<()> { println!("{}", cfg) } - // Emit cfgs like `thread_local_const_init` + // Emit cfgs like `invalid_from_utf8_lint` print_feature_cfgs(); Ok(()) diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index 780f135e211..b7aa2d2abc2 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -151,7 +151,7 @@ rustflags = [ ] ``` -Alternatively, on rust >= 1.56, one can include in `build.rs`: +Alternatively, one can include in `build.rs`: ```rust fn main() { diff --git a/guide/src/getting-started.md b/guide/src/getting-started.md index 59cf5ba6ded..94ab95cb3d6 100644 --- a/guide/src/getting-started.md +++ b/guide/src/getting-started.md @@ -6,7 +6,7 @@ To get started using PyO3 you will need three things: a Rust toolchain, a Python ## Rust -First, make sure you have Rust installed on your system. If you haven't already done so, try following the instructions [here](https://www.rust-lang.org/tools/install). PyO3 runs on both the `stable` and `nightly` versions so you can choose whichever one fits you best. The minimum required Rust version is 1.56. +First, make sure you have Rust installed on your system. If you haven't already done so, try following the instructions [here](https://www.rust-lang.org/tools/install). PyO3 runs on both the `stable` and `nightly` versions so you can choose whichever one fits you best. The minimum required Rust version is 1.63. If you can run `rustc --version` and the version is new enough you're good to go! diff --git a/newsfragments/4129.changed.md b/newsfragments/4129.changed.md new file mode 100644 index 00000000000..6818181ad0c --- /dev/null +++ b/newsfragments/4129.changed.md @@ -0,0 +1 @@ +Raised the MSRV to 1.63 diff --git a/noxfile.py b/noxfile.py index 3d82c8d3746..c8d2ead63d3 100644 --- a/noxfile.py +++ b/noxfile.py @@ -557,22 +557,11 @@ def set_minimal_package_versions(session: nox.Session): "examples/word-count", ) min_pkg_versions = { - "rust_decimal": "1.26.1", - "csv": "1.1.6", - "indexmap": "1.6.2", - "hashbrown": "0.9.1", - "log": "0.4.17", - "once_cell": "1.17.2", - "rayon": "1.6.1", - "rayon-core": "1.10.2", - "regex": "1.7.3", - "proptest": "1.0.0", - "chrono": "0.4.25", - "byteorder": "1.4.3", - "crossbeam-channel": "0.5.8", - "crossbeam-deque": "0.8.3", - "crossbeam-epoch": "0.9.15", - "crossbeam-utils": "0.8.16", + "regex": "1.9.6", + "proptest": "1.2.0", + "trybuild": "1.0.89", + "eyre": "0.6.8", + "allocator-api2": "0.2.10", } # run cargo update first to ensure that everything is at highest diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index eab7376d0ea..5b1cfdaf322 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -145,11 +145,6 @@ pub fn print_feature_cfgs() { let rustc_minor_version = rustc_minor_version().unwrap_or(0); - // Enable use of const initializer for thread_local! on Rust 1.59 and greater - if rustc_minor_version >= 59 { - println!("cargo:rustc-cfg=thread_local_const_init"); - } - // invalid_from_utf8 lint was added in Rust 1.74 if rustc_minor_version >= 74 { println!("cargo:rustc-cfg=invalid_from_utf8_lint"); diff --git a/pyo3-ffi/README.md b/pyo3-ffi/README.md index 4e85e83c88f..283a7072357 100644 --- a/pyo3-ffi/README.md +++ b/pyo3-ffi/README.md @@ -14,7 +14,7 @@ Manual][capi] for up-to-date documentation. PyO3 supports the following software versions: - Python 3.7 and up (CPython and PyPy) - - Rust 1.56 and up + - Rust 1.63 and up # Example: Building Python Native modules diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index a5ab352b6a7..405da889708 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -189,7 +189,7 @@ fn configure_pyo3() -> Result<()> { println!("{}", line); } - // Emit cfgs like `thread_local_const_init` + // Emit cfgs like `invalid_from_utf8_lint` print_feature_cfgs(); Ok(()) diff --git a/pyo3-ffi/src/lib.rs b/pyo3-ffi/src/lib.rs index 7a203362ed5..877d42dce33 100644 --- a/pyo3-ffi/src/lib.rs +++ b/pyo3-ffi/src/lib.rs @@ -51,7 +51,7 @@ //! //! PyO3 supports the following software versions: //! - Python 3.7 and up (CPython and PyPy) -//! - Rust 1.56 and up +//! - Rust 1.63 and up //! //! # Example: Building Python Native modules //! diff --git a/src/coroutine/cancel.rs b/src/coroutine/cancel.rs index 2b10fb9a438..47f5d69430a 100644 --- a/src/coroutine/cancel.rs +++ b/src/coroutine/cancel.rs @@ -1,8 +1,7 @@ use crate::{Py, PyAny, PyObject}; -use parking_lot::Mutex; use std::future::Future; use std::pin::Pin; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Waker}; #[derive(Debug, Default)] @@ -25,12 +24,12 @@ impl CancelHandle { /// Returns whether the associated coroutine has been cancelled. pub fn is_cancelled(&self) -> bool { - self.0.lock().exception.is_some() + self.0.lock().unwrap().exception.is_some() } /// Poll to retrieve the exception thrown in the associated coroutine. pub fn poll_cancelled(&mut self, cx: &mut Context<'_>) -> Poll { - let mut inner = self.0.lock(); + let mut inner = self.0.lock().unwrap(); if let Some(exc) = inner.exception.take() { return Poll::Ready(exc); } @@ -69,7 +68,7 @@ pub struct ThrowCallback(Arc>); impl ThrowCallback { pub(super) fn throw(&self, exc: Py) { - let mut inner = self.0.lock(); + let mut inner = self.0.lock().unwrap(); inner.exception = Some(exc); if let Some(waker) = inner.waker.take() { waker.wake(); diff --git a/src/gil.rs b/src/gil.rs index e2f36037755..0bcb8c086c0 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -2,29 +2,16 @@ use crate::impl_::not_send::{NotSend, NOT_SEND}; use crate::{ffi, Python}; -use parking_lot::{const_mutex, Mutex, Once}; use std::cell::Cell; #[cfg(debug_assertions)] use std::cell::RefCell; #[cfg(not(debug_assertions))] use std::cell::UnsafeCell; -use std::{mem, ptr::NonNull}; +use std::{mem, ptr::NonNull, sync}; -static START: Once = Once::new(); +static START: sync::Once = sync::Once::new(); -cfg_if::cfg_if! { - if #[cfg(thread_local_const_init)] { - use std::thread_local as thread_local_const_init; - } else { - macro_rules! thread_local_const_init { - ($($(#[$attr:meta])* static $name:ident: $ty:ty = const { $init:expr };)*) => ( - thread_local! { $($(#[$attr])* static $name: $ty = $init;)* } - ) - } - } -} - -thread_local_const_init! { +std::thread_local! { /// This is an internal counter in pyo3 monitoring whether this thread has the GIL. /// /// It will be incremented whenever a GILGuard or GILPool is created, and decremented whenever @@ -249,26 +236,26 @@ type PyObjVec = Vec>; /// Thread-safe storage for objects which were inc_ref / dec_ref while the GIL was not held. struct ReferencePool { // .0 is INCREFs, .1 is DECREFs - pointer_ops: Mutex<(PyObjVec, PyObjVec)>, + pointer_ops: sync::Mutex<(PyObjVec, PyObjVec)>, } impl ReferencePool { const fn new() -> Self { Self { - pointer_ops: const_mutex((Vec::new(), Vec::new())), + pointer_ops: sync::Mutex::new((Vec::new(), Vec::new())), } } fn register_incref(&self, obj: NonNull) { - self.pointer_ops.lock().0.push(obj); + self.pointer_ops.lock().unwrap().0.push(obj); } fn register_decref(&self, obj: NonNull) { - self.pointer_ops.lock().1.push(obj); + self.pointer_ops.lock().unwrap().1.push(obj); } fn update_counts(&self, _py: Python<'_>) { - let mut ops = self.pointer_ops.lock(); + let mut ops = self.pointer_ops.lock().unwrap(); if ops.0.is_empty() && ops.1.is_empty() { return; } @@ -523,9 +510,9 @@ mod tests { use super::{gil_is_acquired, GIL_COUNT, OWNED_OBJECTS, POOL}; use crate::types::any::PyAnyMethods; use crate::{ffi, gil, PyObject, Python}; - #[cfg(not(target_arch = "wasm32"))] - use parking_lot::{const_mutex, Condvar, Mutex}; use std::ptr::NonNull; + #[cfg(not(target_arch = "wasm32"))] + use std::sync; fn get_object(py: Python<'_>) -> PyObject { py.eval_bound("object()", None, None).unwrap().unbind() @@ -543,6 +530,7 @@ mod tests { !POOL .pointer_ops .lock() + .unwrap() .0 .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } @@ -551,6 +539,7 @@ mod tests { !POOL .pointer_ops .lock() + .unwrap() .1 .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } @@ -559,6 +548,7 @@ mod tests { fn pool_dec_refs_contains(obj: &PyObject) -> bool { POOL.pointer_ops .lock() + .unwrap() .1 .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } @@ -671,8 +661,8 @@ mod tests { Python::with_gil(|py| { assert_eq!(obj.get_refcnt(py), 1); let non_null = unsafe { NonNull::new_unchecked(obj.as_ptr()) }; - assert!(!POOL.pointer_ops.lock().0.contains(&non_null)); - assert!(!POOL.pointer_ops.lock().1.contains(&non_null)); + assert!(!POOL.pointer_ops.lock().unwrap().0.contains(&non_null)); + assert!(!POOL.pointer_ops.lock().unwrap().1.contains(&non_null)); }); } @@ -770,29 +760,30 @@ mod tests { #[cfg(not(target_arch = "wasm32"))] struct Event { - set: Mutex, - wait: Condvar, + set: sync::Mutex, + wait: sync::Condvar, } #[cfg(not(target_arch = "wasm32"))] impl Event { const fn new() -> Self { Self { - set: const_mutex(false), - wait: Condvar::new(), + set: sync::Mutex::new(false), + wait: sync::Condvar::new(), } } fn set(&self) { - *self.set.lock() = true; + *self.set.lock().unwrap() = true; self.wait.notify_all(); } fn wait(&self) { - let mut set = self.set.lock(); - while !*set { - self.wait.wait(&mut set); - } + drop( + self.wait + .wait_while(self.set.lock().unwrap(), |s| !*s) + .unwrap(), + ); } } @@ -891,16 +882,16 @@ mod tests { // The pointer should appear once in the incref pool, and once in the // decref pool (for the clone being created and also dropped) - assert!(POOL.pointer_ops.lock().0.contains(&ptr)); - assert!(POOL.pointer_ops.lock().1.contains(&ptr)); + assert!(POOL.pointer_ops.lock().unwrap().0.contains(&ptr)); + assert!(POOL.pointer_ops.lock().unwrap().1.contains(&ptr)); (obj, count, ptr) }); Python::with_gil(|py| { // Acquiring the gil clears the pool - assert!(!POOL.pointer_ops.lock().0.contains(&ptr)); - assert!(!POOL.pointer_ops.lock().1.contains(&ptr)); + assert!(!POOL.pointer_ops.lock().unwrap().0.contains(&ptr)); + assert!(!POOL.pointer_ops.lock().unwrap().1.contains(&ptr)); // Overall count is still unchanged assert_eq!(count, obj.get_refcnt(py)); diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index 20560aeb8d5..5f04d888a50 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -5,9 +5,17 @@ use std::{cell::UnsafeCell, marker::PhantomData}; #[cfg(all( not(any(PyPy, GraalPy)), Py_3_9, - not(all(windows, Py_LIMITED_API, not(Py_3_10))) + not(all(windows, Py_LIMITED_API, not(Py_3_10))), + not(target_has_atomic = "64"), ))] use portable_atomic::{AtomicI64, Ordering}; +#[cfg(all( + not(any(PyPy, GraalPy)), + Py_3_9, + not(all(windows, Py_LIMITED_API, not(Py_3_10))), + target_has_atomic = "64", +))] +use std::sync::atomic::{AtomicI64, Ordering}; #[cfg(not(any(PyPy, GraalPy)))] use crate::exceptions::PyImportError; diff --git a/src/lib.rs b/src/lib.rs index e444912a63d..b400f143f5a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -132,7 +132,7 @@ //! //! PyO3 supports the following software versions: //! - Python 3.7 and up (CPython and PyPy) -//! - Rust 1.56 and up +//! - Rust 1.63 and up //! //! # Example: Building a native Python module //! diff --git a/tests/test_coroutine.rs b/tests/test_coroutine.rs index 4abba9f36b4..75b524edf78 100644 --- a/tests/test_coroutine.rs +++ b/tests/test_coroutine.rs @@ -3,6 +3,7 @@ use std::{task::Poll, thread, time::Duration}; use futures::{channel::oneshot, future::poll_fn, FutureExt}; +#[cfg(not(target_has_atomic = "64"))] use portable_atomic::{AtomicBool, Ordering}; use pyo3::{ coroutine::CancelHandle, @@ -10,6 +11,8 @@ use pyo3::{ py_run, types::{IntoPyDict, PyType}, }; +#[cfg(target_has_atomic = "64")] +use std::sync::atomic::{AtomicBool, Ordering}; #[path = "../src/tests/common.rs"] mod common; From d5452bcd8d1b71eaed3437d0f39f687effb8fdbe Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sun, 28 Apr 2024 23:03:51 +0200 Subject: [PATCH 302/349] feature gate deprecated APIs for `PyType`, `PyTypeInfo` and `PySuper` (#4134) --- guide/src/migration.md | 2 +- src/conversion.rs | 3 ++- src/instance.rs | 5 ++--- src/type_object.rs | 30 ++++++++++++------------------ src/types/any.rs | 4 ++-- src/types/pysuper.rs | 13 ++++++------- src/types/typeobject.rs | 20 ++++++++------------ 7 files changed, 33 insertions(+), 44 deletions(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index d855f69d396..9c7bc15002e 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -60,7 +60,7 @@ To migrate, switch all type casts to use `obj.downcast()` instead of `try_from(o Before: -```rust +```rust,ignore # #![allow(deprecated)] # use pyo3::prelude::*; # use pyo3::types::{PyInt, PyList}; diff --git a/src/conversion.rs b/src/conversion.rs index ced209abade..8644db84289 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -3,7 +3,6 @@ use crate::err::{self, PyDowncastError, PyResult}; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::pyclass::boolean_struct::False; -use crate::type_object::PyTypeInfo; use crate::types::any::PyAnyMethods; use crate::types::PyTuple; use crate::{ @@ -435,9 +434,11 @@ pub trait PyTryInto: Sized { fn try_into_exact(&self) -> Result<&T, PyDowncastError<'_>>; } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] mod implementations { use super::*; + use crate::type_object::PyTypeInfo; // TryFrom implies TryInto impl PyTryInto for PyAny diff --git a/src/instance.rs b/src/instance.rs index 02b3fc76241..cc892a2dd44 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -2310,8 +2310,6 @@ a = A() #[cfg(feature = "macros")] mod using_macros { - use crate::PyCell; - use super::*; #[crate::pyclass(crate = "crate")] @@ -2371,9 +2369,10 @@ a = A() } #[test] + #[cfg(feature = "gil-refs")] #[allow(deprecated)] fn cell_tryfrom() { - use crate::PyTryInto; + use crate::{PyCell, PyTryInto}; // More detailed tests of the underlying semantics in pycell.rs Python::with_gil(|py| { let instance: &PyAny = Py::new(py, SomeClass(0)).unwrap().into_ref(py); diff --git a/src/type_object.rs b/src/type_object.rs index 84888bee458..7f35f7d967a 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -66,12 +66,10 @@ pub unsafe trait PyTypeInfo: Sized + HasPyGilRef { /// Returns the safe abstraction over the type object. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyTypeInfo::type_object` will be replaced by `PyTypeInfo::type_object_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyTypeInfo::type_object` will be replaced by `PyTypeInfo::type_object_bound` in a future PyO3 version" )] fn type_object(py: Python<'_>) -> &PyType { // This isn't implemented in terms of `type_object_bound` because this just borrowed the @@ -101,12 +99,10 @@ pub unsafe trait PyTypeInfo: Sized + HasPyGilRef { /// Checks if `object` is an instance of this type or a subclass of this type. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyTypeInfo::is_type_of` will be replaced by `PyTypeInfo::is_type_of_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyTypeInfo::is_type_of` will be replaced by `PyTypeInfo::is_type_of_bound` in a future PyO3 version" )] fn is_type_of(object: &PyAny) -> bool { Self::is_type_of_bound(&object.as_borrowed()) @@ -120,12 +116,10 @@ pub unsafe trait PyTypeInfo: Sized + HasPyGilRef { /// Checks if `object` is an instance of this type. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyTypeInfo::is_exact_type_of` will be replaced by `PyTypeInfo::is_exact_type_of_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyTypeInfo::is_exact_type_of` will be replaced by `PyTypeInfo::is_exact_type_of_bound` in a future PyO3 version" )] fn is_exact_type_of(object: &PyAny) -> bool { Self::is_exact_type_of_bound(&object.as_borrowed()) diff --git a/src/types/any.rs b/src/types/any.rs index 6ba34c86bc6..777a8dcb4c3 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -2726,9 +2726,9 @@ class SimpleClass: #[test] fn test_is_callable() { Python::with_gil(|py| { - assert!(PyList::type_object(py).is_callable()); + assert!(PyList::type_object_bound(py).is_callable()); - let not_callable = 5.to_object(py).into_ref(py); + let not_callable = 5.to_object(py).into_bound(py); assert!(!not_callable.is_callable()); }); } diff --git a/src/types/pysuper.rs b/src/types/pysuper.rs index 0f1a47444d6..7c4d781525a 100644 --- a/src/types/pysuper.rs +++ b/src/types/pysuper.rs @@ -1,7 +1,7 @@ use crate::instance::Bound; use crate::types::any::PyAnyMethods; use crate::types::PyType; -use crate::{ffi, PyNativeType, PyTypeInfo}; +use crate::{ffi, PyTypeInfo}; use crate::{PyAny, PyResult}; /// Represents a Python `super` object. @@ -17,14 +17,13 @@ pyobject_native_type_core!( impl PySuper { /// Deprecated form of `PySuper::new_bound`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PySuper::new` will be replaced by `PySuper::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PySuper::new` will be replaced by `PySuper::new_bound` in a future PyO3 version" )] pub fn new<'py>(ty: &'py PyType, obj: &'py PyAny) -> PyResult<&'py PySuper> { + use crate::PyNativeType; Self::new_bound(&ty.as_borrowed(), &obj.as_borrowed()).map(Bound::into_gil_ref) } diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index d75e39d022d..2261834ef2a 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -15,12 +15,10 @@ pyobject_native_type_core!(PyType, pyobject_native_static_type_object!(ffi::PyTy impl PyType { /// Deprecated form of [`PyType::new_bound`]. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyType::new` will be replaced by `PyType::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyType::new` will be replaced by `PyType::new_bound` in a future PyO3 version" )] pub fn new(py: Python<'_>) -> &PyType { T::type_object_bound(py).into_gil_ref() @@ -44,12 +42,10 @@ impl PyType { /// /// - The pointer must a valid non-null reference to a `PyTypeObject`. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "Use `PyType::from_borrowed_type_ptr` instead" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "Use `PyType::from_borrowed_type_ptr` instead" )] pub unsafe fn from_type_ptr(py: Python<'_>, p: *mut ffi::PyTypeObject) -> &PyType { Self::from_borrowed_type_ptr(py, p).into_gil_ref() From 22c5cff0394f0095b204e9aa6e0f4b31808506a1 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 30 Apr 2024 18:58:53 +0200 Subject: [PATCH 303/349] feature gate deprecated APIs for `PyErr` and exceptions (#4136) --- src/err/mod.rs | 108 ++++++++++++++++----------------------- src/exceptions.rs | 40 ++++++++------- tests/test_exceptions.rs | 7 ++- 3 files changed, 69 insertions(+), 86 deletions(-) diff --git a/src/err/mod.rs b/src/err/mod.rs index a61c8c62d31..4f46f360094 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -24,9 +24,9 @@ use err_state::{PyErrState, PyErrStateLazyFnOutput, PyErrStateNormalized}; /// compatibility with `?` and other Rust errors) this type supports creating exceptions instances /// in a lazy fashion, where the full Python object for the exception is created only when needed. /// -/// Accessing the contained exception in any way, such as with [`value`](PyErr::value), -/// [`get_type`](PyErr::get_type), or [`is_instance`](PyErr::is_instance) will create the full -/// exception object if it was not already created. +/// Accessing the contained exception in any way, such as with [`value_bound`](PyErr::value_bound), +/// [`get_type_bound`](PyErr::get_type_bound), or [`is_instance_bound`](PyErr::is_instance_bound) +/// will create the full exception object if it was not already created. pub struct PyErr { // Safety: can only hand out references when in the "normalized" state. Will never change // after normalization. @@ -136,7 +136,7 @@ impl PyErr { /// /// This exception instance will be initialized lazily. This avoids the need for the Python GIL /// to be held, but requires `args` to be `Send` and `Sync`. If `args` is not `Send` or `Sync`, - /// consider using [`PyErr::from_value`] instead. + /// consider using [`PyErr::from_value_bound`] instead. /// /// If `T` does not inherit from `BaseException`, then a `TypeError` will be returned. /// @@ -192,12 +192,10 @@ impl PyErr { } /// Deprecated form of [`PyErr::from_type_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyErr::from_type` will be replaced by `PyErr::from_type_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyErr::from_type` will be replaced by `PyErr::from_type_bound` in a future PyO3 version" )] pub fn from_type(ty: &PyType, args: A) -> PyErr where @@ -224,12 +222,10 @@ impl PyErr { } /// Deprecated form of [`PyErr::from_value_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyErr::from_value` will be replaced by `PyErr::from_value_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyErr::from_value` will be replaced by `PyErr::from_value_bound` in a future PyO3 version" )] pub fn from_value(obj: &PyAny) -> PyErr { PyErr::from_value_bound(obj.as_borrowed().to_owned()) @@ -284,12 +280,10 @@ impl PyErr { } /// Deprecated form of [`PyErr::get_type_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyErr::get_type` will be replaced by `PyErr::get_type_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyErr::get_type` will be replaced by `PyErr::get_type_bound` in a future PyO3 version" )] pub fn get_type<'py>(&'py self, py: Python<'py>) -> &'py PyType { self.get_type_bound(py).into_gil_ref() @@ -311,12 +305,10 @@ impl PyErr { } /// Deprecated form of [`PyErr::value_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyErr::value` will be replaced by `PyErr::value_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyErr::value` will be replaced by `PyErr::value_bound` in a future PyO3 version" )] pub fn value<'py>(&'py self, py: Python<'py>) -> &'py PyBaseException { self.value_bound(py).as_gil_ref() @@ -355,12 +347,10 @@ impl PyErr { } /// Deprecated form of [`PyErr::traceback_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyErr::traceback` will be replaced by `PyErr::traceback_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyErr::traceback` will be replaced by `PyErr::traceback_bound` in a future PyO3 version" )] pub fn traceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { self.normalized(py).ptraceback(py).map(|b| b.into_gil_ref()) @@ -508,12 +498,10 @@ impl PyErr { } /// Deprecated form of [`PyErr::new_type_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyErr::new_type` will be replaced by `PyErr::new_type_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyErr::new_type` will be replaced by `PyErr::new_type_bound` in a future PyO3 version" )] pub fn new_type( py: Python<'_>, @@ -636,12 +624,10 @@ impl PyErr { } /// Deprecated form of `PyErr::is_instance_bound`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyErr::is_instance` will be replaced by `PyErr::is_instance_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyErr::is_instance` will be replaced by `PyErr::is_instance_bound` in a future PyO3 version" )] #[inline] pub fn is_instance(&self, py: Python<'_>, ty: &PyAny) -> bool { @@ -675,12 +661,10 @@ impl PyErr { } /// Deprecated form of `PyErr::write_unraisable_bound`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyErr::write_unraisable` will be replaced by `PyErr::write_unraisable_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyErr::write_unraisable` will be replaced by `PyErr::write_unraisable_bound` in a future PyO3 version" )] #[inline] pub fn write_unraisable(self, py: Python<'_>, obj: Option<&PyAny>) { @@ -722,12 +706,10 @@ impl PyErr { } /// Deprecated form of [`PyErr::warn_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyErr::warn` will be replaced by `PyErr::warn_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyErr::warn` will be replaced by `PyErr::warn_bound` in a future PyO3 version" )] pub fn warn(py: Python<'_>, category: &PyAny, message: &str, stacklevel: i32) -> PyResult<()> { Self::warn_bound(py, &category.as_borrowed(), message, stacklevel) @@ -771,12 +753,10 @@ impl PyErr { } /// Deprecated form of [`PyErr::warn_explicit_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyErr::warn_explicit` will be replaced by `PyErr::warn_explicit_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyErr::warn_explicit` will be replaced by `PyErr::warn_explicit_bound` in a future PyO3 version" )] pub fn warn_explicit( py: Python<'_>, diff --git a/src/exceptions.rs b/src/exceptions.rs index c650d7af079..367022927c5 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -21,6 +21,7 @@ macro_rules! impl_exception_boilerplate { ($name: ident) => { // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] + #[cfg(feature = "gil-refs")] impl ::std::convert::From<&$name> for $crate::PyErr { #[inline] fn from(err: &$name) -> $crate::PyErr { @@ -650,12 +651,10 @@ impl_windows_native_exception!( impl PyUnicodeDecodeError { /// Deprecated form of [`PyUnicodeDecodeError::new_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyUnicodeDecodeError::new` will be replaced by `PyUnicodeDecodeError::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyUnicodeDecodeError::new` will be replaced by `PyUnicodeDecodeError::new_bound` in a future PyO3 version" )] pub fn new<'p>( py: Python<'p>, @@ -692,12 +691,10 @@ impl PyUnicodeDecodeError { } /// Deprecated form of [`PyUnicodeDecodeError::new_utf8_bound`]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyUnicodeDecodeError::new_utf8` will be replaced by `PyUnicodeDecodeError::new_utf8_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyUnicodeDecodeError::new_utf8` will be replaced by `PyUnicodeDecodeError::new_utf8_bound` in a future PyO3 version" )] pub fn new_utf8<'p>( py: Python<'p>, @@ -808,7 +805,7 @@ macro_rules! test_exception { use super::$exc_ty; $crate::Python::with_gil(|py| { - use std::error::Error; + use $crate::types::PyAnyMethods; let err: $crate::PyErr = { None $( @@ -819,13 +816,19 @@ macro_rules! test_exception { assert!(err.is_instance_of::<$exc_ty>(py)); - let value: &$exc_ty = err.value_bound(py).clone().into_gil_ref().downcast().unwrap(); - assert!(value.source().is_none()); + let value = err.value_bound(py).as_any().downcast::<$exc_ty>().unwrap(); - err.set_cause(py, Some($crate::exceptions::PyValueError::new_err("a cause"))); - assert!(value.source().is_some()); + #[cfg(feature = "gil-refs")] + { + use std::error::Error; + let value = value.as_gil_ref(); + assert!(value.source().is_none()); + + err.set_cause(py, Some($crate::exceptions::PyValueError::new_err("a cause"))); + assert!(value.source().is_some()); + } - assert!($crate::PyErr::from(value).is_instance_of::<$exc_ty>(py)); + assert!($crate::PyErr::from(value.clone()).is_instance_of::<$exc_ty>(py)); }) } }; @@ -1068,6 +1071,7 @@ mod tests { } #[test] + #[cfg(feature = "gil-refs")] fn native_exception_chain() { use std::error::Error; diff --git a/tests/test_exceptions.rs b/tests/test_exceptions.rs index ec2fe156b29..e85355fd40e 100644 --- a/tests/test_exceptions.rs +++ b/tests/test_exceptions.rs @@ -100,10 +100,9 @@ fn test_exception_nosegfault() { #[test] #[cfg(Py_3_8)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] fn test_write_unraisable() { use common::UnraisableCapture; - use pyo3::{exceptions::PyRuntimeError, ffi}; + use pyo3::{exceptions::PyRuntimeError, ffi, types::PyNotImplemented}; Python::with_gil(|py| { let capture = UnraisableCapture::install(py); @@ -111,14 +110,14 @@ fn test_write_unraisable() { assert!(capture.borrow(py).capture.is_none()); let err = PyRuntimeError::new_err("foo"); - err.write_unraisable(py, None); + err.write_unraisable_bound(py, None); let (err, object) = capture.borrow_mut(py).capture.take().unwrap(); assert_eq!(err.to_string(), "RuntimeError: foo"); assert!(object.is_none(py)); let err = PyRuntimeError::new_err("bar"); - err.write_unraisable(py, Some(py.NotImplemented().as_ref(py))); + err.write_unraisable_bound(py, Some(&PyNotImplemented::get_bound(py))); let (err, object) = capture.borrow_mut(py).capture.take().unwrap(); assert_eq!(err.to_string(), "RuntimeError: bar"); From 4616838ee1f89a461ae4560f16e4c1476feac822 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 30 Apr 2024 20:53:40 +0200 Subject: [PATCH 304/349] port `PySequence` tests to `Bound` API (#4139) --- src/types/sequence.rs | 116 +++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 47 deletions(-) diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 2b37b6d14f0..afe4a595964 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -566,15 +566,14 @@ impl<'v> crate::PyTryFrom<'v> for PySequence { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { - use crate::types::{PyList, PySequence, PyTuple}; + use crate::types::{PyAnyMethods, PyList, PySequence, PySequenceMethods, PyTuple}; use crate::{PyObject, Python, ToPyObject}; fn get_object() -> PyObject { // Convenience function for getting a single unique object Python::with_gil(|py| { - let obj = py.eval("object()", None, None).unwrap(); + let obj = py.eval_bound("object()", None, None).unwrap(); obj.to_object(py) }) @@ -584,7 +583,7 @@ mod tests { fn test_numbers_are_not_sequences() { Python::with_gil(|py| { let v = 42i32; - assert!(v.to_object(py).downcast::(py).is_err()); + assert!(v.to_object(py).downcast_bound::(py).is_err()); }); } @@ -592,7 +591,7 @@ mod tests { fn test_strings_are_sequences() { Python::with_gil(|py| { let v = "London Calling"; - assert!(v.to_object(py).downcast::(py).is_ok()); + assert!(v.to_object(py).downcast_bound::(py).is_ok()); }); } @@ -612,7 +611,7 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert_eq!(0, seq.len().unwrap()); let needle = 7i32.to_object(py); @@ -624,11 +623,11 @@ mod tests { fn test_seq_is_empty() { Python::with_gil(|py| { let list = vec![1].to_object(py); - let seq = list.downcast::(py).unwrap(); + let seq = list.downcast_bound::(py).unwrap(); assert!(!seq.is_empty().unwrap()); let vec: Vec = Vec::new(); let empty_list = vec.to_object(py); - let empty_seq = empty_list.downcast::(py).unwrap(); + let empty_seq = empty_list.downcast_bound::(py).unwrap(); assert!(empty_seq.is_empty().unwrap()); }); } @@ -638,7 +637,7 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2, 3, 5, 8]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert_eq!(6, seq.len().unwrap()); let bad_needle = 7i32.to_object(py); @@ -657,7 +656,7 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2, 3, 5, 8]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert_eq!(1, seq.get_item(0).unwrap().extract::().unwrap()); assert_eq!(1, seq.get_item(1).unwrap().extract::().unwrap()); assert_eq!(2, seq.get_item(2).unwrap().extract::().unwrap()); @@ -669,6 +668,8 @@ mod tests { } #[test] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_seq_index_trait() { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2]; @@ -682,6 +683,8 @@ mod tests { #[test] #[should_panic = "index 7 out of range for sequence"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_seq_index_trait_panic() { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2]; @@ -692,6 +695,8 @@ mod tests { } #[test] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_seq_index_trait_ranges() { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2]; @@ -710,6 +715,8 @@ mod tests { #[test] #[should_panic = "range start index 5 out of range for sequence of length 3"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_seq_index_trait_range_panic_start() { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2]; @@ -721,6 +728,8 @@ mod tests { #[test] #[should_panic = "range end index 10 out of range for sequence of length 3"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_seq_index_trait_range_panic_end() { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2]; @@ -732,6 +741,8 @@ mod tests { #[test] #[should_panic = "slice index starts at 2 but ends at 1"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_seq_index_trait_range_panic_wrong_order() { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2]; @@ -744,6 +755,8 @@ mod tests { #[test] #[should_panic = "range start index 8 out of range for sequence of length 3"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_seq_index_trait_range_from_panic() { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2]; @@ -758,19 +771,19 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2, 3, 5, 8]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert!(seq.del_item(10).is_err()); - assert_eq!(1, seq[0].extract::().unwrap()); + assert_eq!(1, seq.get_item(0).unwrap().extract::().unwrap()); assert!(seq.del_item(0).is_ok()); - assert_eq!(1, seq[0].extract::().unwrap()); + assert_eq!(1, seq.get_item(0).unwrap().extract::().unwrap()); assert!(seq.del_item(0).is_ok()); - assert_eq!(2, seq[0].extract::().unwrap()); + assert_eq!(2, seq.get_item(0).unwrap().extract::().unwrap()); assert!(seq.del_item(0).is_ok()); - assert_eq!(3, seq[0].extract::().unwrap()); + assert_eq!(3, seq.get_item(0).unwrap().extract::().unwrap()); assert!(seq.del_item(0).is_ok()); - assert_eq!(5, seq[0].extract::().unwrap()); + assert_eq!(5, seq.get_item(0).unwrap().extract::().unwrap()); assert!(seq.del_item(0).is_ok()); - assert_eq!(8, seq[0].extract::().unwrap()); + assert_eq!(8, seq.get_item(0).unwrap().extract::().unwrap()); assert!(seq.del_item(0).is_ok()); assert_eq!(0, seq.len().unwrap()); assert!(seq.del_item(0).is_err()); @@ -782,10 +795,10 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 2]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); - assert_eq!(2, seq[1].extract::().unwrap()); + let seq = ob.downcast_bound::(py).unwrap(); + assert_eq!(2, seq.get_item(1).unwrap().extract::().unwrap()); assert!(seq.set_item(1, 10).is_ok()); - assert_eq!(10, seq[1].extract::().unwrap()); + assert_eq!(10, seq.get_item(1).unwrap().extract::().unwrap()); }); } @@ -796,9 +809,9 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 2]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert!(seq.set_item(1, &obj).is_ok()); - assert!(seq[1].as_ptr() == obj.as_ptr()); + assert!(seq.get_item(1).unwrap().as_ptr() == obj.as_ptr()); }); Python::with_gil(|py| { @@ -811,7 +824,7 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2, 3, 5, 8]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert_eq!( [1, 2, 3], seq.get_slice(1, 4).unwrap().extract::<[i32; 3]>().unwrap() @@ -832,11 +845,11 @@ mod tests { let v: Vec = vec![1, 1, 2, 3, 5, 8]; let w: Vec = vec![7, 4]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); let ins = w.to_object(py); - seq.set_slice(1, 4, ins.as_ref(py)).unwrap(); + seq.set_slice(1, 4, ins.bind(py)).unwrap(); assert_eq!([1, 7, 4, 5, 8], seq.extract::<[i32; 5]>().unwrap()); - seq.set_slice(3, 100, PyList::empty(py)).unwrap(); + seq.set_slice(3, 100, &PyList::empty_bound(py)).unwrap(); assert_eq!([1, 7, 4], seq.extract::<[i32; 3]>().unwrap()); }); } @@ -846,7 +859,7 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2, 3, 5, 8]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); seq.del_slice(1, 4).unwrap(); assert_eq!([1, 5, 8], seq.extract::<[i32; 3]>().unwrap()); seq.del_slice(1, 100).unwrap(); @@ -859,7 +872,7 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2, 3, 5, 8]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert_eq!(0, seq.index(1i32).unwrap()); assert_eq!(2, seq.index(2i32).unwrap()); assert_eq!(3, seq.index(3i32).unwrap()); @@ -875,7 +888,7 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2, 3, 5, 8]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert_eq!(2, seq.count(1i32).unwrap()); assert_eq!(1, seq.count(2i32).unwrap()); assert_eq!(1, seq.count(3i32).unwrap()); @@ -890,7 +903,7 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2, 3, 5, 8]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); let mut idx = 0; for el in seq.iter().unwrap() { assert_eq!(v[idx], el.unwrap().extract::().unwrap()); @@ -905,7 +918,7 @@ mod tests { Python::with_gil(|py| { let v = vec!["It", "was", "the", "worst", "of", "times"]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); let bad_needle = "blurst".to_object(py); assert!(!seq.contains(bad_needle).unwrap()); @@ -920,7 +933,7 @@ mod tests { Python::with_gil(|py| { let v: Vec = vec![1, 2, 3]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); let concat_seq = seq.concat(seq).unwrap(); assert_eq!(6, concat_seq.len().unwrap()); let concat_v: Vec = vec![1, 2, 3, 1, 2, 3]; @@ -935,7 +948,7 @@ mod tests { Python::with_gil(|py| { let v = "string"; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); let concat_seq = seq.concat(seq).unwrap(); assert_eq!(12, concat_seq.len().unwrap()); let concat_v = "stringstring".to_owned(); @@ -950,7 +963,7 @@ mod tests { Python::with_gil(|py| { let v = vec!["foo", "bar"]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); let repeat_seq = seq.repeat(3).unwrap(); assert_eq!(6, repeat_seq.len().unwrap()); let repeated = ["foo", "bar", "foo", "bar", "foo", "bar"]; @@ -965,14 +978,14 @@ mod tests { Python::with_gil(|py| { let v = vec!["foo", "bar"]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); let rep_seq = seq.in_place_repeat(3).unwrap(); assert_eq!(6, seq.len().unwrap()); - assert!(seq.is(rep_seq)); + assert!(seq.is(&rep_seq)); let conc_seq = seq.in_place_concat(seq).unwrap(); assert_eq!(12, seq.len().unwrap()); - assert!(seq.is(conc_seq)); + assert!(seq.is(&conc_seq)); }); } @@ -981,8 +994,12 @@ mod tests { Python::with_gil(|py| { let v = vec!["foo", "bar"]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); - assert!(seq.to_list().unwrap().eq(PyList::new(py, &v)).unwrap()); + let seq = ob.downcast_bound::(py).unwrap(); + assert!(seq + .to_list() + .unwrap() + .eq(PyList::new_bound(py, &v)) + .unwrap()); }); } @@ -991,11 +1008,11 @@ mod tests { Python::with_gil(|py| { let v = "foo"; let ob = v.to_object(py); - let seq: &PySequence = ob.downcast(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert!(seq .to_list() .unwrap() - .eq(PyList::new(py, ["f", "o", "o"])) + .eq(PyList::new_bound(py, ["f", "o", "o"])) .unwrap()); }); } @@ -1005,7 +1022,7 @@ mod tests { Python::with_gil(|py| { let v = ("foo", "bar"); let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert!(seq .to_tuple() .unwrap() @@ -1019,7 +1036,7 @@ mod tests { Python::with_gil(|py| { let v = vec!["foo", "bar"]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); assert!(seq .to_tuple() .unwrap() @@ -1031,7 +1048,11 @@ mod tests { #[test] fn test_extract_tuple_to_vec() { Python::with_gil(|py| { - let v: Vec = py.eval("(1, 2)", None, None).unwrap().extract().unwrap(); + let v: Vec = py + .eval_bound("(1, 2)", None, None) + .unwrap() + .extract() + .unwrap(); assert!(v == [1, 2]); }); } @@ -1040,7 +1061,7 @@ mod tests { fn test_extract_range_to_vec() { Python::with_gil(|py| { let v: Vec = py - .eval("range(1, 5)", None, None) + .eval_bound("range(1, 5)", None, None) .unwrap() .extract() .unwrap(); @@ -1052,7 +1073,7 @@ mod tests { fn test_extract_bytearray_to_vec() { Python::with_gil(|py| { let v: Vec = py - .eval("bytearray(b'abc')", None, None) + .eval_bound("bytearray(b'abc')", None, None) .unwrap() .extract() .unwrap(); @@ -1065,7 +1086,7 @@ mod tests { Python::with_gil(|py| { let v = vec!["foo", "bar"]; let ob = v.to_object(py); - let seq = ob.downcast::(py).unwrap(); + let seq = ob.downcast_bound::(py).unwrap(); let type_ptr = seq.as_ref(); let seq_from = unsafe { type_ptr.downcast_unchecked::() }; assert!(seq_from.to_list().is_ok()); @@ -1073,6 +1094,7 @@ mod tests { } #[test] + #[cfg(feature = "gil-refs")] #[allow(deprecated)] fn test_seq_try_from() { use crate::PyTryFrom; From 82c00a2fe4bcb732cce075b3416024447755c746 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Tue, 30 Apr 2024 23:49:00 +0200 Subject: [PATCH 305/349] port `PyAny` tests to `Bound` API (#4140) --- src/types/any.rs | 75 +++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/src/types/any.rs b/src/types/any.rs index 777a8dcb4c3..8b0bac44ca6 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -2324,12 +2324,11 @@ impl<'py> Bound<'py, PyAny> { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use crate::{ basic::CompareOp, - types::{any::PyAnyMethods, IntoPyDict, PyAny, PyBool, PyList, PyLong, PyModule}, - Bound, PyNativeType, PyTypeInfo, Python, ToPyObject, + types::{IntoPyDict, PyAny, PyAnyMethods, PyBool, PyList, PyLong, PyModule, PyTypeMethods}, + Bound, PyTypeInfo, Python, ToPyObject, }; #[test] @@ -2407,7 +2406,7 @@ class NonHeapNonDescriptorInt: #[test] fn test_call_for_non_existing_method() { Python::with_gil(|py| { - let a = py.eval("42", None, None).unwrap(); + let a = py.eval_bound("42", None, None).unwrap(); a.call_method0("__str__").unwrap(); // ok assert!(a.call_method("nonexistent_method", (1,), None).is_err()); assert!(a.call_method0("nonexistent_method").is_err()); @@ -2455,7 +2454,7 @@ class SimpleClass: #[test] fn test_type() { Python::with_gil(|py| { - let obj = py.eval("42", None, None).unwrap(); + let obj = py.eval_bound("42", None, None).unwrap(); assert_eq!(obj.get_type().as_type_ptr(), obj.get_type_ptr()); }); } @@ -2463,11 +2462,11 @@ class SimpleClass: #[test] fn test_dir() { Python::with_gil(|py| { - let obj = py.eval("42", None, None).unwrap(); + let obj = py.eval_bound("42", None, None).unwrap(); let dir = py - .eval("dir(42)", None, None) + .eval_bound("dir(42)", None, None) .unwrap() - .downcast::() + .downcast_into::() .unwrap(); let a = obj .dir() @@ -2482,7 +2481,7 @@ class SimpleClass: #[test] fn test_hasattr() { Python::with_gil(|py| { - let x = 5.to_object(py).into_ref(py); + let x = 5.to_object(py).into_bound(py); assert!(x.is_instance_of::()); assert!(x.hasattr("to_bytes").unwrap()); @@ -2509,7 +2508,7 @@ class SimpleClass: Python::with_gil(|py| { let obj = Py::new(py, GetattrFail).unwrap(); - let obj = obj.as_ref(py).as_ref(); + let obj = obj.bind(py).as_ref(); assert!(obj .hasattr("foo") @@ -2521,18 +2520,18 @@ class SimpleClass: #[test] fn test_nan_eq() { Python::with_gil(|py| { - let nan = py.eval("float('nan')", None, None).unwrap(); - assert!(nan.compare(nan).is_err()); + let nan = py.eval_bound("float('nan')", None, None).unwrap(); + assert!(nan.compare(&nan).is_err()); }); } #[test] fn test_any_is_instance_of() { Python::with_gil(|py| { - let x = 5.to_object(py).into_ref(py); + let x = 5.to_object(py).into_bound(py); assert!(x.is_instance_of::()); - let l = vec![x, x].to_object(py).into_ref(py); + let l = vec![&x, &x].to_object(py).into_bound(py); assert!(l.is_instance_of::()); }); } @@ -2540,15 +2539,15 @@ class SimpleClass: #[test] fn test_any_is_instance() { Python::with_gil(|py| { - let l = vec![1u8, 2].to_object(py).into_ref(py); - assert!(l.is_instance(py.get_type::()).unwrap()); + let l = vec![1u8, 2].to_object(py).into_bound(py); + assert!(l.is_instance(&py.get_type_bound::()).unwrap()); }); } #[test] fn test_any_is_exact_instance_of() { Python::with_gil(|py| { - let x = 5.to_object(py).into_ref(py); + let x = 5.to_object(py).into_bound(py); assert!(x.is_exact_instance_of::()); let t = PyBool::new_bound(py, true); @@ -2556,7 +2555,7 @@ class SimpleClass: assert!(!t.is_exact_instance_of::()); assert!(t.is_exact_instance_of::()); - let l = vec![x, x].to_object(py).into_ref(py); + let l = vec![&x, &x].to_object(py).into_bound(py); assert!(l.is_exact_instance_of::()); }); } @@ -2565,11 +2564,9 @@ class SimpleClass: fn test_any_is_exact_instance() { Python::with_gil(|py| { let t = PyBool::new_bound(py, true); - assert!(t - .is_instance(&py.get_type::().as_borrowed()) - .unwrap()); - assert!(!t.is_exact_instance(&py.get_type::().as_borrowed())); - assert!(t.is_exact_instance(&py.get_type::().as_borrowed())); + assert!(t.is_instance(&py.get_type_bound::()).unwrap()); + assert!(!t.is_exact_instance(&py.get_type_bound::())); + assert!(t.is_exact_instance(&py.get_type_bound::())); }); } @@ -2577,7 +2574,7 @@ class SimpleClass: fn test_any_contains() { Python::with_gil(|py| { let v: Vec = vec![1, 1, 2, 3, 5, 8]; - let ob = v.to_object(py).into_ref(py); + let ob = v.to_object(py).into_bound(py); let bad_needle = 7i32.to_object(py); assert!(!ob.contains(&bad_needle).unwrap()); @@ -2589,7 +2586,7 @@ class SimpleClass: assert!(ob.contains(&type_coerced_needle).unwrap()); let n: u32 = 42; - let bad_haystack = n.to_object(py).into_ref(py); + let bad_haystack = n.to_object(py).into_bound(py); let irrelevant_needle = 0i32.to_object(py); assert!(bad_haystack.contains(&irrelevant_needle).is_err()); }); @@ -2603,12 +2600,12 @@ class SimpleClass: Python::with_gil(|py| { for a in list { for b in list { - let a_py = a.to_object(py).into_ref(py); - let b_py = b.to_object(py).into_ref(py); + let a_py = a.to_object(py).into_bound(py); + let b_py = b.to_object(py).into_bound(py); assert_eq!( a.lt(b), - a_py.lt(b_py).unwrap(), + a_py.lt(&b_py).unwrap(), "{} < {} should be {}.", a_py, b_py, @@ -2616,7 +2613,7 @@ class SimpleClass: ); assert_eq!( a.le(b), - a_py.le(b_py).unwrap(), + a_py.le(&b_py).unwrap(), "{} <= {} should be {}.", a_py, b_py, @@ -2624,7 +2621,7 @@ class SimpleClass: ); assert_eq!( a.eq(b), - a_py.eq(b_py).unwrap(), + a_py.eq(&b_py).unwrap(), "{} == {} should be {}.", a_py, b_py, @@ -2632,7 +2629,7 @@ class SimpleClass: ); assert_eq!( a.ne(b), - a_py.ne(b_py).unwrap(), + a_py.ne(&b_py).unwrap(), "{} != {} should be {}.", a_py, b_py, @@ -2640,7 +2637,7 @@ class SimpleClass: ); assert_eq!( a.gt(b), - a_py.gt(b_py).unwrap(), + a_py.gt(&b_py).unwrap(), "{} > {} should be {}.", a_py, b_py, @@ -2648,7 +2645,7 @@ class SimpleClass: ); assert_eq!( a.ge(b), - a_py.ge(b_py).unwrap(), + a_py.ge(&b_py).unwrap(), "{} >= {} should be {}.", a_py, b_py, @@ -2695,10 +2692,10 @@ class SimpleClass: #[test] fn test_rich_compare_type_error() { Python::with_gil(|py| { - let py_int = 1.to_object(py).into_ref(py); - let py_str = "1".to_object(py).into_ref(py); + let py_int = 1.to_object(py).into_bound(py); + let py_str = "1".to_object(py).into_bound(py); - assert!(py_int.rich_compare(py_str, CompareOp::Lt).is_err()); + assert!(py_int.rich_compare(&py_str, CompareOp::Lt).is_err()); assert!(!py_int .rich_compare(py_str, CompareOp::Eq) .unwrap() @@ -2736,13 +2733,13 @@ class SimpleClass: #[test] fn test_is_empty() { Python::with_gil(|py| { - let empty_list: &PyAny = PyList::empty(py); + let empty_list = PyList::empty_bound(py).into_any(); assert!(empty_list.is_empty().unwrap()); - let list: &PyAny = PyList::new(py, vec![1, 2, 3]); + let list = PyList::new_bound(py, vec![1, 2, 3]).into_any(); assert!(!list.is_empty().unwrap()); - let not_container = 5.to_object(py).into_ref(py); + let not_container = 5.to_object(py).into_bound(py); assert!(not_container.is_empty().is_err()); }); } From 2f3a33fda11eb06783378db39ef3515dbb618202 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 1 May 2024 00:00:31 +0200 Subject: [PATCH 306/349] feature gate deprecated APIs for `PyList` (#4127) --- guide/src/migration.md | 4 +- guide/src/types.md | 4 + src/types/list.rs | 190 +++++++++++++++++++++-------------------- 3 files changed, 104 insertions(+), 94 deletions(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index 9c7bc15002e..ad39adfa0e8 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -75,7 +75,7 @@ Python::with_gil(|py| { After: -```rust +```rust,ignore # use pyo3::prelude::*; # use pyo3::types::{PyInt, PyList}; # fn main() -> PyResult<()> { @@ -1089,7 +1089,7 @@ An additional advantage of using Rust's indexing conventions for these types is that these types can now also support Rust's indexing operators as part of a consistent API: -```rust +```rust,ignore #![allow(deprecated)] use pyo3::{Python, types::PyList}; diff --git a/guide/src/types.md b/guide/src/types.md index 4c63d175991..82040de2c43 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -330,8 +330,10 @@ For a `&PyAny` object reference `any` where the underlying object is a Python-na a list: ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # use pyo3::types::PyList; +# #[cfg(feature = "gil-refs")] # Python::with_gil(|py| -> PyResult<()> { #[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. let obj: &PyAny = PyList::empty(py); @@ -390,8 +392,10 @@ To see all Python types exposed by `PyO3` consult the [`pyo3::types`][pyo3::type **Conversions:** ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # use pyo3::types::PyList; +# #[cfg(feature = "gil-refs")] # Python::with_gil(|py| -> PyResult<()> { #[allow(deprecated)] // PyList::empty is part of the deprecated "GIL Refs" API. let list = PyList::empty(py); diff --git a/src/types/list.rs b/src/types/list.rs index 19dbe59510f..56f21feb133 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -58,12 +58,10 @@ impl PyList { /// Deprecated form of [`PyList::new_bound`]. #[inline] #[track_caller] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyList::new` will be replaced by `PyList::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyList::new` will be replaced by `PyList::new_bound` in a future PyO3 version" )] pub fn new(py: Python<'_>, elements: impl IntoIterator) -> &PyList where @@ -113,12 +111,10 @@ impl PyList { /// Deprecated form of [`PyList::empty_bound`]. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyList::empty` will be replaced by `PyList::empty_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyList::empty` will be replaced by `PyList::empty_bound` in a future PyO3 version" )] pub fn empty(py: Python<'_>) -> &PyList { Self::empty_bound(py).into_gil_ref() @@ -721,7 +717,6 @@ impl<'py> IntoIterator for &Bound<'py, PyList> { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use crate::types::any::PyAnyMethods; use crate::types::list::PyListMethods; @@ -733,18 +728,18 @@ mod tests { #[test] fn test_new() { Python::with_gil(|py| { - let list = PyList::new(py, [2, 3, 5, 7]); - assert_eq!(2, list[0].extract::().unwrap()); - assert_eq!(3, list[1].extract::().unwrap()); - assert_eq!(5, list[2].extract::().unwrap()); - assert_eq!(7, list[3].extract::().unwrap()); + let list = PyList::new_bound(py, [2, 3, 5, 7]); + assert_eq!(2, list.get_item(0).unwrap().extract::().unwrap()); + assert_eq!(3, list.get_item(1).unwrap().extract::().unwrap()); + assert_eq!(5, list.get_item(2).unwrap().extract::().unwrap()); + assert_eq!(7, list.get_item(3).unwrap().extract::().unwrap()); }); } #[test] fn test_len() { Python::with_gil(|py| { - let list = PyList::new(py, [1, 2, 3, 4]); + let list = PyList::new_bound(py, [1, 2, 3, 4]); assert_eq!(4, list.len()); }); } @@ -752,7 +747,7 @@ mod tests { #[test] fn test_get_item() { Python::with_gil(|py| { - let list = PyList::new(py, [2, 3, 5, 7]); + let list = PyList::new_bound(py, [2, 3, 5, 7]); assert_eq!(2, list.get_item(0).unwrap().extract::().unwrap()); assert_eq!(3, list.get_item(1).unwrap().extract::().unwrap()); assert_eq!(5, list.get_item(2).unwrap().extract::().unwrap()); @@ -763,7 +758,7 @@ mod tests { #[test] fn test_get_slice() { Python::with_gil(|py| { - let list = PyList::new(py, [2, 3, 5, 7]); + let list = PyList::new_bound(py, [2, 3, 5, 7]); let slice = list.get_slice(1, 3); assert_eq!(2, slice.len()); let slice = list.get_slice(1, 7); @@ -774,12 +769,12 @@ mod tests { #[test] fn test_set_item() { Python::with_gil(|py| { - let list = PyList::new(py, [2, 3, 5, 7]); + let list = PyList::new_bound(py, [2, 3, 5, 7]); let val = 42i32.to_object(py); let val2 = 42i32.to_object(py); - assert_eq!(2, list[0].extract::().unwrap()); + assert_eq!(2, list.get_item(0).unwrap().extract::().unwrap()); list.set_item(0, val).unwrap(); - assert_eq!(42, list[0].extract::().unwrap()); + assert_eq!(42, list.get_item(0).unwrap().extract::().unwrap()); assert!(list.set_item(10, val2).is_err()); }); } @@ -787,15 +782,14 @@ mod tests { #[test] fn test_set_item_refcnt() { Python::with_gil(|py| { - let obj = py.eval("object()", None, None).unwrap(); + let obj = py.eval_bound("object()", None, None).unwrap(); let cnt; { - let _pool = unsafe { crate::GILPool::new() }; let v = vec![2]; let ob = v.to_object(py); - let list: &PyList = ob.downcast(py).unwrap(); + let list = ob.downcast_bound::(py).unwrap(); cnt = obj.get_refcnt(); - list.set_item(0, obj).unwrap(); + list.set_item(0, &obj).unwrap(); } assert_eq!(cnt, obj.get_refcnt()); @@ -805,17 +799,17 @@ mod tests { #[test] fn test_insert() { Python::with_gil(|py| { - let list = PyList::new(py, [2, 3, 5, 7]); + let list = PyList::new_bound(py, [2, 3, 5, 7]); let val = 42i32.to_object(py); let val2 = 43i32.to_object(py); assert_eq!(4, list.len()); - assert_eq!(2, list[0].extract::().unwrap()); + assert_eq!(2, list.get_item(0).unwrap().extract::().unwrap()); list.insert(0, val).unwrap(); list.insert(1000, val2).unwrap(); assert_eq!(6, list.len()); - assert_eq!(42, list[0].extract::().unwrap()); - assert_eq!(2, list[1].extract::().unwrap()); - assert_eq!(43, list[5].extract::().unwrap()); + assert_eq!(42, list.get_item(0).unwrap().extract::().unwrap()); + assert_eq!(2, list.get_item(1).unwrap().extract::().unwrap()); + assert_eq!(43, list.get_item(5).unwrap().extract::().unwrap()); }); } @@ -823,12 +817,11 @@ mod tests { fn test_insert_refcnt() { Python::with_gil(|py| { let cnt; - let obj = py.eval("object()", None, None).unwrap(); + let obj = py.eval_bound("object()", None, None).unwrap(); { - let _pool = unsafe { crate::GILPool::new() }; - let list = PyList::empty(py); + let list = PyList::empty_bound(py); cnt = obj.get_refcnt(); - list.insert(0, obj).unwrap(); + list.insert(0, &obj).unwrap(); } assert_eq!(cnt, obj.get_refcnt()); @@ -838,10 +831,10 @@ mod tests { #[test] fn test_append() { Python::with_gil(|py| { - let list = PyList::new(py, [2]); + let list = PyList::new_bound(py, [2]); list.append(3).unwrap(); - assert_eq!(2, list[0].extract::().unwrap()); - assert_eq!(3, list[1].extract::().unwrap()); + assert_eq!(2, list.get_item(0).unwrap().extract::().unwrap()); + assert_eq!(3, list.get_item(1).unwrap().extract::().unwrap()); }); } @@ -849,12 +842,11 @@ mod tests { fn test_append_refcnt() { Python::with_gil(|py| { let cnt; - let obj = py.eval("object()", None, None).unwrap(); + let obj = py.eval_bound("object()", None, None).unwrap(); { - let _pool = unsafe { crate::GILPool::new() }; - let list = PyList::empty(py); + let list = PyList::empty_bound(py); cnt = obj.get_refcnt(); - list.append(obj).unwrap(); + list.append(&obj).unwrap(); } assert_eq!(cnt, obj.get_refcnt()); }); @@ -864,7 +856,7 @@ mod tests { fn test_iter() { Python::with_gil(|py| { let v = vec![2, 3, 5, 7]; - let list = PyList::new(py, &v); + let list = PyList::new_bound(py, &v); let mut idx = 0; for el in list { assert_eq!(v[idx], el.extract::().unwrap()); @@ -879,7 +871,7 @@ mod tests { Python::with_gil(|py| { let v = vec![2, 3, 5, 7]; let ob = v.to_object(py); - let list: &PyList = ob.downcast(py).unwrap(); + let list = ob.downcast_bound::(py).unwrap(); let mut iter = list.iter(); assert_eq!(iter.size_hint(), (v.len(), Some(v.len()))); @@ -898,7 +890,7 @@ mod tests { Python::with_gil(|py| { let v = vec![2, 3, 5, 7]; let ob = v.to_object(py); - let list: &PyList = ob.downcast(py).unwrap(); + let list = ob.downcast_bound::(py).unwrap(); let mut iter = list.iter().rev(); @@ -924,7 +916,7 @@ mod tests { #[test] fn test_into_iter() { Python::with_gil(|py| { - let list = PyList::new(py, [1, 2, 3, 4]); + let list = PyList::new_bound(py, [1, 2, 3, 4]); for (i, item) in list.iter().enumerate() { assert_eq!((i + 1) as i32, item.extract::().unwrap()); } @@ -978,7 +970,7 @@ mod tests { fn test_extract() { Python::with_gil(|py| { let v = vec![2, 3, 5, 7]; - let list = PyList::new(py, &v); + let list = PyList::new_bound(py, &v); let v2 = list.as_ref().extract::>().unwrap(); assert_eq!(v, v2); }); @@ -988,16 +980,16 @@ mod tests { fn test_sort() { Python::with_gil(|py| { let v = vec![7, 3, 2, 5]; - let list = PyList::new(py, &v); - assert_eq!(7, list[0].extract::().unwrap()); - assert_eq!(3, list[1].extract::().unwrap()); - assert_eq!(2, list[2].extract::().unwrap()); - assert_eq!(5, list[3].extract::().unwrap()); + let list = PyList::new_bound(py, &v); + assert_eq!(7, list.get_item(0).unwrap().extract::().unwrap()); + assert_eq!(3, list.get_item(1).unwrap().extract::().unwrap()); + assert_eq!(2, list.get_item(2).unwrap().extract::().unwrap()); + assert_eq!(5, list.get_item(3).unwrap().extract::().unwrap()); list.sort().unwrap(); - assert_eq!(2, list[0].extract::().unwrap()); - assert_eq!(3, list[1].extract::().unwrap()); - assert_eq!(5, list[2].extract::().unwrap()); - assert_eq!(7, list[3].extract::().unwrap()); + assert_eq!(2, list.get_item(0).unwrap().extract::().unwrap()); + assert_eq!(3, list.get_item(1).unwrap().extract::().unwrap()); + assert_eq!(5, list.get_item(2).unwrap().extract::().unwrap()); + assert_eq!(7, list.get_item(3).unwrap().extract::().unwrap()); }); } @@ -1005,16 +997,16 @@ mod tests { fn test_reverse() { Python::with_gil(|py| { let v = vec![2, 3, 5, 7]; - let list = PyList::new(py, &v); - assert_eq!(2, list[0].extract::().unwrap()); - assert_eq!(3, list[1].extract::().unwrap()); - assert_eq!(5, list[2].extract::().unwrap()); - assert_eq!(7, list[3].extract::().unwrap()); + let list = PyList::new_bound(py, &v); + assert_eq!(2, list.get_item(0).unwrap().extract::().unwrap()); + assert_eq!(3, list.get_item(1).unwrap().extract::().unwrap()); + assert_eq!(5, list.get_item(2).unwrap().extract::().unwrap()); + assert_eq!(7, list.get_item(3).unwrap().extract::().unwrap()); list.reverse().unwrap(); - assert_eq!(7, list[0].extract::().unwrap()); - assert_eq!(5, list[1].extract::().unwrap()); - assert_eq!(3, list[2].extract::().unwrap()); - assert_eq!(2, list[3].extract::().unwrap()); + assert_eq!(7, list.get_item(0).unwrap().extract::().unwrap()); + assert_eq!(5, list.get_item(1).unwrap().extract::().unwrap()); + assert_eq!(3, list.get_item(2).unwrap().extract::().unwrap()); + assert_eq!(2, list.get_item(3).unwrap().extract::().unwrap()); }); } @@ -1022,16 +1014,16 @@ mod tests { fn test_array_into_py() { Python::with_gil(|py| { let array: PyObject = [1, 2].into_py(py); - let list: &PyList = array.downcast(py).unwrap(); - assert_eq!(1, list[0].extract::().unwrap()); - assert_eq!(2, list[1].extract::().unwrap()); + let list = array.downcast_bound::(py).unwrap(); + assert_eq!(1, list.get_item(0).unwrap().extract::().unwrap()); + assert_eq!(2, list.get_item(1).unwrap().extract::().unwrap()); }); } #[test] fn test_list_get_item_invalid_index() { Python::with_gil(|py| { - let list = PyList::new(py, [2, 3, 5, 7]); + let list = PyList::new_bound(py, [2, 3, 5, 7]); let obj = list.get_item(5); assert!(obj.is_err()); assert_eq!( @@ -1044,7 +1036,7 @@ mod tests { #[test] fn test_list_get_item_sanity() { Python::with_gil(|py| { - let list = PyList::new(py, [2, 3, 5, 7]); + let list = PyList::new_bound(py, [2, 3, 5, 7]); let obj = list.get_item(0); assert_eq!(obj.unwrap().extract::().unwrap(), 2); }); @@ -1054,13 +1046,15 @@ mod tests { #[test] fn test_list_get_item_unchecked_sanity() { Python::with_gil(|py| { - let list = PyList::new(py, [2, 3, 5, 7]); + let list = PyList::new_bound(py, [2, 3, 5, 7]); let obj = unsafe { list.get_item_unchecked(0) }; assert_eq!(obj.extract::().unwrap(), 2); }); } #[test] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_list_index_trait() { Python::with_gil(|py| { let list = PyList::new(py, [2, 3, 5]); @@ -1072,6 +1066,8 @@ mod tests { #[test] #[should_panic] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_list_index_trait_panic() { Python::with_gil(|py| { let list = PyList::new(py, [2, 3, 5]); @@ -1080,6 +1076,8 @@ mod tests { } #[test] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_list_index_trait_ranges() { Python::with_gil(|py| { let list = PyList::new(py, [2, 3, 5]); @@ -1096,6 +1094,8 @@ mod tests { #[test] #[should_panic = "range start index 5 out of range for list of length 3"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_list_index_trait_range_panic_start() { Python::with_gil(|py| { let list = PyList::new(py, [2, 3, 5]); @@ -1105,6 +1105,8 @@ mod tests { #[test] #[should_panic = "range end index 10 out of range for list of length 3"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_list_index_trait_range_panic_end() { Python::with_gil(|py| { let list = PyList::new(py, [2, 3, 5]); @@ -1114,6 +1116,8 @@ mod tests { #[test] #[should_panic = "slice index starts at 2 but ends at 1"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_list_index_trait_range_panic_wrong_order() { Python::with_gil(|py| { let list = PyList::new(py, [2, 3, 5]); @@ -1124,6 +1128,8 @@ mod tests { #[test] #[should_panic = "range start index 8 out of range for list of length 3"] + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] fn test_list_index_trait_range_from_panic() { Python::with_gil(|py| { let list = PyList::new(py, [2, 3, 5]); @@ -1134,19 +1140,19 @@ mod tests { #[test] fn test_list_del_item() { Python::with_gil(|py| { - let list = PyList::new(py, [1, 1, 2, 3, 5, 8]); + let list = PyList::new_bound(py, [1, 1, 2, 3, 5, 8]); assert!(list.del_item(10).is_err()); - assert_eq!(1, list[0].extract::().unwrap()); + assert_eq!(1, list.get_item(0).unwrap().extract::().unwrap()); assert!(list.del_item(0).is_ok()); - assert_eq!(1, list[0].extract::().unwrap()); + assert_eq!(1, list.get_item(0).unwrap().extract::().unwrap()); assert!(list.del_item(0).is_ok()); - assert_eq!(2, list[0].extract::().unwrap()); + assert_eq!(2, list.get_item(0).unwrap().extract::().unwrap()); assert!(list.del_item(0).is_ok()); - assert_eq!(3, list[0].extract::().unwrap()); + assert_eq!(3, list.get_item(0).unwrap().extract::().unwrap()); assert!(list.del_item(0).is_ok()); - assert_eq!(5, list[0].extract::().unwrap()); + assert_eq!(5, list.get_item(0).unwrap().extract::().unwrap()); assert!(list.del_item(0).is_ok()); - assert_eq!(8, list[0].extract::().unwrap()); + assert_eq!(8, list.get_item(0).unwrap().extract::().unwrap()); assert!(list.del_item(0).is_ok()); assert_eq!(0, list.len()); assert!(list.del_item(0).is_err()); @@ -1156,11 +1162,11 @@ mod tests { #[test] fn test_list_set_slice() { Python::with_gil(|py| { - let list = PyList::new(py, [1, 1, 2, 3, 5, 8]); - let ins = PyList::new(py, [7, 4]); - list.set_slice(1, 4, ins).unwrap(); + let list = PyList::new_bound(py, [1, 1, 2, 3, 5, 8]); + let ins = PyList::new_bound(py, [7, 4]); + list.set_slice(1, 4, &ins).unwrap(); assert_eq!([1, 7, 4, 5, 8], list.extract::<[i32; 5]>().unwrap()); - list.set_slice(3, 100, PyList::empty(py)).unwrap(); + list.set_slice(3, 100, &PyList::empty_bound(py)).unwrap(); assert_eq!([1, 7, 4], list.extract::<[i32; 3]>().unwrap()); }); } @@ -1168,7 +1174,7 @@ mod tests { #[test] fn test_list_del_slice() { Python::with_gil(|py| { - let list = PyList::new(py, [1, 1, 2, 3, 5, 8]); + let list = PyList::new_bound(py, [1, 1, 2, 3, 5, 8]); list.del_slice(1, 4).unwrap(); assert_eq!([1, 5, 8], list.extract::<[i32; 3]>().unwrap()); list.del_slice(1, 100).unwrap(); @@ -1179,7 +1185,7 @@ mod tests { #[test] fn test_list_contains() { Python::with_gil(|py| { - let list = PyList::new(py, [1, 1, 2, 3, 5, 8]); + let list = PyList::new_bound(py, [1, 1, 2, 3, 5, 8]); assert_eq!(6, list.len()); let bad_needle = 7i32.to_object(py); @@ -1196,7 +1202,7 @@ mod tests { #[test] fn test_list_index() { Python::with_gil(|py| { - let list = PyList::new(py, [1, 1, 2, 3, 5, 8]); + let list = PyList::new_bound(py, [1, 1, 2, 3, 5, 8]); assert_eq!(0, list.index(1i32).unwrap()); assert_eq!(2, list.index(2i32).unwrap()); assert_eq!(3, list.index(3i32).unwrap()); @@ -1233,7 +1239,7 @@ mod tests { fn too_long_iterator() { Python::with_gil(|py| { let iter = FaultyIter(0..usize::MAX, 73); - let _list = PyList::new(py, iter); + let _list = PyList::new_bound(py, iter); }) } @@ -1244,7 +1250,7 @@ mod tests { fn too_short_iterator() { Python::with_gil(|py| { let iter = FaultyIter(0..35, 73); - let _list = PyList::new(py, iter); + let _list = PyList::new_bound(py, iter); }) } @@ -1256,7 +1262,7 @@ mod tests { Python::with_gil(|py| { let iter = FaultyIter(0..0, usize::MAX); - let _list = PyList::new(py, iter); + let _list = PyList::new_bound(py, iter); }) } @@ -1315,7 +1321,7 @@ mod tests { Python::with_gil(|py| { std::panic::catch_unwind(|| { let iter = FaultyIter(0..50, 50); - let _list = PyList::new(py, iter); + let _list = PyList::new_bound(py, iter); }) .unwrap_err(); }); @@ -1330,7 +1336,7 @@ mod tests { #[test] fn test_list_to_tuple() { Python::with_gil(|py| { - let list = PyList::new(py, vec![1, 2, 3]); + let list = PyList::new_bound(py, vec![1, 2, 3]); let tuple = list.to_tuple(); let tuple_expected = PyTuple::new_bound(py, vec![1, 2, 3]); assert!(tuple.eq(tuple_expected).unwrap()); From 261d27d1973caf1b159b10cdc2583f121fbe08a5 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Tue, 30 Apr 2024 19:55:43 -0400 Subject: [PATCH 307/349] feature gate deprecated APIs for `PySlice` (#4141) --- src/types/slice.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/types/slice.rs b/src/types/slice.rs index b6895d09e10..70285c9c251 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -48,12 +48,10 @@ impl PySliceIndices { impl PySlice { /// Deprecated form of `PySlice::new_bound`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PySlice::new` will be replaced by `PySlice::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PySlice::new` will be replaced by `PySlice::new_bound` in a future PyO3 version" )] pub fn new(py: Python<'_>, start: isize, stop: isize, step: isize) -> &PySlice { Self::new_bound(py, start, stop, step).into_gil_ref() @@ -73,12 +71,10 @@ impl PySlice { } /// Deprecated form of `PySlice::full_bound`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PySlice::full` will be replaced by `PySlice::full_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PySlice::full` will be replaced by `PySlice::full_bound` in a future PyO3 version" )] pub fn full(py: Python<'_>) -> &PySlice { PySlice::full_bound(py).into_gil_ref() From dc9a41521af0f1485a152e18fafe2b951f9d98e6 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 1 May 2024 12:57:03 +0200 Subject: [PATCH 308/349] feature gate deprecated APIs for `Py` (#4142) --- guide/src/memory.md | 14 +++ guide/src/python-from-rust/function-calls.md | 2 +- guide/src/types.md | 2 + src/err/mod.rs | 1 + src/instance.rs | 115 ++++++++----------- src/marker.rs | 34 +++--- src/types/any.rs | 5 +- src/types/dict.rs | 6 +- 8 files changed, 88 insertions(+), 91 deletions(-) diff --git a/guide/src/memory.md b/guide/src/memory.md index cd4af4f8f13..a6640e65cf3 100644 --- a/guide/src/memory.md +++ b/guide/src/memory.md @@ -177,9 +177,11 @@ What happens to the memory when the last `Py` is dropped and its reference count reaches zero? It depends whether or not we are holding the GIL. ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { +# #[cfg(feature = "gil-refs")] Python::with_gil(|py| -> PyResult<()> { #[allow(deprecated)] // py.eval() is part of the GIL Refs API let hello: Py = py.eval("\"Hello World!\"", None, None)?.extract()?; @@ -203,9 +205,12 @@ This example wasn't very interesting. We could have just used a GIL-bound we are *not* holding the GIL? ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { +# #[cfg(feature = "gil-refs")] +# { let hello: Py = Python::with_gil(|py| { #[allow(deprecated)] // py.eval() is part of the GIL Refs API py.eval("\"Hello World!\"", None, None)?.extract() @@ -224,6 +229,7 @@ Python::with_gil(|py| // Memory for `hello` is released here. # () ); +# } # Ok(()) # } ``` @@ -237,9 +243,12 @@ We can avoid the delay in releasing memory if we are careful to drop the `Py` while the GIL is held. ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { +# #[cfg(feature = "gil-refs")] +# { #[allow(deprecated)] // py.eval() is part of the GIL Refs API let hello: Py = Python::with_gil(|py| py.eval("\"Hello World!\"", None, None)?.extract())?; @@ -252,6 +261,7 @@ Python::with_gil(|py| { } drop(hello); // Memory released here. }); +# } # Ok(()) # } ``` @@ -263,9 +273,12 @@ that rather than being released immediately, the memory will not be released until the GIL is dropped. ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { +# #[cfg(feature = "gil-refs")] +# { #[allow(deprecated)] // py.eval() is part of the GIL Refs API let hello: Py = Python::with_gil(|py| py.eval("\"Hello World!\"", None, None)?.extract())?; @@ -280,6 +293,7 @@ Python::with_gil(|py| { // Do more stuff... // Memory released here at end of `with_gil()` closure. }); +# } # Ok(()) # } ``` diff --git a/guide/src/python-from-rust/function-calls.md b/guide/src/python-from-rust/function-calls.md index f97de1f24ce..c19d6fafabc 100644 --- a/guide/src/python-from-rust/function-calls.md +++ b/guide/src/python-from-rust/function-calls.md @@ -107,7 +107,7 @@ fn main() -> PyResult<()> {
-During PyO3's [migration from "GIL Refs" to the `Bound` smart pointer](../migration.md#migrating-from-the-gil-refs-api-to-boundt), [`Py::call`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html#method.call) is temporarily named `call_bound` (and `call_method` is temporarily `call_method_bound`). +During PyO3's [migration from "GIL Refs" to the `Bound` smart pointer](../migration.md#migrating-from-the-gil-refs-api-to-boundt), `Py::call` is temporarily named [`Py::call_bound`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html#method.call_bound) (and `call_method` is temporarily `call_method_bound`). (This temporary naming is only the case for the `Py` smart pointer. The methods on the `&PyAny` GIL Ref such as `call` have not been given replacements, and the methods on the `Bound` smart pointer such as [`Bound::call`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call) already use follow the newest API conventions.) diff --git a/guide/src/types.md b/guide/src/types.md index 82040de2c43..d28fb7e15d6 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -353,8 +353,10 @@ let _: Py = obj.extract()?; For a `&PyAny` object reference `any` where the underlying object is a `#[pyclass]`: ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # #[pyclass] #[derive(Clone)] struct MyClass { } +# #[cfg(feature = "gil-refs")] # Python::with_gil(|py| -> PyResult<()> { #[allow(deprecated)] // into_ref is part of the deprecated GIL Refs API let obj: &PyAny = Py::new(py, MyClass {})?.into_ref(py); diff --git a/src/err/mod.rs b/src/err/mod.rs index 4f46f360094..d923761af1d 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -64,6 +64,7 @@ impl<'a> PyDowncastError<'a> { /// Compatibility API to convert the Bound variant `DowncastError` into the /// gil-ref variant + #[cfg(feature = "gil-refs")] pub(crate) fn from_downcast_err(DowncastError { from, to }: DowncastError<'a, 'a>) -> Self { #[allow(deprecated)] let from = unsafe { from.py().from_borrowed_ptr(from.as_ptr()) }; diff --git a/src/instance.rs b/src/instance.rs index cc892a2dd44..11d45df6f78 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,4 +1,4 @@ -use crate::err::{self, PyDowncastError, PyErr, PyResult}; +use crate::err::{self, PyErr, PyResult}; use crate::impl_::pycell::PyClassObject; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; @@ -721,12 +721,12 @@ impl IntoPy for Borrowed<'_, '_, T> { /// /// This type does not auto-dereference to the inner object because you must prove you hold the GIL to access it. /// Instead, call one of its methods to access the inner object: -/// - [`Py::as_ref`], to borrow a GIL-bound reference to the contained object. +/// - [`Py::bind`] or [`Py::into_bound`], to borrow a GIL-bound reference to the contained object. /// - [`Py::borrow`], [`Py::try_borrow`], [`Py::borrow_mut`], or [`Py::try_borrow_mut`], /// to get a (mutable) reference to a contained pyclass, using a scheme similar to std's [`RefCell`]. -/// See the [`PyCell` guide entry](https://pyo3.rs/latest/class.html#pycell-and-interior-mutability) +/// See the [guide entry](https://pyo3.rs/latest/class.html#bound-and-interior-mutability) /// for more information. -/// - You can call methods directly on `Py` with [`Py::call`], [`Py::call_method`] and friends. +/// - You can call methods directly on `Py` with [`Py::call_bound`], [`Py::call_method_bound`] and friends. /// These require passing in the [`Python<'py>`](crate::Python) token but are otherwise similar to the corresponding /// methods on [`PyAny`]. /// @@ -991,12 +991,10 @@ where /// assert!(my_class_cell.try_borrow().is_ok()); /// }); /// ``` - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `obj.bind(py)` instead of `obj.as_ref(py)`" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `obj.bind(py)` instead of `obj.as_ref(py)`" )] pub fn as_ref<'py>(&'py self, _py: Python<'py>) -> &'py T::AsRefTarget { let any = self.as_ptr() as *const PyAny; @@ -1046,12 +1044,10 @@ where /// obj.into_ref(py) /// } /// ``` - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `obj.into_bound(py)` instead of `obj.into_ref(py)`" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `obj.into_bound(py)` instead of `obj.into_ref(py)`" )] pub fn into_ref(self, py: Python<'_>) -> &T::AsRefTarget { #[allow(deprecated)] @@ -1464,12 +1460,10 @@ impl Py { } /// Deprecated form of [`call_bound`][Py::call_bound]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`call` will be replaced by `call_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`call` will be replaced by `call_bound` in a future PyO3 version" )] #[inline] pub fn call(&self, py: Python<'_>, args: A, kwargs: Option<&PyDict>) -> PyResult @@ -1506,12 +1500,10 @@ impl Py { } /// Deprecated form of [`call_method_bound`][Py::call_method_bound]. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`call_method` will be replaced by `call_method_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`call_method` will be replaced by `call_method_bound` in a future PyO3 version" )] #[inline] pub fn call_method( @@ -1779,6 +1771,7 @@ impl std::convert::From> for Py { } // `&PyCell` can be converted to `Py` +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl std::convert::From<&crate::PyCell> for Py where @@ -1844,10 +1837,7 @@ where { /// Extracts `Self` from the source `PyObject`. fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - // TODO update MSRV past 1.59 and use .cloned() to make - // clippy happy - #[allow(clippy::map_clone)] - ob.downcast().map(Clone::clone).map_err(Into::into) + ob.downcast().cloned().map_err(Into::into) } } @@ -1888,21 +1878,22 @@ pub type PyObject = Py; impl PyObject { /// Deprecated form of [`PyObject::downcast_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyObject::downcast` will be replaced by `PyObject::downcast_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyObject::downcast` will be replaced by `PyObject::downcast_bound` in a future PyO3 version" )] #[inline] - pub fn downcast<'py, T>(&'py self, py: Python<'py>) -> Result<&'py T, PyDowncastError<'py>> + pub fn downcast<'py, T>( + &'py self, + py: Python<'py>, + ) -> Result<&'py T, crate::err::PyDowncastError<'py>> where T: PyTypeCheck, { self.downcast_bound::(py) .map(Bound::as_gil_ref) - .map_err(PyDowncastError::from_downcast_err) + .map_err(crate::err::PyDowncastError::from_downcast_err) } /// Downcast this `PyObject` to a concrete Python type or pyclass. /// @@ -1970,12 +1961,10 @@ impl PyObject { /// # Safety /// /// Callers must ensure that the type is valid or risk type confusion. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyObject::downcast_unchecked` will be replaced by `PyObject::downcast_bound_unchecked` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyObject::downcast_unchecked` will be replaced by `PyObject::downcast_bound_unchecked` in a future PyO3 version" )] #[inline] pub unsafe fn downcast_unchecked<'py, T>(&'py self, py: Python<'py>) -> &T @@ -1997,35 +1986,31 @@ impl PyObject { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use super::{Bound, Py, PyObject}; use crate::types::any::PyAnyMethods; use crate::types::{dict::IntoPyDict, PyDict, PyString}; use crate::types::{PyCapsule, PyStringMethods}; - use crate::{ffi, Borrowed, PyAny, PyNativeType, PyResult, Python, ToPyObject}; + use crate::{ffi, Borrowed, PyAny, PyResult, Python, ToPyObject}; #[test] fn test_call() { Python::with_gil(|py| { - let obj = py.get_type::().to_object(py); + let obj = py.get_type_bound::().to_object(py); - let assert_repr = |obj: &PyAny, expected: &str| { - assert_eq!(obj.repr().unwrap().to_str().unwrap(), expected); + let assert_repr = |obj: &Bound<'_, PyAny>, expected: &str| { + assert_eq!(obj.repr().unwrap().to_cow().unwrap(), expected); }; - assert_repr(obj.call0(py).unwrap().as_ref(py), "{}"); - assert_repr(obj.call1(py, ()).unwrap().as_ref(py), "{}"); - assert_repr(obj.call(py, (), None).unwrap().as_ref(py), "{}"); + assert_repr(obj.call0(py).unwrap().bind(py), "{}"); + assert_repr(obj.call1(py, ()).unwrap().bind(py), "{}"); + assert_repr(obj.call_bound(py, (), None).unwrap().bind(py), "{}"); - assert_repr( - obj.call1(py, ((('x', 1),),)).unwrap().as_ref(py), - "{'x': 1}", - ); + assert_repr(obj.call1(py, ((('x', 1),),)).unwrap().bind(py), "{'x': 1}"); assert_repr( obj.call_bound(py, (), Some(&[('x', 1)].into_py_dict_bound(py))) .unwrap() - .as_ref(py), + .bind(py), "{'x': 1}", ); }) @@ -2037,7 +2022,7 @@ mod tests { let obj: PyObject = PyDict::new_bound(py).into(); assert!(obj.call_method0(py, "asdf").is_err()); assert!(obj - .call_method(py, "nonexistent_method", (1,), None) + .call_method_bound(py, "nonexistent_method", (1,), None) .is_err()); assert!(obj.call_method0(py, "nonexistent_method").is_err()); assert!(obj.call_method1(py, "nonexistent_method", (1,)).is_err()); @@ -2083,7 +2068,7 @@ a = A() assert!(instance .getattr(py, "foo")? - .as_ref(py) + .bind(py) .eq(PyString::new_bound(py, "bar"))?); instance.getattr(py, "foo")?; @@ -2109,7 +2094,7 @@ a = A() instance.getattr(py, foo).unwrap_err(); instance.setattr(py, foo, bar)?; - assert!(instance.getattr(py, foo)?.as_ref(py).eq(bar)?); + assert!(instance.getattr(py, foo)?.bind(py).eq(bar)?); Ok(()) }) } @@ -2117,7 +2102,7 @@ a = A() #[test] fn invalid_attr() -> PyResult<()> { Python::with_gil(|py| { - let instance: Py = py.eval("object()", None, None)?.into(); + let instance: Py = py.eval_bound("object()", None, None)?.into(); instance.getattr(py, "foo").unwrap_err(); @@ -2130,7 +2115,7 @@ a = A() #[test] fn test_py2_from_py_object() { Python::with_gil(|py| { - let instance: &PyAny = py.eval("object()", None, None).unwrap(); + let instance = py.eval_bound("object()", None, None).unwrap(); let ptr = instance.as_ptr(); let instance: Bound<'_, PyAny> = instance.extract().unwrap(); assert_eq!(instance.as_ptr(), ptr); @@ -2141,7 +2126,7 @@ a = A() fn test_py2_into_py_object() { Python::with_gil(|py| { let instance = py - .eval("object()", None, None) + .eval_bound("object()", None, None) .unwrap() .as_borrowed() .to_owned(); diff --git a/src/marker.rs b/src/marker.rs index b1f2d399209..29aae69d4f2 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -116,18 +116,17 @@ //! [`SendWrapper`]: https://docs.rs/send_wrapper/latest/send_wrapper/struct.SendWrapper.html //! [`Rc`]: std::rc::Rc //! [`Py`]: crate::Py -use crate::err::{self, PyDowncastError, PyErr, PyResult}; +use crate::err::{self, PyErr, PyResult}; use crate::ffi_ptr_ext::FfiPtrExt; use crate::gil::{GILGuard, SuspendGIL}; use crate::impl_::not_send::NotSend; use crate::py_result_ext::PyResultExt; -use crate::type_object::HasPyGilRef; use crate::types::any::PyAnyMethods; use crate::types::{ PyAny, PyDict, PyEllipsis, PyModule, PyNone, PyNotImplemented, PyString, PyType, }; use crate::version::PythonVersionInfo; -use crate::{ffi, Bound, IntoPy, Py, PyNativeType, PyObject, PyTypeCheck, PyTypeInfo}; +use crate::{ffi, Bound, IntoPy, Py, PyNativeType, PyObject, PyTypeInfo}; #[allow(deprecated)] use crate::{gil::GILPool, FromPyPointer}; use std::ffi::{CStr, CString}; @@ -839,16 +838,17 @@ impl<'py> Python<'py> { } /// Registers the object in the release pool, and tries to downcast to specific type. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `obj.downcast_bound::(py)` instead of `py.checked_cast_as::(obj)`" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `obj.downcast_bound::(py)` instead of `py.checked_cast_as::(obj)`" )] - pub fn checked_cast_as(self, obj: PyObject) -> Result<&'py T, PyDowncastError<'py>> + pub fn checked_cast_as( + self, + obj: PyObject, + ) -> Result<&'py T, crate::err::PyDowncastError<'py>> where - T: PyTypeCheck, + T: crate::PyTypeCheck, { #[allow(deprecated)] obj.into_ref(self).downcast() @@ -860,16 +860,14 @@ impl<'py> Python<'py> { /// # Safety /// /// Callers must ensure that ensure that the cast is valid. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `obj.downcast_bound_unchecked::(py)` instead of `py.cast_as::(obj)`" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `obj.downcast_bound_unchecked::(py)` instead of `py.cast_as::(obj)`" )] pub unsafe fn cast_as(self, obj: PyObject) -> &'py T where - T: HasPyGilRef, + T: crate::type_object::HasPyGilRef, { #[allow(deprecated)] obj.into_ref(self).downcast_unchecked() diff --git a/src/types/any.rs b/src/types/any.rs index 8b0bac44ca6..1854308ae7f 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -2705,17 +2705,16 @@ class SimpleClass: } #[test] - #[allow(deprecated)] fn test_is_ellipsis() { Python::with_gil(|py| { let v = py - .eval("...", None, None) + .eval_bound("...", None, None) .map_err(|e| e.display(py)) .unwrap(); assert!(v.is_ellipsis()); - let not_ellipsis = 5.to_object(py).into_ref(py); + let not_ellipsis = 5.to_object(py).into_bound(py); assert!(!not_ellipsis.is_ellipsis()); }); } diff --git a/src/types/dict.rs b/src/types/dict.rs index 64b3adcad44..68cca1cd981 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -817,8 +817,6 @@ where #[cfg(test)] mod tests { use super::*; - #[cfg(not(any(PyPy, GraalPy)))] - use crate::exceptions; use crate::types::PyTuple; use std::collections::{BTreeMap, HashMap}; @@ -948,7 +946,7 @@ mod tests { #[test] #[allow(deprecated)] - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(all(not(any(PyPy, GraalPy)), feature = "gil-refs"))] fn test_get_item_with_error() { Python::with_gil(|py| { let mut v = HashMap::new(); @@ -967,7 +965,7 @@ mod tests { assert!(dict .get_item_with_error(dict) .unwrap_err() - .is_instance_of::(py)); + .is_instance_of::(py)); }); } From 5534a7bee8baac15885f012fdbc2f793e5890176 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 1 May 2024 08:18:12 -0400 Subject: [PATCH 309/349] feature gate deprecated APIs for `PyBuffer` (#4144) --- src/buffer.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/buffer.rs b/src/buffer.rs index 84b08289771..74ac7fe8e53 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -18,8 +18,10 @@ // DEALINGS IN THE SOFTWARE. //! `PyBuffer` implementation +use crate::Bound; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; use crate::{err, exceptions::PyBufferError, ffi, FromPyObject, PyAny, PyResult, Python}; -use crate::{Bound, PyNativeType}; use std::marker::PhantomData; use std::os::raw; use std::pin::Pin; @@ -190,12 +192,10 @@ impl<'py, T: Element> FromPyObject<'py> for PyBuffer { impl PyBuffer { /// Deprecated form of [`PyBuffer::get_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyBuffer::get` will be replaced by `PyBuffer::get_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyBuffer::get` will be replaced by `PyBuffer::get_bound` in a future PyO3 version" )] pub fn get(obj: &PyAny) -> PyResult> { Self::get_bound(&obj.as_borrowed()) From a454f6e9cc2e67e07943a15bb04f37214c93eca6 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 1 May 2024 19:13:49 +0200 Subject: [PATCH 310/349] feature gate deprecated APIs for `PyFloat` and `PyComplex` (#4145) --- src/conversions/num_complex.rs | 10 ++++------ src/types/complex.rs | 10 ++++------ src/types/float.rs | 18 +++++++++--------- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index a57b2745ec9..12f208aa8d1 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -104,12 +104,10 @@ use std::os::raw::c_double; impl PyComplex { /// Deprecated form of [`PyComplex::from_complex_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyComplex::from_complex` will be replaced by `PyComplex::from_complex_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyComplex::from_complex` will be replaced by `PyComplex::from_complex_bound` in a future PyO3 version" )] pub fn from_complex>(py: Python<'_>, complex: Complex) -> &PyComplex { Self::from_complex_bound(py, complex).into_gil_ref() diff --git a/src/types/complex.rs b/src/types/complex.rs index e711b054fe3..4a0c3e30732 100644 --- a/src/types/complex.rs +++ b/src/types/complex.rs @@ -20,12 +20,10 @@ pyobject_native_type!( impl PyComplex { /// Deprecated form of [`PyComplex::from_doubles_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyComplex::from_doubles` will be replaced by `PyComplex::from_doubles_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyComplex::from_doubles` will be replaced by `PyComplex::from_doubles_bound` in a future PyO3 version" )] pub fn from_doubles(py: Python<'_>, real: c_double, imag: c_double) -> &PyComplex { Self::from_doubles_bound(py, real, imag).into_gil_ref() diff --git a/src/types/float.rs b/src/types/float.rs index 6db1fdae038..3a64694a624 100644 --- a/src/types/float.rs +++ b/src/types/float.rs @@ -26,12 +26,10 @@ pyobject_native_type!( impl PyFloat { /// Deprecated form of [`PyFloat::new_bound`]. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyFloat::new` will be replaced by `PyFloat::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyFloat::new` will be replaced by `PyFloat::new_bound` in a future PyO3 version" )] pub fn new(py: Python<'_>, val: f64) -> &'_ Self { Self::new_bound(py, val).into_gil_ref() @@ -154,9 +152,11 @@ impl<'py> FromPyObject<'py> for f32 { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { - use crate::{types::PyFloat, Python, ToPyObject}; + use crate::{ + types::{PyFloat, PyFloatMethods}, + Python, ToPyObject, + }; macro_rules! num_to_py_object_and_back ( ($func_name:ident, $t1:ty, $t2:ty) => ( @@ -184,7 +184,7 @@ mod tests { Python::with_gil(|py| { let v = 1.23f64; - let obj = PyFloat::new(py, 1.23); + let obj = PyFloat::new_bound(py, 1.23); assert_approx_eq!(v, obj.value()); }); } From 9a808c35c69ffc9b37976cc43c8589a2a7f0fae8 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 1 May 2024 21:05:51 +0200 Subject: [PATCH 311/349] fix `clippy-beta` ci workflow (#4147) --- pyo3-build-config/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 5b1cfdaf322..1aa15d7d62a 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -86,6 +86,8 @@ pub fn get() -> &'static InterpreterConfig { .map(|path| path.exists()) .unwrap_or(false); + // CONFIG_FILE is generated in build.rs, so it's content can vary + #[allow(unknown_lints, clippy::const_is_empty)] if let Some(interpreter_config) = InterpreterConfig::from_cargo_dep_env() { interpreter_config } else if !CONFIG_FILE.is_empty() { @@ -177,6 +179,8 @@ pub mod pyo3_build_script_impl { /// correct value for CARGO_CFG_TARGET_OS). #[cfg(feature = "resolve-config")] pub fn resolve_interpreter_config() -> Result { + // CONFIG_FILE is generated in build.rs, so it's content can vary + #[allow(unknown_lints, clippy::const_is_empty)] if !CONFIG_FILE.is_empty() { let mut interperter_config = InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE))?; interperter_config.generate_import_libs()?; From cd3f3ed67c963b758cb7e399e6f582d027a76707 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 3 May 2024 09:42:30 +0200 Subject: [PATCH 312/349] ci: updates for Rust 1.78 (#4150) * ci: updates for Rust 1.78 * ci: fix clippy * restrict `invalid_pymethods_duplicates` to unlimited api with `full` --- pytests/src/othermod.rs | 4 +- src/pyclass/create_type_object.rs | 6 +- tests/test_compile_error.rs | 2 + tests/ui/abi3_nativetype_inheritance.stderr | 6 +- tests/ui/invalid_argument_attributes.rs | 10 +- tests/ui/invalid_argument_attributes.stderr | 10 +- tests/ui/invalid_cancel_handle.stderr | 10 +- tests/ui/invalid_intern_arg.rs | 4 +- tests/ui/invalid_intern_arg.stderr | 13 ++- tests/ui/invalid_property_args.rs | 6 +- tests/ui/invalid_property_args.stderr | 14 +-- tests/ui/invalid_proto_pymethods.rs | 4 +- tests/ui/invalid_proto_pymethods.stderr | 36 +++++++ tests/ui/invalid_pyfunction_signatures.rs | 1 + tests/ui/invalid_pyfunction_signatures.stderr | 12 +-- tests/ui/invalid_pyfunctions.rs | 18 ++-- tests/ui/invalid_pyfunctions.stderr | 28 ++--- tests/ui/invalid_pymethod_receiver.rs | 2 +- tests/ui/invalid_pymethod_receiver.stderr | 9 +- tests/ui/invalid_pymethods.rs | 10 +- tests/ui/invalid_pymethods.stderr | 21 ++-- tests/ui/invalid_pymethods_duplicates.stderr | 102 ++++++++++++++++++ tests/ui/missing_intopy.stderr | 13 +-- tests/ui/pyclass_send.rs | 2 +- tests/ui/traverse.rs | 12 +-- tests/ui/traverse.stderr | 26 ++--- 26 files changed, 264 insertions(+), 117 deletions(-) diff --git a/pytests/src/othermod.rs b/pytests/src/othermod.rs index 29ca8121890..36ad4b5e23e 100644 --- a/pytests/src/othermod.rs +++ b/pytests/src/othermod.rs @@ -34,8 +34,8 @@ pub fn othermod(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; - m.add("USIZE_MIN", usize::min_value())?; - m.add("USIZE_MAX", usize::max_value())?; + m.add("USIZE_MIN", usize::MIN)?; + m.add("USIZE_MAX", usize::MAX)?; Ok(()) } diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index 52e346212f0..e90c5736e5c 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -145,12 +145,14 @@ impl PyTypeBuilder { #[cfg(all(not(Py_3_9), not(Py_LIMITED_API)))] ffi::Py_bf_getbuffer => { // Safety: slot.pfunc is a valid function pointer - self.buffer_procs.bf_getbuffer = Some(std::mem::transmute(pfunc)); + self.buffer_procs.bf_getbuffer = + Some(std::mem::transmute::<*mut T, ffi::getbufferproc>(pfunc)); } #[cfg(all(not(Py_3_9), not(Py_LIMITED_API)))] ffi::Py_bf_releasebuffer => { // Safety: slot.pfunc is a valid function pointer - self.buffer_procs.bf_releasebuffer = Some(std::mem::transmute(pfunc)); + self.buffer_procs.bf_releasebuffer = + Some(std::mem::transmute::<*mut T, ffi::releasebufferproc>(pfunc)); } _ => {} } diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 44049620598..30e77888cfd 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -13,6 +13,8 @@ fn test_compile_errors() { t.compile_fail("tests/ui/invalid_pyfunction_signatures.rs"); #[cfg(any(not(Py_LIMITED_API), Py_3_11))] t.compile_fail("tests/ui/invalid_pymethods_buffer.rs"); + // The output is not stable across abi3 / not abi3 and features + #[cfg(all(not(Py_LIMITED_API), feature = "full"))] t.compile_fail("tests/ui/invalid_pymethods_duplicates.rs"); t.compile_fail("tests/ui/invalid_pymethod_enum.rs"); t.compile_fail("tests/ui/invalid_pymethod_names.rs"); diff --git a/tests/ui/abi3_nativetype_inheritance.stderr b/tests/ui/abi3_nativetype_inheritance.stderr index 784da4f55ba..f9ca7c61b89 100644 --- a/tests/ui/abi3_nativetype_inheritance.stderr +++ b/tests/ui/abi3_nativetype_inheritance.stderr @@ -1,12 +1,10 @@ -error[E0277]: the trait bound `PyDict: PyClass` is not satisfied +error[E0277]: the trait bound `PyDict: PyClassBaseType` is not satisfied --> tests/ui/abi3_nativetype_inheritance.rs:5:19 | 5 | #[pyclass(extends=PyDict)] | ^^^^^^ the trait `PyClass` is not implemented for `PyDict`, which is required by `PyDict: PyClassBaseType` | - = help: the following other types implement trait `PyClass`: - TestClass - pyo3::coroutine::Coroutine + = help: the trait `PyClassBaseType` is implemented for `PyAny` = note: required for `PyDict` to implement `PyClassBaseType` note: required by a bound in `PyClassImpl::BaseType` --> src/impl_/pyclass.rs diff --git a/tests/ui/invalid_argument_attributes.rs b/tests/ui/invalid_argument_attributes.rs index 311c6c03e0e..6797642d77b 100644 --- a/tests/ui/invalid_argument_attributes.rs +++ b/tests/ui/invalid_argument_attributes.rs @@ -1,18 +1,18 @@ use pyo3::prelude::*; #[pyfunction] -fn invalid_attribute(#[pyo3(get)] param: String) {} +fn invalid_attribute(#[pyo3(get)] _param: String) {} #[pyfunction] -fn from_py_with_no_value(#[pyo3(from_py_with)] param: String) {} +fn from_py_with_no_value(#[pyo3(from_py_with)] _param: String) {} #[pyfunction] -fn from_py_with_string(#[pyo3("from_py_with")] param: String) {} +fn from_py_with_string(#[pyo3("from_py_with")] _param: String) {} #[pyfunction] -fn from_py_with_value_not_a_string(#[pyo3(from_py_with = func)] param: String) {} +fn from_py_with_value_not_a_string(#[pyo3(from_py_with = func)] _param: String) {} #[pyfunction] -fn from_py_with_repeated(#[pyo3(from_py_with = "func", from_py_with = "func")] param: String) {} +fn from_py_with_repeated(#[pyo3(from_py_with = "func", from_py_with = "func")] _param: String) {} fn main() {} diff --git a/tests/ui/invalid_argument_attributes.stderr b/tests/ui/invalid_argument_attributes.stderr index d27c25fd179..e6c42f82a87 100644 --- a/tests/ui/invalid_argument_attributes.stderr +++ b/tests/ui/invalid_argument_attributes.stderr @@ -1,29 +1,29 @@ error: expected `cancel_handle` or `from_py_with` --> tests/ui/invalid_argument_attributes.rs:4:29 | -4 | fn invalid_attribute(#[pyo3(get)] param: String) {} +4 | fn invalid_attribute(#[pyo3(get)] _param: String) {} | ^^^ error: expected `=` --> tests/ui/invalid_argument_attributes.rs:7:45 | -7 | fn from_py_with_no_value(#[pyo3(from_py_with)] param: String) {} +7 | fn from_py_with_no_value(#[pyo3(from_py_with)] _param: String) {} | ^ error: expected `cancel_handle` or `from_py_with` --> tests/ui/invalid_argument_attributes.rs:10:31 | -10 | fn from_py_with_string(#[pyo3("from_py_with")] param: String) {} +10 | fn from_py_with_string(#[pyo3("from_py_with")] _param: String) {} | ^^^^^^^^^^^^^^ error: expected string literal --> tests/ui/invalid_argument_attributes.rs:13:58 | -13 | fn from_py_with_value_not_a_string(#[pyo3(from_py_with = func)] param: String) {} +13 | fn from_py_with_value_not_a_string(#[pyo3(from_py_with = func)] _param: String) {} | ^^^^ error: `from_py_with` may only be specified once per argument --> tests/ui/invalid_argument_attributes.rs:16:56 | -16 | fn from_py_with_repeated(#[pyo3(from_py_with = "func", from_py_with = "func")] param: String) {} +16 | fn from_py_with_repeated(#[pyo3(from_py_with = "func", from_py_with = "func")] _param: String) {} | ^^^^^^^^^^^^ diff --git a/tests/ui/invalid_cancel_handle.stderr b/tests/ui/invalid_cancel_handle.stderr index 41a2c0854b7..f6452611679 100644 --- a/tests/ui/invalid_cancel_handle.stderr +++ b/tests/ui/invalid_cancel_handle.stderr @@ -38,13 +38,17 @@ note: function defined here | ^^^^^^^^^^^^^^^^^^^^^^^^ -------------- = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) -error[E0277]: the trait bound `CancelHandle: PyClass` is not satisfied +error[E0277]: the trait bound `CancelHandle: PyFunctionArgument<'_, '_>` is not satisfied --> tests/ui/invalid_cancel_handle.rs:20:50 | 20 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} | ^^^^ the trait `PyClass` is not implemented for `CancelHandle`, which is required by `CancelHandle: PyFunctionArgument<'_, '_>` | - = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` + = help: the following other types implement trait `PyFunctionArgument<'a, 'py>`: + Option<&'a pyo3::Bound<'py, T>> + &'a pyo3::Bound<'py, T> + &'a pyo3::coroutine::Coroutine + &'a mut pyo3::coroutine::Coroutine = note: required for `CancelHandle` to implement `FromPyObject<'_>` = note: required for `CancelHandle` to implement `FromPyObjectBound<'_, '_>` = note: required for `CancelHandle` to implement `PyFunctionArgument<'_, '_>` @@ -57,7 +61,7 @@ note: required by a bound in `extract_argument` | T: PyFunctionArgument<'a, 'py>, | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `extract_argument` -error[E0277]: the trait bound `CancelHandle: Clone` is not satisfied +error[E0277]: the trait bound `CancelHandle: PyFunctionArgument<'_, '_>` is not satisfied --> tests/ui/invalid_cancel_handle.rs:20:50 | 20 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} diff --git a/tests/ui/invalid_intern_arg.rs b/tests/ui/invalid_intern_arg.rs index 3c7bd592175..eb479431b90 100644 --- a/tests/ui/invalid_intern_arg.rs +++ b/tests/ui/invalid_intern_arg.rs @@ -1,6 +1,6 @@ use pyo3::Python; fn main() { - let foo = if true { "foo" } else { "bar" }; - Python::with_gil(|py| py.import_bound(pyo3::intern!(py, foo)).unwrap()); + let _foo = if true { "foo" } else { "bar" }; + Python::with_gil(|py| py.import_bound(pyo3::intern!(py, _foo)).unwrap()); } diff --git a/tests/ui/invalid_intern_arg.stderr b/tests/ui/invalid_intern_arg.stderr index dce2d85bf09..5d2131bd845 100644 --- a/tests/ui/invalid_intern_arg.stderr +++ b/tests/ui/invalid_intern_arg.stderr @@ -1,8 +1,17 @@ error[E0435]: attempt to use a non-constant value in a constant --> tests/ui/invalid_intern_arg.rs:5:61 | -5 | Python::with_gil(|py| py.import_bound(pyo3::intern!(py, foo)).unwrap()); - | ------------------^^^- +5 | Python::with_gil(|py| py.import_bound(pyo3::intern!(py, _foo)).unwrap()); + | ------------------^^^^- | | | | | non-constant value | help: consider using `let` instead of `static`: `let INTERNED` + +error: lifetime may not live long enough + --> tests/ui/invalid_intern_arg.rs:5:27 + | +5 | Python::with_gil(|py| py.import_bound(pyo3::intern!(py, _foo)).unwrap()); + | --- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ returning this value requires that `'1` must outlive `'2` + | | | + | | return type of closure is pyo3::Bound<'2, pyo3::prelude::PyModule> + | has type `pyo3::Python<'1>` diff --git a/tests/ui/invalid_property_args.rs b/tests/ui/invalid_property_args.rs index 3f335952235..b5eba27eb60 100644 --- a/tests/ui/invalid_property_args.rs +++ b/tests/ui/invalid_property_args.rs @@ -6,7 +6,7 @@ struct ClassWithGetter {} #[pymethods] impl ClassWithGetter { #[getter] - fn getter_with_arg(&self, py: Python<'_>, index: u32) {} + fn getter_with_arg(&self, _py: Python<'_>, _index: u32) {} } #[pyclass] @@ -15,13 +15,13 @@ struct ClassWithSetter {} #[pymethods] impl ClassWithSetter { #[setter] - fn setter_with_no_arg(&mut self, py: Python<'_>) {} + fn setter_with_no_arg(&mut self, _py: Python<'_>) {} } #[pymethods] impl ClassWithSetter { #[setter] - fn setter_with_too_many_args(&mut self, py: Python<'_>, foo: u32, bar: u32) {} + fn setter_with_too_many_args(&mut self, _py: Python<'_>, _foo: u32, _bar: u32) {} } #[pyclass] diff --git a/tests/ui/invalid_property_args.stderr b/tests/ui/invalid_property_args.stderr index a41b6c79b3a..dea2e3fb2b4 100644 --- a/tests/ui/invalid_property_args.stderr +++ b/tests/ui/invalid_property_args.stderr @@ -1,20 +1,20 @@ error: getter function can only have one argument (of type pyo3::Python) - --> tests/ui/invalid_property_args.rs:9:54 + --> tests/ui/invalid_property_args.rs:9:56 | -9 | fn getter_with_arg(&self, py: Python<'_>, index: u32) {} - | ^^^ +9 | fn getter_with_arg(&self, _py: Python<'_>, _index: u32) {} + | ^^^ error: setter function expected to have one argument --> tests/ui/invalid_property_args.rs:18:8 | -18 | fn setter_with_no_arg(&mut self, py: Python<'_>) {} +18 | fn setter_with_no_arg(&mut self, _py: Python<'_>) {} | ^^^^^^^^^^^^^^^^^^ error: setter function can have at most two arguments ([pyo3::Python,] and value) - --> tests/ui/invalid_property_args.rs:24:76 + --> tests/ui/invalid_property_args.rs:24:79 | -24 | fn setter_with_too_many_args(&mut self, py: Python<'_>, foo: u32, bar: u32) {} - | ^^^ +24 | fn setter_with_too_many_args(&mut self, _py: Python<'_>, _foo: u32, _bar: u32) {} + | ^^^ error: `get` and `set` with tuple struct fields require `name` --> tests/ui/invalid_property_args.rs:28:50 diff --git a/tests/ui/invalid_proto_pymethods.rs b/tests/ui/invalid_proto_pymethods.rs index d370c4fddb5..c40790c3168 100644 --- a/tests/ui/invalid_proto_pymethods.rs +++ b/tests/ui/invalid_proto_pymethods.rs @@ -54,11 +54,11 @@ struct EqAndRichcmp; #[pymethods] impl EqAndRichcmp { - fn __eq__(&self, other: &Self) -> bool { + fn __eq__(&self, _other: &Self) -> bool { true } - fn __richcmp__(&self, other: &Self, op: CompareOp) -> bool { + fn __richcmp__(&self, _other: &Self, _op: CompareOp) -> bool { true } } diff --git a/tests/ui/invalid_proto_pymethods.stderr b/tests/ui/invalid_proto_pymethods.stderr index c9f6adff3a1..82c99c2ddc3 100644 --- a/tests/ui/invalid_proto_pymethods.stderr +++ b/tests/ui/invalid_proto_pymethods.stderr @@ -22,6 +22,24 @@ error: `text_signature` cannot be used with magic method `__bool__` 46 | #[pyo3(name = "__bool__", text_signature = "")] | ^^^^^^^^^^^^^^ +error[E0034]: multiple applicable items in scope + --> tests/ui/invalid_proto_pymethods.rs:55:1 + | +55 | #[pymethods] + | ^^^^^^^^^^^^ multiple `__pymethod___richcmp____` found + | +note: candidate #1 is defined in an impl for the type `EqAndRichcmp` + --> tests/ui/invalid_proto_pymethods.rs:55:1 + | +55 | #[pymethods] + | ^^^^^^^^^^^^ +note: candidate #2 is defined in an impl for the type `EqAndRichcmp` + --> tests/ui/invalid_proto_pymethods.rs:55:1 + | +55 | #[pymethods] + | ^^^^^^^^^^^^ + = note: this error originates in the macro `::pyo3::impl_::pyclass::generate_pyclass_richcompare_slot` which comes from the expansion of the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) + error[E0592]: duplicate definitions with name `__pymethod___richcmp____` --> tests/ui/invalid_proto_pymethods.rs:55:1 | @@ -32,3 +50,21 @@ error[E0592]: duplicate definitions with name `__pymethod___richcmp____` | other definition for `__pymethod___richcmp____` | = note: this error originates in the macro `::pyo3::impl_::pyclass::generate_pyclass_richcompare_slot` which comes from the expansion of the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0034]: multiple applicable items in scope + --> tests/ui/invalid_proto_pymethods.rs:55:1 + | +55 | #[pymethods] + | ^^^^^^^^^^^^ multiple `__pymethod___richcmp____` found + | +note: candidate #1 is defined in an impl for the type `EqAndRichcmp` + --> tests/ui/invalid_proto_pymethods.rs:55:1 + | +55 | #[pymethods] + | ^^^^^^^^^^^^ +note: candidate #2 is defined in an impl for the type `EqAndRichcmp` + --> tests/ui/invalid_proto_pymethods.rs:55:1 + | +55 | #[pymethods] + | ^^^^^^^^^^^^ + = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/invalid_pyfunction_signatures.rs b/tests/ui/invalid_pyfunction_signatures.rs index f5a9bee4e6c..86033aa12ea 100644 --- a/tests/ui/invalid_pyfunction_signatures.rs +++ b/tests/ui/invalid_pyfunction_signatures.rs @@ -35,6 +35,7 @@ fn function_with_args_sep_after_args_sep() {} #[pyo3(signature = (**kwargs, *args))] fn function_with_args_after_kwargs(kwargs: Option<&PyDict>, args: &PyTuple) { let _ = args; + let _ = kwargs; } #[pyfunction] diff --git a/tests/ui/invalid_pyfunction_signatures.stderr b/tests/ui/invalid_pyfunction_signatures.stderr index dbca169d8ea..97d0fd3b4af 100644 --- a/tests/ui/invalid_pyfunction_signatures.stderr +++ b/tests/ui/invalid_pyfunction_signatures.stderr @@ -41,19 +41,19 @@ error: `*args` not allowed after `**kwargs` | ^ error: `**kwargs_b` not allowed after `**kwargs_a` - --> tests/ui/invalid_pyfunction_signatures.rs:41:33 + --> tests/ui/invalid_pyfunction_signatures.rs:42:33 | -41 | #[pyo3(signature = (**kwargs_a, **kwargs_b))] +42 | #[pyo3(signature = (**kwargs_a, **kwargs_b))] | ^ error: arguments of type `Python` must not be part of the signature - --> tests/ui/invalid_pyfunction_signatures.rs:47:27 + --> tests/ui/invalid_pyfunction_signatures.rs:48:27 | -47 | #[pyfunction(signature = (py))] +48 | #[pyfunction(signature = (py))] | ^^ error: cannot find attribute `args` in this scope - --> tests/ui/invalid_pyfunction_signatures.rs:57:7 + --> tests/ui/invalid_pyfunction_signatures.rs:58:7 | -57 | #[args(x)] +58 | #[args(x)] | ^^^^ diff --git a/tests/ui/invalid_pyfunctions.rs b/tests/ui/invalid_pyfunctions.rs index 1a95c9e4a34..1c0c45d6b95 100644 --- a/tests/ui/invalid_pyfunctions.rs +++ b/tests/ui/invalid_pyfunctions.rs @@ -2,35 +2,39 @@ use pyo3::prelude::*; use pyo3::types::{PyDict, PyString, PyTuple}; #[pyfunction] -fn generic_function(value: T) {} +fn generic_function(_value: T) {} #[pyfunction] -fn impl_trait_function(impl_trait: impl AsRef) {} +fn impl_trait_function(_impl_trait: impl AsRef) {} #[pyfunction] fn wildcard_argument(_: i32) {} #[pyfunction] -fn destructured_argument((a, b): (i32, i32)) {} +fn destructured_argument((_a, _b): (i32, i32)) {} #[pyfunction] fn function_with_required_after_option(_opt: Option, _x: i32) {} #[pyfunction] #[pyo3(signature=(*args))] -fn function_with_optional_args(args: Option>) {} +fn function_with_optional_args(args: Option>) { + let _ = args; +} #[pyfunction] #[pyo3(signature=(**kwargs))] -fn function_with_required_kwargs(kwargs: Bound<'_, PyDict>) {} +fn function_with_required_kwargs(kwargs: Bound<'_, PyDict>) { + let _ = kwargs; +} #[pyfunction(pass_module)] fn pass_module_but_no_arguments<'py>() {} #[pyfunction(pass_module)] fn first_argument_not_module<'a, 'py>( - string: &str, - module: &'a Bound<'_, PyModule>, + _string: &str, + module: &'a Bound<'py, PyModule>, ) -> PyResult> { module.name() } diff --git a/tests/ui/invalid_pyfunctions.stderr b/tests/ui/invalid_pyfunctions.stderr index 893d7cbec2c..830f17ee877 100644 --- a/tests/ui/invalid_pyfunctions.stderr +++ b/tests/ui/invalid_pyfunctions.stderr @@ -1,14 +1,14 @@ error: Python functions cannot have generic type parameters --> tests/ui/invalid_pyfunctions.rs:5:21 | -5 | fn generic_function(value: T) {} +5 | fn generic_function(_value: T) {} | ^ error: Python functions cannot have `impl Trait` arguments - --> tests/ui/invalid_pyfunctions.rs:8:36 + --> tests/ui/invalid_pyfunctions.rs:8:37 | -8 | fn impl_trait_function(impl_trait: impl AsRef) {} - | ^^^^ +8 | fn impl_trait_function(_impl_trait: impl AsRef) {} + | ^^^^ error: wildcard argument names are not supported --> tests/ui/invalid_pyfunctions.rs:11:22 @@ -19,8 +19,8 @@ error: wildcard argument names are not supported error: destructuring in arguments is not supported --> tests/ui/invalid_pyfunctions.rs:14:26 | -14 | fn destructured_argument((a, b): (i32, i32)) {} - | ^^^^^^ +14 | fn destructured_argument((_a, _b): (i32, i32)) {} + | ^^^^^^^^ error: required arguments after an `Option<_>` argument are ambiguous = help: add a `#[pyo3(signature)]` annotation on this function to unambiguously specify the default values for all optional parameters @@ -32,26 +32,26 @@ error: required arguments after an `Option<_>` argument are ambiguous error: args cannot be optional --> tests/ui/invalid_pyfunctions.rs:21:32 | -21 | fn function_with_optional_args(args: Option>) {} +21 | fn function_with_optional_args(args: Option>) { | ^^^^ error: kwargs must be Option<_> - --> tests/ui/invalid_pyfunctions.rs:25:34 + --> tests/ui/invalid_pyfunctions.rs:27:34 | -25 | fn function_with_required_kwargs(kwargs: Bound<'_, PyDict>) {} +27 | fn function_with_required_kwargs(kwargs: Bound<'_, PyDict>) { | ^^^^^^ error: expected `&PyModule` or `Py` as first argument with `pass_module` - --> tests/ui/invalid_pyfunctions.rs:28:37 + --> tests/ui/invalid_pyfunctions.rs:32:37 | -28 | fn pass_module_but_no_arguments<'py>() {} +32 | fn pass_module_but_no_arguments<'py>() {} | ^^ error[E0277]: the trait bound `&str: From>` is not satisfied - --> tests/ui/invalid_pyfunctions.rs:32:13 + --> tests/ui/invalid_pyfunctions.rs:36:14 | -32 | string: &str, - | ^ the trait `From>` is not implemented for `&str`, which is required by `BoundRef<'_, '_, pyo3::prelude::PyModule>: Into<_>` +36 | _string: &str, + | ^ the trait `From>` is not implemented for `&str`, which is required by `BoundRef<'_, '_, pyo3::prelude::PyModule>: Into<_>` | = help: the following other types implement trait `From`: > diff --git a/tests/ui/invalid_pymethod_receiver.rs b/tests/ui/invalid_pymethod_receiver.rs index 77832e12857..4949ff96022 100644 --- a/tests/ui/invalid_pymethod_receiver.rs +++ b/tests/ui/invalid_pymethod_receiver.rs @@ -5,7 +5,7 @@ struct MyClass {} #[pymethods] impl MyClass { - fn method_with_invalid_self_type(slf: i32, py: Python<'_>, index: u32) {} + fn method_with_invalid_self_type(_slf: i32, _py: Python<'_>, _index: u32) {} } fn main() {} diff --git a/tests/ui/invalid_pymethod_receiver.stderr b/tests/ui/invalid_pymethod_receiver.stderr index a79e289716a..2c8ec045819 100644 --- a/tests/ui/invalid_pymethod_receiver.stderr +++ b/tests/ui/invalid_pymethod_receiver.stderr @@ -1,8 +1,8 @@ -error[E0277]: the trait bound `i32: From>` is not satisfied - --> tests/ui/invalid_pymethod_receiver.rs:8:43 +error[E0277]: the trait bound `i32: TryFrom>` is not satisfied + --> tests/ui/invalid_pymethod_receiver.rs:8:44 | -8 | fn method_with_invalid_self_type(slf: i32, py: Python<'_>, index: u32) {} - | ^^^ the trait `From>` is not implemented for `i32`, which is required by `i32: TryFrom>` +8 | fn method_with_invalid_self_type(_slf: i32, _py: Python<'_>, _index: u32) {} + | ^^^ the trait `From>` is not implemented for `i32`, which is required by `i32: TryFrom>` | = help: the following other types implement trait `From`: > @@ -10,6 +10,5 @@ error[E0277]: the trait bound `i32: From>` is not sati > > > - >> = note: required for `BoundRef<'_, '_, MyClass>` to implement `Into` = note: required for `i32` to implement `TryFrom>` diff --git a/tests/ui/invalid_pymethods.rs b/tests/ui/invalid_pymethods.rs index 00bddfe2fad..41786cd7895 100644 --- a/tests/ui/invalid_pymethods.rs +++ b/tests/ui/invalid_pymethods.rs @@ -6,7 +6,7 @@ struct MyClass {} #[pymethods] impl MyClass { #[classattr] - fn class_attr_with_args(foo: i32) {} + fn class_attr_with_args(_foo: i32) {} } #[pymethods] @@ -164,23 +164,23 @@ impl MyClass { #[pymethods] impl MyClass { - fn generic_method(value: T) {} + fn generic_method(_value: T) {} } #[pymethods] impl MyClass { - fn impl_trait_method_first_arg(impl_trait: impl AsRef) {} + fn impl_trait_method_first_arg(_impl_trait: impl AsRef) {} } #[pymethods] impl MyClass { - fn impl_trait_method_second_arg(&self, impl_trait: impl AsRef) {} + fn impl_trait_method_second_arg(&self, _impl_trait: impl AsRef) {} } #[pymethods] impl MyClass { #[pyo3(pass_module)] - fn method_cannot_pass_module(&self, m: &PyModule) {} + fn method_cannot_pass_module(&self, _m: &PyModule) {} } #[pymethods] diff --git a/tests/ui/invalid_pymethods.stderr b/tests/ui/invalid_pymethods.stderr index 5bcb6aa1be6..9b090e31adc 100644 --- a/tests/ui/invalid_pymethods.stderr +++ b/tests/ui/invalid_pymethods.stderr @@ -1,8 +1,8 @@ error: #[classattr] can only have one argument (of type pyo3::Python) - --> tests/ui/invalid_pymethods.rs:9:34 + --> tests/ui/invalid_pymethods.rs:9:35 | -9 | fn class_attr_with_args(foo: i32) {} - | ^^^ +9 | fn class_attr_with_args(_foo: i32) {} + | ^^^ error: `#[classattr]` does not take any arguments --> tests/ui/invalid_pymethods.rs:14:5 @@ -144,20 +144,20 @@ error: `#[classattr]` does not take any arguments error: Python functions cannot have generic type parameters --> tests/ui/invalid_pymethods.rs:167:23 | -167 | fn generic_method(value: T) {} +167 | fn generic_method(_value: T) {} | ^ error: Python functions cannot have `impl Trait` arguments - --> tests/ui/invalid_pymethods.rs:172:48 + --> tests/ui/invalid_pymethods.rs:172:49 | -172 | fn impl_trait_method_first_arg(impl_trait: impl AsRef) {} - | ^^^^ +172 | fn impl_trait_method_first_arg(_impl_trait: impl AsRef) {} + | ^^^^ error: Python functions cannot have `impl Trait` arguments - --> tests/ui/invalid_pymethods.rs:177:56 + --> tests/ui/invalid_pymethods.rs:177:57 | -177 | fn impl_trait_method_second_arg(&self, impl_trait: impl AsRef) {} - | ^^^^ +177 | fn impl_trait_method_second_arg(&self, _impl_trait: impl AsRef) {} + | ^^^^ error: `pass_module` cannot be used on Python methods --> tests/ui/invalid_pymethods.rs:182:12 @@ -191,5 +191,4 @@ error[E0277]: the trait bound `i32: From>` is not satis > > > - >> = note: required for `BoundRef<'_, '_, PyType>` to implement `Into` diff --git a/tests/ui/invalid_pymethods_duplicates.stderr b/tests/ui/invalid_pymethods_duplicates.stderr index 38bb6f8655b..753c4b1b8dc 100644 --- a/tests/ui/invalid_pymethods_duplicates.stderr +++ b/tests/ui/invalid_pymethods_duplicates.stderr @@ -9,6 +9,40 @@ error[E0119]: conflicting implementations of trait `pyo3::impl_::pyclass::PyClas | = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) +error[E0277]: the trait bound `TwoNew: PyTypeInfo` is not satisfied + --> tests/ui/invalid_pymethods_duplicates.rs:9:6 + | +9 | impl TwoNew { + | ^^^^^^ the trait `PyTypeInfo` is not implemented for `TwoNew` + | + = help: the following other types implement trait `PyTypeInfo`: + PyAny + PyBool + PyByteArray + PyBytes + PyCapsule + PyCode + PyComplex + PyDate + and $N others + +error[E0277]: the trait bound `TwoNew: HasPyGilRef` is not satisfied + --> tests/ui/invalid_pymethods_duplicates.rs:9:6 + | +9 | impl TwoNew { + | ^^^^^^ the trait `PyNativeType` is not implemented for `TwoNew`, which is required by `TwoNew: HasPyGilRef` + | + = help: the trait `HasPyGilRef` is implemented for `pyo3::coroutine::Coroutine` + = note: required for `TwoNew` to implement `HasPyGilRef` +note: required by a bound in `pyo3::PyTypeInfo::NAME` + --> src/type_object.rs + | + | pub unsafe trait PyTypeInfo: Sized + HasPyGilRef { + | ^^^^^^^^^^^ required by this bound in `PyTypeInfo::NAME` + | /// Class name. + | const NAME: &'static str; + | ---- required by a bound in this associated constant + error[E0592]: duplicate definitions with name `__pymethod___new____` --> tests/ui/invalid_pymethods_duplicates.rs:8:1 | @@ -30,3 +64,71 @@ error[E0592]: duplicate definitions with name `__pymethod_func__` | other definition for `__pymethod_func__` | = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `TwoNew: PyClass` is not satisfied + --> tests/ui/invalid_pymethods_duplicates.rs:8:1 + | +8 | #[pymethods] + | ^^^^^^^^^^^^ the trait `PyClass` is not implemented for `TwoNew` + | + = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` +note: required by a bound in `PyClassInitializer` + --> src/pyclass_init.rs + | + | pub struct PyClassInitializer(PyClassInitializerImpl); + | ^^^^^^^ required by this bound in `PyClassInitializer` + = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0599]: no method named `convert` found for struct `TwoNew` in the current scope + --> tests/ui/invalid_pymethods_duplicates.rs:8:1 + | +6 | struct TwoNew {} + | ------------- method `convert` not found for this struct +7 | +8 | #[pymethods] + | ^^^^^^^^^^^^ method not found in `TwoNew` + | + = help: items from traits can only be used if the trait is implemented and in scope + = note: the following trait defines an item `convert`, perhaps you need to implement it: + candidate #1: `IntoPyCallbackOutput` + = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `DuplicateMethod: PyClass` is not satisfied + --> tests/ui/invalid_pymethods_duplicates.rs:26:15 + | +26 | fn func_a(&self) {} + | ^ the trait `PyClass` is not implemented for `DuplicateMethod` + | + = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` +note: required by a bound in `extract_pyclass_ref` + --> src/impl_/extract_argument.rs + | + | pub fn extract_pyclass_ref<'a, 'py: 'a, T: PyClass>( + | ^^^^^^^ required by this bound in `extract_pyclass_ref` + +error[E0277]: the trait bound `DuplicateMethod: PyClass` is not satisfied + --> tests/ui/invalid_pymethods_duplicates.rs:23:1 + | +23 | #[pymethods] + | ^^^^^^^^^^^^ the trait `PyClass` is not implemented for `DuplicateMethod` + | + = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` +note: required by a bound in `PyRef` + --> src/pycell.rs + | + | pub struct PyRef<'p, T: PyClass> { + | ^^^^^^^ required by this bound in `PyRef` + = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `DuplicateMethod: PyClass` is not satisfied + --> tests/ui/invalid_pymethods_duplicates.rs:29:15 + | +29 | fn func_b(&self) {} + | ^ the trait `PyClass` is not implemented for `DuplicateMethod` + | + = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` +note: required by a bound in `extract_pyclass_ref` + --> src/impl_/extract_argument.rs + | + | pub fn extract_pyclass_ref<'a, 'py: 'a, T: PyClass>( + | ^^^^^^^ required by this bound in `extract_pyclass_ref` diff --git a/tests/ui/missing_intopy.stderr b/tests/ui/missing_intopy.stderr index 1d233b28b6c..c0a60143671 100644 --- a/tests/ui/missing_intopy.stderr +++ b/tests/ui/missing_intopy.stderr @@ -1,18 +1,9 @@ -error[E0277]: the trait bound `Blah: IntoPy>` is not satisfied +error[E0277]: the trait bound `Blah: OkWrap` is not satisfied --> tests/ui/missing_intopy.rs:3:1 | 3 | #[pyo3::pyfunction] | ^^^^^^^^^^^^^^^^^^^ the trait `IntoPy>` is not implemented for `Blah`, which is required by `Blah: OkWrap<_>` | - = help: the following other types implement trait `IntoPy`: - >> - >> - >> - >> - >> - >> - >> - >> - and $N others + = help: the trait `OkWrap` is implemented for `Result` = note: required for `Blah` to implement `OkWrap` = note: this error originates in the attribute macro `pyo3::pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/pyclass_send.rs b/tests/ui/pyclass_send.rs index 2747c2cb3bb..a587c071f51 100644 --- a/tests/ui/pyclass_send.rs +++ b/tests/ui/pyclass_send.rs @@ -10,7 +10,7 @@ fn main() { let obj = Python::with_gil(|py| { Bound::new(py, NotThreadSafe { data: Rc::new(5) }) .unwrap() - .unbind(py) + .unbind() }); std::thread::spawn(move || { diff --git a/tests/ui/traverse.rs b/tests/ui/traverse.rs index 0cf7170db21..faa7b5c041d 100644 --- a/tests/ui/traverse.rs +++ b/tests/ui/traverse.rs @@ -7,7 +7,7 @@ struct TraverseTriesToTakePyRef {} #[pymethods] impl TraverseTriesToTakePyRef { - fn __traverse__(slf: PyRef, visit: PyVisit) -> Result<(), PyTraverseError> { + fn __traverse__(_slf: PyRef, _visit: PyVisit) -> Result<(), PyTraverseError> { Ok(()) } } @@ -17,7 +17,7 @@ struct TraverseTriesToTakePyRefMut {} #[pymethods] impl TraverseTriesToTakePyRefMut { - fn __traverse__(slf: PyRefMut, visit: PyVisit) -> Result<(), PyTraverseError> { + fn __traverse__(_slf: PyRefMut, _visit: PyVisit) -> Result<(), PyTraverseError> { Ok(()) } } @@ -27,7 +27,7 @@ struct TraverseTriesToTakeBound {} #[pymethods] impl TraverseTriesToTakeBound { - fn __traverse__(slf: Bound<'_, Self>, visit: PyVisit) -> Result<(), PyTraverseError> { + fn __traverse__(_slf: Bound<'_, Self>, _visit: PyVisit) -> Result<(), PyTraverseError> { Ok(()) } } @@ -37,7 +37,7 @@ struct TraverseTriesToTakeMutSelf {} #[pymethods] impl TraverseTriesToTakeMutSelf { - fn __traverse__(&mut self, visit: PyVisit) -> Result<(), PyTraverseError> { + fn __traverse__(&mut self, _visit: PyVisit) -> Result<(), PyTraverseError> { Ok(()) } } @@ -47,7 +47,7 @@ struct TraverseTriesToTakeSelf {} #[pymethods] impl TraverseTriesToTakeSelf { - fn __traverse__(&self, visit: PyVisit) -> Result<(), PyTraverseError> { + fn __traverse__(&self, _visit: PyVisit) -> Result<(), PyTraverseError> { Ok(()) } } @@ -57,7 +57,7 @@ struct Class; #[pymethods] impl Class { - fn __traverse__(&self, py: Python<'_>, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + fn __traverse__(&self, _py: Python<'_>, _visit: PyVisit<'_>) -> Result<(), PyTraverseError> { Ok(()) } diff --git a/tests/ui/traverse.stderr b/tests/ui/traverse.stderr index 5b1d1b6b2ec..504a6dfa671 100644 --- a/tests/ui/traverse.stderr +++ b/tests/ui/traverse.stderr @@ -1,29 +1,29 @@ error: __traverse__ may not take a receiver other than `&self`. Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. - --> tests/ui/traverse.rs:10:26 + --> tests/ui/traverse.rs:10:27 | -10 | fn __traverse__(slf: PyRef, visit: PyVisit) -> Result<(), PyTraverseError> { - | ^^^^^ +10 | fn __traverse__(_slf: PyRef, _visit: PyVisit) -> Result<(), PyTraverseError> { + | ^^^^^ error: __traverse__ may not take a receiver other than `&self`. Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. - --> tests/ui/traverse.rs:20:26 + --> tests/ui/traverse.rs:20:27 | -20 | fn __traverse__(slf: PyRefMut, visit: PyVisit) -> Result<(), PyTraverseError> { - | ^^^^^^^^ +20 | fn __traverse__(_slf: PyRefMut, _visit: PyVisit) -> Result<(), PyTraverseError> { + | ^^^^^^^^ error: __traverse__ may not take a receiver other than `&self`. Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. - --> tests/ui/traverse.rs:30:26 + --> tests/ui/traverse.rs:30:27 | -30 | fn __traverse__(slf: Bound<'_, Self>, visit: PyVisit) -> Result<(), PyTraverseError> { - | ^^^^^ +30 | fn __traverse__(_slf: Bound<'_, Self>, _visit: PyVisit) -> Result<(), PyTraverseError> { + | ^^^^^ error: __traverse__ may not take a receiver other than `&self`. Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. --> tests/ui/traverse.rs:40:21 | -40 | fn __traverse__(&mut self, visit: PyVisit) -> Result<(), PyTraverseError> { +40 | fn __traverse__(&mut self, _visit: PyVisit) -> Result<(), PyTraverseError> { | ^ error: __traverse__ may not take `Python`. Usually, an implementation of `__traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError>` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. - --> tests/ui/traverse.rs:60:32 + --> tests/ui/traverse.rs:60:33 | -60 | fn __traverse__(&self, py: Python<'_>, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { - | ^^^^^^^^^^ +60 | fn __traverse__(&self, _py: Python<'_>, _visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + | ^^^^^^^^^^ From 7cbb85476c920cb80d2906b08b2a25d96fe4b5c8 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 3 May 2024 12:17:14 +0200 Subject: [PATCH 313/349] fix `check-guide` ci workflow (#4146) --- noxfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index c8d2ead63d3..84676b1ff0c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -415,10 +415,11 @@ def check_guide(session: nox.Session): _run( session, "lychee", - PYO3_DOCS_TARGET, + str(PYO3_DOCS_TARGET), f"--remap=https://pyo3.rs/main/ file://{PYO3_GUIDE_TARGET}/", f"--remap=https://pyo3.rs/latest/ file://{PYO3_GUIDE_TARGET}/", f"--exclude=file://{PYO3_DOCS_TARGET}", + "--exclude=http://www.adobe.com/", *session.posargs, ) From 93cfb51ebbdfb598e4a08539155ac4d9e9194a28 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 3 May 2024 12:02:19 -0400 Subject: [PATCH 314/349] feature gate deprecated APIs for `PyMemoryView` (#4152) --- src/types/memoryview.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/types/memoryview.rs b/src/types/memoryview.rs index c04a98e7886..ffc1c81efa0 100644 --- a/src/types/memoryview.rs +++ b/src/types/memoryview.rs @@ -11,12 +11,10 @@ pyobject_native_type_core!(PyMemoryView, pyobject_native_static_type_object!(ffi impl PyMemoryView { /// Deprecated form of [`PyMemoryView::from_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyMemoryView::from` will be replaced by `PyMemoryView::from_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyMemoryView::from` will be replaced by `PyMemoryView::from_bound` in a future PyO3 version" )] pub fn from(src: &PyAny) -> PyResult<&PyMemoryView> { PyMemoryView::from_bound(&src.as_borrowed()).map(Bound::into_gil_ref) From f3ab62cb7ec17752e776334957b00894ede97d37 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 3 May 2024 13:10:49 -0400 Subject: [PATCH 315/349] feature gate deprecated APIs for `PyModule` (#4151) --- .../python-from-rust/calling-existing-code.md | 12 +++---- src/marker.rs | 2 +- src/types/module.rs | 31 +++++++------------ 3 files changed, 19 insertions(+), 26 deletions(-) diff --git a/guide/src/python-from-rust/calling-existing-code.md b/guide/src/python-from-rust/calling-existing-code.md index 53051d4ce51..572f4b4414f 100644 --- a/guide/src/python-from-rust/calling-existing-code.md +++ b/guide/src/python-from-rust/calling-existing-code.md @@ -2,9 +2,9 @@ If you already have some existing Python code that you need to execute from Rust, the following FAQs can help you select the right PyO3 functionality for your situation: -## Want to access Python APIs? Then use `PyModule::import`. +## Want to access Python APIs? Then use `PyModule::import_bound`. -[`Pymodule::import`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.import) can +[`PyModule::import_bound`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.import_bound) can be used to get handle to a Python module from Rust. You can use this to import and use any Python module available in your environment. @@ -95,9 +95,9 @@ assert userdata.as_tuple() == userdata_as_tuple # } ``` -## You have a Python file or code snippet? Then use `PyModule::from_code`. +## You have a Python file or code snippet? Then use `PyModule::from_code_bound`. -[`PyModule::from_code`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.from_code) +[`PyModule::from_code_bound`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.from_code_bound) can be used to generate a Python module which can then be used just as if it was imported with `PyModule::import`. @@ -171,7 +171,7 @@ fn main() -> PyResult<()> { ``` If `append_to_inittab` cannot be used due to constraints in the program, -an alternative is to create a module using [`PyModule::new`] +an alternative is to create a module using [`PyModule::new_bound`] and insert it manually into `sys.modules`: ```rust @@ -394,4 +394,4 @@ Python::with_gil(|py| -> PyResult<()> { ``` -[`PyModule::new`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.new +[`PyModule::new_bound`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.new_bound diff --git a/src/marker.rs b/src/marker.rs index 29aae69d4f2..c975a4a338e 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -352,7 +352,7 @@ pub use nightly::Ungil; /// # Releasing and freeing memory /// /// The [`Python`] type can be used to create references to variables owned by the Python -/// interpreter, using functions such as [`Python::eval`] and [`PyModule::import`]. These +/// interpreter, using functions such as [`Python::eval`] and `PyModule::import`. These /// references are tied to a [`GILPool`] whose references are not cleared until it is dropped. /// This can cause apparent "memory leaks" if it is kept around for a long time. /// diff --git a/src/types/module.rs b/src/types/module.rs index 11afcf76c83..c8b2cf04551 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -27,12 +27,10 @@ pyobject_native_type_core!(PyModule, pyobject_native_static_type_object!(ffi::Py impl PyModule { /// Deprecated form of [`PyModule::new_bound`]. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyModule::new` will be replaced by `PyModule::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyModule::new` will be replaced by `PyModule::new_bound` in a future PyO3 version" )] pub fn new<'py>(py: Python<'py>, name: &str) -> PyResult<&'py PyModule> { Self::new_bound(py, name).map(Bound::into_gil_ref) @@ -66,12 +64,10 @@ impl PyModule { /// Deprecated form of [`PyModule::import_bound`]. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyModule::import` will be replaced by `PyModule::import_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyModule::import` will be replaced by `PyModule::import_bound` in a future PyO3 version" )] pub fn import(py: Python<'_>, name: N) -> PyResult<&PyModule> where @@ -112,12 +108,10 @@ impl PyModule { /// Deprecated form of [`PyModule::from_code_bound`]. #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyModule::from_code` will be replaced by `PyModule::from_code_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyModule::from_code` will be replaced by `PyModule::from_code_bound` in a future PyO3 version" )] pub fn from_code<'py>( py: Python<'py>, @@ -722,7 +716,6 @@ fn __name__(py: Python<'_>) -> &Bound<'_, PyString> { } #[cfg(test)] -#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))] mod tests { use crate::{ types::{module::PyModuleMethods, string::PyStringMethods, PyModule}, From c08f6c77a60be2c5cdfc5216aa8869ff48738d51 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 3 May 2024 20:15:25 +0200 Subject: [PATCH 316/349] feature gate deprecated APIs for `marshal` (#4149) --- src/marshal.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/marshal.rs b/src/marshal.rs index 9526813976b..3978f4873e1 100644 --- a/src/marshal.rs +++ b/src/marshal.rs @@ -13,12 +13,10 @@ use std::os::raw::c_int; pub const VERSION: i32 = 4; /// Deprecated form of [`dumps_bound`] -#[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`dumps` will be replaced by `dumps_bound` in a future PyO3 version" - ) +#[cfg(feature = "gil-refs")] +#[deprecated( + since = "0.21.0", + note = "`dumps` will be replaced by `dumps_bound` in a future PyO3 version" )] pub fn dumps<'py>( py: Python<'py>, @@ -61,12 +59,10 @@ pub fn dumps_bound<'py>( } /// Deprecated form of [`loads_bound`] -#[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`loads` will be replaced by `loads_bound` in a future PyO3 version" - ) +#[cfg(feature = "gil-refs")] +#[deprecated( + since = "0.21.0", + note = "`loads` will be replaced by `loads_bound` in a future PyO3 version" )] pub fn loads<'py, B>(py: Python<'py>, data: &B) -> PyResult<&'py PyAny> where From d1a0c7278f16925c9075101c763d73bbc5f673c5 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 3 May 2024 15:50:38 -0400 Subject: [PATCH 317/349] feature gate deprecated APIs for `PyCFunction` (#4154) --- src/types/function.rs | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/types/function.rs b/src/types/function.rs index f5305e31886..09c5004b77b 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -1,14 +1,17 @@ +#[cfg(feature = "gil-refs")] use crate::derive_utils::PyFunctionArguments; use crate::ffi_ptr_ext::FfiPtrExt; use crate::py_result_ext::PyResultExt; use crate::types::capsule::PyCapsuleMethods; use crate::types::module::PyModuleMethods; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; use crate::{ ffi, impl_::pymethods::{self, PyMethodDef, PyMethodDefDestructor}, types::{PyCapsule, PyDict, PyModule, PyString, PyTuple}, }; -use crate::{Bound, IntoPy, Py, PyAny, PyNativeType, PyResult, Python}; +use crate::{Bound, IntoPy, Py, PyAny, PyResult, Python}; use std::cell::UnsafeCell; use std::ffi::CStr; @@ -20,12 +23,10 @@ pyobject_native_type_core!(PyCFunction, pyobject_native_static_type_object!(ffi: impl PyCFunction { /// Deprecated form of [`PyCFunction::new_with_keywords_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyCFunction::new_with_keywords` will be replaced by `PyCFunction::new_with_keywords_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyCFunction::new_with_keywords` will be replaced by `PyCFunction::new_with_keywords_bound` in a future PyO3 version" )] pub fn new_with_keywords<'a>( fun: ffi::PyCFunctionWithKeywords, @@ -66,12 +67,10 @@ impl PyCFunction { } /// Deprecated form of [`PyCFunction::new`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyCFunction::new` will be replaced by `PyCFunction::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyCFunction::new` will be replaced by `PyCFunction::new_bound` in a future PyO3 version" )] pub fn new<'a>( fun: ffi::PyCFunction, @@ -104,12 +103,10 @@ impl PyCFunction { } /// Deprecated form of [`PyCFunction::new_closure`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyCFunction::new_closure` will be replaced by `PyCFunction::new_closure_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyCFunction::new_closure` will be replaced by `PyCFunction::new_closure_bound` in a future PyO3 version" )] pub fn new_closure<'a, F, R>( py: Python<'a>, From c10c7429d8b5f16a6c28650b21e214f35cbbfe48 Mon Sep 17 00:00:00 2001 From: Heran Lin Date: Sat, 4 May 2024 15:32:27 +0800 Subject: [PATCH 318/349] docs: Remove out-dated information for pyenv (#4138) --- guide/src/building-and-distribution.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index b7aa2d2abc2..33280a5a180 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -232,7 +232,7 @@ not work when compiling for `abi3`. These are: If you want to embed the Python interpreter inside a Rust program, there are two modes in which this can be done: dynamically and statically. We'll cover each of these modes in the following sections. Each of them affect how you must distribute your program. Instead of learning how to do this yourself, you might want to consider using a project like [PyOxidizer] to ship your application and all of its dependencies in a single file. -PyO3 automatically switches between the two linking modes depending on whether the Python distribution you have configured PyO3 to use ([see above](#configuring-the-python-version)) contains a shared library or a static library. The static library is most often seen in Python distributions compiled from source without the `--enable-shared` configuration option. For example, this is the default for `pyenv` on macOS. +PyO3 automatically switches between the two linking modes depending on whether the Python distribution you have configured PyO3 to use ([see above](#configuring-the-python-version)) contains a shared library or a static library. The static library is most often seen in Python distributions compiled from source without the `--enable-shared` configuration option. ### Dynamically embedding the Python interpreter From e835ff0ec30e0e391fdfcbef017241004ed9d896 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 4 May 2024 09:39:40 +0200 Subject: [PATCH 319/349] handle `#[pyo3(from_py_with = ...)]` on dunder (`__magic__`) methods (#4117) * handle `#[pyo3(from_py_with = ...)]` on dunder (__magic__) methods * add newsfragment --- newsfragments/4117.fixed.md | 1 + pyo3-macros-backend/src/params.rs | 34 +++++++++++++++++--- pyo3-macros-backend/src/pymethod.rs | 42 ++++++++++++++---------- tests/test_class_basics.rs | 11 +++++++ tests/ui/deprecations.rs | 8 +++++ tests/ui/deprecations.stderr | 50 ++++++++++++++++------------- 6 files changed, 103 insertions(+), 43 deletions(-) create mode 100644 newsfragments/4117.fixed.md diff --git a/newsfragments/4117.fixed.md b/newsfragments/4117.fixed.md new file mode 100644 index 00000000000..c3bb4c144b6 --- /dev/null +++ b/newsfragments/4117.fixed.md @@ -0,0 +1 @@ +Correctly handle `#[pyo3(from_py_with = ...)]` attribute on dunder (`__magic__`) method arguments instead of silently ignoring it. diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index d9f77fa07bc..cab9d2a7d29 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -10,7 +10,7 @@ use syn::spanned::Spanned; pub struct Holders { holders: Vec, - gil_refs_checkers: Vec, + gil_refs_checkers: Vec, } impl Holders { @@ -32,14 +32,28 @@ impl Holders { &format!("gil_refs_checker_{}", self.gil_refs_checkers.len()), span, ); - self.gil_refs_checkers.push(gil_refs_checker.clone()); + self.gil_refs_checkers + .push(GilRefChecker::FunctionArg(gil_refs_checker.clone())); + gil_refs_checker + } + + pub fn push_from_py_with_checker(&mut self, span: Span) -> syn::Ident { + let gil_refs_checker = syn::Ident::new( + &format!("gil_refs_checker_{}", self.gil_refs_checkers.len()), + span, + ); + self.gil_refs_checkers + .push(GilRefChecker::FromPyWith(gil_refs_checker.clone())); gil_refs_checker } pub fn init_holders(&self, ctx: &Ctx) -> TokenStream { let Ctx { pyo3_path } = ctx; let holders = &self.holders; - let gil_refs_checkers = &self.gil_refs_checkers; + let gil_refs_checkers = self.gil_refs_checkers.iter().map(|checker| match checker { + GilRefChecker::FunctionArg(ident) => ident, + GilRefChecker::FromPyWith(ident) => ident, + }); quote! { #[allow(clippy::let_unit_value)] #(let mut #holders = #pyo3_path::impl_::extract_argument::FunctionArgumentHolder::INIT;)* @@ -50,11 +64,23 @@ impl Holders { pub fn check_gil_refs(&self) -> TokenStream { self.gil_refs_checkers .iter() - .map(|e| quote_spanned! { e.span() => #e.function_arg(); }) + .map(|checker| match checker { + GilRefChecker::FunctionArg(ident) => { + quote_spanned! { ident.span() => #ident.function_arg(); } + } + GilRefChecker::FromPyWith(ident) => { + quote_spanned! { ident.span() => #ident.from_py_with_arg(); } + } + }) .collect() } } +enum GilRefChecker { + FunctionArg(syn::Ident), + FromPyWith(syn::Ident), +} + /// Return true if the argument list is simply (*args, **kwds). pub fn is_forwarded_args(signature: &FunctionSignature<'_>) -> bool { matches!( diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index aac804316f8..1ef137cfcc8 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -1053,20 +1053,18 @@ impl Ty { ctx: &Ctx, ) -> TokenStream { let Ctx { pyo3_path } = ctx; - let name_str = arg.name().unraw().to_string(); match self { Ty::Object => extract_object( extract_error_mode, holders, - &name_str, + arg, quote! { #ident }, - arg.ty().span(), ctx ), Ty::MaybeNullObject => extract_object( extract_error_mode, holders, - &name_str, + arg, quote! { if #ident.is_null() { #pyo3_path::ffi::Py_None() @@ -1074,23 +1072,20 @@ impl Ty { #ident } }, - arg.ty().span(), ctx ), Ty::NonNullObject => extract_object( extract_error_mode, holders, - &name_str, + arg, quote! { #ident.as_ptr() }, - arg.ty().span(), ctx ), Ty::IPowModulo => extract_object( extract_error_mode, holders, - &name_str, + arg, quote! { #ident.as_ptr() }, - arg.ty().span(), ctx ), Ty::CompareOp => extract_error_mode.handle_error( @@ -1118,24 +1113,37 @@ impl Ty { fn extract_object( extract_error_mode: ExtractErrorMode, holders: &mut Holders, - name: &str, + arg: &FnArg<'_>, source_ptr: TokenStream, - span: Span, ctx: &Ctx, ) -> TokenStream { let Ctx { pyo3_path } = ctx; - let holder = holders.push_holder(Span::call_site()); - let gil_refs_checker = holders.push_gil_refs_checker(span); - let extracted = extract_error_mode.handle_error( + let gil_refs_checker = holders.push_gil_refs_checker(arg.ty().span()); + let name = arg.name().unraw().to_string(); + + let extract = if let Some(from_py_with) = + arg.from_py_with().map(|from_py_with| &from_py_with.value) + { + let from_py_with_checker = holders.push_from_py_with_checker(from_py_with.span()); + quote! { + #pyo3_path::impl_::extract_argument::from_py_with( + #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr).0, + #name, + #pyo3_path::impl_::deprecations::inspect_fn(#from_py_with, &#from_py_with_checker) as fn(_) -> _, + ) + } + } else { + let holder = holders.push_holder(Span::call_site()); quote! { #pyo3_path::impl_::extract_argument::extract_argument( #pyo3_path::impl_::pymethods::BoundRef::ref_from_ptr(py, &#source_ptr).0, &mut #holder, #name ) - }, - ctx, - ); + } + }; + + let extracted = extract_error_mode.handle_error(extract, ctx); quote! { #pyo3_path::impl_::deprecations::inspect_type(#extracted, &#gil_refs_checker) } diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 8c6a1c04915..8ff61bd2d6b 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -290,6 +290,10 @@ fn get_length(obj: &Bound<'_, PyAny>) -> PyResult { Ok(length) } +fn is_even(obj: &Bound<'_, PyAny>) -> PyResult { + obj.extract::().map(|i| i % 2 == 0) +} + #[pyclass] struct ClassWithFromPyWithMethods {} @@ -319,6 +323,10 @@ impl ClassWithFromPyWithMethods { fn staticmethod(#[pyo3(from_py_with = "get_length")] argument: usize) -> usize { argument } + + fn __contains__(&self, #[pyo3(from_py_with = "is_even")] obj: bool) -> bool { + obj + } } #[test] @@ -339,6 +347,9 @@ fn test_pymethods_from_py_with() { if has_gil_refs: assert instance.classmethod_gil_ref(arg) == 2 assert instance.staticmethod(arg) == 2 + + assert 42 in instance + assert 73 not in instance "# ); }) diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index dcc9b7b1d84..96f652d9679 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -38,6 +38,14 @@ impl MyClass { #[setter] fn set_bar_bound(&self, _value: &Bound<'_, PyAny>) {} + + fn __eq__(&self, #[pyo3(from_py_with = "extract_gil_ref")] _other: i32) -> bool { + true + } + + fn __contains__(&self, #[pyo3(from_py_with = "extract_bound")] _value: i32) -> bool { + true + } } fn main() {} diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index e692702f23e..2b75ee23e10 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -16,6 +16,12 @@ error: use of deprecated struct `pyo3::PyCell`: `PyCell` was merged into `Bound` 23 | fn method_gil_ref(_slf: &PyCell) {} | ^^^^^^ +error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor + --> tests/ui/deprecations.rs:42:44 + | +42 | fn __eq__(&self, #[pyo3(from_py_with = "extract_gil_ref")] _other: i32) -> bool { + | ^^^^^^^^^^^^^^^^^ + error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument --> tests/ui/deprecations.rs:18:33 | @@ -47,69 +53,69 @@ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg` | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:53:43 + --> tests/ui/deprecations.rs:61:43 | -53 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { +61 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:63:19 + --> tests/ui/deprecations.rs:71:19 | -63 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { +71 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:69:57 + --> tests/ui/deprecations.rs:77:57 | -69 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +77 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:102:27 + --> tests/ui/deprecations.rs:110:27 | -102 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, +110 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:108:29 + --> tests/ui/deprecations.rs:116:29 | -108 | fn pyfunction_gil_ref(_any: &PyAny) {} +116 | fn pyfunction_gil_ref(_any: &PyAny) {} | ^ error: use of deprecated method `pyo3::deprecations::OptionGilRefs::>::function_arg`: use `Option<&Bound<'_, T>>` instead for this function argument - --> tests/ui/deprecations.rs:111:36 + --> tests/ui/deprecations.rs:119:36 | -111 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} +119 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} | ^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:118:27 + --> tests/ui/deprecations.rs:126:27 | -118 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] +126 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:128:27 + --> tests/ui/deprecations.rs:136:27 | -128 | #[pyo3(from_py_with = "PyAny::len")] usize, +136 | #[pyo3(from_py_with = "PyAny::len")] usize, | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:134:31 + --> tests/ui/deprecations.rs:142:31 | -134 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), +142 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:141:27 + --> tests/ui/deprecations.rs:149:27 | -141 | #[pyo3(from_py_with = "extract_gil_ref")] +149 | #[pyo3(from_py_with = "extract_gil_ref")] | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::>::is_python`: use `wrap_pyfunction_bound!` instead - --> tests/ui/deprecations.rs:154:13 + --> tests/ui/deprecations.rs:162:13 | -154 | let _ = wrap_pyfunction!(double, py); +162 | let _ = wrap_pyfunction!(double, py); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From ef13bc66e9ffb4493085082bde0222eb00ba8b33 Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Sat, 4 May 2024 10:48:15 +0300 Subject: [PATCH 320/349] Add `pyreqwest_impersonate` to example projects (#4123) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 72653e8fc77..5e34f35489d 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,7 @@ about this topic. - [pyheck](https://github.com/kevinheavey/pyheck) _Fast case conversion library, built by wrapping [heck](https://github.com/withoutboats/heck)._ - Quite easy to follow as there's not much code. - [pyre](https://github.com/Project-Dream-Weaver/pyre-http) _Fast Python HTTP server written in Rust._ +- [pyreqwest_impersonate](https://github.com/deedy5/pyreqwest_impersonate) _The fastest python HTTP client that can impersonate web browsers by mimicking their headers and TLS/JA3/JA4/HTTP2 fingerprints._ - [ril-py](https://github.com/Cryptex-github/ril-py) _A performant and high-level image processing library for Python written in Rust._ - [river](https://github.com/online-ml/river) _Online machine learning in python, the computationally heavy statistics algorithms are implemented in Rust._ - [rust-python-coverage](https://github.com/cjermain/rust-python-coverage) _Example PyO3 project with automated test coverage for Rust and Python._ From 7263fa92ef5da663956d7d2b85188b6877b284b2 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 4 May 2024 19:45:27 +0200 Subject: [PATCH 321/349] feature gate deprecated APIs for `PyBool` (#4159) --- src/types/boolobject.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 52e4c886aab..43184e31565 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -16,12 +16,10 @@ pyobject_native_type!(PyBool, ffi::PyObject, pyobject_native_static_type_object! impl PyBool { /// Deprecated form of [`PyBool::new_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyBool::new` will be replaced by `PyBool::new_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`PyBool::new` will be replaced by `PyBool::new_bound` in a future PyO3 version" )] #[inline] pub fn new(py: Python<'_>, val: bool) -> &PyBool { From 72be1cddba022d9fbd8ce3275455966095d5712d Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 8 May 2024 07:46:00 +0200 Subject: [PATCH 322/349] emit `cargo:rustc-check-cfg=CHECK_CFG` for `pyo3`s config names (#4163) --- build.rs | 1 + guide/src/migration.md | 4 ++-- pyo3-build-config/src/impl_.rs | 4 ++-- pyo3-build-config/src/lib.rs | 20 ++++++++++++++++++++ pyo3-ffi/build.rs | 1 + src/marker.rs | 2 +- src/tests/hygiene/pyclass.rs | 10 +++++----- 7 files changed, 32 insertions(+), 10 deletions(-) diff --git a/build.rs b/build.rs index 28592004df2..5d638291f3b 100644 --- a/build.rs +++ b/build.rs @@ -46,6 +46,7 @@ fn configure_pyo3() -> Result<()> { } fn main() { + pyo3_build_config::print_expected_cfgs(); if let Err(e) = configure_pyo3() { eprintln!("error: {}", e.report()); std::process::exit(1) diff --git a/guide/src/migration.md b/guide/src/migration.md index ad39adfa0e8..0a048bf02bc 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -339,12 +339,12 @@ To make PyO3's core functionality continue to work while the GIL Refs API is in PyO3 0.21 has introduced the [`PyBackedStr`]({{#PYO3_DOCS_URL}}/pyo3/pybacked/struct.PyBackedStr.html) and [`PyBackedBytes`]({{#PYO3_DOCS_URL}}/pyo3/pybacked/struct.PyBackedBytes.html) types to help with this case. The easiest way to avoid lifetime challenges from extracting `&str` is to use these. For more complex types like `Vec<&str>`, is now impossible to extract directly from a Python object and `Vec` is the recommended upgrade path. -A key thing to note here is because extracting to these types now ties them to the input lifetime, some extremely common patterns may need to be split into multiple Rust lines. For example, the following snippet of calling `.extract::<&str>()` directly on the result of `.getattr()` needs to be adjusted when deactivating the `gil-refs-migration` feature. +A key thing to note here is because extracting to these types now ties them to the input lifetime, some extremely common patterns may need to be split into multiple Rust lines. For example, the following snippet of calling `.extract::<&str>()` directly on the result of `.getattr()` needs to be adjusted when deactivating the `gil-refs` feature. Before: ```rust -# #[cfg(feature = "gil-refs-migration")] { +# #[cfg(feature = "gil-refs")] { # use pyo3::prelude::*; # use pyo3::types::{PyList, PyType}; # fn example<'py>(py: Python<'py>) -> PyResult<()> { diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 2ee68503faa..35c300da190 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -30,7 +30,7 @@ use crate::{ }; /// Minimum Python version PyO3 supports. -const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; +pub(crate) const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; /// GraalPy may implement the same CPython version over multiple releases. const MINIMUM_SUPPORTED_VERSION_GRAALPY: PythonVersion = PythonVersion { @@ -39,7 +39,7 @@ const MINIMUM_SUPPORTED_VERSION_GRAALPY: PythonVersion = PythonVersion { }; /// Maximum Python version that can be used as minimum required Python version with abi3. -const ABI3_MAX_MINOR: u8 = 12; +pub(crate) const ABI3_MAX_MINOR: u8 = 12; /// Gets an environment variable owned by cargo. /// diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 1aa15d7d62a..24d3ae28124 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -43,6 +43,7 @@ use target_lexicon::OperatingSystem; /// For examples of how to use these attributes, [see PyO3's guide](https://pyo3.rs/latest/building-and-distribution/multiple_python_versions.html). #[cfg(feature = "resolve-config")] pub fn use_pyo3_cfgs() { + print_expected_cfgs(); for cargo_command in get().build_script_outputs() { println!("{}", cargo_command) } @@ -153,6 +154,25 @@ pub fn print_feature_cfgs() { } } +/// Registers `pyo3`s config names as reachable cfg expressions +/// +/// - +/// - +#[doc(hidden)] +pub fn print_expected_cfgs() { + println!("cargo:rustc-check-cfg=cfg(Py_LIMITED_API)"); + println!("cargo:rustc-check-cfg=cfg(PyPy)"); + println!("cargo:rustc-check-cfg=cfg(GraalPy)"); + println!("cargo:rustc-check-cfg=cfg(py_sys_config, values(\"Py_DEBUG\", \"Py_REF_DEBUG\", \"Py_TRACE_REFS\", \"COUNT_ALLOCS\"))"); + println!("cargo:rustc-check-cfg=cfg(invalid_from_utf8_lint)"); + + // allow `Py_3_*` cfgs from the minimum supported version up to the + // maximum minor version (+1 for development for the next) + for i in impl_::MINIMUM_SUPPORTED_VERSION.minor..=impl_::ABI3_MAX_MINOR + 1 { + println!("cargo:rustc-check-cfg=cfg(Py_3_{i})"); + } +} + /// Private exports used in PyO3's build.rs /// /// Please don't use these - they could change at any time. diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index 405da889708..23f03f1a636 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -205,6 +205,7 @@ fn print_config_and_exit(config: &InterpreterConfig) { } fn main() { + pyo3_build_config::print_expected_cfgs(); if let Err(e) = configure_pyo3() { eprintln!("error: {}", e.report()); std::process::exit(1) diff --git a/src/marker.rs b/src/marker.rs index c975a4a338e..2230d776236 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -96,7 +96,7 @@ //! enabled, `Ungil` is defined as the following: //! //! ```rust -//! # #[cfg(FALSE)] +//! # #[cfg(any())] //! # { //! #![feature(auto_traits, negative_impls)] //! diff --git a/src/tests/hygiene/pyclass.rs b/src/tests/hygiene/pyclass.rs index 4d07009cad6..0bdb280d066 100644 --- a/src/tests/hygiene/pyclass.rs +++ b/src/tests/hygiene/pyclass.rs @@ -39,11 +39,11 @@ pub enum Enum { #[pyo3(crate = "crate")] pub struct Foo3 { #[pyo3(get, set)] - #[cfg(FALSE)] + #[cfg(any())] field: i32, #[pyo3(get, set)] - #[cfg(not(FALSE))] + #[cfg(not(any()))] field: u32, } @@ -51,11 +51,11 @@ pub struct Foo3 { #[pyo3(crate = "crate")] pub struct Foo4 { #[pyo3(get, set)] - #[cfg(FALSE)] - #[cfg(not(FALSE))] + #[cfg(any())] + #[cfg(not(any()))] field: i32, #[pyo3(get, set)] - #[cfg(not(FALSE))] + #[cfg(not(any()))] field: u32, } From d803f7f8df8bc9efaed086339b7ac67fc070aa07 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Wed, 8 May 2024 10:04:42 +0200 Subject: [PATCH 323/349] store the `FnArg` ident as a `Cow` instead of a reference (#4157) This allow also storing idents that were generated as part of the macro instead of only ones the were present in the source code. This is needed for example in complex enum tuple variants. --- pyo3-macros-backend/src/method.rs | 21 ++++++++++++++------- pyo3-macros-backend/src/pyclass.rs | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 982cf62946e..cb86e8ec606 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::fmt::Display; use proc_macro2::{Span, TokenStream}; @@ -18,7 +19,7 @@ use crate::{ #[derive(Clone, Debug)] pub struct RegularArg<'a> { - pub name: &'a syn::Ident, + pub name: Cow<'a, syn::Ident>, pub ty: &'a syn::Type, pub from_py_with: Option, pub default_value: Option, @@ -28,14 +29,14 @@ pub struct RegularArg<'a> { /// Pythons *args argument #[derive(Clone, Debug)] pub struct VarargsArg<'a> { - pub name: &'a syn::Ident, + pub name: Cow<'a, syn::Ident>, pub ty: &'a syn::Type, } /// Pythons **kwarg argument #[derive(Clone, Debug)] pub struct KwargsArg<'a> { - pub name: &'a syn::Ident, + pub name: Cow<'a, syn::Ident>, pub ty: &'a syn::Type, } @@ -61,7 +62,7 @@ pub enum FnArg<'a> { } impl<'a> FnArg<'a> { - pub fn name(&self) -> &'a syn::Ident { + pub fn name(&self) -> &syn::Ident { match self { FnArg::Regular(RegularArg { name, .. }) => name, FnArg::VarArgs(VarargsArg { name, .. }) => name, @@ -98,7 +99,10 @@ impl<'a> FnArg<'a> { .. }) = self { - *self = Self::VarArgs(VarargsArg { name, ty }); + *self = Self::VarArgs(VarargsArg { + name: name.clone(), + ty, + }); Ok(self) } else { bail_spanned!(self.name().span() => "args cannot be optional") @@ -113,7 +117,10 @@ impl<'a> FnArg<'a> { .. }) = self { - *self = Self::KwArgs(KwargsArg { name, ty }); + *self = Self::KwArgs(KwargsArg { + name: name.clone(), + ty, + }); Ok(self) } else { bail_spanned!(self.name().span() => "kwargs must be Option<_>") @@ -159,7 +166,7 @@ impl<'a> FnArg<'a> { } Ok(Self::Regular(RegularArg { - name: ident, + name: Cow::Borrowed(ident), ty: &cap.ty, from_py_with, default_value: None, diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index d9c84655b42..f8bfa164d7d 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1153,7 +1153,7 @@ fn complex_enum_struct_variant_new<'a>( for field in &variant.fields { args.push(FnArg::Regular(RegularArg { - name: field.ident, + name: Cow::Borrowed(field.ident), ty: field.ty, from_py_with: None, default_value: None, From 635cb8075cfc3b52e2b403eb5b12db9b0d81c88c Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Thu, 9 May 2024 09:58:44 +0200 Subject: [PATCH 324/349] feature gate APIs using `into_gil_ref` (Part 1) (#4160) --- src/conversions/std/array.rs | 20 ++++---- src/instance.rs | 1 + src/internal_tricks.rs | 5 ++ src/macros.rs | 11 ++-- src/tests/common.rs | 10 ++-- src/tests/hygiene/pymodule.rs | 1 + src/types/bytearray.rs | 1 + src/types/complex.rs | 3 ++ src/types/dict.rs | 55 +++++++++++--------- src/types/frozenset.rs | 53 ++++++++++--------- src/types/iterator.rs | 5 +- src/types/list.rs | 66 +++++++++++++----------- src/types/mapping.rs | 28 +++++++---- src/types/memoryview.rs | 5 +- src/types/mod.rs | 16 ++++-- src/types/module.rs | 95 ++++++++++++++++++----------------- src/types/sequence.rs | 31 ++++++++---- src/types/set.rs | 52 ++++++++++--------- src/types/tuple.rs | 70 +++++++++++++++----------- tests/test_no_imports.rs | 3 +- tests/ui/deprecations.rs | 10 ++-- tests/ui/deprecations.stderr | 48 +++++++++--------- 22 files changed, 336 insertions(+), 253 deletions(-) diff --git a/src/conversions/std/array.rs b/src/conversions/std/array.rs index bf21244e3b2..14ccfd35413 100644 --- a/src/conversions/std/array.rs +++ b/src/conversions/std/array.rs @@ -186,11 +186,11 @@ mod tests { Python::with_gil(|py| { let array: [f32; 4] = [0.0, -16.0, 16.0, 42.0]; let pyobject = array.to_object(py); - let pylist: &PyList = pyobject.extract(py).unwrap(); - assert_eq!(pylist[0].extract::().unwrap(), 0.0); - assert_eq!(pylist[1].extract::().unwrap(), -16.0); - assert_eq!(pylist[2].extract::().unwrap(), 16.0); - assert_eq!(pylist[3].extract::().unwrap(), 42.0); + let pylist = pyobject.downcast_bound::(py).unwrap(); + assert_eq!(pylist.get_item(0).unwrap().extract::().unwrap(), 0.0); + assert_eq!(pylist.get_item(1).unwrap().extract::().unwrap(), -16.0); + assert_eq!(pylist.get_item(2).unwrap().extract::().unwrap(), 16.0); + assert_eq!(pylist.get_item(3).unwrap().extract::().unwrap(), 42.0); }); } @@ -213,11 +213,11 @@ mod tests { Python::with_gil(|py| { let array: [f32; 4] = [0.0, -16.0, 16.0, 42.0]; let pyobject = array.into_py(py); - let pylist: &PyList = pyobject.extract(py).unwrap(); - assert_eq!(pylist[0].extract::().unwrap(), 0.0); - assert_eq!(pylist[1].extract::().unwrap(), -16.0); - assert_eq!(pylist[2].extract::().unwrap(), 16.0); - assert_eq!(pylist[3].extract::().unwrap(), 42.0); + let pylist = pyobject.downcast_bound::(py).unwrap(); + assert_eq!(pylist.get_item(0).unwrap().extract::().unwrap(), 0.0); + assert_eq!(pylist.get_item(1).unwrap().extract::().unwrap(), -16.0); + assert_eq!(pylist.get_item(2).unwrap().extract::().unwrap(), 16.0); + assert_eq!(pylist.get_item(3).unwrap().extract::().unwrap(), 42.0); }); } diff --git a/src/instance.rs b/src/instance.rs index 11d45df6f78..81ceaa95546 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -667,6 +667,7 @@ impl<'py, T> Borrowed<'py, 'py, T> where T: HasPyGilRef, { + #[cfg(feature = "gil-refs")] pub(crate) fn into_gil_ref(self) -> &'py T::AsRefTarget { // Safety: self is a borrow over `'py`. #[allow(deprecated)] diff --git a/src/internal_tricks.rs b/src/internal_tricks.rs index 551ddf5a910..75f23edbbd8 100644 --- a/src/internal_tricks.rs +++ b/src/internal_tricks.rs @@ -46,6 +46,7 @@ pub(crate) fn get_ssize_index(index: usize) -> Py_ssize_t { } /// Implementations used for slice indexing PySequence, PyTuple, and PyList +#[cfg(feature = "gil-refs")] macro_rules! index_impls { ( $ty:ty, @@ -154,6 +155,7 @@ macro_rules! index_impls { #[inline(never)] #[cold] #[track_caller] +#[cfg(feature = "gil-refs")] pub(crate) fn index_len_fail(index: usize, ty_name: &str, len: usize) -> ! { panic!( "index {} out of range for {} of length {}", @@ -164,6 +166,7 @@ pub(crate) fn index_len_fail(index: usize, ty_name: &str, len: usize) -> ! { #[inline(never)] #[cold] #[track_caller] +#[cfg(feature = "gil-refs")] pub(crate) fn slice_start_index_len_fail(index: usize, ty_name: &str, len: usize) -> ! { panic!( "range start index {} out of range for {} of length {}", @@ -174,6 +177,7 @@ pub(crate) fn slice_start_index_len_fail(index: usize, ty_name: &str, len: usize #[inline(never)] #[cold] #[track_caller] +#[cfg(feature = "gil-refs")] pub(crate) fn slice_end_index_len_fail(index: usize, ty_name: &str, len: usize) -> ! { panic!( "range end index {} out of range for {} of length {}", @@ -184,6 +188,7 @@ pub(crate) fn slice_end_index_len_fail(index: usize, ty_name: &str, len: usize) #[inline(never)] #[cold] #[track_caller] +#[cfg(feature = "gil-refs")] pub(crate) fn slice_index_order_fail(index: usize, end: usize) -> ! { panic!("slice index starts at {} but ends at {}", index, end); } diff --git a/src/macros.rs b/src/macros.rs index 0267c2663eb..8bd79408a56 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -120,8 +120,8 @@ macro_rules! py_run_impl { /// Wraps a Rust function annotated with [`#[pyfunction]`](macro@crate::pyfunction). /// -/// This can be used with [`PyModule::add_function`](crate::types::PyModule::add_function) to add free -/// functions to a [`PyModule`](crate::types::PyModule) - see its documentation for more +/// This can be used with [`PyModule::add_function`](crate::types::PyModuleMethods::add_function) to +/// add free functions to a [`PyModule`](crate::types::PyModule) - see its documentation for more /// information. /// /// During the migration from the GIL Ref API to the Bound API, the return type of this macro will @@ -157,8 +157,9 @@ macro_rules! wrap_pyfunction { /// Wraps a Rust function annotated with [`#[pyfunction]`](macro@crate::pyfunction). /// -/// This can be used with [`PyModule::add_function`](crate::types::PyModule::add_function) to add free -/// functions to a [`PyModule`](crate::types::PyModule) - see its documentation for more information. +/// This can be used with [`PyModule::add_function`](crate::types::PyModuleMethods::add_function) to +/// add free functions to a [`PyModule`](crate::types::PyModule) - see its documentation for more +/// information. #[macro_export] macro_rules! wrap_pyfunction_bound { ($function:path) => { @@ -183,7 +184,7 @@ macro_rules! wrap_pyfunction_bound { /// Python module. /// /// Use this together with [`#[pymodule]`](crate::pymodule) and -/// [`PyModule::add_wrapped`](crate::types::PyModule::add_wrapped). +/// [`PyModule::add_wrapped`](crate::types::PyModuleMethods::add_wrapped). #[macro_export] macro_rules! wrap_pymodule { ($module:path) => { diff --git a/src/tests/common.rs b/src/tests/common.rs index 854d73e4d7b..e1f2e7dfc28 100644 --- a/src/tests/common.rs +++ b/src/tests/common.rs @@ -114,15 +114,18 @@ mod inner { } impl<'py> CatchWarnings<'py> { - pub fn enter(py: Python<'py>, f: impl FnOnce(&PyList) -> PyResult) -> PyResult { + pub fn enter( + py: Python<'py>, + f: impl FnOnce(&Bound<'py, PyList>) -> PyResult, + ) -> PyResult { let warnings = py.import_bound("warnings")?; let kwargs = [("record", true)].into_py_dict_bound(py); let catch_warnings = warnings .getattr("catch_warnings")? .call((), Some(&kwargs))?; - let list = catch_warnings.call_method0("__enter__")?.extract()?; + let list = catch_warnings.call_method0("__enter__")?.downcast_into()?; let _guard = Self { catch_warnings }; - f(list) + f(&list) } } @@ -139,6 +142,7 @@ mod inner { macro_rules! assert_warnings { ($py:expr, $body:expr, [$(($category:ty, $message:literal)),+] $(,)? ) => {{ $crate::tests::common::CatchWarnings::enter($py, |w| { + use $crate::types::{PyListMethods, PyStringMethods}; $body; let expected_warnings = [$((<$category as $crate::type_object::PyTypeInfo>::type_object_bound($py), $message)),+]; assert_eq!(w.len(), expected_warnings.len()); diff --git a/src/tests/hygiene/pymodule.rs b/src/tests/hygiene/pymodule.rs index 6229708dd06..32b3632be22 100644 --- a/src/tests/hygiene/pymodule.rs +++ b/src/tests/hygiene/pymodule.rs @@ -14,6 +14,7 @@ fn foo(_py: crate::Python<'_>, _m: &crate::types::PyModule) -> crate::PyResult<( ::std::result::Result::Ok(()) } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] #[crate::pymodule] #[pyo3(crate = "crate")] diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 8bc4bf55d5b..bdf677c9019 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -491,6 +491,7 @@ impl<'a> Borrowed<'a, '_, PyByteArray> { } } +#[cfg(feature = "gil-refs")] impl<'py> TryFrom<&'py PyAny> for &'py PyByteArray { type Error = crate::PyErr; diff --git a/src/types/complex.rs b/src/types/complex.rs index 4a0c3e30732..cd3d5810d04 100644 --- a/src/types/complex.rs +++ b/src/types/complex.rs @@ -59,6 +59,7 @@ mod not_limited_impls { use super::*; use std::ops::{Add, Div, Mul, Neg, Sub}; + #[cfg(feature = "gil-refs")] impl PyComplex { /// Returns `|self|`. pub fn abs(&self) -> c_double { @@ -94,6 +95,7 @@ mod not_limited_impls { } } + #[cfg(feature = "gil-refs")] impl<'py> $trait for &'py PyComplex { type Output = &'py PyComplex; fn $fn(self, other: &'py PyComplex) -> &'py PyComplex { @@ -136,6 +138,7 @@ mod not_limited_impls { bin_ops!(Mul, mul, *, ffi::_Py_c_prod); bin_ops!(Div, div, /, ffi::_Py_c_quot); + #[cfg(feature = "gil-refs")] impl<'py> Neg for &'py PyComplex { type Output = &'py PyComplex; fn neg(self) -> &'py PyComplex { diff --git a/src/types/dict.rs b/src/types/dict.rs index 68cca1cd981..cab6d68124b 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -6,7 +6,9 @@ use crate::instance::{Borrowed, Bound}; use crate::py_result_ext::PyResultExt; use crate::types::any::PyAnyMethods; use crate::types::{PyAny, PyList}; -use crate::{ffi, PyNativeType, Python, ToPyObject}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{ffi, Python, ToPyObject}; /// Represents a Python `dict`. #[repr(transparent)] @@ -56,34 +58,11 @@ pyobject_native_type_core!( ); impl PyDict { - /// Deprecated form of [`new_bound`][PyDict::new_bound]. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyDict::new` will be replaced by `PyDict::new_bound` in a future PyO3 version" - )] - #[inline] - pub fn new(py: Python<'_>) -> &PyDict { - Self::new_bound(py).into_gil_ref() - } - /// Creates a new empty dictionary. pub fn new_bound(py: Python<'_>) -> Bound<'_, PyDict> { unsafe { ffi::PyDict_New().assume_owned(py).downcast_into_unchecked() } } - /// Deprecated form of [`from_sequence_bound`][PyDict::from_sequence_bound]. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyDict::from_sequence` will be replaced by `PyDict::from_sequence_bound` in a future PyO3 version" - )] - #[inline] - #[cfg(not(any(PyPy, GraalPy)))] - pub fn from_sequence(seq: &PyAny) -> PyResult<&PyDict> { - Self::from_sequence_bound(&seq.as_borrowed()).map(Bound::into_gil_ref) - } - /// Creates a new dictionary from the sequence given. /// /// The sequence must consist of `(PyObject, PyObject)`. This is @@ -100,6 +79,30 @@ impl PyDict { })?; Ok(dict) } +} + +#[cfg(feature = "gil-refs")] +impl PyDict { + /// Deprecated form of [`new_bound`][PyDict::new_bound]. + #[deprecated( + since = "0.21.0", + note = "`PyDict::new` will be replaced by `PyDict::new_bound` in a future PyO3 version" + )] + #[inline] + pub fn new(py: Python<'_>) -> &PyDict { + Self::new_bound(py).into_gil_ref() + } + + /// Deprecated form of [`from_sequence_bound`][PyDict::from_sequence_bound]. + #[deprecated( + since = "0.21.0", + note = "`PyDict::from_sequence` will be replaced by `PyDict::from_sequence_bound` in a future PyO3 version" + )] + #[inline] + #[cfg(not(any(PyPy, GraalPy)))] + pub fn from_sequence(seq: &PyAny) -> PyResult<&PyDict> { + Self::from_sequence_bound(&seq.as_borrowed()).map(Bound::into_gil_ref) + } /// Returns a new dictionary that contains the same key-value pairs as self. /// @@ -550,8 +553,10 @@ fn dict_len(dict: &Bound<'_, PyDict>) -> Py_ssize_t { } /// PyO3 implementation of an iterator for a Python `dict` object. +#[cfg(feature = "gil-refs")] pub struct PyDictIterator<'py>(BoundDictIterator<'py>); +#[cfg(feature = "gil-refs")] impl<'py> Iterator for PyDictIterator<'py> { type Item = (&'py PyAny, &'py PyAny); @@ -567,12 +572,14 @@ impl<'py> Iterator for PyDictIterator<'py> { } } +#[cfg(feature = "gil-refs")] impl<'py> ExactSizeIterator for PyDictIterator<'py> { fn len(&self) -> usize { self.0.len() } } +#[cfg(feature = "gil-refs")] impl<'a> IntoIterator for &'a PyDict { type Item = (&'a PyAny, &'a PyAny); type IntoIter = PyDictIterator<'a>; diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 1fbbba44615..78cbf01df67 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -1,11 +1,13 @@ use crate::types::PyIterator; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; use crate::{ err::{self, PyErr, PyResult}, ffi, ffi_ptr_ext::FfiPtrExt, py_result_ext::PyResultExt, types::any::PyAnyMethods, - Bound, PyAny, PyNativeType, PyObject, Python, ToPyObject, + Bound, PyAny, PyObject, Python, ToPyObject, }; use std::ptr; @@ -73,10 +75,32 @@ pyobject_native_type_core!( #checkfunction=ffi::PyFrozenSet_Check ); +impl PyFrozenSet { + /// Creates a new frozenset. + /// + /// May panic when running out of memory. + #[inline] + pub fn new_bound<'a, 'p, T: ToPyObject + 'a>( + py: Python<'p>, + elements: impl IntoIterator, + ) -> PyResult> { + new_from_iter(py, elements) + } + + /// Creates a new empty frozen set + pub fn empty_bound(py: Python<'_>) -> PyResult> { + unsafe { + ffi::PyFrozenSet_New(ptr::null_mut()) + .assume_owned_or_err(py) + .downcast_into_unchecked() + } + } +} + +#[cfg(feature = "gil-refs")] impl PyFrozenSet { /// Deprecated form of [`PyFrozenSet::new_bound`]. #[inline] - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.0", note = "`PyFrozenSet::new` will be replaced by `PyFrozenSet::new_bound` in a future PyO3 version" @@ -88,19 +112,7 @@ impl PyFrozenSet { Self::new_bound(py, elements).map(Bound::into_gil_ref) } - /// Creates a new frozenset. - /// - /// May panic when running out of memory. - #[inline] - pub fn new_bound<'a, 'p, T: ToPyObject + 'a>( - py: Python<'p>, - elements: impl IntoIterator, - ) -> PyResult> { - new_from_iter(py, elements) - } - /// Deprecated form of [`PyFrozenSet::empty_bound`]. - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.0", note = "`PyFrozenSet::empty` will be replaced by `PyFrozenSet::empty_bound` in a future PyO3 version" @@ -109,15 +121,6 @@ impl PyFrozenSet { Self::empty_bound(py).map(Bound::into_gil_ref) } - /// Creates a new empty frozen set - pub fn empty_bound(py: Python<'_>) -> PyResult> { - unsafe { - ffi::PyFrozenSet_New(ptr::null_mut()) - .assume_owned_or_err(py) - .downcast_into_unchecked() - } - } - /// Return the number of items in the set. /// This is equivalent to len(p) on a set. #[inline] @@ -201,8 +204,10 @@ impl<'py> PyFrozenSetMethods<'py> for Bound<'py, PyFrozenSet> { } /// PyO3 implementation of an iterator for a Python `frozenset` object. +#[cfg(feature = "gil-refs")] pub struct PyFrozenSetIterator<'py>(BoundFrozenSetIterator<'py>); +#[cfg(feature = "gil-refs")] impl<'py> Iterator for PyFrozenSetIterator<'py> { type Item = &'py super::PyAny; @@ -217,6 +222,7 @@ impl<'py> Iterator for PyFrozenSetIterator<'py> { } } +#[cfg(feature = "gil-refs")] impl ExactSizeIterator for PyFrozenSetIterator<'_> { #[inline] fn len(&self) -> usize { @@ -224,6 +230,7 @@ impl ExactSizeIterator for PyFrozenSetIterator<'_> { } } +#[cfg(feature = "gil-refs")] impl<'py> IntoIterator for &'py PyFrozenSet { type Item = &'py PyAny; type IntoIter = PyFrozenSetIterator<'py>; diff --git a/src/types/iterator.rs b/src/types/iterator.rs index 53330705869..6131033af7d 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -1,9 +1,9 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::Borrowed; use crate::py_result_ext::PyResultExt; +use crate::{ffi, AsPyPointer, Bound, PyAny, PyErr, PyResult, PyTypeCheck}; #[cfg(feature = "gil-refs")] -use crate::PyDowncastError; -use crate::{ffi, AsPyPointer, Bound, PyAny, PyErr, PyNativeType, PyResult, PyTypeCheck}; +use crate::{PyDowncastError, PyNativeType}; /// A Python iterator object. /// @@ -54,6 +54,7 @@ impl PyIterator { } } +#[cfg(feature = "gil-refs")] impl<'p> Iterator for &'p PyIterator { type Item = PyResult<&'p PyAny>; diff --git a/src/types/list.rs b/src/types/list.rs index 56f21feb133..0d911e03199 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -6,7 +6,9 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::Borrowed; use crate::internal_tricks::get_ssize_index; use crate::types::{PySequence, PyTuple}; -use crate::{Bound, PyAny, PyNativeType, PyObject, Python, ToPyObject}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{Bound, PyAny, PyObject, Python, ToPyObject}; use crate::types::any::PyAnyMethods; use crate::types::sequence::PySequenceMethods; @@ -55,26 +57,10 @@ pub(crate) fn new_from_iter<'py>( } impl PyList { - /// Deprecated form of [`PyList::new_bound`]. - #[inline] - #[track_caller] - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyList::new` will be replaced by `PyList::new_bound` in a future PyO3 version" - )] - pub fn new(py: Python<'_>, elements: impl IntoIterator) -> &PyList - where - T: ToPyObject, - U: ExactSizeIterator, - { - Self::new_bound(py, elements).into_gil_ref() - } - /// Constructs a new list with the given elements. /// /// If you want to create a [`PyList`] with elements of different or unknown types, or from an - /// iterable that doesn't implement [`ExactSizeIterator`], use [`PyList::append`]. + /// iterable that doesn't implement [`ExactSizeIterator`], use [`PyListMethods::append`]. /// /// # Examples /// @@ -109,9 +95,35 @@ impl PyList { new_from_iter(py, &mut iter) } + /// Constructs a new empty list. + pub fn empty_bound(py: Python<'_>) -> Bound<'_, PyList> { + unsafe { + ffi::PyList_New(0) + .assume_owned(py) + .downcast_into_unchecked() + } + } +} + +#[cfg(feature = "gil-refs")] +impl PyList { + /// Deprecated form of [`PyList::new_bound`]. + #[inline] + #[track_caller] + #[deprecated( + since = "0.21.0", + note = "`PyList::new` will be replaced by `PyList::new_bound` in a future PyO3 version" + )] + pub fn new(py: Python<'_>, elements: impl IntoIterator) -> &PyList + where + T: ToPyObject, + U: ExactSizeIterator, + { + Self::new_bound(py, elements).into_gil_ref() + } + /// Deprecated form of [`PyList::empty_bound`]. #[inline] - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.0", note = "`PyList::empty` will be replaced by `PyList::empty_bound` in a future PyO3 version" @@ -120,15 +132,6 @@ impl PyList { Self::empty_bound(py).into_gil_ref() } - /// Constructs a new empty list. - pub fn empty_bound(py: Python<'_>) -> Bound<'_, PyList> { - unsafe { - ffi::PyList_New(0) - .assume_owned(py) - .downcast_into_unchecked() - } - } - /// Returns the length of the list. pub fn len(&self) -> usize { self.as_borrowed().len() @@ -273,6 +276,7 @@ impl PyList { } } +#[cfg(feature = "gil-refs")] index_impls!(PyList, "list", PyList::len, PyList::get_slice); /// Implementation of functionality for [`PyList`]. @@ -586,8 +590,10 @@ impl<'py> PyListMethods<'py> for Bound<'py, PyList> { } /// Used by `PyList::iter()`. +#[cfg(feature = "gil-refs")] pub struct PyListIterator<'a>(BoundListIterator<'a>); +#[cfg(feature = "gil-refs")] impl<'a> Iterator for PyListIterator<'a> { type Item = &'a PyAny; @@ -602,6 +608,7 @@ impl<'a> Iterator for PyListIterator<'a> { } } +#[cfg(feature = "gil-refs")] impl<'a> DoubleEndedIterator for PyListIterator<'a> { #[inline] fn next_back(&mut self) -> Option { @@ -609,14 +616,17 @@ impl<'a> DoubleEndedIterator for PyListIterator<'a> { } } +#[cfg(feature = "gil-refs")] impl<'a> ExactSizeIterator for PyListIterator<'a> { fn len(&self) -> usize { self.0.len() } } +#[cfg(feature = "gil-refs")] impl FusedIterator for PyListIterator<'_> {} +#[cfg(feature = "gil-refs")] impl<'a> IntoIterator for &'a PyList { type Item = &'a PyAny; type IntoIter = PyListIterator<'a>; diff --git a/src/types/mapping.rs b/src/types/mapping.rs index a91dad3679f..aea2b484c3b 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -1,4 +1,4 @@ -use crate::err::{PyDowncastError, PyResult}; +use crate::err::PyResult; use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::Bound; use crate::py_result_ext::PyResultExt; @@ -6,7 +6,9 @@ use crate::sync::GILOnceCell; use crate::type_object::PyTypeInfo; use crate::types::any::PyAnyMethods; use crate::types::{PyAny, PyDict, PySequence, PyType}; -use crate::{ffi, Py, PyNativeType, PyTypeCheck, Python, ToPyObject}; +#[cfg(feature = "gil-refs")] +use crate::{err::PyDowncastError, PyNativeType}; +use crate::{ffi, Py, PyTypeCheck, Python, ToPyObject}; /// Represents a reference to a Python object supporting the mapping protocol. #[repr(transparent)] @@ -14,6 +16,18 @@ pub struct PyMapping(PyAny); pyobject_native_type_named!(PyMapping); pyobject_native_type_extract!(PyMapping); +impl PyMapping { + /// Register a pyclass as a subclass of `collections.abc.Mapping` (from the Python standard + /// library). This is equivalent to `collections.abc.Mapping.register(T)` in Python. + /// This registration is required for a pyclass to be downcastable from `PyAny` to `PyMapping`. + pub fn register(py: Python<'_>) -> PyResult<()> { + let ty = T::type_object_bound(py); + get_mapping_abc(py)?.call_method1("register", (ty,))?; + Ok(()) + } +} + +#[cfg(feature = "gil-refs")] impl PyMapping { /// Returns the number of objects in the mapping. /// @@ -92,15 +106,6 @@ impl PyMapping { pub fn items(&self) -> PyResult<&PySequence> { self.as_borrowed().items().map(Bound::into_gil_ref) } - - /// Register a pyclass as a subclass of `collections.abc.Mapping` (from the Python standard - /// library). This is equvalent to `collections.abc.Mapping.register(T)` in Python. - /// This registration is required for a pyclass to be downcastable from `PyAny` to `PyMapping`. - pub fn register(py: Python<'_>) -> PyResult<()> { - let ty = T::type_object_bound(py); - get_mapping_abc(py)?.call_method1("register", (ty,))?; - Ok(()) - } } /// Implementation of functionality for [`PyMapping`]. @@ -255,6 +260,7 @@ impl PyTypeCheck for PyMapping { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl<'v> crate::PyTryFrom<'v> for PyMapping { /// Downcasting to `PyMapping` requires the concrete class to be a subclass (or registered diff --git a/src/types/memoryview.rs b/src/types/memoryview.rs index ffc1c81efa0..31afb372d7f 100644 --- a/src/types/memoryview.rs +++ b/src/types/memoryview.rs @@ -1,7 +1,9 @@ use crate::err::PyResult; use crate::ffi_ptr_ext::FfiPtrExt; use crate::py_result_ext::PyResultExt; -use crate::{ffi, AsPyPointer, Bound, PyAny, PyNativeType}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{ffi, AsPyPointer, Bound, PyAny}; /// Represents a Python `memoryview`. #[repr(transparent)] @@ -31,6 +33,7 @@ impl PyMemoryView { } } +#[cfg(feature = "gil-refs")] impl<'py> TryFrom<&'py PyAny> for &'py PyMemoryView { type Error = crate::PyErr; diff --git a/src/types/mod.rs b/src/types/mod.rs index d127cbbca20..38c9238961d 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -79,11 +79,17 @@ pub use self::typeobject::{PyType, PyTypeMethods}; /// the Limited API and PyPy, the underlying structures are opaque and that may not be possible. /// In these cases the iterators are implemented by forwarding to [`PyIterator`]. pub mod iter { - pub use super::dict::{BoundDictIterator, PyDictIterator}; - pub use super::frozenset::{BoundFrozenSetIterator, PyFrozenSetIterator}; - pub use super::list::{BoundListIterator, PyListIterator}; - pub use super::set::{BoundSetIterator, PySetIterator}; - pub use super::tuple::{BorrowedTupleIterator, BoundTupleIterator, PyTupleIterator}; + pub use super::dict::BoundDictIterator; + pub use super::frozenset::BoundFrozenSetIterator; + pub use super::list::BoundListIterator; + pub use super::set::BoundSetIterator; + pub use super::tuple::{BorrowedTupleIterator, BoundTupleIterator}; + + #[cfg(feature = "gil-refs")] + pub use super::{ + dict::PyDictIterator, frozenset::PyFrozenSetIterator, list::PyListIterator, + set::PySetIterator, tuple::PyTupleIterator, + }; } /// Python objects that have a base type. diff --git a/src/types/module.rs b/src/types/module.rs index c8b2cf04551..f0ae7385f23 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -6,11 +6,12 @@ use crate::pyclass::PyClass; use crate::types::{ any::PyAnyMethods, list::PyListMethods, PyAny, PyCFunction, PyDict, PyList, PyString, }; -use crate::{exceptions, ffi, Bound, IntoPy, Py, PyNativeType, PyObject, Python}; +use crate::{exceptions, ffi, Bound, IntoPy, Py, PyObject, Python}; use std::ffi::CString; use std::str; -use super::PyStringMethods; +#[cfg(feature = "gil-refs")] +use {super::PyStringMethods, crate::PyNativeType}; /// Represents a Python [`module`][1] object. /// @@ -25,17 +26,6 @@ pub struct PyModule(PyAny); pyobject_native_type_core!(PyModule, pyobject_native_static_type_object!(ffi::PyModule_Type), #checkfunction=ffi::PyModule_Check); impl PyModule { - /// Deprecated form of [`PyModule::new_bound`]. - #[inline] - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyModule::new` will be replaced by `PyModule::new_bound` in a future PyO3 version" - )] - pub fn new<'py>(py: Python<'py>, name: &str) -> PyResult<&'py PyModule> { - Self::new_bound(py, name).map(Bound::into_gil_ref) - } - /// Creates a new module object with the `__name__` attribute set to `name`. /// /// # Examples @@ -62,20 +52,6 @@ impl PyModule { } } - /// Deprecated form of [`PyModule::import_bound`]. - #[inline] - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyModule::import` will be replaced by `PyModule::import_bound` in a future PyO3 version" - )] - pub fn import(py: Python<'_>, name: N) -> PyResult<&PyModule> - where - N: IntoPy>, - { - Self::import_bound(py, name).map(Bound::into_gil_ref) - } - /// Imports the Python module with the specified name. /// /// # Examples @@ -106,22 +82,6 @@ impl PyModule { } } - /// Deprecated form of [`PyModule::from_code_bound`]. - #[inline] - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyModule::from_code` will be replaced by `PyModule::from_code_bound` in a future PyO3 version" - )] - pub fn from_code<'py>( - py: Python<'py>, - code: &str, - file_name: &str, - module_name: &str, - ) -> PyResult<&'py PyModule> { - Self::from_code_bound(py, code, file_name, module_name).map(Bound::into_gil_ref) - } - /// Creates and loads a module named `module_name`, /// containing the Python code passed to `code` /// and pretending to live at `file_name`. @@ -195,6 +155,47 @@ impl PyModule { .downcast_into() } } +} + +#[cfg(feature = "gil-refs")] +impl PyModule { + /// Deprecated form of [`PyModule::new_bound`]. + #[inline] + #[deprecated( + since = "0.21.0", + note = "`PyModule::new` will be replaced by `PyModule::new_bound` in a future PyO3 version" + )] + pub fn new<'py>(py: Python<'py>, name: &str) -> PyResult<&'py PyModule> { + Self::new_bound(py, name).map(Bound::into_gil_ref) + } + + /// Deprecated form of [`PyModule::import_bound`]. + #[inline] + #[deprecated( + since = "0.21.0", + note = "`PyModule::import` will be replaced by `PyModule::import_bound` in a future PyO3 version" + )] + pub fn import(py: Python<'_>, name: N) -> PyResult<&PyModule> + where + N: IntoPy>, + { + Self::import_bound(py, name).map(Bound::into_gil_ref) + } + + /// Deprecated form of [`PyModule::from_code_bound`]. + #[inline] + #[deprecated( + since = "0.21.0", + note = "`PyModule::from_code` will be replaced by `PyModule::from_code_bound` in a future PyO3 version" + )] + pub fn from_code<'py>( + py: Python<'py>, + code: &str, + file_name: &str, + module_name: &str, + ) -> PyResult<&'py PyModule> { + Self::from_code_bound(py, code, file_name, module_name).map(Bound::into_gil_ref) + } /// Returns the module's `__dict__` attribute, which contains the module's symbol table. pub fn dict(&self) -> &PyDict { @@ -433,8 +434,9 @@ pub trait PyModuleMethods<'py>: crate::sealed::Sealed { /// Adds an attribute to the module. /// - /// For adding classes, functions or modules, prefer to use [`PyModule::add_class`], - /// [`PyModule::add_function`] or [`PyModule::add_submodule`] instead, respectively. + /// For adding classes, functions or modules, prefer to use [`PyModuleMethods::add_class`], + /// [`PyModuleMethods::add_function`] or [`PyModuleMethods::add_submodule`] instead, + /// respectively. /// /// # Examples /// @@ -510,7 +512,8 @@ pub trait PyModuleMethods<'py>: crate::sealed::Sealed { /// Adds a function or a (sub)module to a module, using the functions name as name. /// - /// Prefer to use [`PyModule::add_function`] and/or [`PyModule::add_submodule`] instead. + /// Prefer to use [`PyModuleMethods::add_function`] and/or [`PyModuleMethods::add_submodule`] + /// instead. fn add_wrapped(&self, wrapper: &impl Fn(Python<'py>) -> T) -> PyResult<()> where T: IntoPyCallbackOutput; diff --git a/src/types/sequence.rs b/src/types/sequence.rs index afe4a595964..f75d851973d 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -1,4 +1,4 @@ -use crate::err::{self, DowncastError, PyDowncastError, PyErr, PyResult}; +use crate::err::{self, DowncastError, PyErr, PyResult}; use crate::exceptions::PyTypeError; use crate::ffi_ptr_ext::FfiPtrExt; #[cfg(feature = "experimental-inspect")] @@ -9,7 +9,9 @@ use crate::py_result_ext::PyResultExt; use crate::sync::GILOnceCell; use crate::type_object::PyTypeInfo; use crate::types::{any::PyAnyMethods, PyAny, PyList, PyString, PyTuple, PyType}; -use crate::{ffi, FromPyObject, Py, PyNativeType, PyTypeCheck, Python, ToPyObject}; +#[cfg(feature = "gil-refs")] +use crate::{err::PyDowncastError, PyNativeType}; +use crate::{ffi, FromPyObject, Py, PyTypeCheck, Python, ToPyObject}; /// Represents a reference to a Python object supporting the sequence protocol. #[repr(transparent)] @@ -17,6 +19,18 @@ pub struct PySequence(PyAny); pyobject_native_type_named!(PySequence); pyobject_native_type_extract!(PySequence); +impl PySequence { + /// Register a pyclass as a subclass of `collections.abc.Sequence` (from the Python standard + /// library). This is equivalent to `collections.abc.Sequence.register(T)` in Python. + /// This registration is required for a pyclass to be downcastable from `PyAny` to `PySequence`. + pub fn register(py: Python<'_>) -> PyResult<()> { + let ty = T::type_object_bound(py); + get_sequence_abc(py)?.call_method1("register", (ty,))?; + Ok(()) + } +} + +#[cfg(feature = "gil-refs")] impl PySequence { /// Returns the number of objects in sequence. /// @@ -175,15 +189,6 @@ impl PySequence { pub fn to_tuple(&self) -> PyResult<&PyTuple> { self.as_borrowed().to_tuple().map(Bound::into_gil_ref) } - - /// Register a pyclass as a subclass of `collections.abc.Sequence` (from the Python standard - /// library). This is equvalent to `collections.abc.Sequence.register(T)` in Python. - /// This registration is required for a pyclass to be downcastable from `PyAny` to `PySequence`. - pub fn register(py: Python<'_>) -> PyResult<()> { - let ty = T::type_object_bound(py); - get_sequence_abc(py)?.call_method1("register", (ty,))?; - Ok(()) - } } /// Implementation of functionality for [`PySequence`]. @@ -465,16 +470,19 @@ impl<'py> PySequenceMethods<'py> for Bound<'py, PySequence> { } #[inline] +#[cfg(feature = "gil-refs")] fn sequence_len(seq: &PySequence) -> usize { seq.len().expect("failed to get sequence length") } #[inline] +#[cfg(feature = "gil-refs")] fn sequence_slice(seq: &PySequence, start: usize, end: usize) -> &PySequence { seq.get_slice(start, end) .expect("sequence slice operation failed") } +#[cfg(feature = "gil-refs")] index_impls!(PySequence, "sequence", sequence_len, sequence_slice); impl<'py, T> FromPyObject<'py> for Vec @@ -539,6 +547,7 @@ impl PyTypeCheck for PySequence { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl<'v> crate::PyTryFrom<'v> for PySequence { /// Downcasting to `PySequence` requires the concrete class to be a subclass (or registered diff --git a/src/types/set.rs b/src/types/set.rs index 83938f3bf42..1bc4c86be51 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -1,11 +1,12 @@ use crate::types::PyIterator; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; use crate::{ err::{self, PyErr, PyResult}, ffi_ptr_ext::FfiPtrExt, instance::Bound, py_result_ext::PyResultExt, types::any::PyAnyMethods, - PyNativeType, }; use crate::{ffi, PyAny, PyObject, Python, ToPyObject}; use std::ptr; @@ -29,9 +30,31 @@ pyobject_native_type_core!( #checkfunction=ffi::PySet_Check ); +impl PySet { + /// Creates a new set with elements from the given slice. + /// + /// Returns an error if some element is not hashable. + #[inline] + pub fn new_bound<'a, 'p, T: ToPyObject + 'a>( + py: Python<'p>, + elements: impl IntoIterator, + ) -> PyResult> { + new_from_iter(py, elements) + } + + /// Creates a new empty set. + pub fn empty_bound(py: Python<'_>) -> PyResult> { + unsafe { + ffi::PySet_New(ptr::null_mut()) + .assume_owned_or_err(py) + .downcast_into_unchecked() + } + } +} + +#[cfg(feature = "gil-refs")] impl PySet { /// Deprecated form of [`PySet::new_bound`]. - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.0", note = "`PySet::new` will be replaced by `PySet::new_bound` in a future PyO3 version" @@ -44,19 +67,7 @@ impl PySet { Self::new_bound(py, elements).map(Bound::into_gil_ref) } - /// Creates a new set with elements from the given slice. - /// - /// Returns an error if some element is not hashable. - #[inline] - pub fn new_bound<'a, 'p, T: ToPyObject + 'a>( - py: Python<'p>, - elements: impl IntoIterator, - ) -> PyResult> { - new_from_iter(py, elements) - } - /// Deprecated form of [`PySet::empty_bound`]. - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.2", note = "`PySet::empty` will be replaced by `PySet::empty_bound` in a future PyO3 version" @@ -65,15 +76,6 @@ impl PySet { Self::empty_bound(py).map(Bound::into_gil_ref) } - /// Creates a new empty set. - pub fn empty_bound(py: Python<'_>) -> PyResult> { - unsafe { - ffi::PySet_New(ptr::null_mut()) - .assume_owned_or_err(py) - .downcast_into_unchecked() - } - } - /// Removes all elements from the set. #[inline] pub fn clear(&self) { @@ -259,8 +261,10 @@ impl<'py> PySetMethods<'py> for Bound<'py, PySet> { } /// PyO3 implementation of an iterator for a Python `set` object. +#[cfg(feature = "gil-refs")] pub struct PySetIterator<'py>(BoundSetIterator<'py>); +#[cfg(feature = "gil-refs")] impl<'py> Iterator for PySetIterator<'py> { type Item = &'py super::PyAny; @@ -279,12 +283,14 @@ impl<'py> Iterator for PySetIterator<'py> { } } +#[cfg(feature = "gil-refs")] impl ExactSizeIterator for PySetIterator<'_> { fn len(&self) -> usize { self.0.len() } } +#[cfg(feature = "gil-refs")] impl<'py> IntoIterator for &'py PySet { type Item = &'py PyAny; type IntoIter = PySetIterator<'py>; diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 563a81983fa..afe129879f9 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -7,9 +7,11 @@ use crate::inspect::types::TypeInfo; use crate::instance::Borrowed; use crate::internal_tricks::get_ssize_index; use crate::types::{any::PyAnyMethods, sequence::PySequenceMethods, PyList, PySequence}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; use crate::{ - exceptions, Bound, FromPyObject, IntoPy, Py, PyAny, PyErr, PyNativeType, PyObject, PyResult, - Python, ToPyObject, + exceptions, Bound, FromPyObject, IntoPy, Py, PyAny, PyErr, PyObject, PyResult, Python, + ToPyObject, }; #[inline] @@ -57,24 +59,6 @@ pub struct PyTuple(PyAny); pyobject_native_type_core!(PyTuple, pyobject_native_static_type_object!(ffi::PyTuple_Type), #checkfunction=ffi::PyTuple_Check); impl PyTuple { - /// Deprecated form of `PyTuple::new_bound`. - #[track_caller] - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyTuple::new` will be replaced by `PyTuple::new_bound` in a future PyO3 version" - )] - pub fn new( - py: Python<'_>, - elements: impl IntoIterator, - ) -> &PyTuple - where - T: ToPyObject, - U: ExactSizeIterator, - { - Self::new_bound(py, elements).into_gil_ref() - } - /// Constructs a new tuple with the given elements. /// /// If you want to create a [`PyTuple`] with elements of different or unknown types, or from an @@ -114,16 +98,6 @@ impl PyTuple { new_from_iter(py, &mut elements) } - /// Deprecated form of `PyTuple::empty_bound`. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyTuple::empty` will be replaced by `PyTuple::empty_bound` in a future PyO3 version" - )] - pub fn empty(py: Python<'_>) -> &PyTuple { - Self::empty_bound(py).into_gil_ref() - } - /// Constructs an empty tuple (on the Python side, a singleton object). pub fn empty_bound(py: Python<'_>) -> Bound<'_, PyTuple> { unsafe { @@ -132,6 +106,35 @@ impl PyTuple { .downcast_into_unchecked() } } +} + +#[cfg(feature = "gil-refs")] +impl PyTuple { + /// Deprecated form of `PyTuple::new_bound`. + #[track_caller] + #[deprecated( + since = "0.21.0", + note = "`PyTuple::new` will be replaced by `PyTuple::new_bound` in a future PyO3 version" + )] + pub fn new( + py: Python<'_>, + elements: impl IntoIterator, + ) -> &PyTuple + where + T: ToPyObject, + U: ExactSizeIterator, + { + Self::new_bound(py, elements).into_gil_ref() + } + + /// Deprecated form of `PyTuple::empty_bound`. + #[deprecated( + since = "0.21.0", + note = "`PyTuple::empty` will be replaced by `PyTuple::empty_bound` in a future PyO3 version" + )] + pub fn empty(py: Python<'_>) -> &PyTuple { + Self::empty_bound(py).into_gil_ref() + } /// Gets the length of the tuple. pub fn len(&self) -> usize { @@ -236,6 +239,7 @@ impl PyTuple { } } +#[cfg(feature = "gil-refs")] index_impls!(PyTuple, "tuple", PyTuple::len, PyTuple::get_slice); /// Implementation of functionality for [`PyTuple`]. @@ -443,8 +447,10 @@ impl<'a, 'py> Borrowed<'a, 'py, PyTuple> { } /// Used by `PyTuple::iter()`. +#[cfg(feature = "gil-refs")] pub struct PyTupleIterator<'a>(BorrowedTupleIterator<'a, 'a>); +#[cfg(feature = "gil-refs")] impl<'a> Iterator for PyTupleIterator<'a> { type Item = &'a PyAny; @@ -459,6 +465,7 @@ impl<'a> Iterator for PyTupleIterator<'a> { } } +#[cfg(feature = "gil-refs")] impl<'a> DoubleEndedIterator for PyTupleIterator<'a> { #[inline] fn next_back(&mut self) -> Option { @@ -466,14 +473,17 @@ impl<'a> DoubleEndedIterator for PyTupleIterator<'a> { } } +#[cfg(feature = "gil-refs")] impl<'a> ExactSizeIterator for PyTupleIterator<'a> { fn len(&self) -> usize { self.0.len() } } +#[cfg(feature = "gil-refs")] impl FusedIterator for PyTupleIterator<'_> {} +#[cfg(feature = "gil-refs")] impl<'a> IntoIterator for &'a PyTuple { type Item = &'a PyAny; type IntoIter = PyTupleIterator<'a>; diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index 022d61e084d..3509a11f4be 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -10,6 +10,7 @@ fn basic_function(py: pyo3::Python<'_>, x: Option) -> pyo3::PyOb x.unwrap_or_else(|| py.None()) } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] #[pyo3::pymodule] fn basic_module(_py: pyo3::Python<'_>, m: &pyo3::types::PyModule) -> pyo3::PyResult<()> { @@ -108,7 +109,7 @@ impl BasicClass { #[test] fn test_basic() { pyo3::Python::with_gil(|py| { - let module = pyo3::wrap_pymodule!(basic_module)(py); + let module = pyo3::wrap_pymodule!(basic_module_bound)(py); let cls = py.get_type_bound::(); let d = pyo3::types::IntoPyDict::into_py_dict_bound( [ diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index 96f652d9679..ef0b06652e4 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -58,8 +58,8 @@ fn pyfunction_with_module<'py>(module: &Bound<'py, PyModule>) -> PyResult PyResult<&str> { - module.name() +fn pyfunction_with_module_gil_ref(_module: &PyModule) -> PyResult<&str> { + todo!() } #[pyfunction] @@ -68,14 +68,12 @@ fn double(x: usize) -> usize { } #[pymodule] -fn module_gil_ref(m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(double, m)?)?; +fn module_gil_ref(_m: &PyModule) -> PyResult<()> { Ok(()) } #[pymodule] -fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(double, m)?)?; +fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, _m: &PyModule) -> PyResult<()> { Ok(()) } diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index 2b75ee23e10..9c61c26581e 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -53,69 +53,69 @@ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg` | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:61:43 + --> tests/ui/deprecations.rs:61:44 | -61 | fn pyfunction_with_module_gil_ref(module: &PyModule) -> PyResult<&str> { - | ^ +61 | fn pyfunction_with_module_gil_ref(_module: &PyModule) -> PyResult<&str> { + | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument --> tests/ui/deprecations.rs:71:19 | -71 | fn module_gil_ref(m: &PyModule) -> PyResult<()> { - | ^ +71 | fn module_gil_ref(_m: &PyModule) -> PyResult<()> { + | ^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:77:57 + --> tests/ui/deprecations.rs:76:57 | -77 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - | ^ +76 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, _m: &PyModule) -> PyResult<()> { + | ^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:110:27 + --> tests/ui/deprecations.rs:108:27 | -110 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, +108 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:116:29 + --> tests/ui/deprecations.rs:114:29 | -116 | fn pyfunction_gil_ref(_any: &PyAny) {} +114 | fn pyfunction_gil_ref(_any: &PyAny) {} | ^ error: use of deprecated method `pyo3::deprecations::OptionGilRefs::>::function_arg`: use `Option<&Bound<'_, T>>` instead for this function argument - --> tests/ui/deprecations.rs:119:36 + --> tests/ui/deprecations.rs:117:36 | -119 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} +117 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} | ^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:126:27 + --> tests/ui/deprecations.rs:124:27 | -126 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] +124 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:136:27 + --> tests/ui/deprecations.rs:134:27 | -136 | #[pyo3(from_py_with = "PyAny::len")] usize, +134 | #[pyo3(from_py_with = "PyAny::len")] usize, | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:142:31 + --> tests/ui/deprecations.rs:140:31 | -142 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), +140 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:149:27 + --> tests/ui/deprecations.rs:147:27 | -149 | #[pyo3(from_py_with = "extract_gil_ref")] +147 | #[pyo3(from_py_with = "extract_gil_ref")] | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::>::is_python`: use `wrap_pyfunction_bound!` instead - --> tests/ui/deprecations.rs:162:13 + --> tests/ui/deprecations.rs:160:13 | -162 | let _ = wrap_pyfunction!(double, py); +160 | let _ = wrap_pyfunction!(double, py); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From 2d19b7e2a7e29e2f445a8b415a764f40b4242c5c Mon Sep 17 00:00:00 2001 From: David Matos Date: Thu, 9 May 2024 17:37:53 +0200 Subject: [PATCH 325/349] Add `num-rational` support for Python's `fractions.Fraction` type (#4148) * Add `num-rational` support for Python's `fractions.Fraction` type * Add newsfragment * Use Bound instead * Handle objs which atts are incorrect * Add extra test * Add tests for wasm32 arch * add type for wasm32 clipppy --- Cargo.toml | 2 + guide/src/conversions/tables.md | 3 + guide/src/features.md | 4 + newsfragments/4148.added.md | 1 + src/conversions/mod.rs | 1 + src/conversions/num_rational.rs | 277 ++++++++++++++++++++++++++++++++ src/lib.rs | 3 + 7 files changed, 291 insertions(+) create mode 100644 newsfragments/4148.added.md create mode 100644 src/conversions/num_rational.rs diff --git a/Cargo.toml b/Cargo.toml index 9202c69ef92..4c5a083060d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ hashbrown = { version = ">= 0.9, < 0.15", optional = true } indexmap = { version = ">= 1.6, < 3", optional = true } num-bigint = { version = "0.4", optional = true } num-complex = { version = ">= 0.2, < 0.5", optional = true } +num-rational = {version = "0.4.1", optional = true } rust_decimal = { version = "1.0.0", default-features = false, optional = true } serde = { version = "1.0", optional = true } smallvec = { version = "1.0", optional = true } @@ -127,6 +128,7 @@ full = [ "indexmap", "num-bigint", "num-complex", + "num-rational", "rust_decimal", "serde", "smallvec", diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index eb33b17acf7..208e61671ec 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -19,6 +19,7 @@ The table below contains the Python type and the corresponding function argument | `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `PyLong` | | `float` | `f32`, `f64` | `PyFloat` | | `complex` | `num_complex::Complex`[^2] | `PyComplex` | +| `fractions.Fraction`| `num_rational::Ratio`[^8] | - | | `list[T]` | `Vec` | `PyList` | | `dict[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `PyDict` | | `tuple[T, U]` | `(T, U)`, `Vec` | `PyTuple` | @@ -113,3 +114,5 @@ Finally, the following Rust types are also able to convert to Python as return v [^6]: Requires the `chrono-tz` optional feature. [^7]: Requires the `rust_decimal` optional feature. + +[^8]: Requires the `num-rational` optional feature. diff --git a/guide/src/features.md b/guide/src/features.md index 0816770a781..07085a9e89c 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -157,6 +157,10 @@ Adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conver Adds a dependency on [num-complex](https://docs.rs/num-complex) and enables conversions into its [`Complex`](https://docs.rs/num-complex/latest/num_complex/struct.Complex.html) type. +### `num-rational` + +Adds a dependency on [num-rational](https://docs.rs/num-rational) and enables conversions into its [`Ratio`](https://docs.rs/num-rational/latest/num_rational/struct.Ratio.html) type. + ### `rust_decimal` Adds a dependency on [rust_decimal](https://docs.rs/rust_decimal) and enables conversions into its [`Decimal`](https://docs.rs/rust_decimal/latest/rust_decimal/struct.Decimal.html) type. diff --git a/newsfragments/4148.added.md b/newsfragments/4148.added.md new file mode 100644 index 00000000000..16da3d2db37 --- /dev/null +++ b/newsfragments/4148.added.md @@ -0,0 +1 @@ +Conversion between [num-rational](https://github.com/rust-num/num-rational) and Python's fractions.Fraction. diff --git a/src/conversions/mod.rs b/src/conversions/mod.rs index 3d785c02381..53ecf849c07 100644 --- a/src/conversions/mod.rs +++ b/src/conversions/mod.rs @@ -9,6 +9,7 @@ pub mod hashbrown; pub mod indexmap; pub mod num_bigint; pub mod num_complex; +pub mod num_rational; pub mod rust_decimal; pub mod serde; pub mod smallvec; diff --git a/src/conversions/num_rational.rs b/src/conversions/num_rational.rs new file mode 100644 index 00000000000..31eb7ca1c7b --- /dev/null +++ b/src/conversions/num_rational.rs @@ -0,0 +1,277 @@ +#![cfg(feature = "num-rational")] +//! Conversions to and from [num-rational](https://docs.rs/num-rational) types. +//! +//! This is useful for converting between Python's [fractions.Fraction](https://docs.python.org/3/library/fractions.html) into and from a native Rust +//! type. +//! +//! +//! To use this feature, add to your **`Cargo.toml`**: +//! +//! ```toml +//! [dependencies] +#![doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"num-rational\"] }")] +//! num-rational = "0.4.1" +//! ``` +//! +//! # Example +//! +//! Rust code to create a function that adds five to a fraction: +//! +//! ```rust +//! use num_rational::Ratio; +//! use pyo3::prelude::*; +//! +//! #[pyfunction] +//! fn add_five_to_fraction(fraction: Ratio) -> Ratio { +//! fraction + Ratio::new(5, 1) +//! } +//! +//! #[pymodule] +//! fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { +//! m.add_function(wrap_pyfunction!(add_five_to_fraction, m)?)?; +//! Ok(()) +//! } +//! ``` +//! +//! Python code that validates the functionality: +//! ```python +//! from my_module import add_five_to_fraction +//! from fractions import Fraction +//! +//! fraction = Fraction(2,1) +//! fraction_plus_five = add_five_to_fraction(f) +//! assert fraction + 5 == fraction_plus_five +//! ``` + +use crate::ffi; +use crate::sync::GILOnceCell; +use crate::types::any::PyAnyMethods; +use crate::types::PyType; +use crate::{Bound, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; +use std::os::raw::c_char; + +#[cfg(feature = "num-bigint")] +use num_bigint::BigInt; +use num_rational::Ratio; + +static FRACTION_CLS: GILOnceCell> = GILOnceCell::new(); + +fn get_fraction_cls(py: Python<'_>) -> PyResult<&Bound<'_, PyType>> { + FRACTION_CLS.get_or_try_init_type_ref(py, "fractions", "Fraction") +} + +macro_rules! rational_conversion { + ($int: ty) => { + impl<'py> FromPyObject<'py> for Ratio<$int> { + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { + let py = obj.py(); + let py_numerator_obj = unsafe { + Bound::from_owned_ptr_or_err( + py, + ffi::PyObject_GetAttrString( + obj.as_ptr(), + "numerator\0".as_ptr() as *const c_char, + ), + ) + }; + let py_denominator_obj = unsafe { + Bound::from_owned_ptr_or_err( + py, + ffi::PyObject_GetAttrString( + obj.as_ptr(), + "denominator\0".as_ptr() as *const c_char, + ), + ) + }; + let numerator_owned = unsafe { + Bound::from_owned_ptr_or_err( + py, + ffi::PyNumber_Long(py_numerator_obj?.as_ptr()), + )? + }; + let denominator_owned = unsafe { + Bound::from_owned_ptr_or_err( + py, + ffi::PyNumber_Long(py_denominator_obj?.as_ptr()), + )? + }; + let rs_numerator: $int = numerator_owned.extract()?; + let rs_denominator: $int = denominator_owned.extract()?; + Ok(Ratio::new(rs_numerator, rs_denominator)) + } + } + + impl ToPyObject for Ratio<$int> { + fn to_object(&self, py: Python<'_>) -> PyObject { + let fraction_cls = get_fraction_cls(py).expect("failed to load fractions.Fraction"); + let ret = fraction_cls + .call1((self.numer().clone(), self.denom().clone())) + .expect("failed to call fractions.Fraction(value)"); + ret.to_object(py) + } + } + impl IntoPy for Ratio<$int> { + fn into_py(self, py: Python<'_>) -> PyObject { + self.to_object(py) + } + } + }; +} +rational_conversion!(i8); +rational_conversion!(i16); +rational_conversion!(i32); +rational_conversion!(isize); +rational_conversion!(i64); +#[cfg(feature = "num-bigint")] +rational_conversion!(BigInt); +#[cfg(test)] +mod tests { + use super::*; + use crate::types::dict::PyDictMethods; + use crate::types::PyDict; + + #[cfg(not(target_arch = "wasm32"))] + use proptest::prelude::*; + #[test] + fn test_negative_fraction() { + Python::with_gil(|py| { + let locals = PyDict::new_bound(py); + py.run_bound( + "import fractions\npy_frac = fractions.Fraction(-0.125)", + None, + Some(&locals), + ) + .unwrap(); + let py_frac = locals.get_item("py_frac").unwrap().unwrap(); + let roundtripped: Ratio = py_frac.extract().unwrap(); + let rs_frac = Ratio::new(-1, 8); + assert_eq!(roundtripped, rs_frac); + }) + } + #[test] + fn test_obj_with_incorrect_atts() { + Python::with_gil(|py| { + let locals = PyDict::new_bound(py); + py.run_bound( + "not_fraction = \"contains_incorrect_atts\"", + None, + Some(&locals), + ) + .unwrap(); + let py_frac = locals.get_item("not_fraction").unwrap().unwrap(); + assert!(py_frac.extract::>().is_err()); + }) + } + + #[test] + fn test_fraction_with_fraction_type() { + Python::with_gil(|py| { + let locals = PyDict::new_bound(py); + py.run_bound( + "import fractions\npy_frac = fractions.Fraction(fractions.Fraction(10))", + None, + Some(&locals), + ) + .unwrap(); + let py_frac = locals.get_item("py_frac").unwrap().unwrap(); + let roundtripped: Ratio = py_frac.extract().unwrap(); + let rs_frac = Ratio::new(10, 1); + assert_eq!(roundtripped, rs_frac); + }) + } + + #[test] + fn test_fraction_with_decimal() { + Python::with_gil(|py| { + let locals = PyDict::new_bound(py); + py.run_bound( + "import fractions\n\nfrom decimal import Decimal\npy_frac = fractions.Fraction(Decimal(\"1.1\"))", + None, + Some(&locals), + ) + .unwrap(); + let py_frac = locals.get_item("py_frac").unwrap().unwrap(); + let roundtripped: Ratio = py_frac.extract().unwrap(); + let rs_frac = Ratio::new(11, 10); + assert_eq!(roundtripped, rs_frac); + }) + } + + #[test] + fn test_fraction_with_num_den() { + Python::with_gil(|py| { + let locals = PyDict::new_bound(py); + py.run_bound( + "import fractions\npy_frac = fractions.Fraction(10,5)", + None, + Some(&locals), + ) + .unwrap(); + let py_frac = locals.get_item("py_frac").unwrap().unwrap(); + let roundtripped: Ratio = py_frac.extract().unwrap(); + let rs_frac = Ratio::new(10, 5); + assert_eq!(roundtripped, rs_frac); + }) + } + + #[cfg(target_arch = "wasm32")] + #[test] + fn test_int_roundtrip() { + Python::with_gil(|py| { + let rs_frac = Ratio::new(1, 2); + let py_frac: PyObject = rs_frac.into_py(py); + let roundtripped: Ratio = py_frac.extract(py).unwrap(); + assert_eq!(rs_frac, roundtripped); + // float conversion + }) + } + + #[cfg(target_arch = "wasm32")] + #[test] + fn test_big_int_roundtrip() { + Python::with_gil(|py| { + let rs_frac = Ratio::from_float(5.5).unwrap(); + let py_frac: PyObject = rs_frac.clone().into_py(py); + let roundtripped: Ratio = py_frac.extract(py).unwrap(); + assert_eq!(rs_frac, roundtripped); + }) + } + + #[cfg(not(target_arch = "wasm32"))] + proptest! { + #[test] + fn test_int_roundtrip(num in any::(), den in any::()) { + Python::with_gil(|py| { + let rs_frac = Ratio::new(num, den); + let py_frac = rs_frac.into_py(py); + let roundtripped: Ratio = py_frac.extract(py).unwrap(); + assert_eq!(rs_frac, roundtripped); + }) + } + + #[test] + #[cfg(feature = "num-bigint")] + fn test_big_int_roundtrip(num in any::()) { + Python::with_gil(|py| { + let rs_frac = Ratio::from_float(num).unwrap(); + let py_frac = rs_frac.clone().into_py(py); + let roundtripped: Ratio = py_frac.extract(py).unwrap(); + assert_eq!(roundtripped, rs_frac); + }) + } + + } + + #[test] + fn test_infinity() { + Python::with_gil(|py| { + let locals = PyDict::new_bound(py); + let py_bound = py.run_bound( + "import fractions\npy_frac = fractions.Fraction(\"Infinity\")", + None, + Some(&locals), + ); + assert!(py_bound.is_err()); + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index b400f143f5a..3923257f5f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -108,6 +108,7 @@ //! [`BigUint`] types. //! - [`num-complex`]: Enables conversions between Python objects and [num-complex]'s [`Complex`] //! type. +//! - [`num-rational`]: Enables conversions between Python's fractions.Fraction and [num-rational]'s types //! - [`rust_decimal`]: Enables conversions between Python's decimal.Decimal and [rust_decimal]'s //! [`Decimal`] type. //! - [`serde`]: Allows implementing [serde]'s [`Serialize`] and [`Deserialize`] traits for @@ -288,6 +289,7 @@ //! [`maturin`]: https://github.com/PyO3/maturin "Build and publish crates with pyo3, rust-cpython and cffi bindings as well as rust binaries as python packages" //! [`num-bigint`]: ./num_bigint/index.html "Documentation about the `num-bigint` feature." //! [`num-complex`]: ./num_complex/index.html "Documentation about the `num-complex` feature." +//! [`num-rational`]: ./num_rational/index.html "Documentation about the `num-rational` feature." //! [`pyo3-build-config`]: https://docs.rs/pyo3-build-config //! [rust_decimal]: https://docs.rs/rust_decimal //! [`rust_decimal`]: ./rust_decimal/index.html "Documenation about the `rust_decimal` feature." @@ -303,6 +305,7 @@ //! [manual_builds]: https://pyo3.rs/latest/building-and-distribution.html#manual-builds "Manual builds - Building and Distribution - PyO3 user guide" //! [num-bigint]: https://docs.rs/num-bigint //! [num-complex]: https://docs.rs/num-complex +//! [num-rational]: https://docs.rs/num-rational //! [serde]: https://docs.rs/serde //! [setuptools-rust]: https://github.com/PyO3/setuptools-rust "Setuptools plugin for Rust extensions" //! [the guide]: https://pyo3.rs "PyO3 user guide" From 7beb64a8cac4873b151d87c8099c56c69c8f602e Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Thu, 9 May 2024 23:08:23 +0200 Subject: [PATCH 326/349] allow constructor customization of complex enum variants (#4158) * allow `#[pyo3(signature = ...)]` on complex enum variants to specify constructor signature * rename keyword to `constructor` * review feedback * add docs in guide * add newsfragment --- guide/pyclass-parameters.md | 2 + guide/src/class.md | 40 ++++++++++++ newsfragments/4158.added.md | 1 + pyo3-macros-backend/src/attributes.rs | 1 + pyo3-macros-backend/src/pyclass.rs | 62 ++++++++++++++----- pyo3-macros-backend/src/pyfunction.rs | 2 +- .../src/pyfunction/signature.rs | 10 +++ pytests/src/enums.rs | 27 ++++++-- pytests/tests/test_enums.py | 23 +++++++ tests/ui/invalid_pyclass_enum.rs | 7 +++ tests/ui/invalid_pyclass_enum.stderr | 6 ++ 11 files changed, 160 insertions(+), 21 deletions(-) create mode 100644 newsfragments/4158.added.md diff --git a/guide/pyclass-parameters.md b/guide/pyclass-parameters.md index 6951a5b5e15..9bd0534ea5d 100644 --- a/guide/pyclass-parameters.md +++ b/guide/pyclass-parameters.md @@ -2,6 +2,7 @@ | Parameter | Description | | :- | :- | +| `constructor` | This is currently only allowed on [variants of complex enums][params-constructor]. It allows customization of the generated class constructor for each variant. It uses the same syntax and supports the same options as the `signature` attribute of functions and methods. | | `crate = "some::path"` | Path to import the `pyo3` crate, if it's not accessible at `::pyo3`. | | `dict` | Gives instances of this class an empty `__dict__` to store custom attributes. | | `extends = BaseType` | Use a custom baseclass. Defaults to [`PyAny`][params-1] | @@ -39,5 +40,6 @@ struct MyClass {} [params-4]: https://doc.rust-lang.org/std/rc/struct.Rc.html [params-5]: https://doc.rust-lang.org/std/sync/struct.Arc.html [params-6]: https://docs.python.org/3/library/weakref.html +[params-constructor]: https://pyo3.rs/latest/class.html#complex-enums [params-mapping]: https://pyo3.rs/latest/class/protocols.html#mapping--sequence-types [params-sequence]: https://pyo3.rs/latest/class/protocols.html#mapping--sequence-types diff --git a/guide/src/class.md b/guide/src/class.md index b5ef95cb2f7..3fcfaca4bdc 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -1243,6 +1243,46 @@ Python::with_gil(|py| { }) ``` +The constructor of each generated class can be customized using the `#[pyo3(constructor = (...))]` attribute. This uses the same syntax as the [`#[pyo3(signature = (...))]`](function/signature.md) +attribute on function and methods and supports the same options. To apply this attribute simply place it on top of a variant in a `#[pyclass]` complex enum as shown below: + +```rust +# use pyo3::prelude::*; +#[pyclass] +enum Shape { + #[pyo3(constructor = (radius=1.0))] + Circle { radius: f64 }, + #[pyo3(constructor = (*, width, height))] + Rectangle { width: f64, height: f64 }, + #[pyo3(constructor = (side_count, radius=1.0))] + RegularPolygon { side_count: u32, radius: f64 }, + Nothing { }, +} + +# #[cfg(Py_3_10)] +Python::with_gil(|py| { + let cls = py.get_type_bound::(); + pyo3::py_run!(py, cls, r#" + circle = cls.Circle() + assert isinstance(circle, cls) + assert isinstance(circle, cls.Circle) + assert circle.radius == 1.0 + + square = cls.Rectangle(width = 1, height = 1) + assert isinstance(square, cls) + assert isinstance(square, cls.Rectangle) + assert square.width == 1 + assert square.height == 1 + + hexagon = cls.RegularPolygon(6) + assert isinstance(hexagon, cls) + assert isinstance(hexagon, cls.RegularPolygon) + assert hexagon.side_count == 6 + assert hexagon.radius == 1 + "#) +}) +``` + ## Implementation details The `#[pyclass]` macros rely on a lot of conditional code generation: each `#[pyclass]` can optionally have a `#[pymethods]` block. diff --git a/newsfragments/4158.added.md b/newsfragments/4158.added.md new file mode 100644 index 00000000000..42e6d3ff4b4 --- /dev/null +++ b/newsfragments/4158.added.md @@ -0,0 +1 @@ +Added `#[pyo3(constructor = (...))]` to customize the generated constructors for complex enum variants diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs index e91b3b8d9a2..d9c805aa3fa 100644 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -12,6 +12,7 @@ pub mod kw { syn::custom_keyword!(annotation); syn::custom_keyword!(attribute); syn::custom_keyword!(cancel_handle); + syn::custom_keyword!(constructor); syn::custom_keyword!(dict); syn::custom_keyword!(extends); syn::custom_keyword!(freelist); diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index f8bfa164d7d..3023f897645 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -8,6 +8,7 @@ use crate::attributes::{ use crate::deprecations::Deprecations; use crate::konst::{ConstAttributes, ConstSpec}; use crate::method::{FnArg, FnSpec, PyArg, RegularArg}; +use crate::pyfunction::ConstructorAttribute; use crate::pyimpl::{gen_py_const, PyClassMethodsType}; use crate::pymethod::{ impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, MethodAndSlotDef, PropertyType, @@ -620,12 +621,15 @@ struct PyClassEnumVariantNamedField<'a> { } /// `#[pyo3()]` options for pyclass enum variants +#[derive(Default)] struct EnumVariantPyO3Options { name: Option, + constructor: Option, } enum EnumVariantPyO3Option { Name(NameAttribute), + Constructor(ConstructorAttribute), } impl Parse for EnumVariantPyO3Option { @@ -633,6 +637,8 @@ impl Parse for EnumVariantPyO3Option { let lookahead = input.lookahead1(); if lookahead.peek(attributes::kw::name) { input.parse().map(EnumVariantPyO3Option::Name) + } else if lookahead.peek(attributes::kw::constructor) { + input.parse().map(EnumVariantPyO3Option::Constructor) } else { Err(lookahead.error()) } @@ -641,21 +647,33 @@ impl Parse for EnumVariantPyO3Option { impl EnumVariantPyO3Options { fn take_pyo3_options(attrs: &mut Vec) -> Result { - let mut options = EnumVariantPyO3Options { name: None }; + let mut options = EnumVariantPyO3Options::default(); - for option in take_pyo3_options(attrs)? { - match option { - EnumVariantPyO3Option::Name(name) => { + take_pyo3_options(attrs)? + .into_iter() + .try_for_each(|option| options.set_option(option))?; + + Ok(options) + } + + fn set_option(&mut self, option: EnumVariantPyO3Option) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + { ensure_spanned!( - options.name.is_none(), - name.span() => "`name` may only be specified once" + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once") ); - options.name = Some(name); + self.$key = Some($key); } - } + }; } - Ok(options) + match option { + EnumVariantPyO3Option::Constructor(constructor) => set_option!(constructor), + EnumVariantPyO3Option::Name(name) => set_option!(name), + } + Ok(()) } } @@ -689,6 +707,10 @@ fn impl_simple_enum( let variants = simple_enum.variants; let pytypeinfo = impl_pytypeinfo(cls, args, None, ctx); + for variant in &variants { + ensure_spanned!(variant.options.constructor.is_none(), variant.options.constructor.span() => "`constructor` can't be used on a simple enum variant"); + } + let (default_repr, default_repr_slot) = { let variants_repr = variants.iter().map(|variant| { let variant_name = variant.ident; @@ -889,7 +911,7 @@ fn impl_complex_enum( let mut variant_cls_pytypeinfos = vec![]; let mut variant_cls_pyclass_impls = vec![]; let mut variant_cls_impls = vec![]; - for variant in &variants { + for variant in variants { let variant_cls = gen_complex_enum_variant_class_ident(cls, variant.get_ident()); let variant_cls_zst = quote! { @@ -908,11 +930,11 @@ fn impl_complex_enum( let variant_cls_pytypeinfo = impl_pytypeinfo(&variant_cls, &variant_args, None, ctx); variant_cls_pytypeinfos.push(variant_cls_pytypeinfo); - let variant_new = complex_enum_variant_new(cls, variant, ctx)?; - - let (variant_cls_impl, field_getters) = impl_complex_enum_variant_cls(cls, variant, ctx)?; + let (variant_cls_impl, field_getters) = impl_complex_enum_variant_cls(cls, &variant, ctx)?; variant_cls_impls.push(variant_cls_impl); + let variant_new = complex_enum_variant_new(cls, variant, ctx)?; + let pyclass_impl = PyClassImplsBuilder::new( &variant_cls, &variant_args, @@ -1120,7 +1142,7 @@ pub fn gen_complex_enum_variant_attr( fn complex_enum_variant_new<'a>( cls: &'a syn::Ident, - variant: &'a PyClassEnumVariant<'a>, + variant: PyClassEnumVariant<'a>, ctx: &Ctx, ) -> Result { match variant { @@ -1132,7 +1154,7 @@ fn complex_enum_variant_new<'a>( fn complex_enum_struct_variant_new<'a>( cls: &'a syn::Ident, - variant: &'a PyClassEnumStructVariant<'a>, + variant: PyClassEnumStructVariant<'a>, ctx: &Ctx, ) -> Result { let Ctx { pyo3_path } = ctx; @@ -1162,7 +1184,15 @@ fn complex_enum_struct_variant_new<'a>( } args }; - let signature = crate::pyfunction::FunctionSignature::from_arguments(args)?; + + let signature = if let Some(constructor) = variant.options.constructor { + crate::pyfunction::FunctionSignature::from_arguments_and_attribute( + args, + constructor.into_signature(), + )? + } else { + crate::pyfunction::FunctionSignature::from_arguments(args)? + }; let spec = FnSpec { tp: crate::method::FnType::FnNew, diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 7c355533b83..e259f0e2c1e 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -18,7 +18,7 @@ use syn::{ mod signature; -pub use self::signature::{FunctionSignature, SignatureAttribute}; +pub use self::signature::{ConstructorAttribute, FunctionSignature, SignatureAttribute}; #[derive(Clone, Debug)] pub struct PyFunctionArgPyO3Attributes { diff --git a/pyo3-macros-backend/src/pyfunction/signature.rs b/pyo3-macros-backend/src/pyfunction/signature.rs index 3daa79c89f5..b73b96a3d59 100644 --- a/pyo3-macros-backend/src/pyfunction/signature.rs +++ b/pyo3-macros-backend/src/pyfunction/signature.rs @@ -195,6 +195,16 @@ impl ToTokens for SignatureItemPosargsSep { } pub type SignatureAttribute = KeywordAttribute; +pub type ConstructorAttribute = KeywordAttribute; + +impl ConstructorAttribute { + pub fn into_signature(self) -> SignatureAttribute { + SignatureAttribute { + kw: kw::signature(self.kw.span), + value: self.value, + } + } +} #[derive(Default)] pub struct PythonSignature { diff --git a/pytests/src/enums.rs b/pytests/src/enums.rs index 0a1bc49bb63..68a5fc93dfe 100644 --- a/pytests/src/enums.rs +++ b/pytests/src/enums.rs @@ -39,11 +39,26 @@ pub fn do_simple_stuff(thing: &SimpleEnum) -> SimpleEnum { #[pyclass] pub enum ComplexEnum { - Int { i: i32 }, - Float { f: f64 }, - Str { s: String }, + Int { + i: i32, + }, + Float { + f: f64, + }, + Str { + s: String, + }, EmptyStruct {}, - MultiFieldStruct { a: i32, b: f64, c: bool }, + MultiFieldStruct { + a: i32, + b: f64, + c: bool, + }, + #[pyo3(constructor = (a = 42, b = None))] + VariantWithDefault { + a: i32, + b: Option, + }, } #[pyfunction] @@ -58,5 +73,9 @@ pub fn do_complex_stuff(thing: &ComplexEnum) -> ComplexEnum { b: *b, c: *c, }, + ComplexEnum::VariantWithDefault { a, b } => ComplexEnum::VariantWithDefault { + a: 2 * a, + b: b.as_ref().map(|s| s.to_uppercase()), + }, } } diff --git a/pytests/tests/test_enums.py b/pytests/tests/test_enums.py index 04b0cdca431..cd1d7aedaf8 100644 --- a/pytests/tests/test_enums.py +++ b/pytests/tests/test_enums.py @@ -18,6 +18,12 @@ def test_complex_enum_variant_constructors(): multi_field_struct_variant = enums.ComplexEnum.MultiFieldStruct(42, 3.14, True) assert isinstance(multi_field_struct_variant, enums.ComplexEnum.MultiFieldStruct) + variant_with_default_1 = enums.ComplexEnum.VariantWithDefault() + assert isinstance(variant_with_default_1, enums.ComplexEnum.VariantWithDefault) + + variant_with_default_2 = enums.ComplexEnum.VariantWithDefault(25, "Hello") + assert isinstance(variant_with_default_2, enums.ComplexEnum.VariantWithDefault) + @pytest.mark.parametrize( "variant", @@ -27,6 +33,7 @@ def test_complex_enum_variant_constructors(): enums.ComplexEnum.Str("hello"), enums.ComplexEnum.EmptyStruct(), enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + enums.ComplexEnum.VariantWithDefault(), ], ) def test_complex_enum_variant_subclasses(variant: enums.ComplexEnum): @@ -48,6 +55,10 @@ def test_complex_enum_field_getters(): assert multi_field_struct_variant.b == 3.14 assert multi_field_struct_variant.c is True + variant_with_default = enums.ComplexEnum.VariantWithDefault() + assert variant_with_default.a == 42 + assert variant_with_default.b is None + @pytest.mark.parametrize( "variant", @@ -57,6 +68,7 @@ def test_complex_enum_field_getters(): enums.ComplexEnum.Str("hello"), enums.ComplexEnum.EmptyStruct(), enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + enums.ComplexEnum.VariantWithDefault(), ], ) def test_complex_enum_desugared_match(variant: enums.ComplexEnum): @@ -78,6 +90,11 @@ def test_complex_enum_desugared_match(variant: enums.ComplexEnum): assert x == 42 assert y == 3.14 assert z is True + elif isinstance(variant, enums.ComplexEnum.VariantWithDefault): + x = variant.a + y = variant.b + assert x == 42 + assert y is None else: assert False @@ -90,6 +107,7 @@ def test_complex_enum_desugared_match(variant: enums.ComplexEnum): enums.ComplexEnum.Str("hello"), enums.ComplexEnum.EmptyStruct(), enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + enums.ComplexEnum.VariantWithDefault(b="hello"), ], ) def test_complex_enum_pyfunction_in_out_desugared_match(variant: enums.ComplexEnum): @@ -112,5 +130,10 @@ def test_complex_enum_pyfunction_in_out_desugared_match(variant: enums.ComplexEn assert x == 42 assert y == 3.14 assert z is True + elif isinstance(variant, enums.ComplexEnum.VariantWithDefault): + x = variant.a + y = variant.b + assert x == 84 + assert y == "HELLO" else: assert False diff --git a/tests/ui/invalid_pyclass_enum.rs b/tests/ui/invalid_pyclass_enum.rs index 95879c2fbd1..116b8968da8 100644 --- a/tests/ui/invalid_pyclass_enum.rs +++ b/tests/ui/invalid_pyclass_enum.rs @@ -27,4 +27,11 @@ enum NoTupleVariants { TupleVariant(i32), } +#[pyclass] +enum SimpleNoSignature { + #[pyo3(constructor = (a, b))] + A, + B, +} + fn main() {} diff --git a/tests/ui/invalid_pyclass_enum.stderr b/tests/ui/invalid_pyclass_enum.stderr index a03a0ae2814..e9ba9806da8 100644 --- a/tests/ui/invalid_pyclass_enum.stderr +++ b/tests/ui/invalid_pyclass_enum.stderr @@ -31,3 +31,9 @@ error: Tuple variant `TupleVariant` is not yet supported in a complex enum | 27 | TupleVariant(i32), | ^^^^^^^^^^^^ + +error: `constructor` can't be used on a simple enum variant + --> tests/ui/invalid_pyclass_enum.rs:32:12 + | +32 | #[pyo3(constructor = (a, b))] + | ^^^^^^^^^^^ From 21c02484d06354c389c9a3405083a3d4a03e1126 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 10 May 2024 00:21:48 +0200 Subject: [PATCH 327/349] feature gate APIs using `into_gil_ref` (Part 2) (#4166) --- guide/src/conversions/traits.md | 6 ++--- guide/src/exception.md | 4 +-- guide/src/memory.md | 8 ++++++ guide/src/migration.md | 2 +- guide/src/types.md | 2 ++ pytests/src/pyclasses.rs | 26 +----------------- pytests/src/sequence.rs | 2 +- pytests/tests/test_pyclasses.py | 11 -------- src/conversion.rs | 1 + src/conversions/chrono.rs | 2 +- src/conversions/std/osstr.rs | 5 ++-- src/conversions/std/path.rs | 5 ++-- src/derive_utils.rs | 2 +- src/err/mod.rs | 1 + src/exceptions.rs | 6 ++++- src/impl_/deprecations.rs | 32 +++++++--------------- src/impl_/extract_argument.rs | 12 ++++----- src/instance.rs | 4 +-- src/pycell.rs | 4 +-- src/tests/hygiene/pyfunction.rs | 1 + src/types/any.rs | 15 ++++++----- src/types/bytearray.rs | 4 ++- src/types/float.rs | 2 +- src/types/iterator.rs | 4 +-- src/types/memoryview.rs | 4 +-- src/types/mod.rs | 14 +++++++--- src/types/num.rs | 2 +- tests/test_class_basics.rs | 15 +---------- tests/test_compile_error.rs | 10 +++++-- tests/test_methods.rs | 7 ----- tests/test_module.rs | 33 ----------------------- tests/test_proto_methods.rs | 4 +-- tests/ui/deprecations.stderr | 6 ----- tests/ui/invalid_result_conversion.stderr | 2 +- 34 files changed, 93 insertions(+), 165 deletions(-) diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index 65a5d150e79..95d16faaaa6 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -265,7 +265,7 @@ use pyo3::prelude::*; #[derive(FromPyObject)] # #[derive(Debug)] -enum RustyEnum<'a> { +enum RustyEnum<'py> { Int(usize), // input is a positive int String(String), // input is a string IntTuple(usize, usize), // input is a 2-tuple with positive ints @@ -284,7 +284,7 @@ enum RustyEnum<'a> { b: usize, }, #[pyo3(transparent)] - CatchAll(&'a PyAny), // This extraction never fails + CatchAll(Bound<'py, PyAny>), // This extraction never fails } # # use pyo3::types::{PyBytes, PyString}; @@ -394,7 +394,7 @@ enum RustyEnum<'a> { # assert_eq!( # b"text", # match rust_thing { -# RustyEnum::CatchAll(i) => i.downcast::()?.as_bytes(), +# RustyEnum::CatchAll(ref i) => i.downcast::()?.as_bytes(), # other => unreachable!("Error extracting: {:?}", other), # } # ); diff --git a/guide/src/exception.md b/guide/src/exception.md index 3e2f5034897..1a68e24086f 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -128,5 +128,5 @@ defines exceptions for several standard library modules. [`PyErr`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html [`PyResult`]: {{#PYO3_DOCS_URL}}/pyo3/type.PyResult.html [`PyErr::from_value`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html#method.from_value -[`PyAny::is_instance`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.is_instance -[`PyAny::is_instance_of`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.is_instance_of +[`PyAny::is_instance`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.is_instance +[`PyAny::is_instance_of`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.is_instance_of diff --git a/guide/src/memory.md b/guide/src/memory.md index a6640e65cf3..67a78d3be68 100644 --- a/guide/src/memory.md +++ b/guide/src/memory.md @@ -34,9 +34,11 @@ held. (If PyO3 could not assume this, every PyO3 API would need to take a very simple and easy-to-understand programs like this: ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { +# #[cfg(feature = "gil-refs")] Python::with_gil(|py| -> PyResult<()> { #[allow(deprecated)] // py.eval() is part of the GIL Refs API let hello = py @@ -57,9 +59,11 @@ it owns are decreased, releasing them to the Python garbage collector. Most of the time we don't have to think about this, but consider the following: ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { +# #[cfg(feature = "gil-refs")] Python::with_gil(|py| -> PyResult<()> { for _ in 0..10 { #[allow(deprecated)] // py.eval() is part of the GIL Refs API @@ -96,9 +100,11 @@ In general we don't want unbounded memory growth during loops! One workaround is to acquire and release the GIL with each iteration of the loop. ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { +# #[cfg(feature = "gil-refs")] for _ in 0..10 { Python::with_gil(|py| -> PyResult<()> { #[allow(deprecated)] // py.eval() is part of the GIL Refs API @@ -118,9 +124,11 @@ times. Another workaround is to work with the `GILPool` object directly, but this is unsafe. ```rust +# #![allow(unused_imports)] # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { +# #[cfg(feature = "gil-refs")] Python::with_gil(|py| -> PyResult<()> { for _ in 0..10 { #[allow(deprecated)] // `new_pool` is not needed in code not using the GIL Refs API diff --git a/guide/src/migration.md b/guide/src/migration.md index 0a048bf02bc..2317f85185e 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -54,7 +54,7 @@ pyo3 = { version = "0.21", features = ["gil-refs"] } The `PyTryFrom` trait has aged poorly, its `try_from` method now conflicts with `TryFrom::try_from` in the 2021 edition prelude. A lot of its functionality was also duplicated with `PyTypeInfo`. -To tighten up the PyO3 traits as part of the deprecation of the GIL Refs API the `PyTypeInfo` trait has had a simpler companion `PyTypeCheck`. The methods [`PyAny::downcast`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast) and [`PyAny::downcast_exact`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html#method.downcast_exact) no longer use `PyTryFrom` as a bound, instead using `PyTypeCheck` and `PyTypeInfo` respectively. +To tighten up the PyO3 traits as part of the deprecation of the GIL Refs API the `PyTypeInfo` trait has had a simpler companion `PyTypeCheck`. The methods `PyAny::downcast` and `PyAny::downcast_exact` no longer use `PyTryFrom` as a bound, instead using `PyTypeCheck` and `PyTypeInfo` respectively. To migrate, switch all type casts to use `obj.downcast()` instead of `try_from(obj)` (and similar for `downcast_exact`). diff --git a/guide/src/types.md b/guide/src/types.md index d28fb7e15d6..20e5e76a330 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -467,8 +467,10 @@ let _: &mut MyClass = &mut *py_ref_mut; `PyCell` was also accessed like a Python-native type. ```rust +#![allow(unused_imports)] # use pyo3::prelude::*; # #[pyclass] struct MyClass { } +# #[cfg(feature = "gil-refs")] # Python::with_gil(|py| -> PyResult<()> { #[allow(deprecated)] // &PyCell is part of the deprecate GIL Refs API let cell: &PyCell = PyCell::new(py, MyClass {})?; diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index ac817627cfe..6338596b481 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -63,30 +63,6 @@ impl AssertingBaseClass { } } -#[allow(deprecated)] -mod deprecated { - use super::*; - - #[pyclass(subclass)] - #[derive(Clone, Debug)] - pub struct AssertingBaseClassGilRef; - - #[pymethods] - impl AssertingBaseClassGilRef { - #[new] - #[classmethod] - fn new(cls: &PyType, expected_type: &PyType) -> PyResult { - if !cls.is(expected_type) { - return Err(PyValueError::new_err(format!( - "{:?} != {:?}", - cls, expected_type - ))); - } - Ok(Self) - } - } -} - #[pyclass] struct ClassWithoutConstructor; @@ -95,7 +71,7 @@ pub fn pyclasses(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; m.add_class::()?; + Ok(()) } diff --git a/pytests/src/sequence.rs b/pytests/src/sequence.rs index 0e48a161bd3..f552b4048b8 100644 --- a/pytests/src/sequence.rs +++ b/pytests/src/sequence.rs @@ -12,7 +12,7 @@ fn array_to_array_i32(arr: [i32; 3]) -> [i32; 3] { } #[pyfunction] -fn vec_to_vec_pystring(vec: Vec<&PyString>) -> Vec<&PyString> { +fn vec_to_vec_pystring(vec: Vec>) -> Vec> { vec } diff --git a/pytests/tests/test_pyclasses.py b/pytests/tests/test_pyclasses.py index 74f883e3808..efef178d489 100644 --- a/pytests/tests/test_pyclasses.py +++ b/pytests/tests/test_pyclasses.py @@ -65,17 +65,6 @@ def test_new_classmethod(): _ = AssertingSubClass(expected_type=str) -def test_new_classmethod_gil_ref(): - class AssertingSubClass(pyclasses.AssertingBaseClassGilRef): - pass - - # The `AssertingBaseClass` constructor errors if it is not passed the - # relevant subclass. - _ = AssertingSubClass(expected_type=AssertingSubClass) - with pytest.raises(ValueError): - _ = AssertingSubClass(expected_type=str) - - class ClassWithoutConstructorPy: def __new__(cls): raise TypeError("No constructor defined") diff --git a/src/conversion.rs b/src/conversion.rs index 8644db84289..6e116af7303 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -345,6 +345,7 @@ where } #[allow(deprecated)] +#[cfg(feature = "gil-refs")] impl<'py, T> FromPyObject<'py> for &'py crate::PyCell where T: PyClass, diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 544d1cf2663..2e220681951 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -347,7 +347,7 @@ impl FromPyObject<'_> for FixedOffset { /// does not supports microseconds. fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { #[cfg(not(Py_LIMITED_API))] - let ob: &PyTzInfo = ob.extract()?; + let ob = ob.downcast::()?; #[cfg(Py_LIMITED_API)] check_type(ob, &DatetimeTypes::get(ob.py()).tzinfo, "PyTzInfo")?; diff --git a/src/conversions/std/osstr.rs b/src/conversions/std/osstr.rs index b9382688589..4565c3fbd94 100644 --- a/src/conversions/std/osstr.rs +++ b/src/conversions/std/osstr.rs @@ -147,6 +147,7 @@ impl<'a> IntoPy for &'a OsString { #[cfg(test)] mod tests { + use crate::types::{PyAnyMethods, PyStringMethods}; use crate::{types::PyString, IntoPy, PyObject, Python, ToPyObject}; use std::fmt::Debug; use std::{ @@ -179,7 +180,7 @@ mod tests { Python::with_gil(|py| { fn test_roundtrip + Debug>(py: Python<'_>, obj: T) { let pyobject = obj.to_object(py); - let pystring: &PyString = pyobject.extract(py).unwrap(); + let pystring = pyobject.downcast_bound::(py).unwrap(); assert_eq!(pystring.to_string_lossy(), obj.as_ref().to_string_lossy()); let roundtripped_obj: OsString = pystring.extract().unwrap(); assert_eq!(obj.as_ref(), roundtripped_obj.as_os_str()); @@ -200,7 +201,7 @@ mod tests { obj: T, ) { let pyobject = obj.clone().into_py(py); - let pystring: &PyString = pyobject.extract(py).unwrap(); + let pystring = pyobject.downcast_bound::(py).unwrap(); assert_eq!(pystring.to_string_lossy(), obj.as_ref().to_string_lossy()); let roundtripped_obj: OsString = pystring.extract().unwrap(); assert!(obj.as_ref() == roundtripped_obj.as_os_str()); diff --git a/src/conversions/std/path.rs b/src/conversions/std/path.rs index 5d832e89575..d7f3121ea10 100644 --- a/src/conversions/std/path.rs +++ b/src/conversions/std/path.rs @@ -64,6 +64,7 @@ impl<'a> IntoPy for &'a PathBuf { #[cfg(test)] mod tests { + use crate::types::{PyAnyMethods, PyStringMethods}; use crate::{types::PyString, IntoPy, PyObject, Python, ToPyObject}; use std::borrow::Cow; use std::fmt::Debug; @@ -95,7 +96,7 @@ mod tests { Python::with_gil(|py| { fn test_roundtrip + Debug>(py: Python<'_>, obj: T) { let pyobject = obj.to_object(py); - let pystring: &PyString = pyobject.extract(py).unwrap(); + let pystring = pyobject.downcast_bound::(py).unwrap(); assert_eq!(pystring.to_string_lossy(), obj.as_ref().to_string_lossy()); let roundtripped_obj: PathBuf = pystring.extract().unwrap(); assert_eq!(obj.as_ref(), roundtripped_obj.as_path()); @@ -116,7 +117,7 @@ mod tests { obj: T, ) { let pyobject = obj.clone().into_py(py); - let pystring: &PyString = pyobject.extract(py).unwrap(); + let pystring = pyobject.downcast_bound::(py).unwrap(); assert_eq!(pystring.to_string_lossy(), obj.as_ref().to_string_lossy()); let roundtripped_obj: PathBuf = pystring.extract().unwrap(); assert_eq!(obj.as_ref(), roundtripped_obj.as_path()); diff --git a/src/derive_utils.rs b/src/derive_utils.rs index 4ccb38f901b..a47f489ceb8 100644 --- a/src/derive_utils.rs +++ b/src/derive_utils.rs @@ -13,7 +13,7 @@ impl<'a> PyFunctionArguments<'a> { match self { PyFunctionArguments::Python(py) => (py, None), PyFunctionArguments::PyModule(module) => { - let py = module.py(); + let py = crate::PyNativeType::py(module); (py, Some(module)) } } diff --git a/src/err/mod.rs b/src/err/mod.rs index d923761af1d..200c180e9b0 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -1000,6 +1000,7 @@ where } /// Convert `PyDowncastError` to Python `TypeError`. +#[cfg(feature = "gil-refs")] impl<'a> std::convert::From> for PyErr { fn from(err: PyDowncastError<'_>) -> PyErr { let args = PyDowncastErrorArguments { diff --git a/src/exceptions.rs b/src/exceptions.rs index 367022927c5..b44a5c5a3fe 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -32,6 +32,7 @@ macro_rules! impl_exception_boilerplate { $crate::impl_exception_boilerplate_bound!($name); + #[cfg(feature = "gil-refs")] impl ::std::error::Error for $name { fn source(&self) -> ::std::option::Option<&(dyn ::std::error::Error + 'static)> { unsafe { @@ -58,6 +59,7 @@ macro_rules! impl_exception_boilerplate_bound { /// /// [`PyErr`]: https://docs.rs/pyo3/latest/pyo3/struct.PyErr.html "PyErr in pyo3" #[inline] + #[allow(dead_code)] pub fn new_err(args: A) -> $crate::PyErr where A: $crate::PyErrArguments + ::std::marker::Send + ::std::marker::Sync + 'static, @@ -881,7 +883,9 @@ mod tests { use super::*; use crate::types::any::PyAnyMethods; use crate::types::{IntoPyDict, PyDict}; - use crate::{PyErr, PyNativeType}; + use crate::PyErr; + #[cfg(feature = "gil-refs")] + use crate::PyNativeType; import_exception_bound!(socket, gaierror); import_exception_bound!(email.errors, MessageError); diff --git a/src/impl_/deprecations.rs b/src/impl_/deprecations.rs index 9eb1da05c92..650e01ce729 100644 --- a/src/impl_/deprecations.rs +++ b/src/impl_/deprecations.rs @@ -29,39 +29,27 @@ impl GilRefs { } impl GilRefs> { - #[cfg_attr( - not(feature = "gil-refs"), - deprecated(since = "0.21.0", note = "use `wrap_pyfunction_bound!` instead") - )] + #[deprecated(since = "0.21.0", note = "use `wrap_pyfunction_bound!` instead")] pub fn is_python(&self) {} } impl GilRefs { - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `&Bound<'_, T>` instead for this function argument" - ) + #[deprecated( + since = "0.21.0", + note = "use `&Bound<'_, T>` instead for this function argument" )] pub fn function_arg(&self) {} - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor" - ) + #[deprecated( + since = "0.21.0", + note = "use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor" )] pub fn from_py_with_arg(&self) {} } impl OptionGilRefs> { - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Option<&Bound<'_, T>>` instead for this function argument" - ) + #[deprecated( + since = "0.21.0", + note = "use `Option<&Bound<'_, T>>` instead for this function argument" )] pub fn function_arg(&self) {} } diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index 485b8645086..5f652d75122 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -790,10 +790,8 @@ fn push_parameter_list(msg: &mut String, parameter_names: &[&str]) { #[cfg(test)] mod tests { - use crate::{ - types::{IntoPyDict, PyTuple}, - PyAny, Python, - }; + use crate::types::{IntoPyDict, PyTuple}; + use crate::Python; use super::{push_parameter_list, FunctionDescription, NoVarargs, NoVarkeywords}; @@ -809,7 +807,7 @@ mod tests { }; Python::with_gil(|py| { - let args = PyTuple::new_bound(py, Vec::<&PyAny>::new()); + let args = PyTuple::empty_bound(py); let kwargs = [("foo", 0u8)].into_py_dict_bound(py); let err = unsafe { function_description @@ -840,7 +838,7 @@ mod tests { }; Python::with_gil(|py| { - let args = PyTuple::new_bound(py, Vec::<&PyAny>::new()); + let args = PyTuple::empty_bound(py); let kwargs = [(1u8, 1u8)].into_py_dict_bound(py); let err = unsafe { function_description @@ -871,7 +869,7 @@ mod tests { }; Python::with_gil(|py| { - let args = PyTuple::new_bound(py, Vec::<&PyAny>::new()); + let args = PyTuple::empty_bound(py); let mut output = [None, None]; let err = unsafe { function_description.extract_arguments_tuple_dict::( diff --git a/src/instance.rs b/src/instance.rs index 81ceaa95546..e160de3a314 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1283,7 +1283,7 @@ impl Py { } /// Returns whether `self` and `other` point to the same object. To compare - /// the equality of two objects (the `==` operator), use [`eq`](PyAny::eq). + /// the equality of two objects (the `==` operator), use [`eq`](PyAnyMethods::eq). /// /// This is equivalent to the Python expression `self is other`. #[inline] @@ -2142,7 +2142,7 @@ a = A() fn test_is_ellipsis() { Python::with_gil(|py| { let v = py - .eval("...", None, None) + .eval_bound("...", None, None) .map_err(|e| e.display(py)) .unwrap() .to_object(py); diff --git a/src/pycell.rs b/src/pycell.rs index 80ccff0a030..e0088d9b523 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -41,7 +41,7 @@ //! The [`#[pymethods]`](crate::pymethods) proc macro will generate this wrapper function (and more), //! using [`PyCell`] under the hood: //! -//! ```rust +//! ```rust,ignore //! # use pyo3::prelude::*; //! # #[pyclass] //! # struct Number { @@ -148,7 +148,7 @@ //! ``` //! //! It is better to write that function like this: -//! ```rust +//! ```rust,ignore //! # #![allow(deprecated)] //! # use pyo3::prelude::*; //! # #[pyclass] diff --git a/src/tests/hygiene/pyfunction.rs b/src/tests/hygiene/pyfunction.rs index edc8b6e35d3..c1bca213933 100644 --- a/src/tests/hygiene/pyfunction.rs +++ b/src/tests/hygiene/pyfunction.rs @@ -8,6 +8,7 @@ fn do_something(x: i32) -> crate::PyResult { } #[test] +#[cfg(feature = "gil-refs")] fn invoke_wrap_pyfunction() { crate::Python::with_gil(|py| { #[allow(deprecated)] diff --git a/src/types/any.rs b/src/types/any.rs index 1854308ae7f..17837835be1 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1,15 +1,17 @@ use crate::class::basic::CompareOp; use crate::conversion::{AsPyPointer, FromPyObjectBound, IntoPy, ToPyObject}; -use crate::err::{DowncastError, DowncastIntoError, PyDowncastError, PyErr, PyResult}; +use crate::err::{DowncastError, DowncastIntoError, PyErr, PyResult}; use crate::exceptions::{PyAttributeError, PyTypeError}; use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::Bound; use crate::py_result_ext::PyResultExt; -use crate::type_object::{HasPyGilRef, PyTypeCheck, PyTypeInfo}; +use crate::type_object::{PyTypeCheck, PyTypeInfo}; #[cfg(not(any(PyPy, GraalPy)))] use crate::types::PySuper; use crate::types::{PyDict, PyIterator, PyList, PyString, PyTuple, PyType}; -use crate::{err, ffi, Py, PyNativeType, Python}; +use crate::{err, ffi, Py, Python}; +#[cfg(feature = "gil-refs")] +use crate::{err::PyDowncastError, type_object::HasPyGilRef, PyNativeType}; use std::cell::UnsafeCell; use std::cmp::Ordering; use std::os::raw::c_int; @@ -66,6 +68,7 @@ pyobject_native_type_extract!(PyAny); pyobject_native_type_sized!(PyAny, ffi::PyObject); +#[cfg(feature = "gil-refs")] impl PyAny { /// Returns whether `self` and `other` point to the same object. To compare /// the equality of two objects (the `==` operator), use [`eq`](PyAny::eq). @@ -942,7 +945,7 @@ impl PyAny { #[doc(alias = "PyAny")] pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// Returns whether `self` and `other` point to the same object. To compare - /// the equality of two objects (the `==` operator), use [`eq`](PyAny::eq). + /// the equality of two objects (the `==` operator), use [`eq`](PyAnyMethods::eq). /// /// This is equivalent to the Python expression `self is other`. fn is(&self, other: &T) -> bool; @@ -1589,10 +1592,10 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// Downcast this `PyAny` to a concrete Python type or pyclass (but not a subclass of it). /// - /// It is almost always better to use [`PyAny::downcast`] because it accounts for Python + /// It is almost always better to use [`PyAnyMethods::downcast`] because it accounts for Python /// subtyping. Use this method only when you do not want to allow subtypes. /// - /// The advantage of this method over [`PyAny::downcast`] is that it is faster. The implementation + /// The advantage of this method over [`PyAnyMethods::downcast`] is that it is faster. The implementation /// of `downcast_exact` uses the equivalent of the Python expression `type(self) is T`, whereas /// `downcast` uses `isinstance(self, T)`. /// diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index bdf677c9019..ef20509e629 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -3,7 +3,9 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::{Borrowed, Bound}; use crate::py_result_ext::PyResultExt; use crate::types::any::PyAnyMethods; -use crate::{ffi, AsPyPointer, PyAny, PyNativeType, Python}; +#[cfg(feature = "gil-refs")] +use crate::AsPyPointer; +use crate::{ffi, PyAny, PyNativeType, Python}; use std::os::raw::c_char; use std::slice; diff --git a/src/types/float.rs b/src/types/float.rs index 3a64694a624..2ed3d2921b9 100644 --- a/src/types/float.rs +++ b/src/types/float.rs @@ -11,7 +11,7 @@ use super::any::PyAnyMethods; /// Represents a Python `float` object. /// /// You can usually avoid directly working with this type -/// by using [`ToPyObject`] and [`extract`](PyAny::extract) +/// by using [`ToPyObject`] and [`extract`](PyAnyMethods::extract) /// with `f32`/`f64`. #[repr(transparent)] pub struct PyFloat(PyAny); diff --git a/src/types/iterator.rs b/src/types/iterator.rs index 6131033af7d..4562efde3f8 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -1,9 +1,9 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::Borrowed; use crate::py_result_ext::PyResultExt; -use crate::{ffi, AsPyPointer, Bound, PyAny, PyErr, PyResult, PyTypeCheck}; +use crate::{ffi, Bound, PyAny, PyErr, PyResult, PyTypeCheck}; #[cfg(feature = "gil-refs")] -use crate::{PyDowncastError, PyNativeType}; +use crate::{AsPyPointer, PyDowncastError, PyNativeType}; /// A Python iterator object. /// diff --git a/src/types/memoryview.rs b/src/types/memoryview.rs index 31afb372d7f..320b3f9f70b 100644 --- a/src/types/memoryview.rs +++ b/src/types/memoryview.rs @@ -1,9 +1,9 @@ use crate::err::PyResult; use crate::ffi_ptr_ext::FfiPtrExt; use crate::py_result_ext::PyResultExt; +use crate::{ffi, Bound, PyAny}; #[cfg(feature = "gil-refs")] -use crate::PyNativeType; -use crate::{ffi, AsPyPointer, Bound, PyAny}; +use crate::{AsPyPointer, PyNativeType}; /// Represents a Python `memoryview`. #[repr(transparent)] diff --git a/src/types/mod.rs b/src/types/mod.rs index 38c9238961d..2203ccdf2dc 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -130,7 +130,8 @@ macro_rules! pyobject_native_type_base( fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::result::Result<(), ::std::fmt::Error> { - let s = self.repr().or(::std::result::Result::Err(::std::fmt::Error))?; + use $crate::{PyNativeType, types::{PyAnyMethods, PyStringMethods}}; + let s = self.as_borrowed().repr().or(::std::result::Result::Err(::std::fmt::Error))?; f.write_str(&s.to_string_lossy()) } } @@ -139,19 +140,20 @@ macro_rules! pyobject_native_type_base( fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::result::Result<(), ::std::fmt::Error> { - use $crate::PyNativeType; - match self.str() { + use $crate::{PyNativeType, types::{PyAnyMethods, PyStringMethods, PyTypeMethods}}; + match self.as_borrowed().str() { ::std::result::Result::Ok(s) => return f.write_str(&s.to_string_lossy()), ::std::result::Result::Err(err) => err.write_unraisable_bound(self.py(), ::std::option::Option::Some(&self.as_borrowed())), } - match self.get_type().name() { + match self.as_borrowed().get_type().name() { ::std::result::Result::Ok(name) => ::std::write!(f, "", name), ::std::result::Result::Err(_err) => f.write_str(""), } } } + #[cfg(feature = "gil-refs")] impl<$($generics,)*> $crate::ToPyObject for $name { #[inline] @@ -196,6 +198,7 @@ macro_rules! pyobject_native_type_named ( // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] + #[cfg(feature = "gil-refs")] impl<$($generics,)*> $crate::IntoPy<$crate::Py<$name>> for &'_ $name { #[inline] fn into_py(self, py: $crate::Python<'_>) -> $crate::Py<$name> { @@ -205,6 +208,7 @@ macro_rules! pyobject_native_type_named ( // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] + #[cfg(feature = "gil-refs")] impl<$($generics,)*> ::std::convert::From<&'_ $name> for $crate::Py<$name> { #[inline] fn from(other: &$name) -> Self { @@ -215,6 +219,7 @@ macro_rules! pyobject_native_type_named ( // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] + #[cfg(feature = "gil-refs")] impl<'a, $($generics,)*> ::std::convert::From<&'a $name> for &'a $crate::PyAny { fn from(ob: &'a $name) -> Self { unsafe{&*(ob as *const $name as *const $crate::PyAny)} @@ -271,6 +276,7 @@ macro_rules! pyobject_native_type_extract { ($name:ty $(;$generics:ident)*) => { // FIXME https://github.com/PyO3/pyo3/issues/3903 #[allow(unknown_lints, non_local_definitions)] + #[cfg(feature = "gil-refs")] impl<'py, $($generics,)*> $crate::FromPyObject<'py> for &'py $name { #[inline] fn extract_bound(obj: &$crate::Bound<'py, $crate::PyAny>) -> $crate::PyResult { diff --git a/src/types/num.rs b/src/types/num.rs index 26748f7d1c7..924d4b2c593 100644 --- a/src/types/num.rs +++ b/src/types/num.rs @@ -4,7 +4,7 @@ use crate::{ffi, PyAny}; /// /// You can usually avoid directly working with this type /// by using [`ToPyObject`](crate::conversion::ToPyObject) -/// and [`extract`](PyAny::extract) +/// and [`extract`](super::PyAnyMethods::extract) /// with the primitive Rust integer types. #[repr(transparent)] pub struct PyLong(PyAny); diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 8ff61bd2d6b..b7bee2638ca 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -310,15 +310,6 @@ impl ClassWithFromPyWithMethods { argument } - #[classmethod] - #[cfg(feature = "gil-refs")] - fn classmethod_gil_ref( - _cls: &PyType, - #[pyo3(from_py_with = "PyAny::len")] argument: usize, - ) -> usize { - argument - } - #[staticmethod] fn staticmethod(#[pyo3(from_py_with = "get_length")] argument: usize) -> usize { argument @@ -333,19 +324,15 @@ impl ClassWithFromPyWithMethods { fn test_pymethods_from_py_with() { Python::with_gil(|py| { let instance = Py::new(py, ClassWithFromPyWithMethods {}).unwrap(); - let has_gil_refs = cfg!(feature = "gil-refs"); py_run!( py, - instance - has_gil_refs, + instance, r#" arg = {1: 1, 2: 3} assert instance.instance_method(arg) == 2 assert instance.classmethod(arg) == 2 - if has_gil_refs: - assert instance.classmethod_gil_ref(arg) == 2 assert instance.staticmethod(arg) == 2 assert 42 in instance diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 30e77888cfd..975d26009a5 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -20,7 +20,7 @@ fn test_compile_errors() { t.compile_fail("tests/ui/invalid_pymethod_names.rs"); t.compile_fail("tests/ui/invalid_pymodule_args.rs"); t.compile_fail("tests/ui/reject_generics.rs"); - #[cfg(not(feature = "gil-refs"))] + #[cfg(feature = "gil-refs")] t.compile_fail("tests/ui/deprecations.rs"); t.compile_fail("tests/ui/invalid_closure.rs"); t.compile_fail("tests/ui/pyclass_send.rs"); @@ -38,7 +38,13 @@ fn test_compile_errors() { t.compile_fail("tests/ui/invalid_pymethod_receiver.rs"); t.compile_fail("tests/ui/missing_intopy.rs"); // adding extra error conversion impls changes the output - #[cfg(not(any(windows, feature = "eyre", feature = "anyhow", Py_LIMITED_API)))] + #[cfg(not(any( + windows, + feature = "eyre", + feature = "anyhow", + feature = "gil-refs", + Py_LIMITED_API + )))] t.compile_fail("tests/ui/invalid_result_conversion.rs"); t.compile_fail("tests/ui/not_send.rs"); t.compile_fail("tests/ui/not_send2.rs"); diff --git a/tests/test_methods.rs b/tests/test_methods.rs index 2b5396e9ee4..c1610be3dd6 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -77,13 +77,6 @@ impl ClassMethod { Ok(format!("{}.method()!", cls.qualname()?)) } - #[classmethod] - /// Test class method. - #[cfg(feature = "gil-refs")] - fn method_gil_ref(cls: &PyType) -> PyResult { - Ok(format!("{}.method()!", cls.qualname()?)) - } - #[classmethod] fn method_owned(cls: Py) -> PyResult { let qualname = Python::with_gil(|gil| cls.bind(gil).qualname())?; diff --git a/tests/test_module.rs b/tests/test_module.rs index 5760c3ebaf3..b2487cfd8b3 100644 --- a/tests/test_module.rs +++ b/tests/test_module.rs @@ -371,13 +371,6 @@ fn pyfunction_with_module<'py>(module: &Bound<'py, PyModule>) -> PyResult PyResult<&str> { - module.name() -} - #[pyfunction] #[pyo3(pass_module)] fn pyfunction_with_module_owned( @@ -426,28 +419,14 @@ fn pyfunction_with_module_and_args_kwargs<'py>( .map(|s| (s, args.len(), kwargs.map(|d| d.len()))) } -#[pyfunction] -#[pyo3(pass_module)] -#[cfg(feature = "gil-refs")] -fn pyfunction_with_pass_module_in_attribute(module: &PyModule) -> PyResult<&str> { - module.name() -} - #[pymodule] fn module_with_functions_with_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(pyfunction_with_module, m)?)?; - #[cfg(feature = "gil-refs")] - m.add_function(wrap_pyfunction!(pyfunction_with_module_gil_ref, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_owned, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_and_py, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_and_arg, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_and_default_arg, m)?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module_and_args_kwargs, m)?)?; - #[cfg(feature = "gil-refs")] - m.add_function(wrap_pyfunction!( - pyfunction_with_pass_module_in_attribute, - m - )?)?; m.add_function(wrap_pyfunction!(pyfunction_with_module, m)?)?; Ok(()) } @@ -461,12 +440,6 @@ fn test_module_functions_with_module() { m, "m.pyfunction_with_module() == 'module_with_functions_with_module'" ); - #[cfg(feature = "gil-refs")] - py_assert!( - py, - m, - "m.pyfunction_with_module_gil_ref() == 'module_with_functions_with_module'" - ); py_assert!( py, m, @@ -489,12 +462,6 @@ fn test_module_functions_with_module() { "m.pyfunction_with_module_and_args_kwargs(1, x=1, y=2) \ == ('module_with_functions_with_module', 1, 2)" ); - #[cfg(feature = "gil-refs")] - py_assert!( - py, - m, - "m.pyfunction_with_pass_module_in_attribute() == 'module_with_functions_with_module'" - ); }); } diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index c5d7306086d..5f0fa105e1f 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -247,9 +247,9 @@ fn mapping() { } #[derive(FromPyObject)] -enum SequenceIndex<'a> { +enum SequenceIndex<'py> { Integer(isize), - Slice(&'a PySlice), + Slice(Bound<'py, PySlice>), } #[pyclass] diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index 9c61c26581e..dc21b595743 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -10,12 +10,6 @@ note: the lint level is defined here 1 | #![deny(deprecated)] | ^^^^^^^^^^ -error: use of deprecated struct `pyo3::PyCell`: `PyCell` was merged into `Bound`, use that instead; see the migration guide for more info - --> tests/ui/deprecations.rs:23:30 - | -23 | fn method_gil_ref(_slf: &PyCell) {} - | ^^^^^^ - error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor --> tests/ui/deprecations.rs:42:44 | diff --git a/tests/ui/invalid_result_conversion.stderr b/tests/ui/invalid_result_conversion.stderr index 9695c5e6a19..8da8f49fac3 100644 --- a/tests/ui/invalid_result_conversion.stderr +++ b/tests/ui/invalid_result_conversion.stderr @@ -9,10 +9,10 @@ error[E0277]: the trait bound `PyErr: From` is not satisfied > > > - >> >> >> > + > and $N others = note: required for `MyError` to implement `Into` = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From f3c7b90deff8abf0c3bc2dcfd8c08fa7e0e05a91 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Thu, 9 May 2024 23:22:17 +0100 Subject: [PATCH 328/349] remove function pointer wrappers no longer needed for MSRV (#4167) --- pyo3-macros-backend/src/method.rs | 12 +++---- pyo3-macros-backend/src/pyclass.rs | 2 +- pyo3-macros-backend/src/pyimpl.rs | 2 +- pyo3-macros-backend/src/pymethod.rs | 6 ++-- src/impl_/pyclass/lazy_type_object.rs | 2 +- src/impl_/pymethods.rs | 52 +++++++++++---------------- src/pyclass/create_type_object.rs | 4 +-- src/types/function.rs | 22 +++--------- 8 files changed, 39 insertions(+), 63 deletions(-) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index cb86e8ec606..e1a025b819e 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -828,7 +828,7 @@ impl<'a> FnSpec<'a> { CallingConvention::Noargs => quote! { #pyo3_path::impl_::pymethods::PyMethodDef::noargs( #python_name, - #pyo3_path::impl_::pymethods::PyCFunction({ + { unsafe extern "C" fn trampoline( _slf: *mut #pyo3_path::ffi::PyObject, _args: *mut #pyo3_path::ffi::PyObject, @@ -841,14 +841,14 @@ impl<'a> FnSpec<'a> { ) } trampoline - }), + }, #doc, ) }, CallingConvention::Fastcall => quote! { #pyo3_path::impl_::pymethods::PyMethodDef::fastcall_cfunction_with_keywords( #python_name, - #pyo3_path::impl_::pymethods::PyCFunctionFastWithKeywords({ + { unsafe extern "C" fn trampoline( _slf: *mut #pyo3_path::ffi::PyObject, _args: *const *mut #pyo3_path::ffi::PyObject, @@ -865,14 +865,14 @@ impl<'a> FnSpec<'a> { ) } trampoline - }), + }, #doc, ) }, CallingConvention::Varargs => quote! { #pyo3_path::impl_::pymethods::PyMethodDef::cfunction_with_keywords( #python_name, - #pyo3_path::impl_::pymethods::PyCFunctionWithKeywords({ + { unsafe extern "C" fn trampoline( _slf: *mut #pyo3_path::ffi::PyObject, _args: *mut #pyo3_path::ffi::PyObject, @@ -887,7 +887,7 @@ impl<'a> FnSpec<'a> { ) } trampoline - }), + }, #doc, ) }, diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 3023f897645..179fe71bb9e 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1129,7 +1129,7 @@ pub fn gen_complex_enum_variant_attr( #pyo3_path::class::PyMethodDefType::ClassAttribute({ #pyo3_path::class::PyClassAttributeDef::new( #python_name, - #pyo3_path::impl_::pymethods::PyClassAttributeFactory(#cls_type::#wrapper_ident) + #cls_type::#wrapper_ident ) }) }; diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index cf27cf37066..0cb7631a4df 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -200,7 +200,7 @@ pub fn gen_py_const(cls: &syn::Type, spec: &ConstSpec<'_>, ctx: &Ctx) -> MethodA #pyo3_path::class::PyMethodDefType::ClassAttribute({ #pyo3_path::class::PyClassAttributeDef::new( #python_name, - #pyo3_path::impl_::pymethods::PyClassAttributeFactory(#cls::#wrapper_ident) + #cls::#wrapper_ident ) }) }; diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 1ef137cfcc8..3e8c2980700 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -512,7 +512,7 @@ fn impl_py_class_attribute( #pyo3_path::class::PyMethodDefType::ClassAttribute({ #pyo3_path::class::PyClassAttributeDef::new( #python_name, - #pyo3_path::impl_::pymethods::PyClassAttributeFactory(#cls::#wrapper_ident) + #cls::#wrapper_ident ) }) }; @@ -699,7 +699,7 @@ pub fn impl_py_setter_def( #pyo3_path::class::PyMethodDefType::Setter( #pyo3_path::class::PySetterDef::new( #python_name, - #pyo3_path::impl_::pymethods::PySetter(#cls::#wrapper_ident), + #cls::#wrapper_ident, #doc ) ) @@ -831,7 +831,7 @@ pub fn impl_py_getter_def( #pyo3_path::class::PyMethodDefType::Getter( #pyo3_path::class::PyGetterDef::new( #python_name, - #pyo3_path::impl_::pymethods::PyGetter(#cls::#wrapper_ident), + #cls::#wrapper_ident, #doc ) ) diff --git a/src/impl_/pyclass/lazy_type_object.rs b/src/impl_/pyclass/lazy_type_object.rs index efb6ecf37e6..f83fa4c5186 100644 --- a/src/impl_/pyclass/lazy_type_object.rs +++ b/src/impl_/pyclass/lazy_type_object.rs @@ -153,7 +153,7 @@ impl LazyTypeObjectInner { if let PyMethodDefType::ClassAttribute(attr) = def { let key = attr.attribute_c_string().unwrap(); - match (attr.meth.0)(py) { + match (attr.meth)(py) { Ok(val) => items.push((key, val)), Err(err) => { return Err(wrap_in_runtime_error( diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index df89dba7dbd..2e9dd0ac520 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -69,27 +69,13 @@ pub enum PyMethodDefType { #[derive(Copy, Clone, Debug)] pub enum PyMethodType { - PyCFunction(PyCFunction), - PyCFunctionWithKeywords(PyCFunctionWithKeywords), + PyCFunction(ffi::PyCFunction), + PyCFunctionWithKeywords(ffi::PyCFunctionWithKeywords), #[cfg(not(Py_LIMITED_API))] - PyCFunctionFastWithKeywords(PyCFunctionFastWithKeywords), -} - -// These newtype structs serve no purpose other than wrapping which are function pointers - because -// function pointers aren't allowed in const fn, but types wrapping them are! -#[derive(Clone, Copy, Debug)] -pub struct PyCFunction(pub ffi::PyCFunction); -#[derive(Clone, Copy, Debug)] -pub struct PyCFunctionWithKeywords(pub ffi::PyCFunctionWithKeywords); -#[cfg(not(Py_LIMITED_API))] -#[derive(Clone, Copy, Debug)] -pub struct PyCFunctionFastWithKeywords(pub ffi::_PyCFunctionFastWithKeywords); -#[derive(Clone, Copy)] -pub struct PyGetter(pub Getter); -#[derive(Clone, Copy)] -pub struct PySetter(pub Setter); -#[derive(Clone, Copy)] -pub struct PyClassAttributeFactory(pub for<'p> fn(Python<'p>) -> PyResult); + PyCFunctionFastWithKeywords(ffi::_PyCFunctionFastWithKeywords), +} + +pub type PyClassAttributeFactory = for<'p> fn(Python<'p>) -> PyResult; // TODO: it would be nice to use CStr in these types, but then the constructors can't be const fn // until `CStr::from_bytes_with_nul_unchecked` is const fn. @@ -117,14 +103,14 @@ impl PyClassAttributeDef { #[derive(Clone)] pub struct PyGetterDef { pub(crate) name: &'static str, - pub(crate) meth: PyGetter, + pub(crate) meth: Getter, pub(crate) doc: &'static str, } #[derive(Clone)] pub struct PySetterDef { pub(crate) name: &'static str, - pub(crate) meth: PySetter, + pub(crate) meth: Setter, pub(crate) doc: &'static str, } @@ -136,7 +122,11 @@ unsafe impl Sync for PySetterDef {} impl PyMethodDef { /// Define a function with no `*args` and `**kwargs`. - pub const fn noargs(name: &'static str, cfunction: PyCFunction, doc: &'static str) -> Self { + pub const fn noargs( + name: &'static str, + cfunction: ffi::PyCFunction, + doc: &'static str, + ) -> Self { Self { ml_name: name, ml_meth: PyMethodType::PyCFunction(cfunction), @@ -148,7 +138,7 @@ impl PyMethodDef { /// Define a function that can take `*args` and `**kwargs`. pub const fn cfunction_with_keywords( name: &'static str, - cfunction: PyCFunctionWithKeywords, + cfunction: ffi::PyCFunctionWithKeywords, doc: &'static str, ) -> Self { Self { @@ -163,7 +153,7 @@ impl PyMethodDef { #[cfg(not(Py_LIMITED_API))] pub const fn fastcall_cfunction_with_keywords( name: &'static str, - cfunction: PyCFunctionFastWithKeywords, + cfunction: ffi::_PyCFunctionFastWithKeywords, doc: &'static str, ) -> Self { Self { @@ -182,15 +172,13 @@ impl PyMethodDef { /// Convert `PyMethodDef` to Python method definition struct `ffi::PyMethodDef` pub(crate) fn as_method_def(&self) -> PyResult<(ffi::PyMethodDef, PyMethodDefDestructor)> { let meth = match self.ml_meth { - PyMethodType::PyCFunction(meth) => ffi::PyMethodDefPointer { - PyCFunction: meth.0, - }, + PyMethodType::PyCFunction(meth) => ffi::PyMethodDefPointer { PyCFunction: meth }, PyMethodType::PyCFunctionWithKeywords(meth) => ffi::PyMethodDefPointer { - PyCFunctionWithKeywords: meth.0, + PyCFunctionWithKeywords: meth, }, #[cfg(not(Py_LIMITED_API))] PyMethodType::PyCFunctionFastWithKeywords(meth) => ffi::PyMethodDefPointer { - _PyCFunctionFastWithKeywords: meth.0, + _PyCFunctionFastWithKeywords: meth, }, }; @@ -232,7 +220,7 @@ pub(crate) type Setter = impl PyGetterDef { /// Define a getter. - pub const fn new(name: &'static str, getter: PyGetter, doc: &'static str) -> Self { + pub const fn new(name: &'static str, getter: Getter, doc: &'static str) -> Self { Self { name, meth: getter, @@ -243,7 +231,7 @@ impl PyGetterDef { impl PySetterDef { /// Define a setter. - pub const fn new(name: &'static str, setter: PySetter, doc: &'static str) -> Self { + pub const fn new(name: &'static str, setter: Setter, doc: &'static str) -> Self { Self { name, meth: setter, diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index e90c5736e5c..1b3a9fb1296 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -508,7 +508,7 @@ impl GetSetDefBuilder { self.doc = Some(getter.doc); } // TODO: return an error if getter already defined? - self.getter = Some(getter.meth.0) + self.getter = Some(getter.meth) } fn add_setter(&mut self, setter: &PySetterDef) { @@ -517,7 +517,7 @@ impl GetSetDefBuilder { self.doc = Some(setter.doc); } // TODO: return an error if setter already defined? - self.setter = Some(setter.meth.0) + self.setter = Some(setter.meth) } fn as_get_set_def( diff --git a/src/types/function.rs b/src/types/function.rs index 09c5004b77b..a127b4e0574 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -37,11 +37,7 @@ impl PyCFunction { let (py, module) = py_or_module.into_py_and_maybe_module(); Self::internal_new( py, - &PyMethodDef::cfunction_with_keywords( - name, - pymethods::PyCFunctionWithKeywords(fun), - doc, - ), + &PyMethodDef::cfunction_with_keywords(name, fun, doc), module.map(PyNativeType::as_borrowed).as_deref(), ) .map(Bound::into_gil_ref) @@ -57,11 +53,7 @@ impl PyCFunction { ) -> PyResult> { Self::internal_new( py, - &PyMethodDef::cfunction_with_keywords( - name, - pymethods::PyCFunctionWithKeywords(fun), - doc, - ), + &PyMethodDef::cfunction_with_keywords(name, fun, doc), module, ) } @@ -81,7 +73,7 @@ impl PyCFunction { let (py, module) = py_or_module.into_py_and_maybe_module(); Self::internal_new( py, - &PyMethodDef::noargs(name, pymethods::PyCFunction(fun), doc), + &PyMethodDef::noargs(name, fun, doc), module.map(PyNativeType::as_borrowed).as_deref(), ) .map(Bound::into_gil_ref) @@ -95,11 +87,7 @@ impl PyCFunction { doc: &'static str, module: Option<&Bound<'py, PyModule>>, ) -> PyResult> { - Self::internal_new( - py, - &PyMethodDef::noargs(name, pymethods::PyCFunction(fun), doc), - module, - ) + Self::internal_new(py, &PyMethodDef::noargs(name, fun, doc), module) } /// Deprecated form of [`PyCFunction::new_closure`] @@ -153,7 +141,7 @@ impl PyCFunction { { let method_def = pymethods::PyMethodDef::cfunction_with_keywords( name.unwrap_or("pyo3-closure\0"), - pymethods::PyCFunctionWithKeywords(run_closure::), + run_closure::, doc.unwrap_or("\0"), ); let (def, def_destructor) = method_def.as_method_def()?; From 104328ce14a786a290537c6b2d542419a9e9f514 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 10 May 2024 01:54:08 -0400 Subject: [PATCH 329/349] feature gate deprecated more APIs for `Py` (#4169) --- .../python-from-rust/calling-existing-code.md | 10 ++-- src/err/mod.rs | 2 +- src/macros.rs | 4 +- src/marker.rs | 50 ++++++++----------- 4 files changed, 30 insertions(+), 36 deletions(-) diff --git a/guide/src/python-from-rust/calling-existing-code.md b/guide/src/python-from-rust/calling-existing-code.md index 572f4b4414f..9c0f592451f 100644 --- a/guide/src/python-from-rust/calling-existing-code.md +++ b/guide/src/python-from-rust/calling-existing-code.md @@ -24,9 +24,9 @@ fn main() -> PyResult<()> { } ``` -## Want to run just an expression? Then use `eval`. +## Want to run just an expression? Then use `eval_bound`. -[`Python::eval`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval) is +[`Python::eval_bound`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval_bound) is a method to execute a [Python expression](https://docs.python.org/3.7/reference/expressions.html) and return the evaluated value as a `Bound<'py, PyAny>` object. @@ -47,14 +47,14 @@ Python::with_gil(|py| { # } ``` -## Want to run statements? Then use `run`. +## Want to run statements? Then use `run_bound`. -[`Python::run`] is a method to execute one or more +[`Python::run_bound`] is a method to execute one or more [Python statements](https://docs.python.org/3.7/reference/simple_stmts.html). This method returns nothing (like any Python statement), but you can get access to manipulated objects via the `locals` dict. -You can also use the [`py_run!`] macro, which is a shorthand for [`Python::run`]. +You can also use the [`py_run!`] macro, which is a shorthand for [`Python::run_bound`]. Since [`py_run!`] panics on exceptions, we recommend you use this macro only for quickly testing your Python extensions. diff --git a/src/err/mod.rs b/src/err/mod.rs index 200c180e9b0..52e4b6c616a 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -724,7 +724,7 @@ impl PyErr { /// /// The `category` should be one of the `Warning` classes available in /// [`pyo3::exceptions`](crate::exceptions), or a subclass. The Python - /// object can be retrieved using [`Python::get_type()`]. + /// object can be retrieved using [`Python::get_type_bound()`]. /// /// Example: /// ```rust diff --git a/src/macros.rs b/src/macros.rs index 8bd79408a56..d6f25c37308 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -2,10 +2,10 @@ /// /// # Panics /// -/// This macro internally calls [`Python::run`](crate::Python::run) and panics +/// This macro internally calls [`Python::run_bound`](crate::Python::run_bound) and panics /// if it returns `Err`, after printing the error to stdout. /// -/// If you need to handle failures, please use [`Python::run`](crate::marker::Python::run) instead. +/// If you need to handle failures, please use [`Python::run_bound`](crate::marker::Python::run_bound) instead. /// /// # Examples /// ``` diff --git a/src/marker.rs b/src/marker.rs index 2230d776236..ea794856d66 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -126,7 +126,9 @@ use crate::types::{ PyAny, PyDict, PyEllipsis, PyModule, PyNone, PyNotImplemented, PyString, PyType, }; use crate::version::PythonVersionInfo; -use crate::{ffi, Bound, IntoPy, Py, PyNativeType, PyObject, PyTypeInfo}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{ffi, Bound, IntoPy, Py, PyObject, PyTypeInfo}; #[allow(deprecated)] use crate::{gil::GILPool, FromPyPointer}; use std::ffi::{CStr, CString}; @@ -305,7 +307,7 @@ pub use nightly::Ungil; /// A marker token that represents holding the GIL. /// /// It serves three main purposes: -/// - It provides a global API for the Python interpreter, such as [`Python::eval`]. +/// - It provides a global API for the Python interpreter, such as [`Python::eval_bound`]. /// - It can be passed to functions that require a proof of holding the GIL, such as /// [`Py::clone_ref`]. /// - Its lifetime represents the scope of holding the GIL which can be used to create Rust @@ -321,7 +323,7 @@ pub use nightly::Ungil; /// - In a function or method annotated with [`#[pyfunction]`](crate::pyfunction) or [`#[pymethods]`](crate::pymethods) you can declare it /// as a parameter, and PyO3 will pass in the token when Python code calls it. /// - If you already have something with a lifetime bound to the GIL, such as `&`[`PyAny`], you can -/// use its [`.py()`][PyAny::py] method to get a token. +/// use its `.py()` method to get a token. /// - When you need to acquire the GIL yourself, such as when calling Python code from Rust, you /// should call [`Python::with_gil`] to do that and pass your code as a closure to it. /// @@ -352,7 +354,7 @@ pub use nightly::Ungil; /// # Releasing and freeing memory /// /// The [`Python`] type can be used to create references to variables owned by the Python -/// interpreter, using functions such as [`Python::eval`] and `PyModule::import`. These +/// interpreter, using functions such as `Python::eval` and `PyModule::import`. These /// references are tied to a [`GILPool`] whose references are not cleared until it is dropped. /// This can cause apparent "memory leaks" if it is kept around for a long time. /// @@ -552,12 +554,10 @@ impl<'py> Python<'py> { } /// Deprecated version of [`Python::eval_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`Python::eval` will be replaced by `Python::eval_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`Python::eval` will be replaced by `Python::eval_bound` in a future PyO3 version" )] pub fn eval( self, @@ -601,12 +601,10 @@ impl<'py> Python<'py> { } /// Deprecated version of [`Python::run_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`Python::run` will be replaced by `Python::run_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`Python::run` will be replaced by `Python::run_bound` in a future PyO3 version" )] pub fn run( self, @@ -728,12 +726,10 @@ impl<'py> Python<'py> { } /// Gets the Python type object for type `T`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`Python::get_type` will be replaced by `Python::get_type_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`Python::get_type` will be replaced by `Python::get_type_bound` in a future PyO3 version" )] #[inline] pub fn get_type(self) -> &'py PyType @@ -753,12 +749,10 @@ impl<'py> Python<'py> { } /// Deprecated form of [`Python::import_bound`] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`Python::import` will be replaced by `Python::import_bound` in a future PyO3 version" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "`Python::import` will be replaced by `Python::import_bound` in a future PyO3 version" )] pub fn import(self, name: N) -> PyResult<&'py PyModule> where From aef0a05719db45caba0ab90315d1b250a690e35b Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 10 May 2024 12:34:58 +0200 Subject: [PATCH 330/349] deprecate implicit default for trailing optional arguments (#4078) * deprecate "trailing optional arguments" implicit default behaviour * add newsfragment * generate individual deprecation messages per function * add migration guide entry --- guide/src/async-await.md | 1 + guide/src/function/signature.md | 13 +++++ guide/src/migration.md | 35 ++++++++++++ newsfragments/4078.changed.md | 1 + pyo3-macros-backend/src/deprecations.rs | 53 +++++++++++++++++- pyo3-macros-backend/src/method.rs | 7 +++ pyo3-macros-backend/src/pymethod.rs | 4 ++ pytests/src/datetime.rs | 3 ++ src/tests/hygiene/pymethods.rs | 1 + tests/test_arithmetics.rs | 1 + tests/test_mapping.rs | 2 + tests/test_methods.rs | 11 ++++ tests/test_pyfunction.rs | 3 ++ tests/test_sequence.rs | 1 + tests/test_text_signature.rs | 1 + tests/ui/deprecations.rs | 26 +++++++++ tests/ui/deprecations.stderr | 72 ++++++++++++++++--------- 17 files changed, 210 insertions(+), 25 deletions(-) create mode 100644 newsfragments/4078.changed.md diff --git a/guide/src/async-await.md b/guide/src/async-await.md index 06fa1580ad7..27574181804 100644 --- a/guide/src/async-await.md +++ b/guide/src/async-await.md @@ -12,6 +12,7 @@ use futures::channel::oneshot; use pyo3::prelude::*; #[pyfunction] +#[pyo3(signature=(seconds, result=None))] async fn sleep(seconds: f64, result: Option) -> Option { let (tx, rx) = oneshot::channel(); thread::spawn(move || { diff --git a/guide/src/function/signature.md b/guide/src/function/signature.md index b276fc457fb..69949220be6 100644 --- a/guide/src/function/signature.md +++ b/guide/src/function/signature.md @@ -121,9 +121,22 @@ num=-1 ## Trailing optional arguments +
+ +⚠️ Warning: This behaviour is being phased out 🛠️ + +The special casing of trailing optional arguments is deprecated. In a future `pyo3` version, arguments of type `Option<..>` will share the same behaviour as other arguments, they are required unless a default is set using `#[pyo3(signature = (...))]`. + +This is done to better align the Python and Rust definition of such functions and make it more intuitive to rewrite them from Python in Rust. Specifically `def some_fn(a: int, b: Optional[int]): ...` will not automatically default `b` to `none`, but requires an explicit default if desired, where as in current `pyo3` it is handled the other way around. + +During the migration window a `#[pyo3(signature = (...))]` will be required to silence the deprecation warning. After support for trailing optional arguments is fully removed, the signature attribute can be removed if all arguments should be required. +
+ + As a convenience, functions without a `#[pyo3(signature = (...))]` option will treat trailing `Option` arguments as having a default of `None`. In the example below, PyO3 will create `increment` with a signature of `increment(x, amount=None)`. ```rust +#![allow(deprecated)] use pyo3::prelude::*; /// Returns a copy of `x` increased by `amount`. diff --git a/guide/src/migration.md b/guide/src/migration.md index 2317f85185e..875407317bf 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -3,6 +3,41 @@ This guide can help you upgrade code through breaking changes from one PyO3 version to the next. For a detailed list of all changes, see the [CHANGELOG](changelog.md). +## from 0.21.* to 0.22 + +### Deprecation of implicit default for trailing optional arguments +
+Click to expand + +With `pyo3` 0.22 the implicit `None` default for trailing `Option` type argument is deprecated. To migrate, place a `#[pyo3(signature = (...))]` attribute on affected functions or methods and specify the desired behavior. +The migration warning specifies the corresponding signature to keep the current behavior. With 0.23 the signature will be required for any function containing `Option` type parameters to prevent accidental +and unnoticed changes in behavior. With 0.24 this restriction will be lifted again and `Option` type arguments will be treated as any other argument _without_ special handling. + +Before: + +```rust +# #![allow(deprecated, dead_code)] +# use pyo3::prelude::*; +#[pyfunction] +fn increment(x: u64, amount: Option) -> u64 { + x + amount.unwrap_or(1) +} +``` + +After: + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyfunction] +#[pyo3(signature = (x, amount=None))] +fn increment(x: u64, amount: Option) -> u64 { + x + amount.unwrap_or(1) +} +``` + +
+ ## from 0.20.* to 0.21
Click to expand diff --git a/newsfragments/4078.changed.md b/newsfragments/4078.changed.md new file mode 100644 index 00000000000..45f160f5556 --- /dev/null +++ b/newsfragments/4078.changed.md @@ -0,0 +1 @@ +deprecate implicit default for trailing optional arguments diff --git a/pyo3-macros-backend/src/deprecations.rs b/pyo3-macros-backend/src/deprecations.rs index 3f1f34144f6..4db40cc86f7 100644 --- a/pyo3-macros-backend/src/deprecations.rs +++ b/pyo3-macros-backend/src/deprecations.rs @@ -1,4 +1,7 @@ -use crate::utils::Ctx; +use crate::{ + method::{FnArg, FnSpec}, + utils::Ctx, +}; use proc_macro2::{Span, TokenStream}; use quote::{quote_spanned, ToTokens}; @@ -45,3 +48,51 @@ impl<'ctx> ToTokens for Deprecations<'ctx> { } } } + +pub(crate) fn deprecate_trailing_option_default(spec: &FnSpec<'_>) -> TokenStream { + if spec.signature.attribute.is_none() + && spec.signature.arguments.iter().any(|arg| { + if let FnArg::Regular(arg) = arg { + arg.option_wrapped_type.is_some() + } else { + false + } + }) + { + use std::fmt::Write; + let mut deprecation_msg = String::from( + "This function has implicit defaults for the trailing `Option` arguments. \ + These implicit defaults are being phased out. Add `#[pyo3(signature = (", + ); + spec.signature.arguments.iter().for_each(|arg| { + match arg { + FnArg::Regular(arg) => { + if arg.option_wrapped_type.is_some() { + write!(deprecation_msg, "{}=None, ", arg.name) + } else { + write!(deprecation_msg, "{}, ", arg.name) + } + } + FnArg::VarArgs(arg) => write!(deprecation_msg, "{}, ", arg.name), + FnArg::KwArgs(arg) => write!(deprecation_msg, "{}, ", arg.name), + FnArg::Py(_) | FnArg::CancelHandle(_) => Ok(()), + } + .expect("writing to `String` should not fail"); + }); + + //remove trailing space and comma + deprecation_msg.pop(); + deprecation_msg.pop(); + + deprecation_msg + .push_str(")]` to this function to silence this warning and keep the current behavior"); + quote_spanned! { spec.name.span() => + #[deprecated(note = #deprecation_msg)] + #[allow(dead_code)] + const SIGNATURE: () = (); + const _: () = SIGNATURE; + } + } else { + TokenStream::new() + } +} diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index e1a025b819e..c0e38bf8416 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -5,6 +5,7 @@ use proc_macro2::{Span, TokenStream}; use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::{ext::IdentExt, spanned::Spanned, Ident, Result}; +use crate::deprecations::deprecate_trailing_option_default; use crate::utils::Ctx; use crate::{ attributes::{FromPyWithAttribute, TextSignatureAttribute, TextSignatureAttributeValue}, @@ -708,6 +709,8 @@ impl<'a> FnSpec<'a> { quote!(#func_name) }; + let deprecation = deprecate_trailing_option_default(self); + Ok(match self.convention { CallingConvention::Noargs => { let mut holders = Holders::new(); @@ -730,6 +733,7 @@ impl<'a> FnSpec<'a> { py: #pyo3_path::Python<'py>, _slf: *mut #pyo3_path::ffi::PyObject, ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { + #deprecation let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 #init_holders @@ -754,6 +758,7 @@ impl<'a> FnSpec<'a> { _nargs: #pyo3_path::ffi::Py_ssize_t, _kwnames: *mut #pyo3_path::ffi::PyObject ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { + #deprecation let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert @@ -778,6 +783,7 @@ impl<'a> FnSpec<'a> { _args: *mut #pyo3_path::ffi::PyObject, _kwargs: *mut #pyo3_path::ffi::PyObject ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { + #deprecation let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert @@ -805,6 +811,7 @@ impl<'a> FnSpec<'a> { _kwargs: *mut #pyo3_path::ffi::PyObject ) -> #pyo3_path::PyResult<*mut #pyo3_path::ffi::PyObject> { use #pyo3_path::callback::IntoPyCallbackOutput; + #deprecation let _slf_ref = &_slf; let function = #rust_name; // Shadow the function name to avoid #3017 #arg_convert diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 3e8c2980700..208735f2619 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -1,6 +1,7 @@ use std::borrow::Cow; use crate::attributes::{NameAttribute, RenamingRule}; +use crate::deprecations::deprecate_trailing_option_default; use crate::method::{CallingConvention, ExtractErrorMode, PyArg}; use crate::params::{check_arg_for_gil_refs, impl_regular_arg_param, Holders}; use crate::utils::Ctx; @@ -637,7 +638,10 @@ pub fn impl_py_setter_def( ); let extract = check_arg_for_gil_refs(tokens, holders.push_gil_refs_checker(arg.ty.span()), ctx); + + let deprecation = deprecate_trailing_option_default(spec); quote! { + #deprecation #from_py_with let _val = #extract; } diff --git a/pytests/src/datetime.rs b/pytests/src/datetime.rs index d0de99ae406..e26782d04f7 100644 --- a/pytests/src/datetime.rs +++ b/pytests/src/datetime.rs @@ -25,6 +25,7 @@ fn date_from_timestamp(py: Python<'_>, timestamp: i64) -> PyResult( py: Python<'py>, hour: u8, @@ -101,6 +102,7 @@ fn get_delta_tuple<'py>(delta: &Bound<'py, PyDelta>) -> Bound<'py, PyTuple> { #[allow(clippy::too_many_arguments)] #[pyfunction] +#[pyo3(signature=(year, month, day, hour, minute, second, microsecond, tzinfo=None))] fn make_datetime<'py>( py: Python<'py>, year: i32, @@ -159,6 +161,7 @@ fn get_datetime_tuple_fold<'py>(dt: &Bound<'py, PyDateTime>) -> Bound<'py, PyTup } #[pyfunction] +#[pyo3(signature=(ts, tz=None))] fn datetime_from_timestamp<'py>( py: Python<'py>, ts: f64, diff --git a/src/tests/hygiene/pymethods.rs b/src/tests/hygiene/pymethods.rs index 020f983be31..95d670c63a6 100644 --- a/src/tests/hygiene/pymethods.rs +++ b/src/tests/hygiene/pymethods.rs @@ -309,6 +309,7 @@ impl Dummy { 0 } + #[pyo3(signature=(ndigits=::std::option::Option::None))] fn __round__(&self, ndigits: ::std::option::Option) -> u32 { 0 } diff --git a/tests/test_arithmetics.rs b/tests/test_arithmetics.rs index b1efcd0ac55..007f42a79e8 100644 --- a/tests/test_arithmetics.rs +++ b/tests/test_arithmetics.rs @@ -35,6 +35,7 @@ impl UnaryArithmetic { Self::new(self.inner.abs()) } + #[pyo3(signature=(_ndigits=None))] fn __round__(&self, _ndigits: Option) -> Self { Self::new(self.inner.round()) } diff --git a/tests/test_mapping.rs b/tests/test_mapping.rs index 675dd14d63f..784ab8845cd 100644 --- a/tests/test_mapping.rs +++ b/tests/test_mapping.rs @@ -21,6 +21,7 @@ struct Mapping { #[pymethods] impl Mapping { #[new] + #[pyo3(signature=(elements=None))] fn new(elements: Option<&Bound<'_, PyList>>) -> PyResult { if let Some(pylist) = elements { let mut elems = HashMap::with_capacity(pylist.len()); @@ -59,6 +60,7 @@ impl Mapping { } } + #[pyo3(signature=(key, default=None))] fn get(&self, py: Python<'_>, key: &str, default: Option) -> Option { self.index .get(key) diff --git a/tests/test_methods.rs b/tests/test_methods.rs index c1610be3dd6..37f3b2d8bd6 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -187,6 +187,7 @@ impl MethSignature { fn get_optional2(&self, test: Option) -> Option { test } + #[pyo3(signature=(_t1 = None, t2 = None, _t3 = None))] fn get_optional_positional( &self, _t1: Option, @@ -745,11 +746,13 @@ impl MethodWithPyClassArg { fn inplace_add_pyref(&self, mut other: PyRefMut<'_, MethodWithPyClassArg>) { other.value += self.value; } + #[pyo3(signature=(other = None))] fn optional_add(&self, other: Option<&MethodWithPyClassArg>) -> MethodWithPyClassArg { MethodWithPyClassArg { value: self.value + other.map(|o| o.value).unwrap_or(10), } } + #[pyo3(signature=(other = None))] fn optional_inplace_add(&self, other: Option<&mut MethodWithPyClassArg>) { if let Some(other) = other { other.value += self.value; @@ -851,6 +854,7 @@ struct FromSequence { #[pymethods] impl FromSequence { #[new] + #[pyo3(signature=(seq = None))] fn new(seq: Option<&Bound<'_, PySequence>>) -> PyResult { if let Some(seq) = seq { Ok(FromSequence { @@ -1026,6 +1030,7 @@ macro_rules! issue_1506 { issue_1506!( #[pymethods] impl Issue1506 { + #[pyo3(signature = (_arg, _args, _kwargs=None))] fn issue_1506( &self, _py: Python<'_>, @@ -1035,6 +1040,7 @@ issue_1506!( ) { } + #[pyo3(signature = (_arg, _args, _kwargs=None))] fn issue_1506_mut( &mut self, _py: Python<'_>, @@ -1044,6 +1050,7 @@ issue_1506!( ) { } + #[pyo3(signature = (_arg, _args, _kwargs=None))] fn issue_1506_custom_receiver( _slf: Py, _py: Python<'_>, @@ -1053,6 +1060,7 @@ issue_1506!( ) { } + #[pyo3(signature = (_arg, _args, _kwargs=None))] fn issue_1506_custom_receiver_explicit( _slf: Py, _py: Python<'_>, @@ -1063,6 +1071,7 @@ issue_1506!( } #[new] + #[pyo3(signature = (_arg, _args, _kwargs=None))] fn issue_1506_new( _py: Python<'_>, _arg: &Bound<'_, PyAny>, @@ -1081,6 +1090,7 @@ issue_1506!( fn issue_1506_setter(&self, _py: Python<'_>, _value: i32) {} #[staticmethod] + #[pyo3(signature = (_arg, _args, _kwargs=None))] fn issue_1506_static( _py: Python<'_>, _arg: &Bound<'_, PyAny>, @@ -1090,6 +1100,7 @@ issue_1506!( } #[classmethod] + #[pyo3(signature = (_arg, _args, _kwargs=None))] fn issue_1506_class( _cls: &Bound<'_, PyType>, _py: Python<'_>, diff --git a/tests/test_pyfunction.rs b/tests/test_pyfunction.rs index 8a57e2707c9..4a90f3f9d99 100644 --- a/tests/test_pyfunction.rs +++ b/tests/test_pyfunction.rs @@ -182,6 +182,7 @@ fn test_from_py_with_defaults() { // issue 2280 combination of from_py_with and Option did not compile #[pyfunction] + #[pyo3(signature = (int=None))] fn from_py_with_option(#[pyo3(from_py_with = "optional_int")] int: Option) -> i32 { int.unwrap_or(0) } @@ -216,6 +217,7 @@ struct ValueClass { } #[pyfunction] +#[pyo3(signature=(str_arg, int_arg, tuple_arg, option_arg = None, struct_arg = None))] fn conversion_error( str_arg: &str, int_arg: i64, @@ -542,6 +544,7 @@ fn test_some_wrap_arguments() { #[test] fn test_reference_to_bound_arguments() { #[pyfunction] + #[pyo3(signature = (x, y = None))] fn reference_args<'py>( x: &Bound<'py, PyAny>, y: Option<&Bound<'py, PyAny>>, diff --git a/tests/test_sequence.rs b/tests/test_sequence.rs index 3715affc05b..9627f06ca75 100644 --- a/tests/test_sequence.rs +++ b/tests/test_sequence.rs @@ -17,6 +17,7 @@ struct ByteSequence { #[pymethods] impl ByteSequence { #[new] + #[pyo3(signature=(elements = None))] fn new(elements: Option<&Bound<'_, PyList>>) -> PyResult { if let Some(pylist) = elements { let mut elems = Vec::with_capacity(pylist.len()); diff --git a/tests/test_text_signature.rs b/tests/test_text_signature.rs index a9f5a041596..3899878bd56 100644 --- a/tests/test_text_signature.rs +++ b/tests/test_text_signature.rs @@ -142,6 +142,7 @@ fn test_auto_test_signature_function() { } #[pyfunction] + #[pyo3(signature=(a, b=None, c=None))] fn my_function_6(a: i32, b: Option, c: Option) { let _ = (a, b, c); } diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index ef0b06652e4..fc9e8687cae 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -39,6 +39,9 @@ impl MyClass { #[setter] fn set_bar_bound(&self, _value: &Bound<'_, PyAny>) {} + #[setter] + fn set_option(&self, _value: Option) {} + fn __eq__(&self, #[pyo3(from_py_with = "extract_gil_ref")] _other: i32) -> bool { true } @@ -103,6 +106,10 @@ fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { obj.extract() } +fn extract_options(obj: &Bound<'_, PyAny>) -> PyResult> { + obj.extract() +} + #[pyfunction] fn pyfunction_from_py_with( #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, @@ -114,8 +121,27 @@ fn pyfunction_from_py_with( fn pyfunction_gil_ref(_any: &PyAny) {} #[pyfunction] +#[pyo3(signature = (_any))] fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} +#[pyfunction] +#[pyo3(signature = (_i, _any=None))] +fn pyfunction_option_1(_i: u32, _any: Option) {} + +#[pyfunction] +fn pyfunction_option_2(_i: u32, _any: Option) {} + +#[pyfunction] +fn pyfunction_option_3(_i: u32, _any: Option, _foo: Option) {} + +#[pyfunction] +fn pyfunction_option_4( + _i: u32, + #[pyo3(from_py_with = "extract_options")] _any: Option, + _foo: Option, +) { +} + #[derive(Debug, FromPyObject)] pub struct Zap { #[pyo3(item)] diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index dc21b595743..b11c0058ce2 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -10,10 +10,34 @@ note: the lint level is defined here 1 | #![deny(deprecated)] | ^^^^^^^^^^ +error: use of deprecated constant `MyClass::__pymethod_set_set_option__::SIGNATURE`: This function has implicit defaults for the trailing `Option` arguments. These implicit defaults are being phased out. Add `#[pyo3(signature = (_value=None)]` to this function to silence this warning and keep the current behavior + --> tests/ui/deprecations.rs:43:8 + | +43 | fn set_option(&self, _value: Option) {} + | ^^^^^^^^^^ + +error: use of deprecated constant `__pyfunction_pyfunction_option_2::SIGNATURE`: This function has implicit defaults for the trailing `Option` arguments. These implicit defaults are being phased out. Add `#[pyo3(signature = (_i, _any=None)]` to this function to silence this warning and keep the current behavior + --> tests/ui/deprecations.rs:132:4 + | +132 | fn pyfunction_option_2(_i: u32, _any: Option) {} + | ^^^^^^^^^^^^^^^^^^^ + +error: use of deprecated constant `__pyfunction_pyfunction_option_3::SIGNATURE`: This function has implicit defaults for the trailing `Option` arguments. These implicit defaults are being phased out. Add `#[pyo3(signature = (_i, _any=None, _foo=None)]` to this function to silence this warning and keep the current behavior + --> tests/ui/deprecations.rs:135:4 + | +135 | fn pyfunction_option_3(_i: u32, _any: Option, _foo: Option) {} + | ^^^^^^^^^^^^^^^^^^^ + +error: use of deprecated constant `__pyfunction_pyfunction_option_4::SIGNATURE`: This function has implicit defaults for the trailing `Option` arguments. These implicit defaults are being phased out. Add `#[pyo3(signature = (_i, _any=None, _foo=None)]` to this function to silence this warning and keep the current behavior + --> tests/ui/deprecations.rs:138:4 + | +138 | fn pyfunction_option_4( + | ^^^^^^^^^^^^^^^^^^^ + error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:42:44 + --> tests/ui/deprecations.rs:45:44 | -42 | fn __eq__(&self, #[pyo3(from_py_with = "extract_gil_ref")] _other: i32) -> bool { +45 | fn __eq__(&self, #[pyo3(from_py_with = "extract_gil_ref")] _other: i32) -> bool { | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument @@ -47,69 +71,69 @@ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg` | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:61:44 + --> tests/ui/deprecations.rs:64:44 | -61 | fn pyfunction_with_module_gil_ref(_module: &PyModule) -> PyResult<&str> { +64 | fn pyfunction_with_module_gil_ref(_module: &PyModule) -> PyResult<&str> { | ^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:71:19 + --> tests/ui/deprecations.rs:74:19 | -71 | fn module_gil_ref(_m: &PyModule) -> PyResult<()> { +74 | fn module_gil_ref(_m: &PyModule) -> PyResult<()> { | ^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:76:57 + --> tests/ui/deprecations.rs:79:57 | -76 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, _m: &PyModule) -> PyResult<()> { +79 | fn module_gil_ref_with_explicit_py_arg(_py: Python<'_>, _m: &PyModule) -> PyResult<()> { | ^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:108:27 + --> tests/ui/deprecations.rs:115:27 | -108 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, +115 | #[pyo3(from_py_with = "extract_gil_ref")] _gil_ref: i32, | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::function_arg`: use `&Bound<'_, T>` instead for this function argument - --> tests/ui/deprecations.rs:114:29 + --> tests/ui/deprecations.rs:121:29 | -114 | fn pyfunction_gil_ref(_any: &PyAny) {} +121 | fn pyfunction_gil_ref(_any: &PyAny) {} | ^ error: use of deprecated method `pyo3::deprecations::OptionGilRefs::>::function_arg`: use `Option<&Bound<'_, T>>` instead for this function argument - --> tests/ui/deprecations.rs:117:36 + --> tests/ui/deprecations.rs:125:36 | -117 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} +125 | fn pyfunction_option_gil_ref(_any: Option<&PyAny>) {} | ^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:124:27 + --> tests/ui/deprecations.rs:150:27 | -124 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] +150 | #[pyo3(from_py_with = "PyAny::len", item("my_object"))] | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:134:27 + --> tests/ui/deprecations.rs:160:27 | -134 | #[pyo3(from_py_with = "PyAny::len")] usize, +160 | #[pyo3(from_py_with = "PyAny::len")] usize, | ^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:140:31 + --> tests/ui/deprecations.rs:166:31 | -140 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), +166 | Zip(#[pyo3(from_py_with = "extract_gil_ref")] i32), | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor - --> tests/ui/deprecations.rs:147:27 + --> tests/ui/deprecations.rs:173:27 | -147 | #[pyo3(from_py_with = "extract_gil_ref")] +173 | #[pyo3(from_py_with = "extract_gil_ref")] | ^^^^^^^^^^^^^^^^^ error: use of deprecated method `pyo3::deprecations::GilRefs::>::is_python`: use `wrap_pyfunction_bound!` instead - --> tests/ui/deprecations.rs:160:13 + --> tests/ui/deprecations.rs:186:13 | -160 | let _ = wrap_pyfunction!(double, py); +186 | let _ = wrap_pyfunction!(double, py); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this error originates in the macro `wrap_pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) From 1e8e09dce3fccadaf19295c2db98004a49cb0f32 Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 10 May 2024 19:03:57 +0200 Subject: [PATCH 331/349] feature gate `as/into_gil_ref` APIs (Part 3) (#4172) --- guide/src/migration.md | 2 +- guide/src/types.md | 2 + src/conversion.rs | 8 +- src/conversions/std/slice.rs | 4 +- src/conversions/std/string.rs | 4 +- src/impl_/frompyobject.rs | 3 + src/impl_/pyfunction.rs | 13 +- src/impl_/pymethods.rs | 11 +- src/instance.rs | 2 + src/pycell.rs | 214 ++++++++++++------------ src/tests/hygiene/misc.rs | 5 +- src/tests/hygiene/pymodule.rs | 12 +- src/types/string.rs | 64 +++---- tests/test_wrap_pyfunction_deduction.rs | 4 + 14 files changed, 196 insertions(+), 152 deletions(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index 875407317bf..7e0420de22d 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -1649,7 +1649,7 @@ However, for `#[pyproto]` and some functions, you need to manually fix the code. In 0.8 object creation was done with `PyRef::new` and `PyRefMut::new`. In 0.9 these have both been removed. To upgrade code, please use -[`PyCell::new`]({{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyCell.html#method.new) instead. +`PyCell::new` instead. If you need [`PyRef`] or [`PyRefMut`], just call `.borrow()` or `.borrow_mut()` on the newly-created `PyCell`. diff --git a/guide/src/types.md b/guide/src/types.md index 20e5e76a330..2a13c241de1 100644 --- a/guide/src/types.md +++ b/guide/src/types.md @@ -446,8 +446,10 @@ Like PyO3's Python native types, the GIL Ref `&PyCell` implements `Deref` was used to access `&T` and `&mut T` via `PyRef` and `PyRefMut` respectively. ```rust +#![allow(unused_imports)] # use pyo3::prelude::*; # #[pyclass] struct MyClass { } +# #[cfg(feature = "gil-refs")] # Python::with_gil(|py| -> PyResult<()> { #[allow(deprecated)] // &PyCell is part of the deprecated GIL Refs API let cell: &PyCell = PyCell::new(py, MyClass {})?; diff --git a/src/conversion.rs b/src/conversion.rs index 6e116af7303..4df19730b43 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -222,9 +222,7 @@ pub trait FromPyObject<'py>: Sized { /// /// Implementors are encouraged to implement this method and leave `extract` defaulted, as /// this will be most compatible with PyO3's future API. - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - Self::extract(ob.clone().into_gil_ref()) - } + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult; /// Extracts the type hint information for this type when it appears as an argument. /// @@ -350,8 +348,8 @@ impl<'py, T> FromPyObject<'py> for &'py crate::PyCell where T: PyClass, { - fn extract(obj: &'py PyAny) -> PyResult { - obj.downcast().map_err(Into::into) + fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult { + obj.clone().into_gil_ref().downcast().map_err(Into::into) } } diff --git a/src/conversions/std/slice.rs b/src/conversions/std/slice.rs index b3932302ef3..9c9cde06fc7 100644 --- a/src/conversions/std/slice.rs +++ b/src/conversions/std/slice.rs @@ -20,8 +20,8 @@ impl<'a> IntoPy for &'a [u8] { #[cfg(feature = "gil-refs")] impl<'py> crate::FromPyObject<'py> for &'py [u8] { - fn extract(obj: &'py PyAny) -> PyResult { - Ok(obj.downcast::()?.as_bytes()) + fn extract_bound(obj: &crate::Bound<'py, PyAny>) -> PyResult { + Ok(obj.clone().into_gil_ref().downcast::()?.as_bytes()) } #[cfg(feature = "experimental-inspect")] diff --git a/src/conversions/std/string.rs b/src/conversions/std/string.rs index 9c276d1d3d9..5bc05c1a091 100644 --- a/src/conversions/std/string.rs +++ b/src/conversions/std/string.rs @@ -116,8 +116,8 @@ impl<'a> IntoPy for &'a String { /// Accepts Python `str` objects. #[cfg(feature = "gil-refs")] impl<'py> FromPyObject<'py> for &'py str { - fn extract(ob: &'py PyAny) -> PyResult { - ob.downcast::()?.to_str() + fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { + ob.clone().into_gil_ref().downcast::()?.to_str() } #[cfg(feature = "experimental-inspect")] diff --git a/src/impl_/frompyobject.rs b/src/impl_/frompyobject.rs index e38ff3c76b2..1e46efaeae3 100644 --- a/src/impl_/frompyobject.rs +++ b/src/impl_/frompyobject.rs @@ -4,6 +4,7 @@ use crate::{exceptions::PyTypeError, FromPyObject, PyAny, PyErr, PyResult, Pytho pub enum Extractor<'a, 'py, T> { Bound(fn(&'a Bound<'py, PyAny>) -> PyResult), + #[cfg(feature = "gil-refs")] GilRef(fn(&'a PyAny) -> PyResult), } @@ -13,6 +14,7 @@ impl<'a, 'py, T> From) -> PyResult> for Extractor<'a } } +#[cfg(feature = "gil-refs")] impl<'a, T> From PyResult> for Extractor<'a, '_, T> { fn from(value: fn(&'a PyAny) -> PyResult) -> Self { Self::GilRef(value) @@ -23,6 +25,7 @@ impl<'a, 'py, T> Extractor<'a, 'py, T> { pub(crate) fn call(self, obj: &'a Bound<'py, PyAny>) -> PyResult { match self { Extractor::Bound(f) => f(obj), + #[cfg(feature = "gil-refs")] Extractor::GilRef(f) => f(obj.as_gil_ref()), } } diff --git a/src/impl_/pyfunction.rs b/src/impl_/pyfunction.rs index cb838fea6c2..0be5174487f 100644 --- a/src/impl_/pyfunction.rs +++ b/src/impl_/pyfunction.rs @@ -1,6 +1,6 @@ use crate::{ types::{PyCFunction, PyModule}, - Borrowed, Bound, PyNativeType, PyResult, Python, + Borrowed, Bound, PyResult, Python, }; pub use crate::impl_::pymethods::PyMethodDef; @@ -37,14 +37,24 @@ impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for &'_ Borrowed<'_, ' // For Python<'py>, only the GIL Ref form exists to avoid causing type inference to kick in. // The `wrap_pyfunction_bound!` macro is needed for the Bound form. +#[cfg(feature = "gil-refs")] impl<'py> WrapPyFunctionArg<'py, &'py PyCFunction> for Python<'py> { fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult<&'py PyCFunction> { PyCFunction::internal_new(self, method_def, None).map(Bound::into_gil_ref) } } +#[cfg(not(feature = "gil-refs"))] +impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for Python<'py> { + fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { + PyCFunction::internal_new(self, method_def, None) + } +} + +#[cfg(feature = "gil-refs")] impl<'py> WrapPyFunctionArg<'py, &'py PyCFunction> for &'py PyModule { fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult<&'py PyCFunction> { + use crate::PyNativeType; PyCFunction::internal_new(self.py(), method_def, Some(&self.as_borrowed())) .map(Bound::into_gil_ref) } @@ -62,6 +72,7 @@ where } } +#[cfg(feature = "gil-refs")] impl<'py> WrapPyFunctionArg<'py, Bound<'py, PyCFunction>> for OnlyBound> { fn wrap_pyfunction(self, method_def: &PyMethodDef) -> PyResult> { PyCFunction::internal_new(self.0, method_def, None) diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 2e9dd0ac520..44b2af25650 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -5,7 +5,9 @@ use crate::impl_::panic::PanicTrap; use crate::internal_tricks::extract_c_string; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; -use crate::types::{any::PyAnyMethods, PyModule, PyType}; +use crate::types::any::PyAnyMethods; +#[cfg(feature = "gil-refs")] +use crate::types::{PyModule, PyType}; use crate::{ ffi, Borrowed, Bound, DowncastError, Py, PyAny, PyClass, PyClassInitializer, PyErr, PyObject, PyRef, PyRefMut, PyResult, PyTraverseError, PyTypeCheck, PyVisit, Python, @@ -492,6 +494,7 @@ impl<'a, 'py> BoundRef<'a, 'py, PyAny> { // GIL Ref implementations for &'a T ran into trouble with orphan rules, // so explicit implementations are used instead for the two relevant types. +#[cfg(feature = "gil-refs")] impl<'a> From> for &'a PyType { #[inline] fn from(bound: BoundRef<'a, 'a, PyType>) -> Self { @@ -499,6 +502,7 @@ impl<'a> From> for &'a PyType { } } +#[cfg(feature = "gil-refs")] impl<'a> From> for &'a PyModule { #[inline] fn from(bound: BoundRef<'a, 'a, PyModule>) -> Self { @@ -507,6 +511,7 @@ impl<'a> From> for &'a PyModule { } #[allow(deprecated)] +#[cfg(feature = "gil-refs")] impl<'a, 'py, T: PyClass> From> for &'a crate::PyCell { #[inline] fn from(bound: BoundRef<'a, 'py, T>) -> Self { @@ -518,7 +523,7 @@ impl<'a, 'py, T: PyClass> TryFrom> for PyRef<'py, T> { type Error = PyBorrowError; #[inline] fn try_from(value: BoundRef<'a, 'py, T>) -> Result { - value.0.clone().into_gil_ref().try_into() + value.0.try_borrow() } } @@ -526,7 +531,7 @@ impl<'a, 'py, T: PyClass> TryFrom> for PyRe type Error = PyBorrowMutError; #[inline] fn try_from(value: BoundRef<'a, 'py, T>) -> Result { - value.0.clone().into_gil_ref().try_into() + value.0.try_borrow_mut() } } diff --git a/src/instance.rs b/src/instance.rs index e160de3a314..1c510b90be5 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -492,6 +492,7 @@ impl<'py, T> Bound<'py, T> { /// /// This is a helper to be used for migration from the deprecated "GIL Refs" API. #[inline] + #[cfg(feature = "gil-refs")] pub fn as_gil_ref(&'py self) -> &'py T::AsRefTarget where T: HasPyGilRef, @@ -507,6 +508,7 @@ impl<'py, T> Bound<'py, T> { /// /// This is a helper to be used for migration from the deprecated "GIL Refs" API. #[inline] + #[cfg(feature = "gil-refs")] pub fn into_gil_ref(self) -> &'py T::AsRefTarget where T: HasPyGilRef, diff --git a/src/pycell.rs b/src/pycell.rs index e0088d9b523..215ed7bba66 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -201,11 +201,12 @@ use crate::pyclass::{ boolean_struct::{False, True}, PyClass, }; -use crate::pyclass_init::PyClassInitializer; use crate::type_object::{PyLayout, PySizedLayout}; use crate::types::any::PyAnyMethods; use crate::types::PyAny; -use crate::{ffi, Bound, IntoPy, PyErr, PyNativeType, PyObject, PyResult, PyTypeCheck, Python}; +use crate::{ffi, Bound, IntoPy, PyErr, PyNativeType, PyObject, PyTypeCheck, Python}; +#[cfg(feature = "gil-refs")] +use crate::{pyclass_init::PyClassInitializer, PyResult}; use std::fmt; use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; @@ -222,7 +223,7 @@ use self::impl_::{PyClassObject, PyClassObjectLayout}; /// # Examples /// /// This example demonstrates getting a mutable reference of the contained `PyClass`. -/// ```rust +/// ```rust,ignore /// use pyo3::prelude::*; /// /// #[pyclass] @@ -272,12 +273,10 @@ impl PyCell { /// /// In cases where the value in the cell does not need to be accessed immediately after /// creation, consider [`Py::new`](crate::Py::new) as a more efficient alternative. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Bound::new(py, value)` or `Py::new(py, value)` instead of `PyCell::new(py, value)`" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `Bound::new(py, value)` or `Py::new(py, value)` instead of `PyCell::new(py, value)`" )] pub fn new(py: Python<'_>, value: impl Into>) -> PyResult<&Self> { Bound::new(py, value).map(Bound::into_gil_ref) @@ -317,7 +316,7 @@ impl PyCell { /// /// # Examples /// - /// ``` + /// ```ignore /// # use pyo3::prelude::*; /// #[pyclass] /// struct Class {} @@ -347,7 +346,7 @@ impl PyCell { /// /// # Examples /// - /// ``` + /// ```ignore /// # use pyo3::prelude::*; /// #[pyclass] /// struct Class {} @@ -380,7 +379,7 @@ impl PyCell { /// /// # Examples /// - /// ``` + /// ```ignore /// # use pyo3::prelude::*; /// #[pyclass] /// struct Class {} @@ -417,7 +416,7 @@ impl PyCell { /// /// # Examples /// - /// ``` + /// ```ignore /// use std::sync::atomic::{AtomicUsize, Ordering}; /// # use pyo3::prelude::*; /// @@ -561,14 +560,14 @@ impl fmt::Debug for PyCell { } } -/// A wrapper type for an immutably borrowed value from a [`PyCell`]``. +/// A wrapper type for an immutably borrowed value from a [`Bound<'py, T>`]. /// -/// See the [`PyCell`] documentation for more information. +/// See the [`Bound`] documentation for more information. /// /// # Examples /// -/// You can use `PyRef` as an alternative to a `&self` receiver when -/// - you need to access the pointer of the `PyCell`, or +/// You can use [`PyRef`] as an alternative to a `&self` receiver when +/// - you need to access the pointer of the [`Bound`], or /// - you want to get a super class. /// ``` /// # use pyo3::prelude::*; @@ -599,7 +598,7 @@ impl fmt::Debug for PyCell { /// } /// # Python::with_gil(|py| { /// # let sub = Py::new(py, Child::new()).unwrap(); -/// # pyo3::py_run!(py, sub, "assert sub.format() == 'Caterpillar(base: Butterfly, cnt: 5)', sub.format()"); +/// # pyo3::py_run!(py, sub, "assert sub.format() == 'Caterpillar(base: Butterfly, cnt: 4)', sub.format()"); /// # }); /// ``` /// @@ -1004,101 +1003,106 @@ mod tests { #[derive(Copy, Clone, PartialEq, Eq, Debug)] struct SomeClass(i32); - #[test] - fn pycell_replace() { - Python::with_gil(|py| { - #[allow(deprecated)] - let cell = PyCell::new(py, SomeClass(0)).unwrap(); - assert_eq!(*cell.borrow(), SomeClass(0)); - - let previous = cell.replace(SomeClass(123)); - assert_eq!(previous, SomeClass(0)); - assert_eq!(*cell.borrow(), SomeClass(123)); - }) - } - - #[test] - #[should_panic(expected = "Already borrowed: PyBorrowMutError")] - fn pycell_replace_panic() { - Python::with_gil(|py| { - #[allow(deprecated)] - let cell = PyCell::new(py, SomeClass(0)).unwrap(); - let _guard = cell.borrow(); - - cell.replace(SomeClass(123)); - }) - } + #[cfg(feature = "gil-refs")] + mod deprecated { + use super::*; + + #[test] + fn pycell_replace() { + Python::with_gil(|py| { + #[allow(deprecated)] + let cell = PyCell::new(py, SomeClass(0)).unwrap(); + assert_eq!(*cell.borrow(), SomeClass(0)); + + let previous = cell.replace(SomeClass(123)); + assert_eq!(previous, SomeClass(0)); + assert_eq!(*cell.borrow(), SomeClass(123)); + }) + } - #[test] - fn pycell_replace_with() { - Python::with_gil(|py| { - #[allow(deprecated)] - let cell = PyCell::new(py, SomeClass(0)).unwrap(); - assert_eq!(*cell.borrow(), SomeClass(0)); - - let previous = cell.replace_with(|value| { - *value = SomeClass(2); - SomeClass(123) - }); - assert_eq!(previous, SomeClass(2)); - assert_eq!(*cell.borrow(), SomeClass(123)); - }) - } + #[test] + #[should_panic(expected = "Already borrowed: PyBorrowMutError")] + fn pycell_replace_panic() { + Python::with_gil(|py| { + #[allow(deprecated)] + let cell = PyCell::new(py, SomeClass(0)).unwrap(); + let _guard = cell.borrow(); - #[test] - #[should_panic(expected = "Already borrowed: PyBorrowMutError")] - fn pycell_replace_with_panic() { - Python::with_gil(|py| { - #[allow(deprecated)] - let cell = PyCell::new(py, SomeClass(0)).unwrap(); - let _guard = cell.borrow(); + cell.replace(SomeClass(123)); + }) + } - cell.replace_with(|_| SomeClass(123)); - }) - } + #[test] + fn pycell_replace_with() { + Python::with_gil(|py| { + #[allow(deprecated)] + let cell = PyCell::new(py, SomeClass(0)).unwrap(); + assert_eq!(*cell.borrow(), SomeClass(0)); + + let previous = cell.replace_with(|value| { + *value = SomeClass(2); + SomeClass(123) + }); + assert_eq!(previous, SomeClass(2)); + assert_eq!(*cell.borrow(), SomeClass(123)); + }) + } - #[test] - fn pycell_swap() { - Python::with_gil(|py| { - #[allow(deprecated)] - let cell = PyCell::new(py, SomeClass(0)).unwrap(); - #[allow(deprecated)] - let cell2 = PyCell::new(py, SomeClass(123)).unwrap(); - assert_eq!(*cell.borrow(), SomeClass(0)); - assert_eq!(*cell2.borrow(), SomeClass(123)); - - cell.swap(cell2); - assert_eq!(*cell.borrow(), SomeClass(123)); - assert_eq!(*cell2.borrow(), SomeClass(0)); - }) - } + #[test] + #[should_panic(expected = "Already borrowed: PyBorrowMutError")] + fn pycell_replace_with_panic() { + Python::with_gil(|py| { + #[allow(deprecated)] + let cell = PyCell::new(py, SomeClass(0)).unwrap(); + let _guard = cell.borrow(); - #[test] - #[should_panic(expected = "Already borrowed: PyBorrowMutError")] - fn pycell_swap_panic() { - Python::with_gil(|py| { - #[allow(deprecated)] - let cell = PyCell::new(py, SomeClass(0)).unwrap(); - #[allow(deprecated)] - let cell2 = PyCell::new(py, SomeClass(123)).unwrap(); + cell.replace_with(|_| SomeClass(123)); + }) + } - let _guard = cell.borrow(); - cell.swap(cell2); - }) - } + #[test] + fn pycell_swap() { + Python::with_gil(|py| { + #[allow(deprecated)] + let cell = PyCell::new(py, SomeClass(0)).unwrap(); + #[allow(deprecated)] + let cell2 = PyCell::new(py, SomeClass(123)).unwrap(); + assert_eq!(*cell.borrow(), SomeClass(0)); + assert_eq!(*cell2.borrow(), SomeClass(123)); + + cell.swap(cell2); + assert_eq!(*cell.borrow(), SomeClass(123)); + assert_eq!(*cell2.borrow(), SomeClass(0)); + }) + } - #[test] - #[should_panic(expected = "Already borrowed: PyBorrowMutError")] - fn pycell_swap_panic_other_borrowed() { - Python::with_gil(|py| { - #[allow(deprecated)] - let cell = PyCell::new(py, SomeClass(0)).unwrap(); - #[allow(deprecated)] - let cell2 = PyCell::new(py, SomeClass(123)).unwrap(); + #[test] + #[should_panic(expected = "Already borrowed: PyBorrowMutError")] + fn pycell_swap_panic() { + Python::with_gil(|py| { + #[allow(deprecated)] + let cell = PyCell::new(py, SomeClass(0)).unwrap(); + #[allow(deprecated)] + let cell2 = PyCell::new(py, SomeClass(123)).unwrap(); + + let _guard = cell.borrow(); + cell.swap(cell2); + }) + } - let _guard = cell2.borrow(); - cell.swap(cell2); - }) + #[test] + #[should_panic(expected = "Already borrowed: PyBorrowMutError")] + fn pycell_swap_panic_other_borrowed() { + Python::with_gil(|py| { + #[allow(deprecated)] + let cell = PyCell::new(py, SomeClass(0)).unwrap(); + #[allow(deprecated)] + let cell2 = PyCell::new(py, SomeClass(123)).unwrap(); + + let _guard = cell2.borrow(); + cell.swap(cell2); + }) + } } #[test] diff --git a/src/tests/hygiene/misc.rs b/src/tests/hygiene/misc.rs index 24dad7ec196..7a2f58818a1 100644 --- a/src/tests/hygiene/misc.rs +++ b/src/tests/hygiene/misc.rs @@ -41,7 +41,10 @@ fn append_to_inittab() { #[crate::pymodule] #[pyo3(crate = "crate")] #[allow(clippy::unnecessary_wraps)] - fn module_for_inittab(_: crate::Python<'_>, _: &crate::types::PyModule) -> crate::PyResult<()> { + fn module_for_inittab( + _: crate::Python<'_>, + _: &crate::Bound<'_, crate::types::PyModule>, + ) -> crate::PyResult<()> { ::std::result::Result::Ok(()) } crate::append_to_inittab!(module_for_inittab); diff --git a/src/tests/hygiene/pymodule.rs b/src/tests/hygiene/pymodule.rs index 32b3632be22..91f9808bcc4 100644 --- a/src/tests/hygiene/pymodule.rs +++ b/src/tests/hygiene/pymodule.rs @@ -7,6 +7,7 @@ fn do_something(x: i32) -> crate::PyResult { ::std::result::Result::Ok(x) } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] #[crate::pymodule] #[pyo3(crate = "crate")] @@ -14,6 +15,15 @@ fn foo(_py: crate::Python<'_>, _m: &crate::types::PyModule) -> crate::PyResult<( ::std::result::Result::Ok(()) } +#[crate::pymodule] +#[pyo3(crate = "crate")] +fn foo_bound( + _py: crate::Python<'_>, + _m: &crate::Bound<'_, crate::types::PyModule>, +) -> crate::PyResult<()> { + ::std::result::Result::Ok(()) +} + #[cfg(feature = "gil-refs")] #[allow(deprecated)] #[crate::pymodule] @@ -34,7 +44,7 @@ fn my_module_bound(m: &crate::Bound<'_, crate::types::PyModule>) -> crate::PyRes )?; as crate::types::PyModuleMethods>::add_wrapped( m, - crate::wrap_pymodule!(foo), + crate::wrap_pymodule!(foo_bound), )?; ::std::result::Result::Ok(()) diff --git a/src/types/string.rs b/src/types/string.rs index 09c5903547c..4f0025acfe8 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -6,7 +6,9 @@ use crate::py_result_ext::PyResultExt; use crate::types::any::PyAnyMethods; use crate::types::bytes::PyBytesMethods; use crate::types::PyBytes; -use crate::{ffi, Bound, IntoPy, Py, PyAny, PyNativeType, PyResult, Python}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{ffi, Bound, IntoPy, Py, PyAny, PyResult, Python}; use std::borrow::Cow; use std::os::raw::c_char; use std::str; @@ -135,16 +137,6 @@ pub struct PyString(PyAny); pyobject_native_type_core!(PyString, pyobject_native_static_type_object!(ffi::PyUnicode_Type), #checkfunction=ffi::PyUnicode_Check); impl PyString { - /// Deprecated form of [`PyString::new_bound`]. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyString::new` will be replaced by `PyString::new_bound` in a future PyO3 version" - )] - pub fn new<'py>(py: Python<'py>, s: &str) -> &'py Self { - Self::new_bound(py, s).into_gil_ref() - } - /// Creates a new Python string object. /// /// Panics if out of memory. @@ -158,16 +150,6 @@ impl PyString { } } - /// Deprecated form of [`PyString::intern_bound`]. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyString::intern` will be replaced by `PyString::intern_bound` in a future PyO3 version" - )] - pub fn intern<'py>(py: Python<'py>, s: &str) -> &'py Self { - Self::intern_bound(py, s).into_gil_ref() - } - /// Intern the given string /// /// This will return a reference to the same Python string object if called repeatedly with the same string. @@ -188,16 +170,6 @@ impl PyString { } } - /// Deprecated form of [`PyString::from_object_bound`]. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyString::from_object` will be replaced by `PyString::from_object_bound` in a future PyO3 version" - )] - pub fn from_object<'py>(src: &'py PyAny, encoding: &str, errors: &str) -> PyResult<&'py Self> { - Self::from_object_bound(&src.as_borrowed(), encoding, errors).map(Bound::into_gil_ref) - } - /// Attempts to create a Python string from a Python [bytes-like object]. /// /// [bytes-like object]: (https://docs.python.org/3/glossary.html#term-bytes-like-object). @@ -216,6 +188,36 @@ impl PyString { .downcast_into_unchecked() } } +} + +#[cfg(feature = "gil-refs")] +impl PyString { + /// Deprecated form of [`PyString::new_bound`]. + #[deprecated( + since = "0.21.0", + note = "`PyString::new` will be replaced by `PyString::new_bound` in a future PyO3 version" + )] + pub fn new<'py>(py: Python<'py>, s: &str) -> &'py Self { + Self::new_bound(py, s).into_gil_ref() + } + + /// Deprecated form of [`PyString::intern_bound`]. + #[deprecated( + since = "0.21.0", + note = "`PyString::intern` will be replaced by `PyString::intern_bound` in a future PyO3 version" + )] + pub fn intern<'py>(py: Python<'py>, s: &str) -> &'py Self { + Self::intern_bound(py, s).into_gil_ref() + } + + /// Deprecated form of [`PyString::from_object_bound`]. + #[deprecated( + since = "0.21.0", + note = "`PyString::from_object` will be replaced by `PyString::from_object_bound` in a future PyO3 version" + )] + pub fn from_object<'py>(src: &'py PyAny, encoding: &str, errors: &str) -> PyResult<&'py Self> { + Self::from_object_bound(&src.as_borrowed(), encoding, errors).map(Bound::into_gil_ref) + } /// Gets the Python string as a Rust UTF-8 string slice. /// diff --git a/tests/test_wrap_pyfunction_deduction.rs b/tests/test_wrap_pyfunction_deduction.rs index 845cf2a39d7..e205003113e 100644 --- a/tests/test_wrap_pyfunction_deduction.rs +++ b/tests/test_wrap_pyfunction_deduction.rs @@ -5,6 +5,7 @@ use pyo3::{prelude::*, types::PyCFunction}; #[pyfunction] fn f() {} +#[cfg(feature = "gil-refs")] pub fn add_wrapped(wrapper: &impl Fn(Python<'_>) -> PyResult<&PyCFunction>) { let _ = wrapper; } @@ -12,7 +13,10 @@ pub fn add_wrapped(wrapper: &impl Fn(Python<'_>) -> PyResult<&PyCFunction>) { #[test] fn wrap_pyfunction_deduction() { #[allow(deprecated)] + #[cfg(feature = "gil-refs")] add_wrapped(wrap_pyfunction!(f)); + #[cfg(not(feature = "gil-refs"))] + add_wrapped_bound(wrap_pyfunction!(f)); } pub fn add_wrapped_bound(wrapper: &impl Fn(Python<'_>) -> PyResult>) { From 444be3bafaec65c8293e76b6f9086d85ea3e793c Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Fri, 10 May 2024 20:28:30 +0200 Subject: [PATCH 332/349] feature gate deprecated APIs for `Python` (#4173) --- guide/src/memory.md | 3 +- src/conversion.rs | 89 +++++++++++++++-------------------- src/gil.rs | 15 ++++-- src/lib.rs | 1 + src/marker.rs | 112 ++++++++++++++------------------------------ src/prelude.rs | 1 + src/pycell.rs | 2 + 7 files changed, 89 insertions(+), 134 deletions(-) diff --git a/guide/src/memory.md b/guide/src/memory.md index 67a78d3be68..b37d83563e5 100644 --- a/guide/src/memory.md +++ b/guide/src/memory.md @@ -154,8 +154,7 @@ at the end of each loop iteration, before the `with_gil()` closure ends. When doing this, you must be very careful to ensure that once the `GILPool` is dropped you do not retain access to any owned references created after the -`GILPool` was created. Read the -[documentation for `Python::new_pool()`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.new_pool) +`GILPool` was created. Read the documentation for `Python::new_pool()` for more information on safety. This memory management can also be applicable when writing extension modules. diff --git a/src/conversion.rs b/src/conversion.rs index 4df19730b43..95931d30d2e 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -1,14 +1,21 @@ //! Defines conversions between Rust and Python types. -use crate::err::{self, PyDowncastError, PyResult}; +use crate::err::PyResult; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; use crate::types::PyTuple; use crate::{ - ffi, gil, Borrowed, Bound, Py, PyAny, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, + ffi, Borrowed, Bound, Py, PyAny, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, +}; +#[cfg(feature = "gil-refs")] +use { + crate::{ + err::{self, PyDowncastError}, + gil, + }, + std::ptr::NonNull, }; -use std::ptr::NonNull; /// Returns a borrowed pointer to a Python object. /// @@ -385,6 +392,7 @@ where /// If `T` implements `PyTryFrom`, we can convert `&PyAny` to `&T`. /// /// This trait is similar to `std::convert::TryFrom` +#[cfg(feature = "gil-refs")] #[deprecated(since = "0.21.0")] pub trait PyTryFrom<'v>: Sized + PyNativeType { /// Cast from a concrete Python object type to PyObject. @@ -416,6 +424,7 @@ pub trait PyTryFrom<'v>: Sized + PyNativeType { /// Trait implemented by Python object types that allow a checked downcast. /// This trait is similar to `std::convert::TryInto` +#[cfg(feature = "gil-refs")] #[deprecated(since = "0.21.0")] pub trait PyTryInto: Sized { /// Cast from PyObject to a concrete Python object type. @@ -506,6 +515,7 @@ impl IntoPy> for () { /// # Safety /// /// See safety notes on individual functions. +#[cfg(feature = "gil-refs")] #[deprecated(since = "0.21.0")] pub unsafe trait FromPyPointer<'p>: Sized { /// Convert from an arbitrary `PyObject`. @@ -515,12 +525,9 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// Implementations must ensure the object does not get freed during `'p` /// and ensure that `ptr` is of the correct type. /// Note that it must be safe to decrement the reference count of `ptr`. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" - ) + #[deprecated( + since = "0.21.0", + note = "use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" )] unsafe fn from_owned_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self>; /// Convert from an arbitrary `PyObject` or panic. @@ -528,12 +535,9 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" - ) + #[deprecated( + since = "0.21.0", + note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" )] unsafe fn from_owned_ptr_or_panic(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { #[allow(deprecated)] @@ -544,12 +548,9 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" - ) + #[deprecated( + since = "0.21.0", + note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" )] unsafe fn from_owned_ptr(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { #[allow(deprecated)] @@ -560,12 +561,9 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on [`from_owned_ptr_or_opt`](#method.from_owned_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" - ) + #[deprecated( + since = "0.21.0", + note = "use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" )] unsafe fn from_owned_ptr_or_err(py: Python<'p>, ptr: *mut ffi::PyObject) -> PyResult<&'p Self> { #[allow(deprecated)] @@ -576,12 +574,9 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Implementations must ensure the object does not get freed during `'p` and avoid type confusion. - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr_or_opt(py, ptr)` or `Bound::from_borrowed_ptr_or_opt(py, ptr)` instead" - ) + #[deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr_or_opt(py, ptr)` or `Bound::from_borrowed_ptr_or_opt(py, ptr)` instead" )] unsafe fn from_borrowed_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self>; @@ -590,12 +585,9 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" - ) + #[deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" )] unsafe fn from_borrowed_ptr_or_panic(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { #[allow(deprecated)] @@ -606,12 +598,9 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" - ) + #[deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" )] unsafe fn from_borrowed_ptr(py: Python<'p>, ptr: *mut ffi::PyObject) -> &'p Self { #[allow(deprecated)] @@ -622,12 +611,9 @@ pub unsafe trait FromPyPointer<'p>: Sized { /// # Safety /// /// Relies on unsafe fn [`from_borrowed_ptr_or_opt`](#method.from_borrowed_ptr_or_opt). - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr_or_err(py, ptr)` or `Bound::from_borrowed_ptr_or_err(py, ptr)` instead" - ) + #[deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr_or_err(py, ptr)` or `Bound::from_borrowed_ptr_or_err(py, ptr)` instead" )] unsafe fn from_borrowed_ptr_or_err( py: Python<'p>, @@ -638,6 +624,7 @@ pub unsafe trait FromPyPointer<'p>: Sized { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] unsafe impl<'p, T> FromPyPointer<'p> for T where diff --git a/src/gil.rs b/src/gil.rs index 0bcb8c086c0..29c4ffbe389 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -366,12 +366,12 @@ pub struct GILPool { impl GILPool { /// Creates a new [`GILPool`]. This function should only ever be called with the GIL held. /// - /// It is recommended not to use this API directly, but instead to use [`Python::new_pool`], as + /// It is recommended not to use this API directly, but instead to use `Python::new_pool`, as /// that guarantees the GIL is held. /// /// # Safety /// - /// As well as requiring the GIL, see the safety notes on [`Python::new_pool`]. + /// As well as requiring the GIL, see the safety notes on `Python::new_pool`. #[inline] pub unsafe fn new() -> GILPool { increment_gil_count(); @@ -462,6 +462,7 @@ pub unsafe fn register_decref(obj: NonNull) { /// /// # Safety /// The object must be an owned Python reference. +#[cfg(feature = "gil-refs")] pub unsafe fn register_owned(_py: Python<'_>, obj: NonNull) { debug_assert!(gil_is_acquired()); // Ignores the error in case this function called from `atexit`. @@ -507,9 +508,12 @@ fn decrement_gil_count() { mod tests { #[allow(deprecated)] use super::GILPool; - use super::{gil_is_acquired, GIL_COUNT, OWNED_OBJECTS, POOL}; + use super::{gil_is_acquired, GIL_COUNT, POOL}; use crate::types::any::PyAnyMethods; - use crate::{ffi, gil, PyObject, Python}; + use crate::{ffi, PyObject, Python}; + #[cfg(feature = "gil-refs")] + use {super::OWNED_OBJECTS, crate::gil}; + use std::ptr::NonNull; #[cfg(not(target_arch = "wasm32"))] use std::sync; @@ -518,6 +522,7 @@ mod tests { py.eval_bound("object()", None, None).unwrap().unbind() } + #[cfg(feature = "gil-refs")] fn owned_object_count() -> usize { #[cfg(debug_assertions)] let len = OWNED_OBJECTS.with(|owned_objects| owned_objects.borrow().len()); @@ -554,6 +559,7 @@ mod tests { } #[test] + #[cfg(feature = "gil-refs")] #[allow(deprecated)] fn test_owned() { Python::with_gil(|py| { @@ -580,6 +586,7 @@ mod tests { } #[test] + #[cfg(feature = "gil-refs")] #[allow(deprecated)] fn test_owned_nested() { Python::with_gil(|py| { diff --git a/src/lib.rs b/src/lib.rs index 3923257f5f3..2a40445222e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -317,6 +317,7 @@ //! [`Ungil`]: crate::marker::Ungil pub use crate::class::*; pub use crate::conversion::{AsPyPointer, FromPyObject, IntoPy, ToPyObject}; +#[cfg(feature = "gil-refs")] #[allow(deprecated)] pub use crate::conversion::{FromPyPointer, PyTryFrom, PyTryInto}; pub use crate::err::{ diff --git a/src/marker.rs b/src/marker.rs index ea794856d66..b6821deb036 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -130,6 +130,7 @@ use crate::version::PythonVersionInfo; use crate::PyNativeType; use crate::{ffi, Bound, IntoPy, Py, PyObject, PyTypeInfo}; #[allow(deprecated)] +#[cfg(feature = "gil-refs")] use crate::{gil::GILPool, FromPyPointer}; use std::ffi::{CStr, CString}; use std::marker::PhantomData; @@ -354,36 +355,9 @@ pub use nightly::Ungil; /// # Releasing and freeing memory /// /// The [`Python`] type can be used to create references to variables owned by the Python -/// interpreter, using functions such as `Python::eval` and `PyModule::import`. These -/// references are tied to a [`GILPool`] whose references are not cleared until it is dropped. -/// This can cause apparent "memory leaks" if it is kept around for a long time. -/// -/// ```rust -/// use pyo3::prelude::*; -/// use pyo3::types::PyString; -/// -/// # fn main () -> PyResult<()> { -/// Python::with_gil(|py| -> PyResult<()> { -/// for _ in 0..10 { -/// let hello = py.eval_bound("\"Hello World!\"", None, None)?.downcast_into::()?; -/// println!("Python says: {}", hello.to_cow()?); -/// // Normally variables in a loop scope are dropped here, but `hello` is a reference to -/// // something owned by the Python interpreter. Dropping this reference does nothing. -/// } -/// Ok(()) -/// }) -/// // This is where the `hello`'s reference counts start getting decremented. -/// # } -/// ``` -/// -/// The variable `hello` is dropped at the end of each loop iteration, but the lifetime of the -/// pointed-to memory is bound to [`Python::with_gil`]'s [`GILPool`] which will not be dropped until -/// the end of [`Python::with_gil`]'s scope. Only then is each `hello`'s Python reference count -/// decreased. This means that at the last line of the example there are 10 copies of `hello` in -/// Python's memory, not just one at a time as we might expect from Rust's [scoping rules]. +/// interpreter, using functions such as [`Python::eval_bound`] and [`PyModule::import_bound`]. /// -/// See the [Memory Management] chapter of the guide for more information about how PyO3 uses -/// [`GILPool`] to manage memory. +/// See the [Memory Management] chapter of the guide for more information about how PyO3 manages memory. /// /// [scoping rules]: https://doc.rust-lang.org/stable/book/ch04-01-what-is-ownership.html#ownership-rules /// [`Py::clone_ref`]: crate::Py::clone_ref @@ -874,12 +848,10 @@ impl<'py> Python<'py> { /// /// Callers must ensure that ensure that the cast is valid. #[allow(clippy::wrong_self_convention, deprecated)] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `Py::from_owned_ptr(py, ptr)` or `Bound::from_owned_ptr(py, ptr)` instead" )] pub unsafe fn from_owned_ptr(self, ptr: *mut ffi::PyObject) -> &'py T where @@ -897,12 +869,10 @@ impl<'py> Python<'py> { /// /// Callers must ensure that ensure that the cast is valid. #[allow(clippy::wrong_self_convention, deprecated)] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `Py::from_owned_ptr_or_err(py, ptr)` or `Bound::from_owned_ptr_or_err(py, ptr)` instead" )] pub unsafe fn from_owned_ptr_or_err(self, ptr: *mut ffi::PyObject) -> PyResult<&'py T> where @@ -920,12 +890,10 @@ impl<'py> Python<'py> { /// /// Callers must ensure that ensure that the cast is valid. #[allow(clippy::wrong_self_convention, deprecated)] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `Py::from_owned_ptr_or_opt(py, ptr)` or `Bound::from_owned_ptr_or_opt(py, ptr)` instead" )] pub unsafe fn from_owned_ptr_or_opt(self, ptr: *mut ffi::PyObject) -> Option<&'py T> where @@ -942,12 +910,10 @@ impl<'py> Python<'py> { /// /// Callers must ensure that ensure that the cast is valid. #[allow(clippy::wrong_self_convention, deprecated)] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr(py, ptr)` or `Bound::from_borrowed_ptr(py, ptr)` instead" )] pub unsafe fn from_borrowed_ptr(self, ptr: *mut ffi::PyObject) -> &'py T where @@ -964,12 +930,10 @@ impl<'py> Python<'py> { /// /// Callers must ensure that ensure that the cast is valid. #[allow(clippy::wrong_self_convention, deprecated)] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr_or_err(py, ptr)` or `Bound::from_borrowed_ptr_or_err(py, ptr)` instead" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr_or_err(py, ptr)` or `Bound::from_borrowed_ptr_or_err(py, ptr)` instead" )] pub unsafe fn from_borrowed_ptr_or_err(self, ptr: *mut ffi::PyObject) -> PyResult<&'py T> where @@ -986,12 +950,10 @@ impl<'py> Python<'py> { /// /// Callers must ensure that ensure that the cast is valid. #[allow(clippy::wrong_self_convention, deprecated)] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "use `Py::from_borrowed_ptr_or_opt(py, ptr)` or `Bound::from_borrowed_ptr_or_opt(py, ptr)` instead" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "use `Py::from_borrowed_ptr_or_opt(py, ptr)` or `Bound::from_borrowed_ptr_or_opt(py, ptr)` instead" )] pub unsafe fn from_borrowed_ptr_or_opt(self, ptr: *mut ffi::PyObject) -> Option<&'py T> where @@ -1103,12 +1065,10 @@ impl<'py> Python<'py> { /// /// [`.python()`]: crate::GILPool::python #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "code not using the GIL Refs API can safely remove use of `Python::new_pool`" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "code not using the GIL Refs API can safely remove use of `Python::new_pool`" )] #[allow(deprecated)] pub unsafe fn new_pool(self) -> GILPool { @@ -1172,12 +1132,10 @@ impl Python<'_> { /// }); /// ``` #[inline] - #[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "code not using the GIL Refs API can safely remove use of `Python::with_pool`" - ) + #[cfg(feature = "gil-refs")] + #[deprecated( + since = "0.21.0", + note = "code not using the GIL Refs API can safely remove use of `Python::with_pool`" )] #[allow(deprecated)] pub fn with_pool(&self, f: F) -> R diff --git a/src/prelude.rs b/src/prelude.rs index ef42b2706e9..7aa45f6ccc2 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -9,6 +9,7 @@ //! ``` pub use crate::conversion::{FromPyObject, IntoPy, ToPyObject}; +#[cfg(feature = "gil-refs")] #[allow(deprecated)] pub use crate::conversion::{PyTryFrom, PyTryInto}; pub use crate::err::{PyErr, PyResult}; diff --git a/src/pycell.rs b/src/pycell.rs index 215ed7bba66..dc5ccc45ea1 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -518,6 +518,7 @@ impl ToPyObject for &PyCell { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl AsRef for PyCell { fn as_ref(&self) -> &PyAny { @@ -528,6 +529,7 @@ impl AsRef for PyCell { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl Deref for PyCell { type Target = PyAny; From 033caa8fd1fefbf51fba97f85f5fbcb191c264bb Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sat, 11 May 2024 15:48:17 +0200 Subject: [PATCH 333/349] split more impl blocks (#4175) --- src/types/boolobject.rs | 36 ++++++++++--------- src/types/bytearray.rs | 80 ++++++++++++++++++++--------------------- src/types/bytes.rs | 77 ++++++++++++++++++++------------------- src/types/capsule.rs | 70 ++++++++++++++++++------------------ src/types/complex.rs | 26 ++++++++------ src/types/datetime.rs | 8 +++++ src/types/float.rs | 31 ++++++++-------- src/types/slice.rs | 47 ++++++++++++------------ src/types/traceback.rs | 6 ++-- src/types/typeobject.rs | 56 +++++++++++++++-------------- 10 files changed, 235 insertions(+), 202 deletions(-) diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 43184e31565..9b5aa659fdf 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -1,9 +1,11 @@ #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; use crate::{ exceptions::PyTypeError, ffi, ffi_ptr_ext::FfiPtrExt, instance::Bound, - types::typeobject::PyTypeMethods, Borrowed, FromPyObject, IntoPy, PyAny, PyNativeType, - PyObject, PyResult, Python, ToPyObject, + types::typeobject::PyTypeMethods, Borrowed, FromPyObject, IntoPy, PyAny, PyObject, PyResult, + Python, ToPyObject, }; use super::any::PyAnyMethods; @@ -15,20 +17,6 @@ pub struct PyBool(PyAny); pyobject_native_type!(PyBool, ffi::PyObject, pyobject_native_static_type_object!(ffi::PyBool_Type), #checkfunction=ffi::PyBool_Check); impl PyBool { - /// Deprecated form of [`PyBool::new_bound`] - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyBool::new` will be replaced by `PyBool::new_bound` in a future PyO3 version" - )] - #[inline] - pub fn new(py: Python<'_>, val: bool) -> &PyBool { - #[allow(deprecated)] - unsafe { - py.from_borrowed_ptr(if val { ffi::Py_True() } else { ffi::Py_False() }) - } - } - /// Depending on `val`, returns `true` or `false`. /// /// # Note @@ -42,6 +30,22 @@ impl PyBool { .downcast_unchecked() } } +} + +#[cfg(feature = "gil-refs")] +impl PyBool { + /// Deprecated form of [`PyBool::new_bound`] + #[deprecated( + since = "0.21.0", + note = "`PyBool::new` will be replaced by `PyBool::new_bound` in a future PyO3 version" + )] + #[inline] + pub fn new(py: Python<'_>, val: bool) -> &PyBool { + #[allow(deprecated)] + unsafe { + py.from_borrowed_ptr(if val { ffi::Py_True() } else { ffi::Py_False() }) + } + } /// Gets whether this boolean is `true`. #[inline] diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index ef20509e629..ec3d7eafbfd 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -3,9 +3,9 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::{Borrowed, Bound}; use crate::py_result_ext::PyResultExt; use crate::types::any::PyAnyMethods; +use crate::{ffi, PyAny, Python}; #[cfg(feature = "gil-refs")] -use crate::AsPyPointer; -use crate::{ffi, PyAny, PyNativeType, Python}; +use crate::{AsPyPointer, PyNativeType}; use std::os::raw::c_char; use std::slice; @@ -16,16 +16,6 @@ pub struct PyByteArray(PyAny); pyobject_native_type_core!(PyByteArray, pyobject_native_static_type_object!(ffi::PyByteArray_Type), #checkfunction=ffi::PyByteArray_Check); impl PyByteArray { - /// Deprecated form of [`PyByteArray::new_bound`] - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyByteArray::new` will be replaced by `PyByteArray::new_bound` in a future PyO3 version" - )] - pub fn new<'py>(py: Python<'py>, src: &[u8]) -> &'py PyByteArray { - Self::new_bound(py, src).into_gil_ref() - } - /// Creates a new Python bytearray object. /// /// The byte string is initialized by copying the data from the `&[u8]`. @@ -39,19 +29,6 @@ impl PyByteArray { } } - /// Deprecated form of [`PyByteArray::new_bound_with`] - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyByteArray::new_with` will be replaced by `PyByteArray::new_bound_with` in a future PyO3 version" - )] - pub fn new_with(py: Python<'_>, len: usize, init: F) -> PyResult<&PyByteArray> - where - F: FnOnce(&mut [u8]) -> PyResult<()>, - { - Self::new_bound_with(py, len, init).map(Bound::into_gil_ref) - } - /// Creates a new Python `bytearray` object with an `init` closure to write its contents. /// Before calling `init` the bytearray is zero-initialised. /// * If Python raises a MemoryError on the allocation, `new_with` will return @@ -101,16 +78,6 @@ impl PyByteArray { } } - /// Deprecated form of [`PyByteArray::from_bound`] - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyByteArray::from` will be replaced by `PyByteArray::from_bound` in a future PyO3 version" - )] - pub fn from(src: &PyAny) -> PyResult<&PyByteArray> { - PyByteArray::from_bound(&src.as_borrowed()).map(Bound::into_gil_ref) - } - /// Creates a new Python `bytearray` object from another Python object that /// implements the buffer protocol. pub fn from_bound<'py>(src: &Bound<'py, PyAny>) -> PyResult> { @@ -120,6 +87,39 @@ impl PyByteArray { .downcast_into_unchecked() } } +} + +#[cfg(feature = "gil-refs")] +impl PyByteArray { + /// Deprecated form of [`PyByteArray::new_bound`] + #[deprecated( + since = "0.21.0", + note = "`PyByteArray::new` will be replaced by `PyByteArray::new_bound` in a future PyO3 version" + )] + pub fn new<'py>(py: Python<'py>, src: &[u8]) -> &'py PyByteArray { + Self::new_bound(py, src).into_gil_ref() + } + + /// Deprecated form of [`PyByteArray::new_bound_with`] + #[deprecated( + since = "0.21.0", + note = "`PyByteArray::new_with` will be replaced by `PyByteArray::new_bound_with` in a future PyO3 version" + )] + pub fn new_with(py: Python<'_>, len: usize, init: F) -> PyResult<&PyByteArray> + where + F: FnOnce(&mut [u8]) -> PyResult<()>, + { + Self::new_bound_with(py, len, init).map(Bound::into_gil_ref) + } + + /// Deprecated form of [`PyByteArray::from_bound`] + #[deprecated( + since = "0.21.0", + note = "`PyByteArray::from` will be replaced by `PyByteArray::from_bound` in a future PyO3 version" + )] + pub fn from(src: &PyAny) -> PyResult<&PyByteArray> { + PyByteArray::from_bound(&src.as_borrowed()).map(Bound::into_gil_ref) + } /// Gets the length of the bytearray. #[inline] @@ -300,7 +300,7 @@ pub trait PyByteArrayMethods<'py>: crate::sealed::Sealed { /// /// # Safety /// - /// See the safety requirements of [`PyByteArray::as_bytes`] and [`PyByteArray::as_bytes_mut`]. + /// See the safety requirements of [`PyByteArrayMethods::as_bytes`] and [`PyByteArrayMethods::as_bytes_mut`]. fn data(&self) -> *mut u8; /// Extracts a slice of the `ByteArray`'s entire buffer. @@ -311,7 +311,7 @@ pub trait PyByteArrayMethods<'py>: crate::sealed::Sealed { /// undefined. /// /// These mutations may occur in Python code as well as from Rust: - /// - Calling methods like [`PyByteArray::as_bytes_mut`] and [`PyByteArray::resize`] will + /// - Calling methods like [`PyByteArrayMethods::as_bytes_mut`] and [`PyByteArrayMethods::resize`] will /// invalidate the slice. /// - Actions like dropping objects or raising exceptions can invoke `__del__`methods or signal /// handlers, which may execute arbitrary Python code. This means that if Python code has a @@ -405,7 +405,7 @@ pub trait PyByteArrayMethods<'py>: crate::sealed::Sealed { /// # Safety /// /// Any other accesses of the `bytearray`'s buffer invalidate the slice. If it is used - /// afterwards, the behavior is undefined. The safety requirements of [`PyByteArray::as_bytes`] + /// afterwards, the behavior is undefined. The safety requirements of [`PyByteArrayMethods::as_bytes`] /// apply to this function as well. #[allow(clippy::mut_from_ref)] unsafe fn as_bytes_mut(&self) -> &mut [u8]; @@ -432,8 +432,8 @@ pub trait PyByteArrayMethods<'py>: crate::sealed::Sealed { /// Resizes the bytearray object to the new length `len`. /// - /// Note that this will invalidate any pointers obtained by [PyByteArray::data], as well as - /// any (unsafe) slices obtained from [PyByteArray::as_bytes] and [PyByteArray::as_bytes_mut]. + /// Note that this will invalidate any pointers obtained by [PyByteArrayMethods::data], as well as + /// any (unsafe) slices obtained from [PyByteArrayMethods::as_bytes] and [PyByteArrayMethods::as_bytes_mut]. fn resize(&self, len: usize) -> PyResult<()>; } diff --git a/src/types/bytes.rs b/src/types/bytes.rs index f3da65c7c97..1d6a2f8ec7d 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -1,7 +1,9 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::instance::{Borrowed, Bound}; use crate::types::any::PyAnyMethods; -use crate::{ffi, Py, PyAny, PyNativeType, PyResult, Python}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{ffi, Py, PyAny, PyResult, Python}; use std::ops::Index; use std::os::raw::c_char; use std::slice::SliceIndex; @@ -16,16 +18,6 @@ pub struct PyBytes(PyAny); pyobject_native_type_core!(PyBytes, pyobject_native_static_type_object!(ffi::PyBytes_Type), #checkfunction=ffi::PyBytes_Check); impl PyBytes { - /// Deprecated form of [`PyBytes::new_bound`]. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyBytes::new` will be replaced by `PyBytes::new_bound` in a future PyO3 version" - )] - pub fn new<'p>(py: Python<'p>, s: &[u8]) -> &'p PyBytes { - Self::new_bound(py, s).into_gil_ref() - } - /// Creates a new Python bytestring object. /// The bytestring is initialized by copying the data from the `&[u8]`. /// @@ -40,19 +32,6 @@ impl PyBytes { } } - /// Deprecated form of [`PyBytes::new_bound_with`]. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyBytes::new_with` will be replaced by `PyBytes::new_bound_with` in a future PyO3 version" - )] - pub fn new_with(py: Python<'_>, len: usize, init: F) -> PyResult<&PyBytes> - where - F: FnOnce(&mut [u8]) -> PyResult<()>, - { - Self::new_bound_with(py, len, init).map(Bound::into_gil_ref) - } - /// Creates a new Python `bytes` object with an `init` closure to write its contents. /// Before calling `init` the bytes' contents are zero-initialised. /// * If Python raises a MemoryError on the allocation, `new_with` will return @@ -95,19 +74,6 @@ impl PyBytes { } } - /// Deprecated form of [`PyBytes::bound_from_ptr`]. - /// - /// # Safety - /// See [`PyBytes::bound_from_ptr`]. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyBytes::from_ptr` will be replaced by `PyBytes::bound_from_ptr` in a future PyO3 version" - )] - pub unsafe fn from_ptr(py: Python<'_>, ptr: *const u8, len: usize) -> &PyBytes { - Self::bound_from_ptr(py, ptr, len).into_gil_ref() - } - /// Creates a new Python byte string object from a raw pointer and length. /// /// Panics if out of memory. @@ -123,6 +89,42 @@ impl PyBytes { .assume_owned(py) .downcast_into_unchecked() } +} + +#[cfg(feature = "gil-refs")] +impl PyBytes { + /// Deprecated form of [`PyBytes::new_bound`]. + #[deprecated( + since = "0.21.0", + note = "`PyBytes::new` will be replaced by `PyBytes::new_bound` in a future PyO3 version" + )] + pub fn new<'p>(py: Python<'p>, s: &[u8]) -> &'p PyBytes { + Self::new_bound(py, s).into_gil_ref() + } + + /// Deprecated form of [`PyBytes::new_bound_with`]. + #[deprecated( + since = "0.21.0", + note = "`PyBytes::new_with` will be replaced by `PyBytes::new_bound_with` in a future PyO3 version" + )] + pub fn new_with(py: Python<'_>, len: usize, init: F) -> PyResult<&PyBytes> + where + F: FnOnce(&mut [u8]) -> PyResult<()>, + { + Self::new_bound_with(py, len, init).map(Bound::into_gil_ref) + } + + /// Deprecated form of [`PyBytes::bound_from_ptr`]. + /// + /// # Safety + /// See [`PyBytes::bound_from_ptr`]. + #[deprecated( + since = "0.21.0", + note = "`PyBytes::from_ptr` will be replaced by `PyBytes::bound_from_ptr` in a future PyO3 version" + )] + pub unsafe fn from_ptr(py: Python<'_>, ptr: *const u8, len: usize) -> &PyBytes { + Self::bound_from_ptr(py, ptr, len).into_gil_ref() + } /// Gets the Python string as a byte slice. #[inline] @@ -172,6 +174,7 @@ impl Py { } /// This is the same way [Vec] is indexed. +#[cfg(feature = "gil-refs")] impl> Index for PyBytes { type Output = I::Output; diff --git a/src/types/capsule.rs b/src/types/capsule.rs index c2b73a0d0f7..2851970ba10 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -1,11 +1,12 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::py_result_ext::PyResultExt; -use crate::{ffi, PyAny, PyNativeType}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{ffi, PyAny}; use crate::{Bound, Python}; use crate::{PyErr, PyResult}; use std::ffi::{CStr, CString}; use std::os::raw::{c_char, c_int, c_void}; - /// Represents a Python Capsule /// as described in [Capsules](https://docs.python.org/3/c-api/capsule.html#capsules): /// > This subtype of PyObject represents an opaque value, useful for C extension @@ -46,20 +47,6 @@ pub struct PyCapsule(PyAny); pyobject_native_type_core!(PyCapsule, pyobject_native_static_type_object!(ffi::PyCapsule_Type), #checkfunction=ffi::PyCapsule_CheckExact); impl PyCapsule { - /// Deprecated form of [`PyCapsule::new_bound`]. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyCapsule::new` will be replaced by `PyCapsule::new_bound` in a future PyO3 version" - )] - pub fn new( - py: Python<'_>, - value: T, - name: Option, - ) -> PyResult<&Self> { - Self::new_bound(py, value, name).map(Bound::into_gil_ref) - } - /// Constructs a new capsule whose contents are `value`, associated with `name`. /// `name` is the identifier for the capsule; if it is stored as an attribute of a module, /// the name should be in the format `"modulename.attribute"`. @@ -99,24 +86,6 @@ impl PyCapsule { Self::new_bound_with_destructor(py, value, name, |_, _| {}) } - /// Deprecated form of [`PyCapsule::new_bound_with_destructor`]. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyCapsule::new_with_destructor` will be replaced by `PyCapsule::new_bound_with_destructor` in a future PyO3 version" - )] - pub fn new_with_destructor< - T: 'static + Send + AssertNotZeroSized, - F: FnOnce(T, *mut c_void) + Send, - >( - py: Python<'_>, - value: T, - name: Option, - destructor: F, - ) -> PyResult<&'_ Self> { - Self::new_bound_with_destructor(py, value, name, destructor).map(Bound::into_gil_ref) - } - /// Constructs a new capsule whose contents are `value`, associated with `name`. /// /// Also provides a destructor: when the `PyCapsule` is destroyed, it will be passed the original object, @@ -172,6 +141,39 @@ impl PyCapsule { Ok(&*ptr.cast::()) } } +} + +#[cfg(feature = "gil-refs")] +impl PyCapsule { + /// Deprecated form of [`PyCapsule::new_bound`]. + #[deprecated( + since = "0.21.0", + note = "`PyCapsule::new` will be replaced by `PyCapsule::new_bound` in a future PyO3 version" + )] + pub fn new( + py: Python<'_>, + value: T, + name: Option, + ) -> PyResult<&Self> { + Self::new_bound(py, value, name).map(Bound::into_gil_ref) + } + + /// Deprecated form of [`PyCapsule::new_bound_with_destructor`]. + #[deprecated( + since = "0.21.0", + note = "`PyCapsule::new_with_destructor` will be replaced by `PyCapsule::new_bound_with_destructor` in a future PyO3 version" + )] + pub fn new_with_destructor< + T: 'static + Send + AssertNotZeroSized, + F: FnOnce(T, *mut c_void) + Send, + >( + py: Python<'_>, + value: T, + name: Option, + destructor: F, + ) -> PyResult<&'_ Self> { + Self::new_bound_with_destructor(py, value, name, destructor).map(Bound::into_gil_ref) + } /// Sets the context pointer in the capsule. /// diff --git a/src/types/complex.rs b/src/types/complex.rs index cd3d5810d04..65b08cc9c5c 100644 --- a/src/types/complex.rs +++ b/src/types/complex.rs @@ -1,4 +1,6 @@ -use crate::{ffi, types::any::PyAnyMethods, Bound, PyAny, PyNativeType, Python}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{ffi, types::any::PyAnyMethods, Bound, PyAny, Python}; use std::os::raw::c_double; /// Represents a Python [`complex`](https://docs.python.org/3/library/functions.html#complex) object. @@ -19,15 +21,6 @@ pyobject_native_type!( ); impl PyComplex { - /// Deprecated form of [`PyComplex::from_doubles_bound`] - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PyComplex::from_doubles` will be replaced by `PyComplex::from_doubles_bound` in a future PyO3 version" - )] - pub fn from_doubles(py: Python<'_>, real: c_double, imag: c_double) -> &PyComplex { - Self::from_doubles_bound(py, real, imag).into_gil_ref() - } /// Creates a new `PyComplex` from the given real and imaginary values. pub fn from_doubles_bound( py: Python<'_>, @@ -41,6 +34,19 @@ impl PyComplex { .downcast_into_unchecked() } } +} + +#[cfg(feature = "gil-refs")] +impl PyComplex { + /// Deprecated form of [`PyComplex::from_doubles_bound`] + #[deprecated( + since = "0.21.0", + note = "`PyComplex::from_doubles` will be replaced by `PyComplex::from_doubles_bound` in a future PyO3 version" + )] + pub fn from_doubles(py: Python<'_>, real: c_double, imag: c_double) -> &PyComplex { + Self::from_doubles_bound(py, real, imag).into_gil_ref() + } + /// Returns the real part of the complex number. pub fn real(&self) -> c_double { self.as_borrowed().real() diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 12db67ab961..cdf3b011e6c 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -22,6 +22,7 @@ use crate::ffi::{ PyDateTime_TIME_GET_MINUTE, PyDateTime_TIME_GET_SECOND, }; use crate::ffi_ptr_ext::FfiPtrExt; +#[cfg(feature = "gil-refs")] use crate::instance::PyNativeType; use crate::py_result_ext::PyResultExt; use crate::types::any::PyAnyMethods; @@ -249,6 +250,7 @@ impl PyDate { } } +#[cfg(feature = "gil-refs")] impl PyDateAccess for PyDate { fn get_year(&self) -> i32 { self.as_borrowed().get_year() @@ -461,6 +463,7 @@ impl PyDateTime { } } +#[cfg(feature = "gil-refs")] impl PyDateAccess for PyDateTime { fn get_year(&self) -> i32 { self.as_borrowed().get_year() @@ -489,6 +492,7 @@ impl PyDateAccess for Bound<'_, PyDateTime> { } } +#[cfg(feature = "gil-refs")] impl PyTimeAccess for PyDateTime { fn get_hour(&self) -> u8 { self.as_borrowed().get_hour() @@ -533,6 +537,7 @@ impl PyTimeAccess for Bound<'_, PyDateTime> { } } +#[cfg(feature = "gil-refs")] impl<'py> PyTzInfoAccess<'py> for &'py PyDateTime { fn get_tzinfo_bound(&self) -> Option> { self.as_borrowed().get_tzinfo_bound() @@ -688,6 +693,7 @@ impl PyTime { } } +#[cfg(feature = "gil-refs")] impl PyTimeAccess for PyTime { fn get_hour(&self) -> u8 { self.as_borrowed().get_hour() @@ -732,6 +738,7 @@ impl PyTimeAccess for Bound<'_, PyTime> { } } +#[cfg(feature = "gil-refs")] impl<'py> PyTzInfoAccess<'py> for &'py PyTime { fn get_tzinfo_bound(&self) -> Option> { self.as_borrowed().get_tzinfo_bound() @@ -878,6 +885,7 @@ impl PyDelta { } } +#[cfg(feature = "gil-refs")] impl PyDeltaAccess for PyDelta { fn get_days(&self) -> i32 { self.as_borrowed().get_days() diff --git a/src/types/float.rs b/src/types/float.rs index 2ed3d2921b9..8499d1e54aa 100644 --- a/src/types/float.rs +++ b/src/types/float.rs @@ -1,13 +1,14 @@ +use super::any::PyAnyMethods; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; use crate::{ - ffi, ffi_ptr_ext::FfiPtrExt, instance::Bound, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, - PyObject, PyResult, Python, ToPyObject, + ffi, ffi_ptr_ext::FfiPtrExt, instance::Bound, FromPyObject, IntoPy, PyAny, PyErr, PyObject, + PyResult, Python, ToPyObject, }; use std::os::raw::c_double; -use super::any::PyAnyMethods; - /// Represents a Python `float` object. /// /// You can usually avoid directly working with this type @@ -23,10 +24,21 @@ pyobject_native_type!( #checkfunction=ffi::PyFloat_Check ); +impl PyFloat { + /// Creates a new Python `float` object. + pub fn new_bound(py: Python<'_>, val: c_double) -> Bound<'_, PyFloat> { + unsafe { + ffi::PyFloat_FromDouble(val) + .assume_owned(py) + .downcast_into_unchecked() + } + } +} + +#[cfg(feature = "gil-refs")] impl PyFloat { /// Deprecated form of [`PyFloat::new_bound`]. #[inline] - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.0", note = "`PyFloat::new` will be replaced by `PyFloat::new_bound` in a future PyO3 version" @@ -35,15 +47,6 @@ impl PyFloat { Self::new_bound(py, val).into_gil_ref() } - /// Creates a new Python `float` object. - pub fn new_bound(py: Python<'_>, val: c_double) -> Bound<'_, PyFloat> { - unsafe { - ffi::PyFloat_FromDouble(val) - .assume_owned(py) - .downcast_into_unchecked() - } - } - /// Gets the value of this float. pub fn value(&self) -> c_double { self.as_borrowed().value() diff --git a/src/types/slice.rs b/src/types/slice.rs index 70285c9c251..7daa2c030b6 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -2,7 +2,9 @@ use crate::err::{PyErr, PyResult}; use crate::ffi; use crate::ffi_ptr_ext::FfiPtrExt; use crate::types::any::PyAnyMethods; -use crate::{Bound, PyAny, PyNativeType, PyObject, Python, ToPyObject}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{Bound, PyAny, PyObject, Python, ToPyObject}; /// Represents a Python `slice`. /// @@ -17,7 +19,7 @@ pyobject_native_type!( #checkfunction=ffi::PySlice_Check ); -/// Return value from [`PySlice::indices`]. +/// Return value from [`PySliceMethods::indices`]. #[derive(Debug, Eq, PartialEq)] pub struct PySliceIndices { /// Start of the slice @@ -47,16 +49,6 @@ impl PySliceIndices { } impl PySlice { - /// Deprecated form of `PySlice::new_bound`. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PySlice::new` will be replaced by `PySlice::new_bound` in a future PyO3 version" - )] - pub fn new(py: Python<'_>, start: isize, stop: isize, step: isize) -> &PySlice { - Self::new_bound(py, start, stop, step).into_gil_ref() - } - /// Constructs a new slice with the given elements. pub fn new_bound(py: Python<'_>, start: isize, stop: isize, step: isize) -> Bound<'_, PySlice> { unsafe { @@ -70,16 +62,6 @@ impl PySlice { } } - /// Deprecated form of `PySlice::full_bound`. - #[cfg(feature = "gil-refs")] - #[deprecated( - since = "0.21.0", - note = "`PySlice::full` will be replaced by `PySlice::full_bound` in a future PyO3 version" - )] - pub fn full(py: Python<'_>) -> &PySlice { - PySlice::full_bound(py).into_gil_ref() - } - /// Constructs a new full slice that is equivalent to `::`. pub fn full_bound(py: Python<'_>) -> Bound<'_, PySlice> { unsafe { @@ -88,6 +70,27 @@ impl PySlice { .downcast_into_unchecked() } } +} + +#[cfg(feature = "gil-refs")] +impl PySlice { + /// Deprecated form of `PySlice::new_bound`. + #[deprecated( + since = "0.21.0", + note = "`PySlice::new` will be replaced by `PySlice::new_bound` in a future PyO3 version" + )] + pub fn new(py: Python<'_>, start: isize, stop: isize, step: isize) -> &PySlice { + Self::new_bound(py, start, stop, step).into_gil_ref() + } + + /// Deprecated form of `PySlice::full_bound`. + #[deprecated( + since = "0.21.0", + note = "`PySlice::full` will be replaced by `PySlice::full_bound` in a future PyO3 version" + )] + pub fn full(py: Python<'_>) -> &PySlice { + PySlice::full_bound(py).into_gil_ref() + } /// Retrieves the start, stop, and step indices from the slice object, /// assuming a sequence of length `length`, and stores the length of the diff --git a/src/types/traceback.rs b/src/types/traceback.rs index c4cedd791f6..dbbdb6a85ea 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -1,7 +1,8 @@ use crate::err::{error_on_minusone, PyResult}; use crate::types::{any::PyAnyMethods, string::PyStringMethods, PyString}; -use crate::{ffi, Bound}; -use crate::{PyAny, PyNativeType}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{ffi, Bound, PyAny}; /// Represents a Python traceback. #[repr(transparent)] @@ -13,6 +14,7 @@ pyobject_native_type_core!( #checkfunction=ffi::PyTraceBack_Check ); +#[cfg(feature = "gil-refs")] impl PyTraceback { /// Formats the traceback as a string. /// diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 2261834ef2a..e6c4a2180d9 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -1,21 +1,47 @@ use crate::err::{self, PyResult}; use crate::instance::Borrowed; use crate::types::any::PyAnyMethods; -use crate::{ffi, Bound, PyAny, PyNativeType, PyTypeInfo, Python}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{ffi, Bound, PyAny, PyTypeInfo, Python}; use std::borrow::Cow; #[cfg(not(any(Py_LIMITED_API, PyPy)))] use std::ffi::CStr; - /// Represents a reference to a Python `type object`. #[repr(transparent)] pub struct PyType(PyAny); pyobject_native_type_core!(PyType, pyobject_native_static_type_object!(ffi::PyType_Type), #checkfunction=ffi::PyType_Check); +impl PyType { + /// Creates a new type object. + #[inline] + pub fn new_bound(py: Python<'_>) -> Bound<'_, PyType> { + T::type_object_bound(py) + } + + /// Converts the given FFI pointer into `Bound`, to use in safe code. + /// + /// The function creates a new reference from the given pointer, and returns + /// it as a `Bound`. + /// + /// # Safety + /// - The pointer must be a valid non-null reference to a `PyTypeObject` + #[inline] + pub unsafe fn from_borrowed_type_ptr( + py: Python<'_>, + p: *mut ffi::PyTypeObject, + ) -> Bound<'_, PyType> { + Borrowed::from_ptr_unchecked(py, p.cast()) + .downcast_unchecked() + .to_owned() + } +} + +#[cfg(feature = "gil-refs")] impl PyType { /// Deprecated form of [`PyType::new_bound`]. #[inline] - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.0", note = "`PyType::new` will be replaced by `PyType::new_bound` in a future PyO3 version" @@ -24,12 +50,6 @@ impl PyType { T::type_object_bound(py).into_gil_ref() } - /// Creates a new type object. - #[inline] - pub fn new_bound(py: Python<'_>) -> Bound<'_, PyType> { - T::type_object_bound(py) - } - /// Retrieves the underlying FFI pointer associated with this Python object. #[inline] pub fn as_type_ptr(&self) -> *mut ffi::PyTypeObject { @@ -42,7 +62,6 @@ impl PyType { /// /// - The pointer must a valid non-null reference to a `PyTypeObject`. #[inline] - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.0", note = "Use `PyType::from_borrowed_type_ptr` instead" @@ -51,23 +70,6 @@ impl PyType { Self::from_borrowed_type_ptr(py, p).into_gil_ref() } - /// Converts the given FFI pointer into `Bound`, to use in safe code. - /// - /// The function creates a new reference from the given pointer, and returns - /// it as a `Bound`. - /// - /// # Safety - /// - The pointer must be a valid non-null reference to a `PyTypeObject` - #[inline] - pub unsafe fn from_borrowed_type_ptr( - py: Python<'_>, - p: *mut ffi::PyTypeObject, - ) -> Bound<'_, PyType> { - Borrowed::from_ptr_unchecked(py, p.cast()) - .downcast_unchecked() - .to_owned() - } - /// Gets the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name) of the `PyType`. pub fn qualname(&self) -> PyResult { self.as_borrowed().qualname() From c5f9001985c21ec9c5616c3ee09f2f1e5ca746af Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sat, 11 May 2024 16:48:45 +0200 Subject: [PATCH 334/349] Remove deferred reference count increments and make the global reference pool optional (#4095) * Add feature controlling the global reference pool to enable avoiding its overhead. * Document reference-pool feature in the performance guide. * Invert semantics of feature to disable reference pool so the new behaviour becomes opt-in * Remove delayed reference count increments as we cannot prevent reference count errors as long as these are available * Adjust tests to be compatible with disable-reference-pool feature * Adjust tests to be compatible with py-clone feature * Adjust the GIL benchmark to the updated reference pool semantics. * Further extend and clarify the documentation of the py-clone and disable-reference-pool features * Replace disable-reference-pool feature by pyo3_disable_reference_pool conditional compilation flag Such a flag is harder to use and thereby also harder to abuse. This seems appropriate as this is purely a performance-oriented change which show only be enabled by leaf crates and brings with it additional highly implicit sources of process aborts. * Add pyo3_leak_on_drop_without_reference_pool to turn aborts into leaks when the global reference pool is disabled and the GIL is not held --- Cargo.toml | 4 + examples/Cargo.toml | 2 +- guide/src/class.md | 4 +- guide/src/faq.md | 7 +- guide/src/features.md | 10 ++ guide/src/memory.md | 9 +- guide/src/migration.md | 13 +- guide/src/performance.md | 44 ++++++ newsfragments/4095.added.md | 1 + newsfragments/4095.changed.md | 1 + pyo3-benches/benches/bench_gil.rs | 12 +- pyo3-build-config/src/lib.rs | 2 + src/conversions/std/option.rs | 2 +- src/err/err_state.rs | 14 +- src/err/mod.rs | 2 +- src/gil.rs | 242 ++++++------------------------ src/instance.rs | 32 +++- src/marker.rs | 4 +- src/pybacked.rs | 9 +- src/tests/hygiene/pyclass.rs | 2 +- src/types/capsule.rs | 8 +- src/types/iterator.rs | 2 +- src/types/sequence.rs | 2 +- tests/test_buffer_protocol.rs | 2 +- tests/test_bytes.rs | 2 + tests/test_class_basics.rs | 14 +- tests/test_class_conversion.rs | 4 + tests/test_gc.rs | 2 + tests/test_methods.rs | 3 + tests/test_no_imports.rs | 3 + tests/test_sequence.rs | 3 + tests/test_serde.rs | 5 +- 32 files changed, 226 insertions(+), 240 deletions(-) create mode 100644 newsfragments/4095.added.md create mode 100644 newsfragments/4095.changed.md diff --git a/Cargo.toml b/Cargo.toml index 4c5a083060d..aba8fd1bc98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,9 @@ auto-initialize = [] # Allows use of the deprecated "GIL Refs" APIs. gil-refs = [] +# Enables `Clone`ing references to Python objects `Py` which panics if the GIL is not held. +py-clone = [] + # Optimizes PyObject to Vec conversion and so on. nightly = [] @@ -129,6 +132,7 @@ full = [ "num-bigint", "num-complex", "num-rational", + "py-clone", "rust_decimal", "serde", "smallvec", diff --git a/examples/Cargo.toml b/examples/Cargo.toml index e54b3b5cde2..81557e7f534 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -10,5 +10,5 @@ pyo3 = { path = "..", features = ["auto-initialize", "extension-module"] } [[example]] name = "decorator" path = "decorator/src/lib.rs" -crate_type = ["cdylib"] +crate-type = ["cdylib"] doc-scrape-examples = true diff --git a/guide/src/class.md b/guide/src/class.md index 3fcfaca4bdc..91a6fb2c495 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -249,7 +249,7 @@ fn return_myclass() -> Py { let obj = return_myclass(); -Python::with_gil(|py| { +Python::with_gil(move |py| { let bound = obj.bind(py); // Py::bind returns &Bound<'py, MyClass> let obj_ref = bound.borrow(); // Get PyRef assert_eq!(obj_ref.num, 1); @@ -280,6 +280,8 @@ let py_counter: Py = Python::with_gil(|py| { }); py_counter.get().value.fetch_add(1, Ordering::Relaxed); + +Python::with_gil(move |_py| drop(py_counter)); ``` Frozen classes are likely to become the default thereby guiding the PyO3 ecosystem towards a more deliberate application of interior mutability. Eventually, this should enable further optimizations of PyO3's internals and avoid downstream code paying the cost of interior mutability when it is not actually required. diff --git a/guide/src/faq.md b/guide/src/faq.md index 19f9b5d50ab..b79641a3803 100644 --- a/guide/src/faq.md +++ b/guide/src/faq.md @@ -127,12 +127,10 @@ If you don't want that cloning to happen, a workaround is to allocate the field ```rust # use pyo3::prelude::*; #[pyclass] -#[derive(Clone)] struct Inner {/* fields omitted */} #[pyclass] struct Outer { - #[pyo3(get)] inner: Py, } @@ -144,6 +142,11 @@ impl Outer { inner: Py::new(py, Inner {})?, }) } + + #[getter] + fn inner(&self, py: Python<'_>) -> Py { + self.inner.clone_ref(py) + } } ``` This time `a` and `b` *are* the same object: diff --git a/guide/src/features.md b/guide/src/features.md index 07085a9e89c..6a25d40cedc 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -75,6 +75,14 @@ This feature is a backwards-compatibility feature to allow continued use of the This feature and the APIs it enables is expected to be removed in a future PyO3 version. +### `py-clone` + +This feature was introduced to ease migration. It was found that delayed reference counts cannot be made sound and hence `Clon`ing an instance of `Py` must panic without the GIL being held. To avoid migrations introducing new panics without warning, the `Clone` implementation itself is now gated behind this feature. + +### `pyo3_disable_reference_pool` + +This is a performance-oriented conditional compilation flag, e.g. [set via `$RUSTFLAGS`][set-configuration-options], which disabled the global reference pool and the assocaited overhead for the crossing the Python-Rust boundary. However, if enabled, `Drop`ping an instance of `Py` without the GIL being held will abort the process. + ### `macros` This feature enables a dependency on the `pyo3-macros` crate, which provides the procedural macros portion of PyO3's API: @@ -195,3 +203,5 @@ struct User { ### `smallvec` Adds a dependency on [smallvec](https://docs.rs/smallvec) and enables conversions into its [`SmallVec`](https://docs.rs/smallvec/latest/smallvec/struct.SmallVec.html) type. + +[set-configuration-options]: https://doc.rust-lang.org/reference/conditional-compilation.html#set-configuration-options diff --git a/guide/src/memory.md b/guide/src/memory.md index b37d83563e5..38a31f4d0ef 100644 --- a/guide/src/memory.md +++ b/guide/src/memory.md @@ -212,7 +212,8 @@ This example wasn't very interesting. We could have just used a GIL-bound we are *not* holding the GIL? ```rust -# #![allow(unused_imports)] +# #![allow(unused_imports, dead_code)] +# #[cfg(not(pyo3_disable_reference_pool))] { # use pyo3::prelude::*; # use pyo3::types::PyString; # fn main() -> PyResult<()> { @@ -239,12 +240,14 @@ Python::with_gil(|py| # } # Ok(()) # } +# } ``` When `hello` is dropped *nothing* happens to the pointed-to memory on Python's heap because nothing _can_ happen if we're not holding the GIL. Fortunately, -the memory isn't leaked. PyO3 keeps track of the memory internally and will -release it the next time we acquire the GIL. +the memory isn't leaked. If the `pyo3_disable_reference_pool` conditional compilation flag +is not enabled, PyO3 keeps track of the memory internally and will release it +the next time we acquire the GIL. We can avoid the delay in releasing memory if we are careful to drop the `Py` while the GIL is held. diff --git a/guide/src/migration.md b/guide/src/migration.md index 7e0420de22d..10b62002a02 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -35,7 +35,16 @@ fn increment(x: u64, amount: Option) -> u64 { x + amount.unwrap_or(1) } ``` +
+ +### `Py::clone` is now gated behind the `py-clone` feature +
+Click to expand +If you rely on `impl Clone for Py` to fulfil trait requirements imposed by existing Rust code written without PyO3-based code in mind, the newly introduced feature `py-clone` must be enabled. +However, take care to note that the behaviour is different from previous versions. If `Clone` was called without the GIL being held, we tried to delay the application of these reference count increments until PyO3-based code would re-acquire it. This turned out to be impossible to implement in a sound manner and hence was removed. Now, if `Clone` is called without the GIL being held, we panic instead for which calling code might not be prepared. + +Related to this, we also added a `pyo3_disable_reference_pool` conditional compilation flag which removes the infrastructure necessary to apply delayed reference count decrements implied by `impl Drop for Py`. They do not appear to be a soundness hazard as they should lead to memory leaks in the worst case. However, the global synchronization adds significant overhead to cross the Python-Rust boundary. Enabling this feature will remove these costs and make the `Drop` implementation abort the process if called without the GIL being held instead.
## from 0.20.* to 0.21 @@ -676,7 +685,7 @@ drop(second); The replacement is [`Python::with_gil`](https://docs.rs/pyo3/0.18.3/pyo3/marker/struct.Python.html#method.with_gil) which is more cumbersome but enforces the proper nesting by design, e.g. -```rust +```rust,ignore # #![allow(dead_code)] # use pyo3::prelude::*; @@ -701,7 +710,7 @@ let second = Python::with_gil(|py| Object::new(py)); drop(first); drop(second); -// Or it ensure releasing the inner lock before the outer one. +// Or it ensures releasing the inner lock before the outer one. Python::with_gil(|py| { let first = Object::new(py); let second = Python::with_gil(|py| Object::new(py)); diff --git a/guide/src/performance.md b/guide/src/performance.md index c47a91deee5..b3d160fe6b1 100644 --- a/guide/src/performance.md +++ b/guide/src/performance.md @@ -96,3 +96,47 @@ impl PartialEq for FooBound<'_> { } } ``` + +## Disable the global reference pool + +PyO3 uses global mutable state to keep track of deferred reference count updates implied by `impl Drop for Py` being called without the GIL being held. The necessary synchronization to obtain and apply these reference count updates when PyO3-based code next acquires the GIL is somewhat expensive and can become a significant part of the cost of crossing the Python-Rust boundary. + +This functionality can be avoided by setting the `pyo3_disable_reference_pool` conditional compilation flag. This removes the global reference pool and the associated costs completely. However, it does _not_ remove the `Drop` implementation for `Py` which is necessary to interoperate with existing Rust code written without PyO3-based code in mind. To stay compatible with the wider Rust ecosystem in these cases, we keep the implementation but abort when `Drop` is called without the GIL being held. If `pyo3_leak_on_drop_without_reference_pool` is additionally enabled, objects dropped without the GIL being held will be leaked instead which is always sound but might have determinal effects like resource exhaustion in the long term. + +This limitation is important to keep in mind when this setting is used, especially when embedding Python code into a Rust application as it is quite easy to accidentally drop a `Py` (or types containing it like `PyErr`, `PyBackedStr` or `PyBackedBytes`) returned from `Python::with_gil` without making sure to re-acquire the GIL beforehand. For example, the following code + +```rust,ignore +# use pyo3::prelude::*; +# use pyo3::types::PyList; +let numbers: Py = Python::with_gil(|py| PyList::empty_bound(py).unbind()); + +Python::with_gil(|py| { + numbers.bind(py).append(23).unwrap(); +}); + +Python::with_gil(|py| { + numbers.bind(py).append(42).unwrap(); +}); +``` + +will abort if the list not explicitly disposed via + +```rust +# use pyo3::prelude::*; +# use pyo3::types::PyList; +let numbers: Py = Python::with_gil(|py| PyList::empty_bound(py).unbind()); + +Python::with_gil(|py| { + numbers.bind(py).append(23).unwrap(); +}); + +Python::with_gil(|py| { + numbers.bind(py).append(42).unwrap(); +}); + +Python::with_gil(move |py| { + drop(numbers); +}); +``` + +[conditional-compilation]: https://doc.rust-lang.org/reference/conditional-compilation.html diff --git a/newsfragments/4095.added.md b/newsfragments/4095.added.md new file mode 100644 index 00000000000..c9940f70f12 --- /dev/null +++ b/newsfragments/4095.added.md @@ -0,0 +1 @@ +Add `pyo3_disable_reference_pool` conditional compilation flag to avoid the overhead of the global reference pool at the cost of known limitations as explained in the performance section of the guide. diff --git a/newsfragments/4095.changed.md b/newsfragments/4095.changed.md new file mode 100644 index 00000000000..7f155ae04ef --- /dev/null +++ b/newsfragments/4095.changed.md @@ -0,0 +1 @@ +`Clone`ing pointers into the Python heap has been moved behind the `py-clone` feature, as it must panic without the GIL being held as a soundness fix. diff --git a/pyo3-benches/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs index 59b9ff9686f..cede8836f35 100644 --- a/pyo3-benches/benches/bench_gil.rs +++ b/pyo3-benches/benches/bench_gil.rs @@ -1,4 +1,4 @@ -use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; @@ -9,14 +9,8 @@ fn bench_clean_acquire_gil(b: &mut Bencher<'_>) { fn bench_dirty_acquire_gil(b: &mut Bencher<'_>) { let obj = Python::with_gil(|py| py.None()); - b.iter_batched( - || { - // Clone and drop an object so that the GILPool has work to do. - let _ = obj.clone(); - }, - |_| Python::with_gil(|_| {}), - BatchSize::NumBatches(1), - ); + // Drop the returned clone of the object so that the reference pool has work to do. + b.iter(|| Python::with_gil(|py| obj.clone_ref(py))); } fn criterion_benchmark(c: &mut Criterion) { diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 24d3ae28124..54aff4d10de 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -165,6 +165,8 @@ pub fn print_expected_cfgs() { println!("cargo:rustc-check-cfg=cfg(GraalPy)"); println!("cargo:rustc-check-cfg=cfg(py_sys_config, values(\"Py_DEBUG\", \"Py_REF_DEBUG\", \"Py_TRACE_REFS\", \"COUNT_ALLOCS\"))"); println!("cargo:rustc-check-cfg=cfg(invalid_from_utf8_lint)"); + println!("cargo:rustc-check-cfg=cfg(pyo3_disable_reference_pool)"); + println!("cargo:rustc-check-cfg=cfg(pyo3_leak_on_drop_without_reference_pool)"); // allow `Py_3_*` cfgs from the minimum supported version up to the // maximum minor version (+1 for development for the next) diff --git a/src/conversions/std/option.rs b/src/conversions/std/option.rs index 2fa082ba16a..13527315e70 100644 --- a/src/conversions/std/option.rs +++ b/src/conversions/std/option.rs @@ -61,7 +61,7 @@ mod tests { assert_eq!(option.as_ptr(), std::ptr::null_mut()); let none = py.None(); - option = Some(none.clone()); + option = Some(none.clone_ref(py)); let ref_cnt = none.get_refcnt(py); assert_eq!(option.as_ptr(), none.as_ptr()); diff --git a/src/err/err_state.rs b/src/err/err_state.rs index 9f85296f661..14345b275c9 100644 --- a/src/err/err_state.rs +++ b/src/err/err_state.rs @@ -5,7 +5,6 @@ use crate::{ Bound, IntoPy, Py, PyAny, PyObject, PyTypeInfo, Python, }; -#[derive(Clone)] pub(crate) struct PyErrStateNormalized { #[cfg(not(Py_3_12))] ptype: Py, @@ -63,6 +62,19 @@ impl PyErrStateNormalized { ptraceback: Py::from_owned_ptr_or_opt(py, ptraceback), } } + + pub fn clone_ref(&self, py: Python<'_>) -> Self { + Self { + #[cfg(not(Py_3_12))] + ptype: self.ptype.clone_ref(py), + pvalue: self.pvalue.clone_ref(py), + #[cfg(not(Py_3_12))] + ptraceback: self + .ptraceback + .as_ref() + .map(|ptraceback| ptraceback.clone_ref(py)), + } + } } pub(crate) struct PyErrStateLazyFnOutput { diff --git a/src/err/mod.rs b/src/err/mod.rs index 52e4b6c616a..29d4ac36294 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -837,7 +837,7 @@ impl PyErr { /// ``` #[inline] pub fn clone_ref(&self, py: Python<'_>) -> PyErr { - PyErr::from_state(PyErrState::Normalized(self.normalized(py).clone())) + PyErr::from_state(PyErrState::Normalized(self.normalized(py).clone_ref(py))) } /// Return the cause (either an exception instance, or None, set by `raise ... from ...`) diff --git a/src/gil.rs b/src/gil.rs index 29c4ffbe389..d4b92a4cff4 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -1,6 +1,8 @@ //! Interaction with Python's global interpreter lock use crate::impl_::not_send::{NotSend, NOT_SEND}; +#[cfg(pyo3_disable_reference_pool)] +use crate::impl_::panic::PanicTrap; use crate::{ffi, Python}; use std::cell::Cell; #[cfg(debug_assertions)] @@ -233,42 +235,32 @@ impl Drop for GILGuard { // Vector of PyObject type PyObjVec = Vec>; +#[cfg(not(pyo3_disable_reference_pool))] /// Thread-safe storage for objects which were inc_ref / dec_ref while the GIL was not held. struct ReferencePool { - // .0 is INCREFs, .1 is DECREFs - pointer_ops: sync::Mutex<(PyObjVec, PyObjVec)>, + pending_decrefs: sync::Mutex, } +#[cfg(not(pyo3_disable_reference_pool))] impl ReferencePool { const fn new() -> Self { Self { - pointer_ops: sync::Mutex::new((Vec::new(), Vec::new())), + pending_decrefs: sync::Mutex::new(Vec::new()), } } - fn register_incref(&self, obj: NonNull) { - self.pointer_ops.lock().unwrap().0.push(obj); - } - fn register_decref(&self, obj: NonNull) { - self.pointer_ops.lock().unwrap().1.push(obj); + self.pending_decrefs.lock().unwrap().push(obj); } fn update_counts(&self, _py: Python<'_>) { - let mut ops = self.pointer_ops.lock().unwrap(); - if ops.0.is_empty() && ops.1.is_empty() { + let mut pending_decrefs = self.pending_decrefs.lock().unwrap(); + if pending_decrefs.is_empty() { return; } - let (increfs, decrefs) = mem::take(&mut *ops); - drop(ops); - - // Always increase reference counts first - as otherwise objects which have a - // nonzero total reference count might be incorrectly dropped by Python during - // this update. - for ptr in increfs { - unsafe { ffi::Py_INCREF(ptr.as_ptr()) }; - } + let decrefs = mem::take(&mut *pending_decrefs); + drop(pending_decrefs); for ptr in decrefs { unsafe { ffi::Py_DECREF(ptr.as_ptr()) }; @@ -276,8 +268,10 @@ impl ReferencePool { } } +#[cfg(not(pyo3_disable_reference_pool))] unsafe impl Sync for ReferencePool {} +#[cfg(not(pyo3_disable_reference_pool))] static POOL: ReferencePool = ReferencePool::new(); /// A guard which can be used to temporarily release the GIL and restore on `Drop`. @@ -302,6 +296,7 @@ impl Drop for SuspendGIL { ffi::PyEval_RestoreThread(self.tstate); // Update counts of PyObjects / Py that were cloned or dropped while the GIL was released. + #[cfg(not(pyo3_disable_reference_pool))] POOL.update_counts(Python::assume_gil_acquired()); } } @@ -376,6 +371,7 @@ impl GILPool { pub unsafe fn new() -> GILPool { increment_gil_count(); // Update counts of PyObjects / Py that have been cloned or dropped since last acquisition + #[cfg(not(pyo3_disable_reference_pool))] POOL.update_counts(Python::assume_gil_acquired()); GILPool { start: OWNED_OBJECTS @@ -434,11 +430,13 @@ impl Drop for GILPool { /// /// # Safety /// The object must be an owned Python reference. +#[cfg(feature = "py-clone")] +#[track_caller] pub unsafe fn register_incref(obj: NonNull) { if gil_is_acquired() { ffi::Py_INCREF(obj.as_ptr()) } else { - POOL.register_incref(obj); + panic!("Cannot clone pointer into Python heap without the GIL being held."); } } @@ -450,11 +448,21 @@ pub unsafe fn register_incref(obj: NonNull) { /// /// # Safety /// The object must be an owned Python reference. +#[track_caller] pub unsafe fn register_decref(obj: NonNull) { if gil_is_acquired() { ffi::Py_DECREF(obj.as_ptr()) } else { + #[cfg(not(pyo3_disable_reference_pool))] POOL.register_decref(obj); + #[cfg(all( + pyo3_disable_reference_pool, + not(pyo3_leak_on_drop_without_reference_pool) + ))] + { + let _trap = PanicTrap::new("Aborting the process to avoid panic-from-drop."); + panic!("Cannot drop pointer into Python heap without the GIL being held."); + } } } @@ -508,15 +516,18 @@ fn decrement_gil_count() { mod tests { #[allow(deprecated)] use super::GILPool; - use super::{gil_is_acquired, GIL_COUNT, POOL}; + #[cfg(not(pyo3_disable_reference_pool))] + use super::POOL; + use super::{gil_is_acquired, GIL_COUNT}; + #[cfg(not(pyo3_disable_reference_pool))] + use crate::ffi; use crate::types::any::PyAnyMethods; - use crate::{ffi, PyObject, Python}; + use crate::{PyObject, Python}; #[cfg(feature = "gil-refs")] use {super::OWNED_OBJECTS, crate::gil}; + #[cfg(not(pyo3_disable_reference_pool))] use std::ptr::NonNull; - #[cfg(not(target_arch = "wasm32"))] - use std::sync; fn get_object(py: Python<'_>) -> PyObject { py.eval_bound("object()", None, None).unwrap().unbind() @@ -531,30 +542,20 @@ mod tests { len } - fn pool_inc_refs_does_not_contain(obj: &PyObject) -> bool { - !POOL - .pointer_ops - .lock() - .unwrap() - .0 - .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) - } - + #[cfg(not(pyo3_disable_reference_pool))] fn pool_dec_refs_does_not_contain(obj: &PyObject) -> bool { !POOL - .pointer_ops + .pending_decrefs .lock() .unwrap() - .1 .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } - #[cfg(not(target_arch = "wasm32"))] + #[cfg(all(not(pyo3_disable_reference_pool), not(target_arch = "wasm32")))] fn pool_dec_refs_contains(obj: &PyObject) -> bool { - POOL.pointer_ops + POOL.pending_decrefs .lock() .unwrap() - .1 .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } @@ -629,20 +630,20 @@ mod tests { let reference = obj.clone_ref(py); assert_eq!(obj.get_refcnt(py), 2); - assert!(pool_inc_refs_does_not_contain(&obj)); + #[cfg(not(pyo3_disable_reference_pool))] assert!(pool_dec_refs_does_not_contain(&obj)); // With the GIL held, reference count will be decreased immediately. drop(reference); assert_eq!(obj.get_refcnt(py), 1); - assert!(pool_inc_refs_does_not_contain(&obj)); + #[cfg(not(pyo3_disable_reference_pool))] assert!(pool_dec_refs_does_not_contain(&obj)); }); } #[test] - #[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled + #[cfg(all(not(pyo3_disable_reference_pool), not(target_arch = "wasm32")))] // We are building wasm Python with pthreads disabled fn test_pyobject_drop_without_gil_doesnt_decrease_refcnt() { let obj = Python::with_gil(|py| { let obj = get_object(py); @@ -650,7 +651,6 @@ mod tests { let reference = obj.clone_ref(py); assert_eq!(obj.get_refcnt(py), 2); - assert!(pool_inc_refs_does_not_contain(&obj)); assert!(pool_dec_refs_does_not_contain(&obj)); // Drop reference in a separate thread which doesn't have the GIL. @@ -659,7 +659,6 @@ mod tests { // The reference count should not have changed (the GIL has always // been held by this thread), it is remembered to release later. assert_eq!(obj.get_refcnt(py), 2); - assert!(pool_inc_refs_does_not_contain(&obj)); assert!(pool_dec_refs_contains(&obj)); obj }); @@ -667,9 +666,7 @@ mod tests { // Next time the GIL is acquired, the reference is released Python::with_gil(|py| { assert_eq!(obj.get_refcnt(py), 1); - let non_null = unsafe { NonNull::new_unchecked(obj.as_ptr()) }; - assert!(!POOL.pointer_ops.lock().unwrap().0.contains(&non_null)); - assert!(!POOL.pointer_ops.lock().unwrap().1.contains(&non_null)); + assert!(pool_dec_refs_does_not_contain(&obj)); }); } @@ -725,19 +722,16 @@ mod tests { assert!(!gil_is_acquired()); } + #[cfg(feature = "py-clone")] #[test] + #[should_panic] fn test_allow_threads_updates_refcounts() { Python::with_gil(|py| { // Make a simple object with 1 reference let obj = get_object(py); assert!(obj.get_refcnt(py) == 1); - // Clone the object without the GIL to use internal tracking - let escaped_ref = py.allow_threads(|| obj.clone()); - // But after the block the refcounts are updated - assert!(obj.get_refcnt(py) == 2); - drop(escaped_ref); - assert!(obj.get_refcnt(py) == 1); - drop(obj); + // Clone the object without the GIL which should panic + py.allow_threads(|| obj.clone()); }); } @@ -752,6 +746,7 @@ mod tests { }) } + #[cfg(feature = "py-clone")] #[test] fn test_clone_with_gil() { Python::with_gil(|py| { @@ -765,147 +760,8 @@ mod tests { }) } - #[cfg(not(target_arch = "wasm32"))] - struct Event { - set: sync::Mutex, - wait: sync::Condvar, - } - - #[cfg(not(target_arch = "wasm32"))] - impl Event { - const fn new() -> Self { - Self { - set: sync::Mutex::new(false), - wait: sync::Condvar::new(), - } - } - - fn set(&self) { - *self.set.lock().unwrap() = true; - self.wait.notify_all(); - } - - fn wait(&self) { - drop( - self.wait - .wait_while(self.set.lock().unwrap(), |s| !*s) - .unwrap(), - ); - } - } - - #[test] - #[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled - fn test_clone_without_gil() { - use crate::{Py, PyAny}; - use std::{sync::Arc, thread}; - - // Some events for synchronizing - static GIL_ACQUIRED: Event = Event::new(); - static OBJECT_CLONED: Event = Event::new(); - static REFCNT_CHECKED: Event = Event::new(); - - Python::with_gil(|py| { - let obj: Arc> = Arc::new(get_object(py)); - let thread_obj = Arc::clone(&obj); - - let count = obj.get_refcnt(py); - println!( - "1: The object has been created and its reference count is {}", - count - ); - - let handle = thread::spawn(move || { - Python::with_gil(move |py| { - println!("3. The GIL has been acquired on another thread."); - GIL_ACQUIRED.set(); - - // Wait while the main thread registers obj in POOL - OBJECT_CLONED.wait(); - println!("5. Checking refcnt"); - assert_eq!(thread_obj.get_refcnt(py), count); - - REFCNT_CHECKED.set(); - }) - }); - - let cloned = py.allow_threads(|| { - println!("2. The GIL has been released."); - - // Wait until the GIL has been acquired on the thread. - GIL_ACQUIRED.wait(); - - println!("4. The other thread is now hogging the GIL, we clone without it held"); - // Cloning without GIL should not update reference count - let cloned = Py::clone(&*obj); - OBJECT_CLONED.set(); - cloned - }); - - REFCNT_CHECKED.wait(); - - println!("6. The main thread has acquired the GIL again and processed the pool."); - - // Total reference count should be one higher - assert_eq!(obj.get_refcnt(py), count + 1); - - // Clone dropped - drop(cloned); - // Ensure refcount of the arc is 1 - handle.join().unwrap(); - - // Overall count is now back to the original, and should be no pending change - assert_eq!(Arc::try_unwrap(obj).unwrap().get_refcnt(py), count); - }); - } - - #[test] - #[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled - fn test_clone_in_other_thread() { - use crate::Py; - use std::{sync::Arc, thread}; - - // Some events for synchronizing - static OBJECT_CLONED: Event = Event::new(); - - let (obj, count, ptr) = Python::with_gil(|py| { - let obj = Arc::new(get_object(py)); - let count = obj.get_refcnt(py); - let thread_obj = Arc::clone(&obj); - - // Start a thread which does not have the GIL, and clone it - let t = thread::spawn(move || { - // Cloning without GIL should not update reference count - #[allow(clippy::redundant_clone)] - let _ = Py::clone(&*thread_obj); - OBJECT_CLONED.set(); - }); - - OBJECT_CLONED.wait(); - assert_eq!(count, obj.get_refcnt(py)); - - t.join().unwrap(); - let ptr = NonNull::new(obj.as_ptr()).unwrap(); - - // The pointer should appear once in the incref pool, and once in the - // decref pool (for the clone being created and also dropped) - assert!(POOL.pointer_ops.lock().unwrap().0.contains(&ptr)); - assert!(POOL.pointer_ops.lock().unwrap().1.contains(&ptr)); - - (obj, count, ptr) - }); - - Python::with_gil(|py| { - // Acquiring the gil clears the pool - assert!(!POOL.pointer_ops.lock().unwrap().0.contains(&ptr)); - assert!(!POOL.pointer_ops.lock().unwrap().1.contains(&ptr)); - - // Overall count is still unchanged - assert_eq!(count, obj.get_refcnt(py)); - }); - } - #[test] + #[cfg(not(pyo3_disable_reference_pool))] fn test_update_counts_does_not_deadlock() { // update_counts can run arbitrary Python code during Py_DECREF. // if the locking is implemented incorrectly, it will deadlock. diff --git a/src/instance.rs b/src/instance.rs index 1c510b90be5..7835b9c79b6 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -81,10 +81,11 @@ where /// struct Foo {/* fields omitted */} /// /// # fn main() -> PyResult<()> { - /// Python::with_gil(|py| -> PyResult> { + /// let foo: Py = Python::with_gil(|py| -> PyResult<_> { /// let foo: Bound<'_, Foo> = Bound::new(py, Foo {})?; /// Ok(foo.into()) /// })?; + /// # Python::with_gil(move |_py| drop(foo)); /// # Ok(()) /// # } /// ``` @@ -865,7 +866,9 @@ impl IntoPy for Borrowed<'_, '_, T> { /// // All of these are valid syntax /// let second = Py::clone_ref(&first, py); /// let third = first.clone_ref(py); +/// #[cfg(feature = "py-clone")] /// let fourth = Py::clone(&first); +/// #[cfg(feature = "py-clone")] /// let fifth = first.clone(); /// /// // Disposing of our original `Py` just decrements the reference count. @@ -873,7 +876,9 @@ impl IntoPy for Borrowed<'_, '_, T> { /// /// // They all point to the same object /// assert!(second.is(&third)); +/// #[cfg(feature = "py-clone")] /// assert!(fourth.is(&fifth)); +/// #[cfg(feature = "py-clone")] /// assert!(second.is(&fourth)); /// }); /// # } @@ -935,10 +940,11 @@ where /// struct Foo {/* fields omitted */} /// /// # fn main() -> PyResult<()> { - /// Python::with_gil(|py| -> PyResult> { + /// let foo = Python::with_gil(|py| -> PyResult<_> { /// let foo: Py = Py::new(py, Foo {})?; /// Ok(foo) /// })?; + /// # Python::with_gil(move |_py| drop(foo)); /// # Ok(()) /// # } /// ``` @@ -1244,6 +1250,7 @@ where /// }); /// /// cell.get().value.fetch_add(1, Ordering::Relaxed); + /// # Python::with_gil(move |_py| drop(cell)); /// ``` #[inline] pub fn get(&self) -> &T @@ -1804,9 +1811,12 @@ where } /// If the GIL is held this increments `self`'s reference count. -/// Otherwise this registers the [`Py`]`` instance to have its reference count -/// incremented the next time PyO3 acquires the GIL. +/// Otherwise, it will panic. +/// +/// Only available if the `py-clone` feature is enabled. +#[cfg(feature = "py-clone")] impl Clone for Py { + #[track_caller] fn clone(&self) -> Self { unsafe { gil::register_incref(self.0); @@ -1815,8 +1825,16 @@ impl Clone for Py { } } -/// Dropping a `Py` instance decrements the reference count on the object by 1. +/// Dropping a `Py` instance decrements the reference count +/// on the object by one if the GIL is held. +/// +/// Otherwise and by default, this registers the underlying pointer to have its reference count +/// decremented the next time PyO3 acquires the GIL. +/// +/// However, if the `pyo3_disable_reference_pool` conditional compilation flag +/// is enabled, it will abort the process. impl Drop for Py { + #[track_caller] fn drop(&mut self) { unsafe { gil::register_decref(self.0); @@ -2039,7 +2057,9 @@ mod tests { Py::from(native) }); - assert_eq!(Python::with_gil(|py| dict.get_refcnt(py)), 1); + Python::with_gil(move |py| { + assert_eq!(dict.get_refcnt(py), 1); + }); } #[test] diff --git a/src/marker.rs b/src/marker.rs index b6821deb036..072b882c875 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -1178,7 +1178,6 @@ impl<'unbound> Python<'unbound> { mod tests { use super::*; use crate::types::{IntoPyDict, PyList}; - use std::sync::Arc; #[test] fn test_eval() { @@ -1264,11 +1263,12 @@ mod tests { }); } + #[cfg(not(pyo3_disable_reference_pool))] #[test] fn test_allow_threads_pass_stuff_in() { let list = Python::with_gil(|py| PyList::new_bound(py, vec!["foo", "bar"]).unbind()); let mut v = vec![1, 2, 3]; - let a = Arc::new(String::from("foo")); + let a = std::sync::Arc::new(String::from("foo")); Python::with_gil(|py| { py.allow_threads(|| { diff --git a/src/pybacked.rs b/src/pybacked.rs index e0bacb86144..ea5face516b 100644 --- a/src/pybacked.rs +++ b/src/pybacked.rs @@ -13,7 +13,7 @@ use crate::{ /// A wrapper around `str` where the storage is owned by a Python `bytes` or `str` object. /// /// This type gives access to the underlying data via a `Deref` implementation. -#[derive(Clone)] +#[cfg_attr(feature = "py-clone", derive(Clone))] pub struct PyBackedStr { #[allow(dead_code)] // only held so that the storage is not dropped storage: Py, @@ -88,7 +88,7 @@ impl FromPyObject<'_> for PyBackedStr { /// A wrapper around `[u8]` where the storage is either owned by a Python `bytes` object, or a Rust `Box<[u8]>`. /// /// This type gives access to the underlying data via a `Deref` implementation. -#[derive(Clone)] +#[cfg_attr(feature = "py-clone", derive(Clone))] pub struct PyBackedBytes { #[allow(dead_code)] // only held so that the storage is not dropped storage: PyBackedBytesStorage, @@ -96,7 +96,7 @@ pub struct PyBackedBytes { } #[allow(dead_code)] -#[derive(Clone)] +#[cfg_attr(feature = "py-clone", derive(Clone))] enum PyBackedBytesStorage { Python(Py), Rust(Arc<[u8]>), @@ -336,6 +336,7 @@ mod test { is_sync::(); } + #[cfg(feature = "py-clone")] #[test] fn test_backed_str_clone() { Python::with_gil(|py| { @@ -398,6 +399,7 @@ mod test { }) } + #[cfg(feature = "py-clone")] #[test] fn test_backed_bytes_from_bytes_clone() { Python::with_gil(|py| { @@ -410,6 +412,7 @@ mod test { }); } + #[cfg(feature = "py-clone")] #[test] fn test_backed_bytes_from_bytearray_clone() { Python::with_gil(|py| { diff --git a/src/tests/hygiene/pyclass.rs b/src/tests/hygiene/pyclass.rs index 0bdb280d066..34b30a8c6f4 100644 --- a/src/tests/hygiene/pyclass.rs +++ b/src/tests/hygiene/pyclass.rs @@ -25,7 +25,7 @@ pub struct Bar { a: u8, #[pyo3(get, set)] b: Foo, - #[pyo3(get, set)] + #[pyo3(set)] c: ::std::option::Option>, } diff --git a/src/types/capsule.rs b/src/types/capsule.rs index 2851970ba10..9b9445cdffe 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -489,7 +489,7 @@ mod tests { cap.into() }); - Python::with_gil(|py| { + Python::with_gil(move |py| { let f = unsafe { cap.bind(py).reference:: u32>() }; assert_eq!(f(123), 123); }); @@ -553,7 +553,7 @@ mod tests { cap.into() }); - Python::with_gil(|py| { + Python::with_gil(move |py| { let ctx: &Vec = unsafe { cap.bind(py).reference() }; assert_eq!(ctx, &[1, 2, 3, 4]); }) @@ -572,7 +572,7 @@ mod tests { cap.into() }); - Python::with_gil(|py| { + Python::with_gil(move |py| { let ctx_ptr: *mut c_void = cap.bind(py).context().unwrap(); let ctx = unsafe { *Box::from_raw(ctx_ptr.cast::<&Vec>()) }; assert_eq!(ctx, &vec![1_u8, 2, 3, 4]); @@ -589,7 +589,7 @@ mod tests { context.send(true).unwrap(); } - Python::with_gil(|py| { + Python::with_gil(move |py| { let name = CString::new("foo").unwrap(); let cap = PyCapsule::new_bound_with_destructor(py, 0, Some(name), destructor).unwrap(); cap.set_context(Box::into_raw(Box::new(tx)).cast()).unwrap(); diff --git a/src/types/iterator.rs b/src/types/iterator.rs index 4562efde3f8..1835f484adf 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -195,7 +195,7 @@ mod tests { ); }); - Python::with_gil(|py| { + Python::with_gil(move |py| { assert_eq!(count, obj.get_refcnt(py)); }); } diff --git a/src/types/sequence.rs b/src/types/sequence.rs index f75d851973d..a5765ebc8b2 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -823,7 +823,7 @@ mod tests { assert!(seq.get_item(1).unwrap().as_ptr() == obj.as_ptr()); }); - Python::with_gil(|py| { + Python::with_gil(move |py| { assert_eq!(1, obj.get_refcnt(py)); }); } diff --git a/tests/test_buffer_protocol.rs b/tests/test_buffer_protocol.rs index dca900808a8..700bcdcd206 100644 --- a/tests/test_buffer_protocol.rs +++ b/tests/test_buffer_protocol.rs @@ -122,7 +122,7 @@ fn test_releasebuffer_unraisable_error() { let capture = UnraisableCapture::install(py); let instance = Py::new(py, ReleaseBufferError {}).unwrap(); - let env = [("ob", instance.clone())].into_py_dict_bound(py); + let env = [("ob", instance.clone_ref(py))].into_py_dict_bound(py); assert!(capture.borrow(py).capture.is_none()); diff --git a/tests/test_bytes.rs b/tests/test_bytes.rs index a3f1e2fcafe..5adca3f154a 100644 --- a/tests/test_bytes.rs +++ b/tests/test_bytes.rs @@ -48,4 +48,6 @@ fn test_py_as_bytes() { let data = Python::with_gil(|py| pyobj.as_bytes(py)); assert_eq!(data, b"abc"); + + Python::with_gil(move |_py| drop(pyobj)); } diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index b7bee2638ca..d0e745377c8 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -172,6 +172,7 @@ fn empty_class_in_module() { }); } +#[cfg(feature = "py-clone")] #[pyclass] struct ClassWithObjectField { // It used to be that PyObject was not supported with (get, set) @@ -180,6 +181,7 @@ struct ClassWithObjectField { value: PyObject, } +#[cfg(feature = "py-clone")] #[pymethods] impl ClassWithObjectField { #[new] @@ -188,6 +190,7 @@ impl ClassWithObjectField { } } +#[cfg(feature = "py-clone")] #[test] fn class_with_object_field() { Python::with_gil(|py| { @@ -229,7 +232,7 @@ impl UnsendableChild { } fn test_unsendable() -> PyResult<()> { - let obj = Python::with_gil(|py| -> PyResult<_> { + let (keep_obj_here, obj) = Python::with_gil(|py| -> PyResult<_> { let obj: Py = PyType::new_bound::(py).call1((5,))?.extract()?; // Accessing the value inside this thread should not panic @@ -241,14 +244,13 @@ fn test_unsendable() -> PyResult<()> { .is_err(); assert!(!caught_panic); - Ok(obj) - })?; - let keep_obj_here = obj.clone(); + Ok((obj.clone_ref(py), obj)) + })?; let caught_panic = std::thread::spawn(move || { // This access must panic - Python::with_gil(|py| { + Python::with_gil(move |py| { obj.borrow(py); }); }) @@ -549,6 +551,8 @@ fn access_frozen_class_without_gil() { }); assert_eq!(py_counter.get().value.load(Ordering::Relaxed), 1); + + Python::with_gil(move |_py| drop(py_counter)); } #[test] diff --git a/tests/test_class_conversion.rs b/tests/test_class_conversion.rs index ede8928f865..a46132b9586 100644 --- a/tests/test_class_conversion.rs +++ b/tests/test_class_conversion.rs @@ -54,12 +54,14 @@ impl SubClass { } } +#[cfg(feature = "py-clone")] #[pyclass] struct PolymorphicContainer { #[pyo3(get, set)] inner: Py, } +#[cfg(feature = "py-clone")] #[test] fn test_polymorphic_container_stores_base_class() { Python::with_gil(|py| { @@ -76,6 +78,7 @@ fn test_polymorphic_container_stores_base_class() { }); } +#[cfg(feature = "py-clone")] #[test] fn test_polymorphic_container_stores_sub_class() { Python::with_gil(|py| { @@ -103,6 +106,7 @@ fn test_polymorphic_container_stores_sub_class() { }); } +#[cfg(feature = "py-clone")] #[test] fn test_polymorphic_container_does_not_accept_other_types() { Python::with_gil(|py| { diff --git a/tests/test_gc.rs b/tests/test_gc.rs index 11d9221c7ad..b95abd4adea 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -445,6 +445,7 @@ impl DropDuringTraversal { } } +#[cfg(not(pyo3_disable_reference_pool))] #[test] fn drop_during_traversal_with_gil() { let drop_called = Arc::new(AtomicBool::new(false)); @@ -476,6 +477,7 @@ fn drop_during_traversal_with_gil() { assert!(drop_called.load(Ordering::Relaxed)); } +#[cfg(not(pyo3_disable_reference_pool))] #[test] fn drop_during_traversal_without_gil() { let drop_called = Arc::new(AtomicBool::new(false)); diff --git a/tests/test_methods.rs b/tests/test_methods.rs index 37f3b2d8bd6..615e2dba0af 100644 --- a/tests/test_methods.rs +++ b/tests/test_methods.rs @@ -874,6 +874,7 @@ fn test_from_sequence() { }); } +#[cfg(feature = "py-clone")] #[pyclass] struct r#RawIdents { #[pyo3(get, set)] @@ -882,6 +883,7 @@ struct r#RawIdents { r#subsubtype: PyObject, } +#[cfg(feature = "py-clone")] #[pymethods] impl r#RawIdents { #[new] @@ -946,6 +948,7 @@ impl r#RawIdents { } } +#[cfg(feature = "py-clone")] #[test] fn test_raw_idents() { Python::with_gil(|py| { diff --git a/tests/test_no_imports.rs b/tests/test_no_imports.rs index 3509a11f4be..89d54f4e057 100644 --- a/tests/test_no_imports.rs +++ b/tests/test_no_imports.rs @@ -143,12 +143,14 @@ fn test_basic() { }); } +#[cfg(feature = "py-clone")] #[pyo3::pyclass] struct NewClassMethod { #[pyo3(get)] cls: pyo3::PyObject, } +#[cfg(feature = "py-clone")] #[pyo3::pymethods] impl NewClassMethod { #[new] @@ -160,6 +162,7 @@ impl NewClassMethod { } } +#[cfg(feature = "py-clone")] #[test] fn test_new_class_method() { pyo3::Python::with_gil(|py| { diff --git a/tests/test_sequence.rs b/tests/test_sequence.rs index 9627f06ca75..8adba35c86a 100644 --- a/tests/test_sequence.rs +++ b/tests/test_sequence.rs @@ -248,12 +248,14 @@ fn test_inplace_repeat() { // Check that #[pyo3(get, set)] works correctly for Vec +#[cfg(feature = "py-clone")] #[pyclass] struct GenericList { #[pyo3(get, set)] items: Vec, } +#[cfg(feature = "py-clone")] #[test] fn test_generic_list_get() { Python::with_gil(|py| { @@ -266,6 +268,7 @@ fn test_generic_list_get() { }); } +#[cfg(feature = "py-clone")] #[test] fn test_generic_list_set() { Python::with_gil(|py| { diff --git a/tests/test_serde.rs b/tests/test_serde.rs index f1d5bee4bee..9e97946b5f2 100644 --- a/tests/test_serde.rs +++ b/tests/test_serde.rs @@ -11,7 +11,7 @@ mod test_serde { } #[pyclass] - #[derive(Debug, Clone, Serialize, Deserialize)] + #[derive(Debug, Serialize, Deserialize)] struct User { username: String, group: Option>, @@ -27,7 +27,8 @@ mod test_serde { }; let friend2 = User { username: "friend 2".into(), - ..friend1.clone() + group: None, + friends: vec![], }; let user = Python::with_gil(|py| { From 57500d9b090ae5249bb69943f22ceb22a03e1dea Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sat, 11 May 2024 12:48:38 -0400 Subject: [PATCH 335/349] Updates comments regarding the reference pool that were inaccurate (#4176) --- src/gil.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/gil.rs b/src/gil.rs index d4b92a4cff4..db8311ab2fc 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -236,7 +236,7 @@ impl Drop for GILGuard { type PyObjVec = Vec>; #[cfg(not(pyo3_disable_reference_pool))] -/// Thread-safe storage for objects which were inc_ref / dec_ref while the GIL was not held. +/// Thread-safe storage for objects which were dec_ref while the GIL was not held. struct ReferencePool { pending_decrefs: sync::Mutex, } @@ -422,11 +422,8 @@ impl Drop for GILPool { } } -/// Registers a Python object pointer inside the release pool, to have its reference count increased -/// the next time the GIL is acquired in pyo3. -/// -/// If the GIL is held, the reference count will be increased immediately instead of being queued -/// for later. +/// Increments the reference count of a Python object if the GIL is held. If +/// the GIL is not held, this function will panic. /// /// # Safety /// The object must be an owned Python reference. From 10152a7078b6cd3470bd8400144629b94f74c59e Mon Sep 17 00:00:00 2001 From: Icxolu <10486322+Icxolu@users.noreply.github.com> Date: Sun, 12 May 2024 20:30:08 +0200 Subject: [PATCH 336/349] feature gate `PyCell` (#4177) * feature gate `PyCell` * feature gate `HasPyGilRef` completely * bump version --- Cargo.toml | 10 +-- guide/src/class.md | 1 + guide/src/migration.md | 3 +- pyo3-build-config/Cargo.toml | 2 +- pyo3-ffi/Cargo.toml | 4 +- pyo3-macros-backend/Cargo.toml | 5 +- pyo3-macros-backend/src/module.rs | 7 +- pyo3-macros-backend/src/pyclass.rs | 10 ++- pyo3-macros/Cargo.toml | 5 +- pyproject.toml | 2 +- src/conversion.rs | 7 +- src/err/mod.rs | 9 ++- src/exceptions.rs | 1 + src/impl_/deprecations.rs | 1 + src/impl_/pyclass.rs | 10 ++- src/instance.rs | 22 +++--- src/lib.rs | 12 ++-- src/macros.rs | 1 + src/prelude.rs | 2 + src/pycell.rs | 76 +++++++++++--------- src/pycell/impl_.rs | 3 + src/pyclass.rs | 13 ++++ src/type_object.rs | 74 ++++++++++++++++++- src/types/any.rs | 2 +- src/types/mod.rs | 3 + tests/test_compile_error.rs | 4 +- tests/ui/deprecations.stderr | 6 ++ tests/ui/invalid_intern_arg.stderr | 4 +- tests/ui/invalid_pymethods_duplicates.stderr | 17 ----- tests/ui/wrong_aspyref_lifetimes.stderr | 2 +- 30 files changed, 220 insertions(+), 98 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aba8fd1bc98..921e551751c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.21.2" +version = "0.22.0-dev" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -20,10 +20,10 @@ libc = "0.2.62" memoffset = "0.9" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.21.2" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.22.0-dev" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.21.2", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.22.0-dev", optional = true } indoc = { version = "2.0.1", optional = true } unindent = { version = "0.2.1", optional = true } @@ -62,7 +62,7 @@ rayon = "1.6.1" futures = "0.3.28" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.21.2", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.22.0-dev", features = ["resolve-config"] } [features] default = ["macros"] @@ -106,7 +106,7 @@ generate-import-lib = ["pyo3-ffi/generate-import-lib"] auto-initialize = [] # Allows use of the deprecated "GIL Refs" APIs. -gil-refs = [] +gil-refs = ["pyo3-macros/gil-refs"] # Enables `Clone`ing references to Python objects `Py` which panics if the GIL is not held. py-clone = [] diff --git a/guide/src/class.md b/guide/src/class.md index 91a6fb2c495..ce86ec40e5f 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -1307,6 +1307,7 @@ struct MyClass { impl pyo3::types::DerefToPyAny for MyClass {} # #[allow(deprecated)] +# #[cfg(feature = "gil-refs")] unsafe impl pyo3::type_object::HasPyGilRef for MyClass { type AsRefTarget = pyo3::PyCell; } diff --git a/guide/src/migration.md b/guide/src/migration.md index 10b62002a02..f56db2a5fc7 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -1609,7 +1609,7 @@ For more, see [the constructor section](class.md#constructor) of this guide.
Click to expand -PyO3 0.9 introduces [`PyCell`], which is a [`RefCell`]-like object wrapper +PyO3 0.9 introduces `PyCell`, which is a [`RefCell`]-like object wrapper for ensuring Rust's rules regarding aliasing of references are upheld. For more detail, see the [Rust Book's section on Rust's rules of references](https://doc.rust-lang.org/book/ch04-02-references-and-borrowing.html#the-rules-of-references) @@ -1788,7 +1788,6 @@ impl PySequenceProtocol for ByteSequence { [`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html [`PyAny`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html -[`PyCell`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyCell.html [`PyBorrowMutError`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyBorrowMutError.html [`PyRef`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html [`PyRefMut`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 60bf9a13ef9..600237f8646 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.21.2" +version = "0.22.0-dev" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 8f7767254f1..865da93926a 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.21.2" +version = "0.22.0-dev" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,7 +38,7 @@ abi3-py312 = ["abi3", "pyo3-build-config/abi3-py312"] generate-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.2", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.22.0-dev", features = ["resolve-config"] } [lints] workspace = true diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index c2ffd53b0fc..7bc0f6a2da1 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.21.2" +version = "0.22.0-dev" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -16,7 +16,7 @@ edition = "2021" [dependencies] heck = "0.5" proc-macro2 = { version = "1", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.21.2", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.22.0-dev", features = ["resolve-config"] } quote = { version = "1", default-features = false } [dependencies.syn] @@ -29,3 +29,4 @@ workspace = true [features] experimental-async = [] +gil-refs = [] diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 626cde121e6..756037263e3 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -384,6 +384,11 @@ fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn let Ctx { pyo3_path } = ctx; let mut stmts: Vec = Vec::new(); + #[cfg(feature = "gil-refs")] + let imports = quote!(use #pyo3_path::{PyNativeType, types::PyModuleMethods};); + #[cfg(not(feature = "gil-refs"))] + let imports = quote!(use #pyo3_path::types::PyModuleMethods;); + for mut stmt in func.block.stmts.drain(..) { if let syn::Stmt::Item(Item::Fn(func)) = &mut stmt { if let Some(pyfn_args) = get_pyfn_attr(&mut func.attrs)? { @@ -394,7 +399,7 @@ fn process_functions_in_module(options: &PyModuleOptions, func: &mut syn::ItemFn #wrapped_function { #[allow(unknown_lints, unused_imports, redundant_imports)] - use #pyo3_path::{PyNativeType, types::PyModuleMethods}; + #imports #module_name.as_borrowed().add_function(#pyo3_path::wrap_pyfunction!(#name, #module_name.as_borrowed())?)?; } }; diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 179fe71bb9e..4e71a711802 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1307,11 +1307,19 @@ fn impl_pytypeinfo( quote! { ::core::option::Option::None } }; - quote! { + #[cfg(feature = "gil-refs")] + let has_py_gil_ref = quote! { #[allow(deprecated)] unsafe impl #pyo3_path::type_object::HasPyGilRef for #cls { type AsRefTarget = #pyo3_path::PyCell; } + }; + + #[cfg(not(feature = "gil-refs"))] + let has_py_gil_ref = TokenStream::new(); + + quote! { + #has_py_gil_ref unsafe impl #pyo3_path::type_object::PyTypeInfo for #cls { const NAME: &'static str = #cls_name; diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 690924c76a5..e4b550cfb8e 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.21.2" +version = "0.22.0-dev" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -17,12 +17,13 @@ proc-macro = true multiple-pymethods = [] experimental-async = ["pyo3-macros-backend/experimental-async"] experimental-declarative-modules = [] +gil-refs = ["pyo3-macros-backend/gil-refs"] [dependencies] proc-macro2 = { version = "1", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.21.2" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.22.0-dev" } [lints] workspace = true diff --git a/pyproject.toml b/pyproject.toml index 9a70116f301..a007ee6dc7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.towncrier] filename = "CHANGELOG.md" -version = "0.21.2" +version = "0.22.0-dev" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" diff --git a/src/conversion.rs b/src/conversion.rs index 95931d30d2e..44dbc3c7eed 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -5,14 +5,12 @@ use crate::inspect::types::TypeInfo; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; use crate::types::PyTuple; -use crate::{ - ffi, Borrowed, Bound, Py, PyAny, PyClass, PyNativeType, PyObject, PyRef, PyRefMut, Python, -}; +use crate::{ffi, Borrowed, Bound, Py, PyAny, PyClass, PyObject, PyRef, PyRefMut, Python}; #[cfg(feature = "gil-refs")] use { crate::{ err::{self, PyDowncastError}, - gil, + gil, PyNativeType, }, std::ptr::NonNull, }; @@ -221,6 +219,7 @@ pub trait FromPyObject<'py>: Sized { /// /// Implementors are encouraged to implement `extract_bound` and leave this method as the /// default implementation, which will forward calls to `extract_bound`. + #[cfg(feature = "gil-refs")] fn extract(ob: &'py PyAny) -> PyResult { Self::extract_bound(&ob.as_borrowed()) } diff --git a/src/err/mod.rs b/src/err/mod.rs index 29d4ac36294..6bfe1a6cc99 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -3,11 +3,13 @@ use crate::panic::PanicException; use crate::type_object::PyTypeInfo; use crate::types::any::PyAnyMethods; use crate::types::{string::PyStringMethods, typeobject::PyTypeMethods, PyTraceback, PyType}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; use crate::{ exceptions::{self, PyBaseException}, ffi, }; -use crate::{Borrowed, IntoPy, Py, PyAny, PyNativeType, PyObject, Python, ToPyObject}; +use crate::{Borrowed, IntoPy, Py, PyAny, PyObject, Python, ToPyObject}; use std::borrow::Cow; use std::cell::UnsafeCell; use std::ffi::CString; @@ -47,11 +49,13 @@ pub type PyResult = Result; /// Error that indicates a failure to convert a PyAny to a more specific Python type. #[derive(Debug)] +#[cfg(feature = "gil-refs")] pub struct PyDowncastError<'a> { from: &'a PyAny, to: Cow<'static, str>, } +#[cfg(feature = "gil-refs")] impl<'a> PyDowncastError<'a> { /// Create a new `PyDowncastError` representing a failure to convert the object /// `from` into the type named in `to`. @@ -64,7 +68,6 @@ impl<'a> PyDowncastError<'a> { /// Compatibility API to convert the Bound variant `DowncastError` into the /// gil-ref variant - #[cfg(feature = "gil-refs")] pub(crate) fn from_downcast_err(DowncastError { from, to }: DowncastError<'a, 'a>) -> Self { #[allow(deprecated)] let from = unsafe { from.py().from_borrowed_ptr(from.as_ptr()) }; @@ -1012,8 +1015,10 @@ impl<'a> std::convert::From> for PyErr { } } +#[cfg(feature = "gil-refs")] impl<'a> std::error::Error for PyDowncastError<'a> {} +#[cfg(feature = "gil-refs")] impl<'a> std::fmt::Display for PyDowncastError<'a> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { display_downcast_error(f, &self.from.as_borrowed(), &self.to) diff --git a/src/exceptions.rs b/src/exceptions.rs index b44a5c5a3fe..d6a6e859e3b 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -146,6 +146,7 @@ macro_rules! import_exception_bound { // FIXME remove this: was necessary while `PyTypeInfo` requires `HasPyGilRef`, // should change in 0.22. + #[cfg(feature = "gil-refs")] unsafe impl $crate::type_object::HasPyGilRef for $name { type AsRefTarget = $crate::PyAny; } diff --git a/src/impl_/deprecations.rs b/src/impl_/deprecations.rs index 650e01ce729..eb5caa8dffb 100644 --- a/src/impl_/deprecations.rs +++ b/src/impl_/deprecations.rs @@ -19,6 +19,7 @@ pub struct NotAGilRef(std::marker::PhantomData); pub trait IsGilRef {} +#[cfg(feature = "gil-refs")] impl IsGilRef for &'_ T {} impl GilRefs { diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 1302834ca4b..3ec2e329e1a 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; use crate::{ exceptions::{PyAttributeError, PyNotImplementedError, PyRuntimeError, PyValueError}, ffi, @@ -7,8 +9,7 @@ use crate::{ pyclass_init::PyObjectInit, types::any::PyAnyMethods, types::PyBool, - Borrowed, Py, PyAny, PyClass, PyErr, PyMethodDefType, PyNativeType, PyResult, PyTypeInfo, - Python, + Borrowed, Py, PyAny, PyClass, PyErr, PyMethodDefType, PyResult, PyTypeInfo, Python, }; use std::{ borrow::Cow, @@ -168,7 +169,12 @@ pub trait PyClassImpl: Sized + 'static { /// The closest native ancestor. This is `PyAny` by default, and when you declare /// `#[pyclass(extends=PyDict)]`, it's `PyDict`. + #[cfg(feature = "gil-refs")] type BaseNativeType: PyTypeInfo + PyNativeType; + /// The closest native ancestor. This is `PyAny` by default, and when you declare + /// `#[pyclass(extends=PyDict)]`, it's `PyDict`. + #[cfg(not(feature = "gil-refs"))] + type BaseNativeType: PyTypeInfo; /// This handles following two situations: /// 1. In case `T` is `Send`, stub `ThreadChecker` is used and does nothing. diff --git a/src/instance.rs b/src/instance.rs index 7835b9c79b6..82b05e782ff 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -2,6 +2,7 @@ use crate::err::{self, PyErr, PyResult}; use crate::impl_::pycell::PyClassObject; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; +#[cfg(feature = "gil-refs")] use crate::type_object::HasPyGilRef; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; use crate::types::{DerefToPyAny, PyDict, PyString, PyTuple}; @@ -24,6 +25,7 @@ use std::ptr::NonNull; /// # Safety /// /// This trait must only be implemented for types which cannot be accessed without the GIL. +#[cfg(feature = "gil-refs")] pub unsafe trait PyNativeType: Sized { /// The form of this which is stored inside a `Py` smart pointer. type AsRefSource: HasPyGilRef; @@ -666,11 +668,11 @@ impl<'a, 'py, T> From<&'a Bound<'py, T>> for Borrowed<'a, 'py, T> { } } +#[cfg(feature = "gil-refs")] impl<'py, T> Borrowed<'py, 'py, T> where T: HasPyGilRef, { - #[cfg(feature = "gil-refs")] pub(crate) fn into_gil_ref(self) -> &'py T::AsRefTarget { // Safety: self is a borrow over `'py`. #[allow(deprecated)] @@ -953,6 +955,7 @@ where } } +#[cfg(feature = "gil-refs")] impl Py where T: HasPyGilRef, @@ -1000,7 +1003,6 @@ where /// assert!(my_class_cell.try_borrow().is_ok()); /// }); /// ``` - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.0", note = "use `obj.bind(py)` instead of `obj.as_ref(py)`" @@ -1053,7 +1055,6 @@ where /// obj.into_ref(py) /// } /// ``` - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.0", note = "use `obj.into_bound(py)` instead of `obj.into_ref(py)`" @@ -1118,8 +1119,7 @@ where /// /// For frozen classes, the simpler [`get`][Self::get] is available. /// - /// Equivalent to `self.as_ref(py).borrow()` - - /// see [`PyCell::borrow`](crate::pycell::PyCell::borrow). + /// Equivalent to `self.bind(py).borrow()` - see [`Bound::borrow`]. /// /// # Examples /// @@ -1157,8 +1157,7 @@ where /// /// This borrow lasts while the returned [`PyRefMut`] exists. /// - /// Equivalent to `self.as_ref(py).borrow_mut()` - - /// see [`PyCell::borrow_mut`](crate::pycell::PyCell::borrow_mut). + /// Equivalent to `self.bind(py).borrow_mut()` - see [`Bound::borrow_mut`]. /// /// # Examples /// @@ -1202,8 +1201,7 @@ where /// /// For frozen classes, the simpler [`get`][Self::get] is available. /// - /// Equivalent to `self.as_ref(py).borrow_mut()` - - /// see [`PyCell::try_borrow`](crate::pycell::PyCell::try_borrow). + /// Equivalent to `self.bind(py).try_borrow()` - see [`Bound::try_borrow`]. #[inline] pub fn try_borrow<'py>(&'py self, py: Python<'py>) -> Result, PyBorrowError> { self.bind(py).try_borrow() @@ -1215,8 +1213,7 @@ where /// /// This is the non-panicking variant of [`borrow_mut`](#method.borrow_mut). /// - /// Equivalent to `self.as_ref(py).try_borrow_mut()` - - /// see [`PyCell::try_borrow_mut`](crate::pycell::PyCell::try_borrow_mut). + /// Equivalent to `self.bind(py).try_borrow_mut()` - see [`Bound::try_borrow_mut`]. #[inline] pub fn try_borrow_mut<'py>( &'py self, @@ -1742,6 +1739,7 @@ unsafe impl crate::AsPyPointer for Py { } } +#[cfg(feature = "gil-refs")] impl std::convert::From<&'_ T> for PyObject where T: PyNativeType, @@ -1866,6 +1864,7 @@ where /// /// However for GIL lifetime reasons, cause() cannot be implemented for `Py`. /// Use .as_ref() to get the GIL-scoped error if you need to inspect the cause. +#[cfg(feature = "gil-refs")] impl std::error::Error for Py where T: std::error::Error + PyTypeInfo, @@ -1876,7 +1875,6 @@ where impl std::fmt::Display for Py where T: PyTypeInfo, - T::AsRefTarget: std::fmt::Display, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { Python::with_gil(|py| std::fmt::Display::fmt(self.bind(py), f)) diff --git a/src/lib.rs b/src/lib.rs index 2a40445222e..b1d8ae6c7cf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -320,15 +320,18 @@ pub use crate::conversion::{AsPyPointer, FromPyObject, IntoPy, ToPyObject}; #[cfg(feature = "gil-refs")] #[allow(deprecated)] pub use crate::conversion::{FromPyPointer, PyTryFrom, PyTryInto}; -pub use crate::err::{ - DowncastError, DowncastIntoError, PyDowncastError, PyErr, PyErrArguments, PyResult, ToPyErr, -}; +#[cfg(feature = "gil-refs")] +pub use crate::err::PyDowncastError; +pub use crate::err::{DowncastError, DowncastIntoError, PyErr, PyErrArguments, PyResult, ToPyErr}; #[allow(deprecated)] pub use crate::gil::GILPool; #[cfg(not(any(PyPy, GraalPy)))] pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter}; -pub use crate::instance::{Borrowed, Bound, Py, PyNativeType, PyObject}; +#[cfg(feature = "gil-refs")] +pub use crate::instance::PyNativeType; +pub use crate::instance::{Borrowed, Bound, Py, PyObject}; pub use crate::marker::Python; +#[cfg(feature = "gil-refs")] #[allow(deprecated)] pub use crate::pycell::PyCell; pub use crate::pycell::{PyRef, PyRefMut}; @@ -443,6 +446,7 @@ mod conversions; pub mod coroutine; #[macro_use] #[doc(hidden)] +#[cfg(feature = "gil-refs")] pub mod derive_utils; mod err; pub mod exceptions; diff --git a/src/macros.rs b/src/macros.rs index d6f25c37308..6dde89e51a0 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -105,6 +105,7 @@ macro_rules! py_run_impl { ($py:expr, *$dict:expr, $code:expr) => {{ use ::std::option::Option::*; #[allow(unused_imports)] + #[cfg(feature = "gil-refs")] use $crate::PyNativeType; if let ::std::result::Result::Err(e) = $py.run_bound($code, None, Some(&$dict.as_borrowed())) { e.print($py); diff --git a/src/prelude.rs b/src/prelude.rs index 7aa45f6ccc2..4052f7c2d0b 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -15,11 +15,13 @@ pub use crate::conversion::{PyTryFrom, PyTryInto}; pub use crate::err::{PyErr, PyResult}; pub use crate::instance::{Borrowed, Bound, Py, PyObject}; pub use crate::marker::Python; +#[cfg(feature = "gil-refs")] #[allow(deprecated)] pub use crate::pycell::PyCell; pub use crate::pycell::{PyRef, PyRefMut}; pub use crate::pyclass_init::PyClassInitializer; pub use crate::types::{PyAny, PyModule}; +#[cfg(feature = "gil-refs")] pub use crate::PyNativeType; #[cfg(feature = "macros")] diff --git a/src/pycell.rs b/src/pycell.rs index dc5ccc45ea1..f15f5a54431 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -14,13 +14,13 @@ //! - However, methods and functions in Rust usually *do* need `&mut` references. While PyO3 can //! use the [`Python<'py>`](crate::Python) token to guarantee thread-safe access to them, it cannot //! statically guarantee uniqueness of `&mut` references. As such those references have to be tracked -//! dynamically at runtime, using [`PyCell`] and the other types defined in this module. This works +//! dynamically at runtime, using `PyCell` and the other types defined in this module. This works //! similar to std's [`RefCell`](std::cell::RefCell) type. //! //! # When *not* to use PyCell //! //! Usually you can use `&mut` references as method and function receivers and arguments, and you -//! won't need to use [`PyCell`] directly: +//! won't need to use `PyCell` directly: //! //! ```rust //! use pyo3::prelude::*; @@ -39,7 +39,7 @@ //! ``` //! //! The [`#[pymethods]`](crate::pymethods) proc macro will generate this wrapper function (and more), -//! using [`PyCell`] under the hood: +//! using `PyCell` under the hood: //! //! ```rust,ignore //! # use pyo3::prelude::*; @@ -76,7 +76,7 @@ //! # When to use PyCell //! ## Using pyclasses from Rust //! -//! However, we *do* need [`PyCell`] if we want to call its methods from Rust: +//! However, we *do* need `PyCell` if we want to call its methods from Rust: //! ```rust //! # use pyo3::prelude::*; //! # @@ -115,7 +115,7 @@ //! ``` //! ## Dealing with possibly overlapping mutable references //! -//! It is also necessary to use [`PyCell`] if you can receive mutable arguments that may overlap. +//! It is also necessary to use `PyCell` if you can receive mutable arguments that may overlap. //! Suppose the following function that swaps the values of two `Number`s: //! ``` //! # use pyo3::prelude::*; @@ -193,28 +193,30 @@ //! [guide]: https://pyo3.rs/latest/class.html#pycell-and-interior-mutability "PyCell and interior mutability" //! [Interior Mutability]: https://doc.rust-lang.org/book/ch15-05-interior-mutability.html "RefCell and the Interior Mutability Pattern - The Rust Programming Language" -use crate::conversion::{AsPyPointer, ToPyObject}; +use crate::conversion::AsPyPointer; use crate::exceptions::PyRuntimeError; use crate::ffi_ptr_ext::FfiPtrExt; -use crate::impl_::pyclass::PyClassImpl; -use crate::pyclass::{ - boolean_struct::{False, True}, - PyClass, -}; -use crate::type_object::{PyLayout, PySizedLayout}; +use crate::pyclass::{boolean_struct::False, PyClass}; use crate::types::any::PyAnyMethods; -use crate::types::PyAny; -use crate::{ffi, Bound, IntoPy, PyErr, PyNativeType, PyObject, PyTypeCheck, Python}; #[cfg(feature = "gil-refs")] -use crate::{pyclass_init::PyClassInitializer, PyResult}; +use crate::{ + conversion::ToPyObject, + impl_::pyclass::PyClassImpl, + pyclass::boolean_struct::True, + pyclass_init::PyClassInitializer, + type_object::{PyLayout, PySizedLayout}, + types::PyAny, + PyNativeType, PyResult, PyTypeCheck, +}; +use crate::{ffi, Bound, IntoPy, PyErr, PyObject, Python}; use std::fmt; use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; pub(crate) mod impl_; -use impl_::PyClassBorrowChecker; - -use self::impl_::{PyClassObject, PyClassObjectLayout}; +#[cfg(feature = "gil-refs")] +use self::impl_::PyClassObject; +use impl_::{PyClassBorrowChecker, PyClassObjectLayout}; /// A container type for (mutably) accessing [`PyClass`] values /// @@ -223,7 +225,7 @@ use self::impl_::{PyClassObject, PyClassObjectLayout}; /// # Examples /// /// This example demonstrates getting a mutable reference of the contained `PyClass`. -/// ```rust,ignore +/// ```rust /// use pyo3::prelude::*; /// /// #[pyclass] @@ -252,28 +254,27 @@ use self::impl_::{PyClassObject, PyClassObjectLayout}; /// ``` /// For more information on how, when and why (not) to use `PyCell` please see the /// [module-level documentation](self). -#[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`PyCell` was merged into `Bound`, use that instead; see the migration guide for more info" - ) +#[cfg(feature = "gil-refs")] +#[deprecated( + since = "0.21.0", + note = "`PyCell` was merged into `Bound`, use that instead; see the migration guide for more info" )] #[repr(transparent)] pub struct PyCell(PyClassObject); +#[cfg(feature = "gil-refs")] #[allow(deprecated)] unsafe impl PyNativeType for PyCell { type AsRefSource = T; } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl PyCell { /// Makes a new `PyCell` on the Python heap and return the reference to it. /// /// In cases where the value in the cell does not need to be accessed immediately after /// creation, consider [`Py::new`](crate::Py::new) as a more efficient alternative. - #[cfg(feature = "gil-refs")] #[deprecated( since = "0.21.0", note = "use `Bound::new(py, value)` or `Py::new(py, value)` instead of `PyCell::new(py, value)`" @@ -316,7 +317,7 @@ impl PyCell { /// /// # Examples /// - /// ```ignore + /// ``` /// # use pyo3::prelude::*; /// #[pyclass] /// struct Class {} @@ -346,7 +347,7 @@ impl PyCell { /// /// # Examples /// - /// ```ignore + /// ``` /// # use pyo3::prelude::*; /// #[pyclass] /// struct Class {} @@ -379,7 +380,7 @@ impl PyCell { /// /// # Examples /// - /// ```ignore + /// ``` /// # use pyo3::prelude::*; /// #[pyclass] /// struct Class {} @@ -416,7 +417,7 @@ impl PyCell { /// /// # Examples /// - /// ```ignore + /// ``` /// use std::sync::atomic::{AtomicUsize, Ordering}; /// # use pyo3::prelude::*; /// @@ -487,11 +488,14 @@ impl PyCell { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] unsafe impl PyLayout for PyCell {} +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl PySizedLayout for PyCell {} +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl PyTypeCheck for PyCell where @@ -503,7 +507,7 @@ where ::type_check(object) } } - +#[cfg(feature = "gil-refs")] #[allow(deprecated)] unsafe impl AsPyPointer for PyCell { fn as_ptr(&self) -> *mut ffi::PyObject { @@ -511,6 +515,7 @@ unsafe impl AsPyPointer for PyCell { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl ToPyObject for &PyCell { fn to_object(&self, py: Python<'_>) -> PyObject { @@ -542,6 +547,7 @@ impl Deref for PyCell { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl fmt::Debug for PyCell { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -768,6 +774,7 @@ impl IntoPy for &'_ PyRef<'_, T> { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl<'a, T: PyClass> std::convert::TryFrom<&'a PyCell> for crate::PyRef<'a, T> { type Error = PyBorrowError; @@ -788,7 +795,7 @@ impl fmt::Debug for PyRef<'_, T> { } } -/// A wrapper type for a mutably borrowed value from a[`PyCell`]``. +/// A wrapper type for a mutably borrowed value from a [`Bound<'py, T>`]. /// /// See the [module-level documentation](self) for more information. pub struct PyRefMut<'p, T: PyClass> { @@ -928,6 +935,7 @@ unsafe impl<'a, T: PyClass> AsPyPointer for PyRefMut<'a, T> { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl<'a, T: PyClass> std::convert::TryFrom<&'a PyCell> for crate::PyRefMut<'a, T> @@ -944,7 +952,7 @@ impl + fmt::Debug> fmt::Debug for PyRefMut<'_, T> { } } -/// An error type returned by [`PyCell::try_borrow`]. +/// An error type returned by [`Bound::try_borrow`]. /// /// If this error is allowed to bubble up into Python code it will raise a `RuntimeError`. pub struct PyBorrowError { @@ -969,7 +977,7 @@ impl From for PyErr { } } -/// An error type returned by [`PyCell::try_borrow_mut`]. +/// An error type returned by [`Bound::try_borrow_mut`]. /// /// If this error is allowed to bubble up into Python code it will raise a `RuntimeError`. pub struct PyBorrowMutError { diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 1bdd44b9e9c..5404464caba 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -74,6 +74,7 @@ pub trait PyClassBorrowChecker { /// Increments immutable borrow count, if possible fn try_borrow(&self) -> Result<(), PyBorrowError>; + #[cfg(feature = "gil-refs")] fn try_borrow_unguarded(&self) -> Result<(), PyBorrowError>; /// Decrements immutable borrow count @@ -96,6 +97,7 @@ impl PyClassBorrowChecker for EmptySlot { } #[inline] + #[cfg(feature = "gil-refs")] fn try_borrow_unguarded(&self) -> Result<(), PyBorrowError> { Ok(()) } @@ -130,6 +132,7 @@ impl PyClassBorrowChecker for BorrowChecker { } } + #[cfg(feature = "gil-refs")] fn try_borrow_unguarded(&self) -> Result<(), PyBorrowError> { let flag = self.0.get(); if flag != BorrowFlag::HAS_MUTABLE_BORROW { diff --git a/src/pyclass.rs b/src/pyclass.rs index b9b01cac26a..162ae0d3119 100644 --- a/src/pyclass.rs +++ b/src/pyclass.rs @@ -16,6 +16,7 @@ pub use self::gc::{PyTraverseError, PyVisit}; /// The `#[pyclass]` attribute implements this trait for your Rust struct - /// you shouldn't implement this trait directly. #[allow(deprecated)] +#[cfg(feature = "gil-refs")] pub trait PyClass: PyTypeInfo> + PyClassImpl { /// Whether the pyclass is frozen. /// @@ -23,6 +24,18 @@ pub trait PyClass: PyTypeInfo> + PyClassImpl { type Frozen: Frozen; } +/// Types that can be used as Python classes. +/// +/// The `#[pyclass]` attribute implements this trait for your Rust struct - +/// you shouldn't implement this trait directly. +#[cfg(not(feature = "gil-refs"))] +pub trait PyClass: PyTypeInfo + PyClassImpl { + /// Whether the pyclass is frozen. + /// + /// This can be enabled via `#[pyclass(frozen)]`. + type Frozen: Frozen; +} + /// Operators for the `__richcmp__` method #[derive(Debug, Clone, Copy)] pub enum CompareOp { diff --git a/src/type_object.rs b/src/type_object.rs index 7f35f7d967a..871e8366865 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -3,7 +3,9 @@ use crate::ffi_ptr_ext::FfiPtrExt; use crate::types::any::PyAnyMethods; use crate::types::{PyAny, PyType}; -use crate::{ffi, Bound, PyNativeType, Python}; +#[cfg(feature = "gil-refs")] +use crate::PyNativeType; +use crate::{ffi, Bound, Python}; /// `T: PyLayout` represents that `T` is a concrete representation of `U` in the Python heap. /// E.g., `PyClassObject` is a concrete representation of all `pyclass`es, and `ffi::PyObject` @@ -29,11 +31,13 @@ pub trait PySizedLayout: PyLayout + Sized {} /// /// - `Py::as_ref` will hand out references to `Self::AsRefTarget`. /// - `Self::AsRefTarget` must have the same layout as `UnsafeCell`. +#[cfg(feature = "gil-refs")] pub unsafe trait HasPyGilRef { /// Utility type to make Py::as_ref work. type AsRefTarget: PyNativeType; } +#[cfg(feature = "gil-refs")] unsafe impl HasPyGilRef for T where T: PyNativeType, @@ -54,6 +58,7 @@ where /// /// Implementations must provide an implementation for `type_object_raw` which infallibly produces a /// non-null pointer to the corresponding Python type object. +#[cfg(feature = "gil-refs")] pub unsafe trait PyTypeInfo: Sized + HasPyGilRef { /// Class name. const NAME: &'static str; @@ -132,7 +137,62 @@ pub unsafe trait PyTypeInfo: Sized + HasPyGilRef { } } +/// Python type information. +/// All Python native types (e.g., `PyDict`) and `#[pyclass]` structs implement this trait. +/// +/// This trait is marked unsafe because: +/// - specifying the incorrect layout can lead to memory errors +/// - the return value of type_object must always point to the same PyTypeObject instance +/// +/// It is safely implemented by the `pyclass` macro. +/// +/// # Safety +/// +/// Implementations must provide an implementation for `type_object_raw` which infallibly produces a +/// non-null pointer to the corresponding Python type object. +#[cfg(not(feature = "gil-refs"))] +pub unsafe trait PyTypeInfo: Sized { + /// Class name. + const NAME: &'static str; + + /// Module name, if any. + const MODULE: Option<&'static str>; + + /// Returns the PyTypeObject instance for this type. + fn type_object_raw(py: Python<'_>) -> *mut ffi::PyTypeObject; + + /// Returns the safe abstraction over the type object. + #[inline] + fn type_object_bound(py: Python<'_>) -> Bound<'_, PyType> { + // Making the borrowed object `Bound` is necessary for soundness reasons. It's an extreme + // edge case, but arbitrary Python code _could_ change the __class__ of an object and cause + // the type object to be freed. + // + // By making `Bound` we assume ownership which is then safe against races. + unsafe { + Self::type_object_raw(py) + .cast::() + .assume_borrowed_unchecked(py) + .to_owned() + .downcast_into_unchecked() + } + } + + /// Checks if `object` is an instance of this type or a subclass of this type. + #[inline] + fn is_type_of_bound(object: &Bound<'_, PyAny>) -> bool { + unsafe { ffi::PyObject_TypeCheck(object.as_ptr(), Self::type_object_raw(object.py())) != 0 } + } + + /// Checks if `object` is an instance of this type. + #[inline] + fn is_exact_type_of_bound(object: &Bound<'_, PyAny>) -> bool { + unsafe { ffi::Py_TYPE(object.as_ptr()) == Self::type_object_raw(object.py()) } + } +} + /// Implemented by types which can be used as a concrete Python type inside `Py` smart pointers. +#[cfg(feature = "gil-refs")] pub trait PyTypeCheck: HasPyGilRef { /// Name of self. This is used in error messages, for example. const NAME: &'static str; @@ -143,6 +203,18 @@ pub trait PyTypeCheck: HasPyGilRef { fn type_check(object: &Bound<'_, PyAny>) -> bool; } +/// Implemented by types which can be used as a concrete Python type inside `Py` smart pointers. +#[cfg(not(feature = "gil-refs"))] +pub trait PyTypeCheck { + /// Name of self. This is used in error messages, for example. + const NAME: &'static str; + + /// Checks if `object` is an instance of `Self`, which may include a subtype. + /// + /// This should be equivalent to the Python expression `isinstance(object, Self)`. + fn type_check(object: &Bound<'_, PyAny>) -> bool; +} + impl PyTypeCheck for T where T: PyTypeInfo, diff --git a/src/types/any.rs b/src/types/any.rs index 17837835be1..ba5ea01b1a3 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1646,7 +1646,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// Extracts some type from the Python object. /// /// This is a wrapper function around - /// [`FromPyObject::extract()`](crate::FromPyObject::extract). + /// [`FromPyObject::extract_bound()`](crate::FromPyObject::extract_bound). fn extract<'a, T>(&'a self) -> PyResult where T: FromPyObjectBound<'a, 'py>; diff --git a/src/types/mod.rs b/src/types/mod.rs index 2203ccdf2dc..12dabda7463 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -122,10 +122,12 @@ pub trait DerefToPyAny { #[macro_export] macro_rules! pyobject_native_type_base( ($name:ty $(;$generics:ident)* ) => { + #[cfg(feature = "gil-refs")] unsafe impl<$($generics,)*> $crate::PyNativeType for $name { type AsRefSource = Self; } + #[cfg(feature = "gil-refs")] impl<$($generics,)*> ::std::fmt::Debug for $name { fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::result::Result<(), ::std::fmt::Error> @@ -136,6 +138,7 @@ macro_rules! pyobject_native_type_base( } } + #[cfg(feature = "gil-refs")] impl<$($generics,)*> ::std::fmt::Display for $name { fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::result::Result<(), ::std::fmt::Error> diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 975d26009a5..d31c558f096 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -14,7 +14,7 @@ fn test_compile_errors() { #[cfg(any(not(Py_LIMITED_API), Py_3_11))] t.compile_fail("tests/ui/invalid_pymethods_buffer.rs"); // The output is not stable across abi3 / not abi3 and features - #[cfg(all(not(Py_LIMITED_API), feature = "full"))] + #[cfg(all(not(Py_LIMITED_API), feature = "full", not(feature = "gil-refs")))] t.compile_fail("tests/ui/invalid_pymethods_duplicates.rs"); t.compile_fail("tests/ui/invalid_pymethod_enum.rs"); t.compile_fail("tests/ui/invalid_pymethod_names.rs"); @@ -27,12 +27,14 @@ fn test_compile_errors() { t.compile_fail("tests/ui/invalid_argument_attributes.rs"); t.compile_fail("tests/ui/invalid_frompy_derive.rs"); t.compile_fail("tests/ui/static_ref.rs"); + #[cfg(not(feature = "gil-refs"))] t.compile_fail("tests/ui/wrong_aspyref_lifetimes.rs"); t.compile_fail("tests/ui/invalid_pyfunctions.rs"); t.compile_fail("tests/ui/invalid_pymethods.rs"); // output changes with async feature #[cfg(all(Py_LIMITED_API, feature = "experimental-async"))] t.compile_fail("tests/ui/abi3_nativetype_inheritance.rs"); + #[cfg(not(feature = "gil-refs"))] t.compile_fail("tests/ui/invalid_intern_arg.rs"); t.compile_fail("tests/ui/invalid_frozen_pyclass_borrow.rs"); t.compile_fail("tests/ui/invalid_pymethod_receiver.rs"); diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index b11c0058ce2..d014a06bbcc 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -34,6 +34,12 @@ error: use of deprecated constant `__pyfunction_pyfunction_option_4::SIGNATURE`: 138 | fn pyfunction_option_4( | ^^^^^^^^^^^^^^^^^^^ +error: use of deprecated struct `pyo3::PyCell`: `PyCell` was merged into `Bound`, use that instead; see the migration guide for more info + --> tests/ui/deprecations.rs:23:30 + | +23 | fn method_gil_ref(_slf: &PyCell) {} + | ^^^^^^ + error: use of deprecated method `pyo3::deprecations::GilRefs::::from_py_with_arg`: use `&Bound<'_, PyAny>` as the argument for this `from_py_with` extractor --> tests/ui/deprecations.rs:45:44 | diff --git a/tests/ui/invalid_intern_arg.stderr b/tests/ui/invalid_intern_arg.stderr index 5d2131bd845..7d1aad1ae28 100644 --- a/tests/ui/invalid_intern_arg.stderr +++ b/tests/ui/invalid_intern_arg.stderr @@ -13,5 +13,5 @@ error: lifetime may not live long enough 5 | Python::with_gil(|py| py.import_bound(pyo3::intern!(py, _foo)).unwrap()); | --- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ returning this value requires that `'1` must outlive `'2` | | | - | | return type of closure is pyo3::Bound<'2, pyo3::prelude::PyModule> - | has type `pyo3::Python<'1>` + | | return type of closure is pyo3::Bound<'2, PyModule> + | has type `Python<'1>` diff --git a/tests/ui/invalid_pymethods_duplicates.stderr b/tests/ui/invalid_pymethods_duplicates.stderr index 753c4b1b8dc..db301336e4f 100644 --- a/tests/ui/invalid_pymethods_duplicates.stderr +++ b/tests/ui/invalid_pymethods_duplicates.stderr @@ -26,23 +26,6 @@ error[E0277]: the trait bound `TwoNew: PyTypeInfo` is not satisfied PyDate and $N others -error[E0277]: the trait bound `TwoNew: HasPyGilRef` is not satisfied - --> tests/ui/invalid_pymethods_duplicates.rs:9:6 - | -9 | impl TwoNew { - | ^^^^^^ the trait `PyNativeType` is not implemented for `TwoNew`, which is required by `TwoNew: HasPyGilRef` - | - = help: the trait `HasPyGilRef` is implemented for `pyo3::coroutine::Coroutine` - = note: required for `TwoNew` to implement `HasPyGilRef` -note: required by a bound in `pyo3::PyTypeInfo::NAME` - --> src/type_object.rs - | - | pub unsafe trait PyTypeInfo: Sized + HasPyGilRef { - | ^^^^^^^^^^^ required by this bound in `PyTypeInfo::NAME` - | /// Class name. - | const NAME: &'static str; - | ---- required by a bound in this associated constant - error[E0592]: duplicate definitions with name `__pymethod___new____` --> tests/ui/invalid_pymethods_duplicates.rs:8:1 | diff --git a/tests/ui/wrong_aspyref_lifetimes.stderr b/tests/ui/wrong_aspyref_lifetimes.stderr index 30e63bb8261..f2f43d99f25 100644 --- a/tests/ui/wrong_aspyref_lifetimes.stderr +++ b/tests/ui/wrong_aspyref_lifetimes.stderr @@ -5,4 +5,4 @@ error: lifetime may not live long enough | --- ^^^^^^^^^^^^^ returning this value requires that `'1` must outlive `'2` | | | | | return type of closure is &'2 pyo3::Bound<'_, PyDict> - | has type `pyo3::Python<'1>` + | has type `Python<'1>` From 7790dab48046f01cbee967b054a05de767d3586d Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Wed, 15 May 2024 07:11:49 -0400 Subject: [PATCH 337/349] emit rustc-check-cfg only on rust 1.80+ (#4168) --- pyo3-build-config/src/lib.rs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 54aff4d10de..6d4ec759653 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -18,7 +18,6 @@ use std::{ use std::{env, process::Command, str::FromStr}; -#[cfg(feature = "resolve-config")] use once_cell::sync::OnceCell; pub use impl_::{ @@ -135,17 +134,6 @@ fn resolve_cross_compile_config_path() -> Option { /// so this function is unstable. #[doc(hidden)] pub fn print_feature_cfgs() { - fn rustc_minor_version() -> Option { - let rustc = env::var_os("RUSTC")?; - let output = Command::new(rustc).arg("--version").output().ok()?; - let version = core::str::from_utf8(&output.stdout).ok()?; - let mut pieces = version.split('.'); - if pieces.next() != Some("rustc 1") { - return None; - } - pieces.next()?.parse().ok() - } - let rustc_minor_version = rustc_minor_version().unwrap_or(0); // invalid_from_utf8 lint was added in Rust 1.74 @@ -160,6 +148,11 @@ pub fn print_feature_cfgs() { /// - #[doc(hidden)] pub fn print_expected_cfgs() { + if rustc_minor_version().map_or(false, |version| version < 80) { + // rustc 1.80.0 stabilized `rustc-check-cfg` feature, don't emit before + return; + } + println!("cargo:rustc-check-cfg=cfg(Py_LIMITED_API)"); println!("cargo:rustc-check-cfg=cfg(PyPy)"); println!("cargo:rustc-check-cfg=cfg(GraalPy)"); @@ -233,6 +226,20 @@ pub mod pyo3_build_script_impl { } } +fn rustc_minor_version() -> Option { + static RUSTC_MINOR_VERSION: OnceCell> = OnceCell::new(); + *RUSTC_MINOR_VERSION.get_or_init(|| { + let rustc = env::var_os("RUSTC")?; + let output = Command::new(rustc).arg("--version").output().ok()?; + let version = core::str::from_utf8(&output.stdout).ok()?; + let mut pieces = version.split('.'); + if pieces.next() != Some("rustc 1") { + return None; + } + pieces.next()?.parse().ok() + }) +} + #[cfg(test)] mod tests { use super::*; From 8de1787580c26c1d19c7a99028139340a8498cae Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Thu, 16 May 2024 17:55:05 -0400 Subject: [PATCH 338/349] Change `GILGuard` to be able to represent a GIL that was already held (#4187) See #4181 --- src/gil.rs | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/src/gil.rs b/src/gil.rs index db8311ab2fc..a1d9feba494 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -148,20 +148,26 @@ where } /// RAII type that represents the Global Interpreter Lock acquisition. -pub(crate) struct GILGuard { - gstate: ffi::PyGILState_STATE, - #[allow(deprecated)] // TODO: remove this with the gil-refs feature in 0.22 - pool: mem::ManuallyDrop, +pub(crate) enum GILGuard { + /// Indicates the GIL was already held with this GILGuard was acquired. + Assumed, + /// Indicates that we actually acquired the GIL when this GILGuard was acquired + Ensured { + gstate: ffi::PyGILState_STATE, + #[allow(deprecated)] // TODO: remove this with the gil-refs feature in 0.22 + pool: mem::ManuallyDrop, + }, } impl GILGuard { /// PyO3 internal API for acquiring the GIL. The public API is Python::with_gil. /// - /// If the GIL was already acquired via PyO3, this returns `None`. Otherwise, - /// the GIL will be acquired and a new `GILPool` created. - pub(crate) fn acquire() -> Option { + /// If the GIL was already acquired via PyO3, this returns + /// `GILGuard::Assumed`. Otherwise, the GIL will be acquired and + /// `GILGuard::Ensured` will be returned. + pub(crate) fn acquire() -> Self { if gil_is_acquired() { - return None; + return GILGuard::Assumed; } // Maybe auto-initialize the GIL: @@ -207,27 +213,30 @@ impl GILGuard { /// This can be called in "unsafe" contexts where the normal interpreter state /// checking performed by `GILGuard::acquire` may fail. This includes calling /// as part of multi-phase interpreter initialization. - pub(crate) fn acquire_unchecked() -> Option { + pub(crate) fn acquire_unchecked() -> Self { if gil_is_acquired() { - return None; + return GILGuard::Assumed; } let gstate = unsafe { ffi::PyGILState_Ensure() }; // acquire GIL #[allow(deprecated)] let pool = unsafe { mem::ManuallyDrop::new(GILPool::new()) }; - Some(GILGuard { gstate, pool }) + GILGuard::Ensured { gstate, pool } } } /// The Drop implementation for `GILGuard` will release the GIL. impl Drop for GILGuard { fn drop(&mut self) { - unsafe { - // Drop the objects in the pool before attempting to release the thread state - mem::ManuallyDrop::drop(&mut self.pool); - - ffi::PyGILState_Release(self.gstate); + match self { + GILGuard::Assumed => {} + GILGuard::Ensured { gstate, pool } => unsafe { + // Drop the objects in the pool before attempting to release the thread state + mem::ManuallyDrop::drop(pool); + + ffi::PyGILState_Release(*gstate); + }, } } } From 88f2f6f4d56f2bac1220d1f0d0ac912b8c160c4a Mon Sep 17 00:00:00 2001 From: newcomertv Date: Fri, 17 May 2024 04:59:00 +0200 Subject: [PATCH 339/349] feat: support pyclass on tuple enums (#4072) * feat: support pyclass on tuple enums * cargo fmt * changelog * ruff format * rebase with adaptation for FnArg refactor * fix class.md from pr comments * add enum tuple variant getitem implementation * fmt * progress toward getitem and len impl on derive pyclass for complex enum tuple * working getitem and len slots for complex tuple enum pyclass derivation * refactor code generation * address PR concerns - take py from function argument on get_item - make more general slot def implementation - remove unnecessary function arguments - add testcases for uncovered cases including future feature match_args * add tracking issue * fmt * ruff * remove me * support match_args for tuple enum * integrate FnArg now takes Cow * fix empty and single element tuples * use impl_py_slot_def for cimplex tuple enum slots * reverse erroneous doc change * Address latest comments * formatting suggestion * fix : - clippy beta - better compile error (+related doc and test) --------- Co-authored-by: Chris Arderne --- guide/src/class.md | 21 +- newsfragments/4072.added.md | 1 + pyo3-macros-backend/src/pyclass.rs | 333 ++++++++++++++++++++++++-- pyo3-macros-backend/src/pymethod.rs | 5 +- pytests/src/enums.rs | 42 ++++ pytests/tests/test_enums.py | 64 +++++ pytests/tests/test_enums_match.py | 99 ++++++++ tests/ui/invalid_pyclass_enum.rs | 6 - tests/ui/invalid_pyclass_enum.stderr | 14 +- tests/ui/invalid_pymethod_enum.rs | 16 ++ tests/ui/invalid_pymethod_enum.stderr | 25 ++ 11 files changed, 581 insertions(+), 45 deletions(-) create mode 100644 newsfragments/4072.added.md diff --git a/guide/src/class.md b/guide/src/class.md index ce86ec40e5f..57a5cf6d467 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -52,15 +52,18 @@ enum HttpResponse { // ... } -// PyO3 also supports enums with non-unit variants +// PyO3 also supports enums with Struct and Tuple variants // These complex enums have sligtly different behavior from the simple enums above // They are meant to work with instance checks and match statement patterns +// The variants can be mixed and matched +// Struct variants have named fields while tuple enums generate generic names for fields in order _0, _1, _2, ... +// Apart from this both types are functionally identical #[pyclass] enum Shape { Circle { radius: f64 }, Rectangle { width: f64, height: f64 }, - RegularPolygon { side_count: u32, radius: f64 }, - Nothing {}, + RegularPolygon(u32, f64), + Nothing(), } ``` @@ -1180,7 +1183,7 @@ enum BadSubclass { An enum is complex if it has any non-unit (struct or tuple) variants. -Currently PyO3 supports only struct variants in a complex enum. Support for unit and tuple variants is planned. +PyO3 supports only struct and tuple variants in a complex enum. Unit variants aren't supported at present (the recommendation is to use an empty tuple enum instead). PyO3 adds a class attribute for each variant, which may be used to construct values and in match patterns. PyO3 also provides getter methods for all fields of each variant. @@ -1190,14 +1193,14 @@ PyO3 adds a class attribute for each variant, which may be used to construct val enum Shape { Circle { radius: f64 }, Rectangle { width: f64, height: f64 }, - RegularPolygon { side_count: u32, radius: f64 }, + RegularPolygon(u32, f64), Nothing { }, } # #[cfg(Py_3_10)] Python::with_gil(|py| { let circle = Shape::Circle { radius: 10.0 }.into_py(py); - let square = Shape::RegularPolygon { side_count: 4, radius: 10.0 }.into_py(py); + let square = Shape::RegularPolygon(4, 10.0).into_py(py); let cls = py.get_type_bound::(); pyo3::py_run!(py, circle square cls, r#" assert isinstance(circle, cls) @@ -1206,8 +1209,8 @@ Python::with_gil(|py| { assert isinstance(square, cls) assert isinstance(square, cls.RegularPolygon) - assert square.side_count == 4 - assert square.radius == 10.0 + assert square[0] == 4 # Gets _0 field + assert square[1] == 10.0 # Gets _1 field def count_vertices(cls, shape): match shape: @@ -1215,7 +1218,7 @@ Python::with_gil(|py| { return 0 case cls.Rectangle(): return 4 - case cls.RegularPolygon(side_count=n): + case cls.RegularPolygon(n): return n case cls.Nothing(): return 0 diff --git a/newsfragments/4072.added.md b/newsfragments/4072.added.md new file mode 100644 index 00000000000..23207c849d8 --- /dev/null +++ b/newsfragments/4072.added.md @@ -0,0 +1 @@ +Support `#[pyclass]` on enums that have tuple variants. \ No newline at end of file diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 4e71a711802..47c52c84518 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -12,7 +12,7 @@ use crate::pyfunction::ConstructorAttribute; use crate::pyimpl::{gen_py_const, PyClassMethodsType}; use crate::pymethod::{ impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, MethodAndSlotDef, PropertyType, - SlotDef, __INT__, __REPR__, __RICHCMP__, + SlotDef, __GETITEM__, __INT__, __LEN__, __REPR__, __RICHCMP__, }; use crate::utils::Ctx; use crate::utils::{self, apply_renaming_rule, PythonDoc}; @@ -504,10 +504,10 @@ impl<'a> PyClassComplexEnum<'a> { let variant = match &variant.fields { Fields::Unit => { bail_spanned!(variant.span() => format!( - "Unit variant `{ident}` is not yet supported in a complex enum\n\ - = help: change to a struct variant with no fields: `{ident} {{ }}`\n\ - = note: the enum is complex because of non-unit variant `{witness}`", - ident=ident, witness=witness)) + "Unit variant `{ident}` is not yet supported in a complex enum\n\ + = help: change to an empty tuple variant instead: `{ident}()`\n\ + = note: the enum is complex because of non-unit variant `{witness}`", + ident=ident, witness=witness)) } Fields::Named(fields) => { let fields = fields @@ -526,12 +526,21 @@ impl<'a> PyClassComplexEnum<'a> { options, }) } - Fields::Unnamed(_) => { - bail_spanned!(variant.span() => format!( - "Tuple variant `{ident}` is not yet supported in a complex enum\n\ - = help: change to a struct variant with named fields: `{ident} {{ /* fields */ }}`\n\ - = note: the enum is complex because of non-unit variant `{witness}`", - ident=ident, witness=witness)) + Fields::Unnamed(types) => { + let fields = types + .unnamed + .iter() + .map(|field| PyClassEnumVariantUnnamedField { + ty: &field.ty, + span: field.span(), + }) + .collect(); + + PyClassEnumVariant::Tuple(PyClassEnumTupleVariant { + ident, + fields, + options, + }) } }; @@ -553,7 +562,7 @@ impl<'a> PyClassComplexEnum<'a> { enum PyClassEnumVariant<'a> { // TODO(mkovaxx): Unit(PyClassEnumUnitVariant<'a>), Struct(PyClassEnumStructVariant<'a>), - // TODO(mkovaxx): Tuple(PyClassEnumTupleVariant<'a>), + Tuple(PyClassEnumTupleVariant<'a>), } trait EnumVariant { @@ -581,12 +590,14 @@ impl<'a> EnumVariant for PyClassEnumVariant<'a> { fn get_ident(&self) -> &syn::Ident { match self { PyClassEnumVariant::Struct(struct_variant) => struct_variant.ident, + PyClassEnumVariant::Tuple(tuple_variant) => tuple_variant.ident, } } fn get_options(&self) -> &EnumVariantPyO3Options { match self { PyClassEnumVariant::Struct(struct_variant) => &struct_variant.options, + PyClassEnumVariant::Tuple(tuple_variant) => &tuple_variant.options, } } } @@ -614,12 +625,23 @@ struct PyClassEnumStructVariant<'a> { options: EnumVariantPyO3Options, } +struct PyClassEnumTupleVariant<'a> { + ident: &'a syn::Ident, + fields: Vec>, + options: EnumVariantPyO3Options, +} + struct PyClassEnumVariantNamedField<'a> { ident: &'a syn::Ident, ty: &'a syn::Type, span: Span, } +struct PyClassEnumVariantUnnamedField<'a> { + ty: &'a syn::Type, + span: Span, +} + /// `#[pyo3()]` options for pyclass enum variants #[derive(Default)] struct EnumVariantPyO3Options { @@ -930,17 +952,19 @@ fn impl_complex_enum( let variant_cls_pytypeinfo = impl_pytypeinfo(&variant_cls, &variant_args, None, ctx); variant_cls_pytypeinfos.push(variant_cls_pytypeinfo); - let (variant_cls_impl, field_getters) = impl_complex_enum_variant_cls(cls, &variant, ctx)?; + let (variant_cls_impl, field_getters, mut slots) = + impl_complex_enum_variant_cls(cls, &variant, ctx)?; variant_cls_impls.push(variant_cls_impl); let variant_new = complex_enum_variant_new(cls, variant, ctx)?; + slots.push(variant_new); let pyclass_impl = PyClassImplsBuilder::new( &variant_cls, &variant_args, methods_type, field_getters, - vec![variant_new], + slots, ) .impl_all(ctx)?; @@ -970,19 +994,52 @@ fn impl_complex_enum_variant_cls( enum_name: &syn::Ident, variant: &PyClassEnumVariant<'_>, ctx: &Ctx, -) -> Result<(TokenStream, Vec)> { +) -> Result<(TokenStream, Vec, Vec)> { match variant { PyClassEnumVariant::Struct(struct_variant) => { impl_complex_enum_struct_variant_cls(enum_name, struct_variant, ctx) } + PyClassEnumVariant::Tuple(tuple_variant) => { + impl_complex_enum_tuple_variant_cls(enum_name, tuple_variant, ctx) + } } } +fn impl_complex_enum_variant_match_args( + ctx: &Ctx, + variant_cls_type: &syn::Type, + field_names: &mut Vec, +) -> (MethodAndMethodDef, syn::ImplItemConst) { + let match_args_const_impl: syn::ImplItemConst = { + let args_tp = field_names.iter().map(|_| { + quote! { &'static str } + }); + parse_quote! { + const __match_args__: ( #(#args_tp,)* ) = ( + #(stringify!(#field_names),)* + ); + } + }; + + let spec = ConstSpec { + rust_ident: format_ident!("__match_args__"), + attributes: ConstAttributes { + is_class_attr: true, + name: None, + deprecations: Deprecations::new(ctx), + }, + }; + + let variant_match_args = gen_py_const(variant_cls_type, &spec, ctx); + + (variant_match_args, match_args_const_impl) +} + fn impl_complex_enum_struct_variant_cls( enum_name: &syn::Ident, variant: &PyClassEnumStructVariant<'_>, ctx: &Ctx, -) -> Result<(TokenStream, Vec)> { +) -> Result<(TokenStream, Vec, Vec)> { let Ctx { pyo3_path } = ctx; let variant_ident = &variant.ident; let variant_cls = gen_complex_enum_variant_class_ident(enum_name, variant.ident); @@ -1015,6 +1072,11 @@ fn impl_complex_enum_struct_variant_cls( field_getter_impls.push(field_getter_impl); } + let (variant_match_args, match_args_const_impl) = + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names); + + field_getters.push(variant_match_args); + let cls_impl = quote! { #[doc(hidden)] #[allow(non_snake_case)] @@ -1024,11 +1086,190 @@ fn impl_complex_enum_struct_variant_cls( #pyo3_path::PyClassInitializer::from(base_value).add_subclass(#variant_cls) } + #match_args_const_impl + #(#field_getter_impls)* } }; - Ok((cls_impl, field_getters)) + Ok((cls_impl, field_getters, Vec::new())) +} + +fn impl_complex_enum_tuple_variant_field_getters( + ctx: &Ctx, + variant: &PyClassEnumTupleVariant<'_>, + enum_name: &syn::Ident, + variant_cls_type: &syn::Type, + variant_ident: &&Ident, + field_names: &mut Vec, + fields_types: &mut Vec, +) -> Result<(Vec, Vec)> { + let Ctx { pyo3_path } = ctx; + + let mut field_getters = vec![]; + let mut field_getter_impls = vec![]; + + for (index, field) in variant.fields.iter().enumerate() { + let field_name = format_ident!("_{}", index); + let field_type = field.ty; + + let field_getter = + complex_enum_variant_field_getter(variant_cls_type, &field_name, field.span, ctx)?; + + // Generate the match arms needed to destructure the tuple and access the specific field + let field_access_tokens: Vec<_> = (0..variant.fields.len()) + .map(|i| { + if i == index { + quote! { val } + } else { + quote! { _ } + } + }) + .collect(); + + let field_getter_impl: syn::ImplItemFn = parse_quote! { + fn #field_name(slf: #pyo3_path::PyRef) -> #pyo3_path::PyResult<#field_type> { + match &*slf.into_super() { + #enum_name::#variant_ident ( #(#field_access_tokens), *) => Ok(val.clone()), + _ => unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), + } + } + }; + + field_names.push(field_name); + fields_types.push(field_type.clone()); + field_getters.push(field_getter); + field_getter_impls.push(field_getter_impl); + } + + Ok((field_getters, field_getter_impls)) +} + +fn impl_complex_enum_tuple_variant_len( + ctx: &Ctx, + + variant_cls_type: &syn::Type, + num_fields: usize, +) -> Result<(MethodAndSlotDef, syn::ImplItemFn)> { + let Ctx { pyo3_path } = ctx; + + let mut len_method_impl: syn::ImplItemFn = parse_quote! { + fn __len__(slf: #pyo3_path::PyRef) -> #pyo3_path::PyResult { + Ok(#num_fields) + } + }; + + let variant_len = + generate_default_protocol_slot(variant_cls_type, &mut len_method_impl, &__LEN__, ctx)?; + + Ok((variant_len, len_method_impl)) +} + +fn impl_complex_enum_tuple_variant_getitem( + ctx: &Ctx, + variant_cls: &syn::Ident, + variant_cls_type: &syn::Type, + num_fields: usize, +) -> Result<(MethodAndSlotDef, syn::ImplItemFn)> { + let Ctx { pyo3_path } = ctx; + + let match_arms: Vec<_> = (0..num_fields) + .map(|i| { + let field_access = format_ident!("_{}", i); + quote! { + #i => Ok( + #pyo3_path::IntoPy::into_py( + #variant_cls::#field_access(slf)? + , py) + ) + + } + }) + .collect(); + + let mut get_item_method_impl: syn::ImplItemFn = parse_quote! { + fn __getitem__(slf: #pyo3_path::PyRef, idx: usize) -> #pyo3_path::PyResult< #pyo3_path::PyObject> { + let py = slf.py(); + match idx { + #( #match_arms, )* + _ => Err(pyo3::exceptions::PyIndexError::new_err("tuple index out of range")), + } + } + }; + + let variant_getitem = generate_default_protocol_slot( + variant_cls_type, + &mut get_item_method_impl, + &__GETITEM__, + ctx, + )?; + + Ok((variant_getitem, get_item_method_impl)) +} + +fn impl_complex_enum_tuple_variant_cls( + enum_name: &syn::Ident, + variant: &PyClassEnumTupleVariant<'_>, + ctx: &Ctx, +) -> Result<(TokenStream, Vec, Vec)> { + let Ctx { pyo3_path } = ctx; + let variant_ident = &variant.ident; + let variant_cls = gen_complex_enum_variant_class_ident(enum_name, variant.ident); + let variant_cls_type = parse_quote!(#variant_cls); + + let mut slots = vec![]; + + // represents the index of the field + let mut field_names: Vec = vec![]; + let mut field_types: Vec = vec![]; + + let (mut field_getters, field_getter_impls) = impl_complex_enum_tuple_variant_field_getters( + ctx, + variant, + enum_name, + &variant_cls_type, + variant_ident, + &mut field_names, + &mut field_types, + )?; + + let num_fields = variant.fields.len(); + + let (variant_len, len_method_impl) = + impl_complex_enum_tuple_variant_len(ctx, &variant_cls_type, num_fields)?; + + slots.push(variant_len); + + let (variant_getitem, getitem_method_impl) = + impl_complex_enum_tuple_variant_getitem(ctx, &variant_cls, &variant_cls_type, num_fields)?; + + slots.push(variant_getitem); + + let (variant_match_args, match_args_method_impl) = + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names); + + field_getters.push(variant_match_args); + + let cls_impl = quote! { + #[doc(hidden)] + #[allow(non_snake_case)] + impl #variant_cls { + fn __pymethod_constructor__(py: #pyo3_path::Python<'_>, #(#field_names : #field_types,)*) -> #pyo3_path::PyClassInitializer<#variant_cls> { + let base_value = #enum_name::#variant_ident ( #(#field_names,)* ); + #pyo3_path::PyClassInitializer::from(base_value).add_subclass(#variant_cls) + } + + #len_method_impl + + #getitem_method_impl + + #match_args_method_impl + + #(#field_getter_impls)* + } + }; + + Ok((cls_impl, field_getters, slots)) } fn gen_complex_enum_variant_class_ident(enum_: &syn::Ident, variant: &syn::Ident) -> syn::Ident { @@ -1149,6 +1390,9 @@ fn complex_enum_variant_new<'a>( PyClassEnumVariant::Struct(struct_variant) => { complex_enum_struct_variant_new(cls, struct_variant, ctx) } + PyClassEnumVariant::Tuple(tuple_variant) => { + complex_enum_tuple_variant_new(cls, tuple_variant, ctx) + } } } @@ -1209,6 +1453,61 @@ fn complex_enum_struct_variant_new<'a>( crate::pymethod::impl_py_method_def_new(&variant_cls_type, &spec, ctx) } +fn complex_enum_tuple_variant_new<'a>( + cls: &'a syn::Ident, + variant: PyClassEnumTupleVariant<'a>, + ctx: &Ctx, +) -> Result { + let Ctx { pyo3_path } = ctx; + + let variant_cls: Ident = format_ident!("{}_{}", cls, variant.ident); + let variant_cls_type: syn::Type = parse_quote!(#variant_cls); + + let arg_py_ident: syn::Ident = parse_quote!(py); + let arg_py_type: syn::Type = parse_quote!(#pyo3_path::Python<'_>); + + let args = { + let mut args = vec![FnArg::Py(PyArg { + name: &arg_py_ident, + ty: &arg_py_type, + })]; + + for (i, field) in variant.fields.iter().enumerate() { + args.push(FnArg::Regular(RegularArg { + name: std::borrow::Cow::Owned(format_ident!("_{}", i)), + ty: field.ty, + from_py_with: None, + default_value: None, + option_wrapped_type: None, + })); + } + args + }; + + let signature = if let Some(constructor) = variant.options.constructor { + crate::pyfunction::FunctionSignature::from_arguments_and_attribute( + args, + constructor.into_signature(), + )? + } else { + crate::pyfunction::FunctionSignature::from_arguments(args)? + }; + + let spec = FnSpec { + tp: crate::method::FnType::FnNew, + name: &format_ident!("__pymethod_constructor__"), + python_name: format_ident!("__new__"), + signature, + convention: crate::method::CallingConvention::TpNew, + text_signature: None, + asyncness: None, + unsafety: None, + deprecations: Deprecations::new(ctx), + }; + + crate::pymethod::impl_py_method_def_new(&variant_cls_type, &spec, ctx) +} + fn complex_enum_variant_field_getter<'a>( variant_cls_type: &'a syn::Type, field_name: &'a syn::Ident, diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 208735f2619..f5b11af3c27 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -934,7 +934,7 @@ const __ANEXT__: SlotDef = SlotDef::new("Py_am_anext", "unaryfunc").return_speci ), TokenGenerator(|_| quote! { async_iter_tag }), ); -const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc").ret_ty(Ty::PySsizeT); +pub const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc").ret_ty(Ty::PySsizeT); const __CONTAINS__: SlotDef = SlotDef::new("Py_sq_contains", "objobjproc") .arguments(&[Ty::Object]) .ret_ty(Ty::Int); @@ -944,7 +944,8 @@ const __INPLACE_CONCAT__: SlotDef = SlotDef::new("Py_sq_concat", "binaryfunc").arguments(&[Ty::Object]); const __INPLACE_REPEAT__: SlotDef = SlotDef::new("Py_sq_repeat", "ssizeargfunc").arguments(&[Ty::PySsizeT]); -const __GETITEM__: SlotDef = SlotDef::new("Py_mp_subscript", "binaryfunc").arguments(&[Ty::Object]); +pub const __GETITEM__: SlotDef = + SlotDef::new("Py_mp_subscript", "binaryfunc").arguments(&[Ty::Object]); const __POS__: SlotDef = SlotDef::new("Py_nb_positive", "unaryfunc"); const __NEG__: SlotDef = SlotDef::new("Py_nb_negative", "unaryfunc"); diff --git a/pytests/src/enums.rs b/pytests/src/enums.rs index 68a5fc93dfe..964f0d431c3 100644 --- a/pytests/src/enums.rs +++ b/pytests/src/enums.rs @@ -8,8 +8,13 @@ use pyo3::{ pub fn enums(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_wrapped(wrap_pyfunction_bound!(do_simple_stuff))?; m.add_wrapped(wrap_pyfunction_bound!(do_complex_stuff))?; + m.add_wrapped(wrap_pyfunction_bound!(do_tuple_stuff))?; + m.add_wrapped(wrap_pyfunction_bound!(do_mixed_complex_stuff))?; Ok(()) } @@ -79,3 +84,40 @@ pub fn do_complex_stuff(thing: &ComplexEnum) -> ComplexEnum { }, } } + +#[pyclass] +enum SimpleTupleEnum { + Int(i32), + Str(String), +} + +#[pyclass] +pub enum TupleEnum { + #[pyo3(constructor = (_0 = 1, _1 = 1.0, _2 = true))] + FullWithDefault(i32, f64, bool), + Full(i32, f64, bool), + EmptyTuple(), +} + +#[pyfunction] +pub fn do_tuple_stuff(thing: &TupleEnum) -> TupleEnum { + match thing { + TupleEnum::FullWithDefault(a, b, c) => TupleEnum::FullWithDefault(*a, *b, *c), + TupleEnum::Full(a, b, c) => TupleEnum::Full(*a, *b, *c), + TupleEnum::EmptyTuple() => TupleEnum::EmptyTuple(), + } +} + +#[pyclass] +pub enum MixedComplexEnum { + Nothing {}, + Empty(), +} + +#[pyfunction] +pub fn do_mixed_complex_stuff(thing: &MixedComplexEnum) -> MixedComplexEnum { + match thing { + MixedComplexEnum::Nothing {} => MixedComplexEnum::Empty(), + MixedComplexEnum::Empty() => MixedComplexEnum::Nothing {}, + } +} diff --git a/pytests/tests/test_enums.py b/pytests/tests/test_enums.py index cd1d7aedaf8..cd4f7e124c9 100644 --- a/pytests/tests/test_enums.py +++ b/pytests/tests/test_enums.py @@ -137,3 +137,67 @@ def test_complex_enum_pyfunction_in_out_desugared_match(variant: enums.ComplexEn assert y == "HELLO" else: assert False + + +def test_tuple_enum_variant_constructors(): + tuple_variant = enums.TupleEnum.Full(42, 3.14, False) + assert isinstance(tuple_variant, enums.TupleEnum.Full) + + empty_tuple_variant = enums.TupleEnum.EmptyTuple() + assert isinstance(empty_tuple_variant, enums.TupleEnum.EmptyTuple) + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.FullWithDefault(), + enums.TupleEnum.Full(42, 3.14, False), + enums.TupleEnum.EmptyTuple(), + ], +) +def test_tuple_enum_variant_subclasses(variant: enums.TupleEnum): + assert isinstance(variant, enums.TupleEnum) + + +def test_tuple_enum_defaults(): + variant = enums.TupleEnum.FullWithDefault() + assert variant._0 == 1 + assert variant._1 == 1.0 + assert variant._2 is True + + +def test_tuple_enum_field_getters(): + tuple_variant = enums.TupleEnum.Full(42, 3.14, False) + assert tuple_variant._0 == 42 + assert tuple_variant._1 == 3.14 + assert tuple_variant._2 is False + + +def test_tuple_enum_index_getter(): + tuple_variant = enums.TupleEnum.Full(42, 3.14, False) + assert len(tuple_variant) == 3 + assert tuple_variant[0] == 42 + + +@pytest.mark.parametrize( + "variant", + [enums.MixedComplexEnum.Nothing()], +) +def test_mixed_complex_enum_pyfunction_instance_nothing( + variant: enums.MixedComplexEnum, +): + assert isinstance(variant, enums.MixedComplexEnum.Nothing) + assert isinstance( + enums.do_mixed_complex_stuff(variant), enums.MixedComplexEnum.Empty + ) + + +@pytest.mark.parametrize( + "variant", + [enums.MixedComplexEnum.Empty()], +) +def test_mixed_complex_enum_pyfunction_instance_empty(variant: enums.MixedComplexEnum): + assert isinstance(variant, enums.MixedComplexEnum.Empty) + assert isinstance( + enums.do_mixed_complex_stuff(variant), enums.MixedComplexEnum.Nothing + ) diff --git a/pytests/tests/test_enums_match.py b/pytests/tests/test_enums_match.py index 4d55bbbe351..6c4b5f6aa07 100644 --- a/pytests/tests/test_enums_match.py +++ b/pytests/tests/test_enums_match.py @@ -57,3 +57,102 @@ def test_complex_enum_pyfunction_in_out(variant: enums.ComplexEnum): assert z is True case _: assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.ComplexEnum.MultiFieldStruct(42, 3.14, True), + ], +) +def test_complex_enum_partial_match(variant: enums.ComplexEnum): + match variant: + case enums.ComplexEnum.MultiFieldStruct(a): + assert a == 42 + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.Full(42, 3.14, True), + enums.TupleEnum.EmptyTuple(), + ], +) +def test_tuple_enum_match_statement(variant: enums.TupleEnum): + match variant: + case enums.TupleEnum.Full(_0=x, _1=y, _2=z): + assert x == 42 + assert y == 3.14 + assert z is True + case enums.TupleEnum.EmptyTuple(): + assert True + case _: + print(variant) + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.SimpleTupleEnum.Int(42), + enums.SimpleTupleEnum.Str("hello"), + ], +) +def test_simple_tuple_enum_match_statement(variant: enums.SimpleTupleEnum): + match variant: + case enums.SimpleTupleEnum.Int(x): + assert x == 42 + case enums.SimpleTupleEnum.Str(x): + assert x == "hello" + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.Full(42, 3.14, True), + ], +) +def test_tuple_enum_match_match_args(variant: enums.TupleEnum): + match variant: + case enums.TupleEnum.Full(x, y, z): + assert x == 42 + assert y == 3.14 + assert z is True + assert True + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.TupleEnum.Full(42, 3.14, True), + ], +) +def test_tuple_enum_partial_match(variant: enums.TupleEnum): + match variant: + case enums.TupleEnum.Full(a): + assert a == 42 + case _: + assert False + + +@pytest.mark.parametrize( + "variant", + [ + enums.MixedComplexEnum.Nothing(), + enums.MixedComplexEnum.Empty(), + ], +) +def test_mixed_complex_enum_match_statement(variant: enums.MixedComplexEnum): + match variant: + case enums.MixedComplexEnum.Nothing(): + assert True + case enums.MixedComplexEnum.Empty(): + assert True + case _: + assert False diff --git a/tests/ui/invalid_pyclass_enum.rs b/tests/ui/invalid_pyclass_enum.rs index 116b8968da8..e98010fea32 100644 --- a/tests/ui/invalid_pyclass_enum.rs +++ b/tests/ui/invalid_pyclass_enum.rs @@ -21,12 +21,6 @@ enum NoUnitVariants { UnitVariant, } -#[pyclass] -enum NoTupleVariants { - StructVariant { field: i32 }, - TupleVariant(i32), -} - #[pyclass] enum SimpleNoSignature { #[pyo3(constructor = (a, b))] diff --git a/tests/ui/invalid_pyclass_enum.stderr b/tests/ui/invalid_pyclass_enum.stderr index e9ba9806da8..7e3b6ffa425 100644 --- a/tests/ui/invalid_pyclass_enum.stderr +++ b/tests/ui/invalid_pyclass_enum.stderr @@ -17,23 +17,15 @@ error: #[pyclass] can't be used on enums without any variants | ^^ error: Unit variant `UnitVariant` is not yet supported in a complex enum - = help: change to a struct variant with no fields: `UnitVariant { }` + = help: change to an empty tuple variant instead: `UnitVariant()` = note: the enum is complex because of non-unit variant `StructVariant` --> tests/ui/invalid_pyclass_enum.rs:21:5 | 21 | UnitVariant, | ^^^^^^^^^^^ -error: Tuple variant `TupleVariant` is not yet supported in a complex enum - = help: change to a struct variant with named fields: `TupleVariant { /* fields */ }` - = note: the enum is complex because of non-unit variant `StructVariant` - --> tests/ui/invalid_pyclass_enum.rs:27:5 - | -27 | TupleVariant(i32), - | ^^^^^^^^^^^^ - error: `constructor` can't be used on a simple enum variant - --> tests/ui/invalid_pyclass_enum.rs:32:12 + --> tests/ui/invalid_pyclass_enum.rs:26:12 | -32 | #[pyo3(constructor = (a, b))] +26 | #[pyo3(constructor = (a, b))] | ^^^^^^^^^^^ diff --git a/tests/ui/invalid_pymethod_enum.rs b/tests/ui/invalid_pymethod_enum.rs index 9b596e087ff..5c41d19d4e7 100644 --- a/tests/ui/invalid_pymethod_enum.rs +++ b/tests/ui/invalid_pymethod_enum.rs @@ -16,4 +16,20 @@ impl ComplexEnum { } } +#[pyclass] +enum TupleEnum { + Int(i32), + Str(String), +} + +#[pymethods] +impl TupleEnum { + fn mutate_in_place(&mut self) { + *self = match self { + TupleEnum::Int(int) => TupleEnum::Str(int.to_string()), + TupleEnum::Str(string) => TupleEnum::Int(string.len() as i32), + } + } +} + fn main() {} diff --git a/tests/ui/invalid_pymethod_enum.stderr b/tests/ui/invalid_pymethod_enum.stderr index 6cf6fe89bdf..bc377d2a055 100644 --- a/tests/ui/invalid_pymethod_enum.stderr +++ b/tests/ui/invalid_pymethod_enum.stderr @@ -22,3 +22,28 @@ note: required by a bound in `PyRefMut` | pub struct PyRefMut<'p, T: PyClass> { | ^^^^^^^^^^^^^^ required by this bound in `PyRefMut` = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0271]: type mismatch resolving `::Frozen == False` + --> tests/ui/invalid_pymethod_enum.rs:27:24 + | +27 | fn mutate_in_place(&mut self) { + | ^ expected `False`, found `True` + | +note: required by a bound in `extract_pyclass_ref_mut` + --> src/impl_/extract_argument.rs + | + | pub fn extract_pyclass_ref_mut<'a, 'py: 'a, T: PyClass>( + | ^^^^^^^^^^^^^^ required by this bound in `extract_pyclass_ref_mut` + +error[E0271]: type mismatch resolving `::Frozen == False` + --> tests/ui/invalid_pymethod_enum.rs:25:1 + | +25 | #[pymethods] + | ^^^^^^^^^^^^ expected `False`, found `True` + | +note: required by a bound in `PyRefMut` + --> src/pycell.rs + | + | pub struct PyRefMut<'p, T: PyClass> { + | ^^^^^^^^^^^^^^ required by this bound in `PyRefMut` + = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) From 1c64a03ea084852572872c0d6b5fd029f116c807 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 17 May 2024 00:25:41 -0400 Subject: [PATCH 340/349] Move GIL counting from GILPool to GILGuard (#4188) --- src/gil.rs | 26 ++++++++++++++++++++------ src/impl_/trampoline.rs | 27 +++++++++++++++------------ src/marker.rs | 9 ++++----- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/gil.rs b/src/gil.rs index a1d9feba494..b624e8c5fcd 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -167,6 +167,7 @@ impl GILGuard { /// `GILGuard::Ensured` will be returned. pub(crate) fn acquire() -> Self { if gil_is_acquired() { + increment_gil_count(); return GILGuard::Assumed; } @@ -215,6 +216,7 @@ impl GILGuard { /// as part of multi-phase interpreter initialization. pub(crate) fn acquire_unchecked() -> Self { if gil_is_acquired() { + increment_gil_count(); return GILGuard::Assumed; } @@ -222,8 +224,21 @@ impl GILGuard { #[allow(deprecated)] let pool = unsafe { mem::ManuallyDrop::new(GILPool::new()) }; + increment_gil_count(); + GILGuard::Ensured { gstate, pool } } + /// Acquires the `GILGuard` while assuming that the GIL is already held. + pub(crate) unsafe fn assume() -> Self { + increment_gil_count(); + GILGuard::Assumed + } + + /// Gets the Python token associated with this [`GILGuard`]. + #[inline] + pub fn python(&self) -> Python<'_> { + unsafe { Python::assume_gil_acquired() } + } } /// The Drop implementation for `GILGuard` will release the GIL. @@ -238,6 +253,7 @@ impl Drop for GILGuard { ffi::PyGILState_Release(*gstate); }, } + decrement_gil_count(); } } @@ -378,7 +394,6 @@ impl GILPool { /// As well as requiring the GIL, see the safety notes on `Python::new_pool`. #[inline] pub unsafe fn new() -> GILPool { - increment_gil_count(); // Update counts of PyObjects / Py that have been cloned or dropped since last acquisition #[cfg(not(pyo3_disable_reference_pool))] POOL.update_counts(Python::assume_gil_acquired()); @@ -427,7 +442,6 @@ impl Drop for GILPool { } } } - decrement_gil_count(); } } @@ -687,19 +701,19 @@ mod tests { assert_eq!(get_gil_count(), 1); let pool = unsafe { GILPool::new() }; - assert_eq!(get_gil_count(), 2); + assert_eq!(get_gil_count(), 1); let pool2 = unsafe { GILPool::new() }; - assert_eq!(get_gil_count(), 3); + assert_eq!(get_gil_count(), 1); drop(pool); - assert_eq!(get_gil_count(), 2); + assert_eq!(get_gil_count(), 1); Python::with_gil(|_| { // nested with_gil doesn't update gil count assert_eq!(get_gil_count(), 2); }); - assert_eq!(get_gil_count(), 2); + assert_eq!(get_gil_count(), 1); drop(pool2); assert_eq!(get_gil_count(), 1); diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index db493817cba..f485258e5e5 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -9,8 +9,7 @@ use std::{ panic::{self, UnwindSafe}, }; -#[allow(deprecated)] -use crate::gil::GILPool; +use crate::gil::GILGuard; use crate::{ callback::PyCallbackOutput, ffi, ffi_ptr_ext::FfiPtrExt, impl_::panic::PanicTrap, methods::IPowModulo, panic::PanicException, types::PyModule, Py, PyResult, Python, @@ -171,17 +170,19 @@ trampoline!( /// /// Panics during execution are trapped so that they don't propagate through any /// outer FFI boundary. +/// +/// The GIL must already be held when this is called. #[inline] -pub(crate) fn trampoline(body: F) -> R +pub(crate) unsafe fn trampoline(body: F) -> R where F: for<'py> FnOnce(Python<'py>) -> PyResult + UnwindSafe, R: PyCallbackOutput, { let trap = PanicTrap::new("uncaught panic at ffi boundary"); - // Necessary to construct a pool until PyO3 0.22 when the GIL Refs API is fully disabled - #[allow(deprecated)] - let pool = unsafe { GILPool::new() }; - let py = pool.python(); + + // SAFETY: This function requires the GIL to already be held. + let guard = GILGuard::assume(); + let py = guard.python(); let out = panic_result_into_callback_output( py, panic::catch_unwind(move || -> PyResult<_> { body(py) }), @@ -218,17 +219,19 @@ where /// /// # Safety /// -/// ctx must be either a valid ffi::PyObject or NULL +/// - ctx must be either a valid ffi::PyObject or NULL +/// - The GIL must already be held when this is called. #[inline] unsafe fn trampoline_unraisable(body: F, ctx: *mut ffi::PyObject) where F: for<'py> FnOnce(Python<'py>) -> PyResult<()> + UnwindSafe, { let trap = PanicTrap::new("uncaught panic at ffi boundary"); - // Necessary to construct a pool until PyO3 0.22 when the GIL Refs API is fully disabled - #[allow(deprecated)] - let pool = GILPool::new(); - let py = pool.python(); + + // SAFETY: The GIL is already held. + let guard = GILGuard::assume(); + let py = guard.python(); + if let Err(py_err) = panic::catch_unwind(move || body(py)) .unwrap_or_else(|payload| Err(PanicException::from_panic_payload(payload))) { diff --git a/src/marker.rs b/src/marker.rs index 072b882c875..e8e93c2419d 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -411,10 +411,10 @@ impl Python<'_> { where F: for<'py> FnOnce(Python<'py>) -> R, { - let _guard = GILGuard::acquire(); + let guard = GILGuard::acquire(); // SAFETY: Either the GIL was already acquired or we just created a new `GILGuard`. - f(unsafe { Python::assume_gil_acquired() }) + f(guard.python()) } /// Like [`Python::with_gil`] except Python interpreter state checking is skipped. @@ -445,10 +445,9 @@ impl Python<'_> { where F: for<'py> FnOnce(Python<'py>) -> R, { - let _guard = GILGuard::acquire_unchecked(); + let guard = GILGuard::acquire_unchecked(); - // SAFETY: Either the GIL was already acquired or we just created a new `GILGuard`. - f(Python::assume_gil_acquired()) + f(guard.python()) } } From fff053bde740d1e2bcc1bbb2dde8f7ed979c98bd Mon Sep 17 00:00:00 2001 From: Bruno Kolenbrander <59372212+mejrs@users.noreply.github.com> Date: Fri, 17 May 2024 12:55:41 +0200 Subject: [PATCH 341/349] Emit a better error for abi3 inheritance (#4185) * Emit a better error for abi3 inheritance * Update tests/test_compile_error.rs --------- Co-authored-by: David Hewitt --- pyo3-build-config/src/lib.rs | 5 +++++ src/impl_/pyclass.rs | 7 ++++++ tests/test_compile_error.rs | 3 +++ tests/ui/abi3_inheritance.rs | 10 +++++++++ tests/ui/abi3_inheritance.stderr | 24 +++++++++++++++++++++ tests/ui/abi3_nativetype_inheritance.stderr | 1 + 6 files changed, 50 insertions(+) create mode 100644 tests/ui/abi3_inheritance.rs create mode 100644 tests/ui/abi3_inheritance.stderr diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 6d4ec759653..2b5e76e4b95 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -140,6 +140,10 @@ pub fn print_feature_cfgs() { if rustc_minor_version >= 74 { println!("cargo:rustc-cfg=invalid_from_utf8_lint"); } + + if rustc_minor_version >= 78 { + println!("cargo:rustc-cfg=diagnostic_namespace"); + } } /// Registers `pyo3`s config names as reachable cfg expressions @@ -160,6 +164,7 @@ pub fn print_expected_cfgs() { println!("cargo:rustc-check-cfg=cfg(invalid_from_utf8_lint)"); println!("cargo:rustc-check-cfg=cfg(pyo3_disable_reference_pool)"); println!("cargo:rustc-check-cfg=cfg(pyo3_leak_on_drop_without_reference_pool)"); + println!("cargo:rustc-check-cfg=cfg(diagnostic_namespace)"); // allow `Py_3_*` cfgs from the minimum supported version up to the // maximum minor version (+1 for development for the next) diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 3ec2e329e1a..a3e466670a4 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1102,6 +1102,13 @@ impl PyClassThreadChecker for ThreadCheckerImpl { } /// Trait denoting that this class is suitable to be used as a base type for PyClass. + +#[cfg_attr( + all(diagnostic_namespace, feature = "abi3"), + diagnostic::on_unimplemented( + note = "with the `abi3` feature enabled, PyO3 does not support subclassing native types" + ) +)] pub trait PyClassBaseType: Sized { type LayoutAsBase: PyClassObjectLayout; type BaseNativeType; diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index d31c558f096..2b32de2fcfa 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -64,4 +64,7 @@ fn test_compile_errors() { #[cfg(any(not(Py_LIMITED_API), Py_3_10))] // to avoid PyFunctionArgument for &str t.compile_fail("tests/ui/invalid_cancel_handle.rs"); t.pass("tests/ui/pymodule_missing_docs.rs"); + #[cfg(all(Py_LIMITED_API, not(feature = "experimental-async")))] + // output changes with async feature + t.compile_fail("tests/ui/abi3_inheritance.rs"); } diff --git a/tests/ui/abi3_inheritance.rs b/tests/ui/abi3_inheritance.rs new file mode 100644 index 00000000000..60972e4cf7a --- /dev/null +++ b/tests/ui/abi3_inheritance.rs @@ -0,0 +1,10 @@ +use pyo3::exceptions::PyException; +use pyo3::prelude::*; + +#[pyclass(extends=PyException)] +#[derive(Clone)] +struct MyException { + code: u32, +} + +fn main() {} diff --git a/tests/ui/abi3_inheritance.stderr b/tests/ui/abi3_inheritance.stderr new file mode 100644 index 00000000000..756df2eb75e --- /dev/null +++ b/tests/ui/abi3_inheritance.stderr @@ -0,0 +1,24 @@ +error[E0277]: the trait bound `PyException: PyClassBaseType` is not satisfied + --> tests/ui/abi3_inheritance.rs:4:19 + | +4 | #[pyclass(extends=PyException)] + | ^^^^^^^^^^^ the trait `PyClass` is not implemented for `PyException`, which is required by `PyException: PyClassBaseType` + | + = note: with the `abi3` feature enabled, PyO3 does not support subclassing native types + = help: the trait `PyClassBaseType` is implemented for `PyAny` + = note: required for `PyException` to implement `PyClassBaseType` +note: required by a bound in `PyClassImpl::BaseType` + --> src/impl_/pyclass.rs + | + | type BaseType: PyTypeInfo + PyClassBaseType; + | ^^^^^^^^^^^^^^^ required by this bound in `PyClassImpl::BaseType` + +error[E0277]: the trait bound `PyException: PyClass` is not satisfied + --> tests/ui/abi3_inheritance.rs:4:1 + | +4 | #[pyclass(extends=PyException)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `PyClass` is not implemented for `PyException`, which is required by `PyException: PyClassBaseType` + | + = help: the trait `PyClass` is implemented for `MyException` + = note: required for `PyException` to implement `PyClassBaseType` + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/abi3_nativetype_inheritance.stderr b/tests/ui/abi3_nativetype_inheritance.stderr index f9ca7c61b89..5cd985ccfe5 100644 --- a/tests/ui/abi3_nativetype_inheritance.stderr +++ b/tests/ui/abi3_nativetype_inheritance.stderr @@ -4,6 +4,7 @@ error[E0277]: the trait bound `PyDict: PyClassBaseType` is not satisfied 5 | #[pyclass(extends=PyDict)] | ^^^^^^ the trait `PyClass` is not implemented for `PyDict`, which is required by `PyDict: PyClassBaseType` | + = note: with the `abi3` feature enabled, PyO3 does not support subclassing native types = help: the trait `PyClassBaseType` is implemented for `PyAny` = note: required for `PyDict` to implement `PyClassBaseType` note: required by a bound in `PyClassImpl::BaseType` From fe79f548174eb8108813f202bb0df9428ddfd806 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 17 May 2024 07:31:52 -0400 Subject: [PATCH 342/349] feature gate deprecated APIs for `GILPool` (#4181) --- src/gil.rs | 77 +++++++++++++++++++++++++------------------ src/impl_/not_send.rs | 1 + src/lib.rs | 1 + src/marker.rs | 4 +-- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/src/gil.rs b/src/gil.rs index b624e8c5fcd..8689cde2c9d 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -1,5 +1,6 @@ //! Interaction with Python's global interpreter lock +#[cfg(feature = "gil-refs")] use crate::impl_::not_send::{NotSend, NOT_SEND}; #[cfg(pyo3_disable_reference_pool)] use crate::impl_::panic::PanicTrap; @@ -127,19 +128,16 @@ where ffi::Py_InitializeEx(0); - // Safety: the GIL is already held because of the Py_IntializeEx call. - #[allow(deprecated)] // TODO: remove this with the GIL Refs feature in 0.22 - let pool = GILPool::new(); - - // Import the threading module - this ensures that it will associate this thread as the "main" - // thread, which is important to avoid an `AssertionError` at finalization. - pool.python().import_bound("threading").unwrap(); + let result = { + let guard = GILGuard::assume(); + let py = guard.python(); + // Import the threading module - this ensures that it will associate this thread as the "main" + // thread, which is important to avoid an `AssertionError` at finalization. + py.import_bound("threading").unwrap(); - // Execute the closure. - let result = f(pool.python()); - - // Drop the pool before finalizing. - drop(pool); + // Execute the closure. + f(py) + }; // Finalize the Python interpreter. ffi::Py_Finalize(); @@ -154,7 +152,8 @@ pub(crate) enum GILGuard { /// Indicates that we actually acquired the GIL when this GILGuard was acquired Ensured { gstate: ffi::PyGILState_STATE, - #[allow(deprecated)] // TODO: remove this with the gil-refs feature in 0.22 + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] pool: mem::ManuallyDrop, }, } @@ -221,12 +220,23 @@ impl GILGuard { } let gstate = unsafe { ffi::PyGILState_Ensure() }; // acquire GIL + #[cfg(feature = "gil-refs")] #[allow(deprecated)] - let pool = unsafe { mem::ManuallyDrop::new(GILPool::new()) }; + { + let pool = unsafe { mem::ManuallyDrop::new(GILPool::new()) }; + increment_gil_count(); + GILGuard::Ensured { gstate, pool } + } - increment_gil_count(); + #[cfg(not(feature = "gil-refs"))] + { + increment_gil_count(); + // Update counts of PyObjects / Py that have been cloned or dropped since last acquisition + #[cfg(not(pyo3_disable_reference_pool))] + POOL.update_counts(unsafe { Python::assume_gil_acquired() }); - GILGuard::Ensured { gstate, pool } + GILGuard::Ensured { gstate } + } } /// Acquires the `GILGuard` while assuming that the GIL is already held. pub(crate) unsafe fn assume() -> Self { @@ -246,12 +256,17 @@ impl Drop for GILGuard { fn drop(&mut self) { match self { GILGuard::Assumed => {} + #[cfg(feature = "gil-refs")] GILGuard::Ensured { gstate, pool } => unsafe { // Drop the objects in the pool before attempting to release the thread state mem::ManuallyDrop::drop(pool); ffi::PyGILState_Release(*gstate); }, + #[cfg(not(feature = "gil-refs"))] + GILGuard::Ensured { gstate } => unsafe { + ffi::PyGILState_Release(*gstate); + }, } decrement_gil_count(); } @@ -368,12 +383,10 @@ impl Drop for LockGIL { /// /// [Memory Management]: https://pyo3.rs/main/memory.html#gil-bound-memory -#[cfg_attr( - not(feature = "gil-refs"), - deprecated( - since = "0.21.0", - note = "`GILPool` has no function if PyO3's deprecated GIL Refs API is not used" - ) +#[cfg(feature = "gil-refs")] +#[deprecated( + since = "0.21.0", + note = "`GILPool` has no function if PyO3's deprecated GIL Refs API is not used" )] pub struct GILPool { /// Initial length of owned objects and anys. @@ -382,6 +395,7 @@ pub struct GILPool { _not_send: NotSend, } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl GILPool { /// Creates a new [`GILPool`]. This function should only ever be called with the GIL held. @@ -419,6 +433,7 @@ impl GILPool { } } +#[cfg(feature = "gil-refs")] #[allow(deprecated)] impl Drop for GILPool { fn drop(&mut self) { @@ -534,19 +549,15 @@ fn decrement_gil_count() { #[cfg(test)] mod tests { - #[allow(deprecated)] - use super::GILPool; #[cfg(not(pyo3_disable_reference_pool))] - use super::POOL; - use super::{gil_is_acquired, GIL_COUNT}; - #[cfg(not(pyo3_disable_reference_pool))] - use crate::ffi; + use super::{gil_is_acquired, POOL}; + #[cfg(feature = "gil-refs")] + #[allow(deprecated)] + use super::{GILPool, GIL_COUNT, OWNED_OBJECTS}; use crate::types::any::PyAnyMethods; - use crate::{PyObject, Python}; #[cfg(feature = "gil-refs")] - use {super::OWNED_OBJECTS, crate::gil}; - - #[cfg(not(pyo3_disable_reference_pool))] + use crate::{ffi, gil}; + use crate::{PyObject, Python}; use std::ptr::NonNull; fn get_object(py: Python<'_>) -> PyObject { @@ -691,6 +702,7 @@ mod tests { } #[test] + #[cfg(feature = "gil-refs")] #[allow(deprecated)] fn test_gil_counts() { // Check with_gil and GILPool both increase counts correctly @@ -782,6 +794,7 @@ mod tests { #[test] #[cfg(not(pyo3_disable_reference_pool))] + #[cfg(feature = "gil-refs")] fn test_update_counts_does_not_deadlock() { // update_counts can run arbitrary Python code during Py_DECREF. // if the locking is implemented incorrectly, it will deadlock. diff --git a/src/impl_/not_send.rs b/src/impl_/not_send.rs index 97c2984aff8..382e07a14ee 100644 --- a/src/impl_/not_send.rs +++ b/src/impl_/not_send.rs @@ -6,4 +6,5 @@ use crate::Python; /// Workaround for lack of !Send on stable (). pub(crate) struct NotSend(PhantomData<*mut Python<'static>>); +#[cfg(feature = "gil-refs")] pub(crate) const NOT_SEND: NotSend = NotSend(PhantomData); diff --git a/src/lib.rs b/src/lib.rs index b1d8ae6c7cf..a9c5bd0b731 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -323,6 +323,7 @@ pub use crate::conversion::{FromPyPointer, PyTryFrom, PyTryInto}; #[cfg(feature = "gil-refs")] pub use crate::err::PyDowncastError; pub use crate::err::{DowncastError, DowncastIntoError, PyErr, PyErrArguments, PyResult, ToPyErr}; +#[cfg(feature = "gil-refs")] #[allow(deprecated)] pub use crate::gil::GILPool; #[cfg(not(any(PyPy, GraalPy)))] diff --git a/src/marker.rs b/src/marker.rs index e8e93c2419d..1f4655d9656 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -126,12 +126,10 @@ use crate::types::{ PyAny, PyDict, PyEllipsis, PyModule, PyNone, PyNotImplemented, PyString, PyType, }; use crate::version::PythonVersionInfo; -#[cfg(feature = "gil-refs")] -use crate::PyNativeType; use crate::{ffi, Bound, IntoPy, Py, PyObject, PyTypeInfo}; #[allow(deprecated)] #[cfg(feature = "gil-refs")] -use crate::{gil::GILPool, FromPyPointer}; +use crate::{gil::GILPool, FromPyPointer, PyNativeType}; use std::ffi::{CStr, CString}; use std::marker::PhantomData; use std::os::raw::c_int; From ac273a16122deadad0cabd09bff1457ddf68e277 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 19 May 2024 09:39:29 -0400 Subject: [PATCH 343/349] docs: minor updates to pyenv installs (#4189) --- Contributing.md | 4 ---- guide/src/getting-started.md | 9 ++------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/Contributing.md b/Contributing.md index 111e814ac8f..1503f803e80 100644 --- a/Contributing.md +++ b/Contributing.md @@ -23,10 +23,6 @@ To work and develop PyO3, you need Python & Rust installed on your system. * [virtualenv](https://virtualenv.pypa.io/en/latest/) can also be used with or without Pyenv to use specific installed Python versions. * [`nox`][nox] is used to automate many of our CI tasks. -### Caveats - -* When using pyenv on macOS, installing a Python version using `--enable-shared` is required to make it work. i.e `env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.7.12` - ### Help users identify bugs The [PyO3 Discord server](https://discord.gg/33kcChzH7f) is very active with users who are new to PyO3, and often completely new to Rust. Helping them debug is a great way to get experience with the PyO3 codebase. diff --git a/guide/src/getting-started.md b/guide/src/getting-started.md index 94ab95cb3d6..5dbffaba99c 100644 --- a/guide/src/getting-started.md +++ b/guide/src/getting-started.md @@ -18,19 +18,14 @@ To use PyO3, you need at least Python 3.7. While you can simply use the default While you can use any virtualenv manager you like, we recommend the use of `pyenv` in particular if you want to develop or test for multiple different Python versions, so that is what the examples in this book will use. The installation instructions for `pyenv` can be found [here](https://github.com/pyenv/pyenv#getting-pyenv). (Note: To get the `pyenv activate` and `pyenv virtualenv` commands, you will also need to install the [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv) plugin. The [pyenv installer](https://github.com/pyenv/pyenv-installer#installation--update--uninstallation) will install both together.) -If you intend to run Python from Rust (for example in unit tests) you should set the following environment variable when installing a new Python version using `pyenv`: -```bash -PYTHON_CONFIGURE_OPTS="--enable-shared" -``` +It can be useful to keep the sources used when installing using `pyenv` so that future debugging can see the original source files. This can be done by passing the `--keep` flag as part of the `pyenv install` command. For example: ```bash -env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.12 +pyenv install 3.12 --keep ``` -You can read more about `pyenv`'s configuration options [here](https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#building-with---enable-shared). - ### Building There are a number of build and Python package management systems such as [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](./building-and-distribution.md#manual-builds). We recommend the use of `maturin`, which you can install [here](https://maturin.rs/installation.html). It is developed to work with PyO3 and provides the most "batteries included" experience, especially if you are aiming to publish to PyPI. `maturin` is just a Python package, so you can add it in the same way you already install Python packages. From 674708cb4c7fc41b7a5328f4e51a797b42388d9e Mon Sep 17 00:00:00 2001 From: Adam Reichold Date: Sun, 19 May 2024 15:40:55 +0200 Subject: [PATCH 344/349] Remove OWNED_OBJECTS thread local when GILPool is disabled. (#4193) --- src/gil.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gil.rs b/src/gil.rs index 8689cde2c9d..6f97011b71c 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -6,9 +6,9 @@ use crate::impl_::not_send::{NotSend, NOT_SEND}; use crate::impl_::panic::PanicTrap; use crate::{ffi, Python}; use std::cell::Cell; -#[cfg(debug_assertions)] +#[cfg(all(feature = "gil-refs", debug_assertions))] use std::cell::RefCell; -#[cfg(not(debug_assertions))] +#[cfg(all(feature = "gil-refs", not(debug_assertions)))] use std::cell::UnsafeCell; use std::{mem, ptr::NonNull, sync}; @@ -27,9 +27,9 @@ std::thread_local! { static GIL_COUNT: Cell = const { Cell::new(0) }; /// Temporarily hold objects that will be released when the GILPool drops. - #[cfg(debug_assertions)] + #[cfg(all(feature = "gil-refs", debug_assertions))] static OWNED_OBJECTS: RefCell = const { RefCell::new(Vec::new()) }; - #[cfg(not(debug_assertions))] + #[cfg(all(feature = "gil-refs", not(debug_assertions)))] static OWNED_OBJECTS: UnsafeCell = const { UnsafeCell::new(Vec::new()) }; } From 3e4b3c5c52e06d6003a4a7be200ad8feb21e50ad Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sun, 19 May 2024 16:13:11 -0400 Subject: [PATCH 345/349] docs: attempt to clarify magic methods supported by PyO3 (#4190) * docs: attempt to clarify magic methods supported by PyO3 * Update guide/src/class/protocols.md Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> --------- Co-authored-by: Icxolu <10486322+Icxolu@users.noreply.github.com> --- guide/src/class/protocols.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 3b12fd531c3..4e5f6010e6d 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -1,20 +1,27 @@ -# Magic methods and slots +# Class customizations -Python's object model defines several protocols for different object behavior, such as the sequence, mapping, and number protocols. You may be familiar with implementing these protocols in Python classes by "magic" methods, such as `__str__` or `__repr__`. Because of the double-underscores surrounding their name, these are also known as "dunder" methods. +Python's object model defines several protocols for different object behavior, such as the sequence, mapping, and number protocols. Python classes support these protocols by implementing "magic" methods, such as `__str__` or `__repr__`. Because of the double-underscores surrounding their name, these are also known as "dunder" methods. -In the Python C-API which PyO3 is implemented upon, many of these magic methods have to be placed into special "slots" on the class type object, as covered in the previous section. +PyO3 makes it possible for every magic method to be implemented in `#[pymethods]` just as they would be done in a regular Python class, with a few notable differences: +- `__new__` and `__init__` are replaced by the [`#[new]` attribute](../class.md#constructor). +- `__del__` is not yet supported, but may be in the future. +- `__buffer__` and `__release_buffer__` are currently not supported and instead PyO3 supports [`__getbuffer__` and `__releasebuffer__`](#buffer-objects) methods (these predate [PEP 688](https://peps.python.org/pep-0688/#python-level-buffer-protocol)), again this may change in the future. +- PyO3 adds [`__traverse__` and `__clear__`](#garbage-collector-integration) methods for controlling garbage collection. +- The Python C-API which PyO3 is implemented upon requires many magic methods to have a specific function signature in C and be placed into special "slots" on the class type object. This limits the allowed argument and return types for these methods. They are listed in detail in the section below. -If a function name in `#[pymethods]` is a recognised magic method, it will be automatically placed into the correct slot in the Python type object. The function name is taken from the usual rules for naming `#[pymethods]`: the `#[pyo3(name = "...")]` attribute is used if present, otherwise the Rust function name is used. +If a magic method is not on the list above (for example `__init_subclass__`), then it should just work in PyO3. If this is not the case, please file a bug report. -The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html). Some of the slots do not have a magic method in Python, which leads to a few additional magic methods defined only in PyO3: - - Magic methods for garbage collection - - Magic methods for the buffer protocol +## Magic Methods handled by PyO3 + +If a function name in `#[pymethods]` is a magic method which is known to need special handling, it will be automatically placed into the correct slot in the Python type object. The function name is taken from the usual rules for naming `#[pymethods]`: the `#[pyo3(name = "...")]` attribute is used if present, otherwise the Rust function name is used. + +The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html). When PyO3 handles a magic method, a couple of changes apply compared to other `#[pymethods]`: - The Rust function signature is restricted to match the magic method. - The `#[pyo3(signature = (...)]` and `#[pyo3(text_signature = "...")]` attributes are not allowed. -The following sections list of all magic methods PyO3 currently handles. The +The following sections list all magic methods for which PyO3 implements the necessary special handling. The given signatures should be interpreted as follows: - All methods take a receiver as first argument, shown as ``. It can be `&self`, `&mut self` or a `Bound` reference like `self_: PyRef<'_, Self>` and @@ -31,7 +38,6 @@ given signatures should be interpreted as follows: checked by the Python interpreter. For example, `__str__` needs to return a string object. This is indicated by `object (Python type)`. - ### Basic object customization - `__str__() -> object (str)` From 81ba9a8cd529a7b70b63a983628e424629c998ee Mon Sep 17 00:00:00 2001 From: Cheuk Ting Ho Date: Tue, 21 May 2024 19:24:06 +0100 Subject: [PATCH 346/349] Include import hook in getting-started.md (#4198) --- guide/src/getting-started.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/guide/src/getting-started.md b/guide/src/getting-started.md index 5dbffaba99c..ede48d50c33 100644 --- a/guide/src/getting-started.md +++ b/guide/src/getting-started.md @@ -176,3 +176,7 @@ $ python ``` For more instructions on how to use Python code from Rust, see the [Python from Rust](python-from-rust.md) page. + +## Maturin Import Hook + +In development, any changes in the code would require running `maturin develop` before testing. To streamline the development process, you may want to install [Maturin Import Hook](https://github.com/PyO3/maturin-import-hook) which will run `maturin develop` automatically when the library with code changes is being imported. From 2c654b2906b7267c5c6a6d5cd75f0340676ee99c Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 21 May 2024 15:27:20 -0400 Subject: [PATCH 347/349] ci: adjust test to avoid type inference (#4199) --- src/pybacked.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pybacked.rs b/src/pybacked.rs index ea5face516b..ed68ea52ec7 100644 --- a/src/pybacked.rs +++ b/src/pybacked.rs @@ -291,9 +291,9 @@ mod test { #[test] fn py_backed_bytes_empty() { Python::with_gil(|py| { - let b = PyBytes::new_bound(py, &[]); + let b = PyBytes::new_bound(py, b""); let py_backed_bytes = b.extract::().unwrap(); - assert_eq!(&*py_backed_bytes, &[]); + assert_eq!(&*py_backed_bytes, b""); }); } From d21045cbc1a2df7edff12bc7d7911457ce3d408d Mon Sep 17 00:00:00 2001 From: Cheuk Ting Ho Date: Sat, 25 May 2024 23:39:48 +0100 Subject: [PATCH 348/349] adding new getter for type obj (#4197) * adding new getter for type obj * fixing limited api build * fix formating ssues from clippy * add changelog info * Update newsfragments/4197.added.md Co-authored-by: David Hewitt * Update src/types/typeobject.rs Co-authored-by: David Hewitt * Update src/types/typeobject.rs Co-authored-by: David Hewitt * Update src/types/typeobject.rs Co-authored-by: David Hewitt * Update src/types/typeobject.rs Co-authored-by: David Hewitt * using uncheck downcast * fix formating * move import * Update src/types/typeobject.rs Co-authored-by: Matt Hooks * Update src/types/typeobject.rs Co-authored-by: Matt Hooks --------- Co-authored-by: David Hewitt Co-authored-by: Matt Hooks --- newsfragments/4197.added.md | 1 + src/types/typeobject.rs | 97 ++++++++++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 newsfragments/4197.added.md diff --git a/newsfragments/4197.added.md b/newsfragments/4197.added.md new file mode 100644 index 00000000000..5652028cb76 --- /dev/null +++ b/newsfragments/4197.added.md @@ -0,0 +1 @@ +Add `PyTypeMethods::mro` and `PyTypeMethods::bases`. diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index e6c4a2180d9..9c2d5c5f2c4 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -1,6 +1,7 @@ use crate::err::{self, PyResult}; use crate::instance::Borrowed; use crate::types::any::PyAnyMethods; +use crate::types::PyTuple; #[cfg(feature = "gil-refs")] use crate::PyNativeType; use crate::{ffi, Bound, PyAny, PyTypeInfo, Python}; @@ -127,6 +128,16 @@ pub trait PyTypeMethods<'py>: crate::sealed::Sealed { fn is_subclass_of(&self) -> PyResult where T: PyTypeInfo; + + /// Return the method resolution order for this type. + /// + /// Equivalent to the Python expression `self.__mro__`. + fn mro(&self) -> Bound<'py, PyTuple>; + + /// Return Python bases + /// + /// Equivalent to the Python expression `self.__bases__`. + fn bases(&self) -> Bound<'py, PyTuple>; } impl<'py> PyTypeMethods<'py> for Bound<'py, PyType> { @@ -177,6 +188,48 @@ impl<'py> PyTypeMethods<'py> for Bound<'py, PyType> { { self.is_subclass(&T::type_object_bound(self.py())) } + + fn mro(&self) -> Bound<'py, PyTuple> { + #[cfg(any(Py_LIMITED_API, PyPy))] + let mro = self + .getattr(intern!(self.py(), "__mro__")) + .expect("Cannot get `__mro__` from object.") + .extract() + .expect("Unexpected type in `__mro__` attribute."); + + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + let mro = unsafe { + use crate::ffi_ptr_ext::FfiPtrExt; + (*self.as_type_ptr()) + .tp_mro + .assume_borrowed(self.py()) + .to_owned() + .downcast_into_unchecked() + }; + + mro + } + + fn bases(&self) -> Bound<'py, PyTuple> { + #[cfg(any(Py_LIMITED_API, PyPy))] + let bases = self + .getattr(intern!(self.py(), "__bases__")) + .expect("Cannot get `__bases__` from object.") + .extract() + .expect("Unexpected type in `__bases__` attribute."); + + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + let bases = unsafe { + use crate::ffi_ptr_ext::FfiPtrExt; + (*self.as_type_ptr()) + .tp_bases + .assume_borrowed(self.py()) + .to_owned() + .downcast_into_unchecked() + }; + + bases + } } impl<'a> Borrowed<'a, '_, PyType> { @@ -215,8 +268,8 @@ impl<'a> Borrowed<'a, '_, PyType> { #[cfg(test)] mod tests { - use crate::types::typeobject::PyTypeMethods; - use crate::types::{PyBool, PyLong}; + use crate::types::{PyAnyMethods, PyBool, PyInt, PyLong, PyTuple, PyTypeMethods}; + use crate::PyAny; use crate::Python; #[test] @@ -237,4 +290,44 @@ mod tests { .unwrap()); }); } + + #[test] + fn test_mro() { + Python::with_gil(|py| { + assert!(py + .get_type_bound::() + .mro() + .eq(PyTuple::new_bound( + py, + [ + py.get_type_bound::(), + py.get_type_bound::(), + py.get_type_bound::() + ] + )) + .unwrap()); + }); + } + + #[test] + fn test_bases_bool() { + Python::with_gil(|py| { + assert!(py + .get_type_bound::() + .bases() + .eq(PyTuple::new_bound(py, [py.get_type_bound::()])) + .unwrap()); + }); + } + + #[test] + fn test_bases_object() { + Python::with_gil(|py| { + assert!(py + .get_type_bound::() + .bases() + .eq(PyTuple::empty_bound(py)) + .unwrap()); + }); + } } From 388d1760b5d6545c94925dafe0d640200b9fded2 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 25 May 2024 23:41:26 +0100 Subject: [PATCH 349/349] ci: start testing on 3.13-dev (#4184) * ci: start testing on 3.13-dev * ffi fixes for 3.13 beta 1 * support 3.13 * move gevent to be binary-only * adjust for div_ceil * fixup pytests --- .github/workflows/ci.yml | 1 + newsfragments/4184.packaging.md | 1 + noxfile.py | 12 +-- pyo3-ffi/build.rs | 2 +- pyo3-ffi/src/cpython/code.rs | 11 ++- pyo3-ffi/src/cpython/compile.rs | 4 +- pyo3-ffi/src/cpython/import.rs | 2 +- pyo3-ffi/src/cpython/initconfig.rs | 4 + pyo3-ffi/src/cpython/longobject.rs | 74 ++++++++++++++++ pyo3-ffi/src/cpython/mod.rs | 2 + pyo3-ffi/src/cpython/object.rs | 2 + pyo3-ffi/src/longobject.rs | 27 +----- pytests/noxfile.py | 15 ++-- pytests/pyproject.toml | 1 - pytests/tests/test_misc.py | 17 ++-- src/conversions/num_bigint.rs | 134 ++++++++++++++++++++++------- src/conversions/std/num.rs | 99 +++++++++++++++------ 17 files changed, 302 insertions(+), 106 deletions(-) create mode 100644 newsfragments/4184.packaging.md create mode 100644 pyo3-ffi/src/cpython/longobject.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c245f33483..5bf2b7ea1e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -224,6 +224,7 @@ jobs: "3.10", "3.11", "3.12", + "3.13-dev", "pypy3.7", "pypy3.8", "pypy3.9", diff --git a/newsfragments/4184.packaging.md b/newsfragments/4184.packaging.md new file mode 100644 index 00000000000..c12302a7029 --- /dev/null +++ b/newsfragments/4184.packaging.md @@ -0,0 +1 @@ +Support Python 3.13. diff --git a/noxfile.py b/noxfile.py index 84676b1ff0c..2383e2f865f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -30,7 +30,7 @@ PYO3_GUIDE_SRC = PYO3_DIR / "guide" / "src" PYO3_GUIDE_TARGET = PYO3_TARGET / "guide" PYO3_DOCS_TARGET = PYO3_TARGET / "doc" -PY_VERSIONS = ("3.7", "3.8", "3.9", "3.10", "3.11", "3.12") +PY_VERSIONS = ("3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13") PYPY_VERSIONS = ("3.7", "3.8", "3.9", "3.10") @@ -631,11 +631,11 @@ def test_version_limits(session: nox.Session): config_file.set("CPython", "3.6") _run_cargo(session, "check", env=env, expect_error=True) - assert "3.13" not in PY_VERSIONS - config_file.set("CPython", "3.13") + assert "3.14" not in PY_VERSIONS + config_file.set("CPython", "3.14") _run_cargo(session, "check", env=env, expect_error=True) - # 3.13 CPython should build with forward compatibility + # 3.14 CPython should build with forward compatibility env["PYO3_USE_ABI3_FORWARD_COMPATIBILITY"] = "1" _run_cargo(session, "check", env=env) @@ -734,7 +734,9 @@ def update_ui_tests(session: nox.Session): def _build_docs_for_ffi_check(session: nox.Session) -> None: # pyo3-ffi-check needs to scrape docs of pyo3-ffi - _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") + env = os.environ.copy() + env["PYO3_PYTHON"] = sys.executable + _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps", env=env) @lru_cache() diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index 23f03f1a636..0f4931d6dc7 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -17,7 +17,7 @@ const SUPPORTED_VERSIONS_CPYTHON: SupportedVersions = SupportedVersions { min: PythonVersion { major: 3, minor: 7 }, max: PythonVersion { major: 3, - minor: 12, + minor: 13, }, }; diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index 74586eac595..86e862f21ab 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -20,7 +20,11 @@ pub const _PY_MONITORING_EVENTS: usize = 17; #[repr(C)] #[derive(Clone, Copy)] pub struct _Py_LocalMonitors { - pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS], + pub tools: [u8; if cfg!(Py_3_13) { + _PY_MONITORING_LOCAL_EVENTS + } else { + _PY_MONITORING_UNGROUPED_EVENTS + }], } #[cfg(Py_3_12)] @@ -102,6 +106,9 @@ pub struct PyCodeObject { pub co_extra: *mut c_void, } +#[cfg(Py_3_13)] +opaque_struct!(_PyExecutorArray); + #[cfg(all(not(any(PyPy, GraalPy)), Py_3_8, not(Py_3_11)))] #[repr(C)] #[derive(Copy, Clone)] @@ -176,6 +183,8 @@ pub struct PyCodeObject { pub _co_code: *mut PyObject, #[cfg(not(Py_3_12))] pub _co_linearray: *mut c_char, + #[cfg(Py_3_13)] + pub co_executors: *mut _PyExecutorArray, #[cfg(Py_3_12)] pub _co_cached: *mut _PyCoCached, #[cfg(Py_3_12)] diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 8bce9dacb3b..79f06c92003 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -30,7 +30,7 @@ pub struct PyCompilerFlags { // skipped non-limited _PyCompilerFlags_INIT -#[cfg(all(Py_3_12, not(any(PyPy, GraalPy))))] +#[cfg(all(Py_3_12, not(any(Py_3_13, PyPy, GraalPy))))] #[repr(C)] #[derive(Copy, Clone)] pub struct _PyCompilerSrcLocation { @@ -42,7 +42,7 @@ pub struct _PyCompilerSrcLocation { // skipped SRC_LOCATION_FROM_AST -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_13)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyFutureFeatures { diff --git a/pyo3-ffi/src/cpython/import.rs b/pyo3-ffi/src/cpython/import.rs index aafd71a8355..697d68a419c 100644 --- a/pyo3-ffi/src/cpython/import.rs +++ b/pyo3-ffi/src/cpython/import.rs @@ -57,7 +57,7 @@ pub struct _frozen { pub size: c_int, #[cfg(Py_3_11)] pub is_package: c_int, - #[cfg(Py_3_11)] + #[cfg(all(Py_3_11, not(Py_3_13)))] pub get_code: Option *mut PyObject>, } diff --git a/pyo3-ffi/src/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs index 17fe7559e1b..32931415888 100644 --- a/pyo3-ffi/src/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -141,6 +141,8 @@ pub struct PyConfig { pub safe_path: c_int, #[cfg(Py_3_12)] pub int_max_str_digits: c_int, + #[cfg(Py_3_13)] + pub cpu_count: c_int, pub pathconfig_warnings: c_int, #[cfg(Py_3_10)] pub program_name: *mut wchar_t, @@ -165,6 +167,8 @@ pub struct PyConfig { pub run_command: *mut wchar_t, pub run_module: *mut wchar_t, pub run_filename: *mut wchar_t, + #[cfg(Py_3_13)] + pub sys_path_0: *mut wchar_t, pub _install_importlib: c_int, pub _init_main: c_int, #[cfg(all(Py_3_9, not(Py_3_12)))] diff --git a/pyo3-ffi/src/cpython/longobject.rs b/pyo3-ffi/src/cpython/longobject.rs new file mode 100644 index 00000000000..45acaae577d --- /dev/null +++ b/pyo3-ffi/src/cpython/longobject.rs @@ -0,0 +1,74 @@ +use crate::longobject::*; +use crate::object::*; +#[cfg(Py_3_13)] +use crate::pyport::Py_ssize_t; +use libc::size_t; +#[cfg(Py_3_13)] +use std::os::raw::c_void; +use std::os::raw::{c_int, c_uchar}; + +#[cfg(Py_3_13)] +extern "C" { + pub fn PyLong_FromUnicodeObject(u: *mut PyObject, base: c_int) -> *mut PyObject; +} + +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_DEFAULTS: c_int = -1; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_BIG_ENDIAN: c_int = 0; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_LITTLE_ENDIAN: c_int = 1; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_NATIVE_ENDIAN: c_int = 3; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_UNSIGNED_BUFFER: c_int = 4; +#[cfg(Py_3_13)] +pub const Py_ASNATIVEBYTES_REJECT_NEGATIVE: c_int = 8; + +extern "C" { + // skipped _PyLong_Sign + + #[cfg(Py_3_13)] + pub fn PyLong_AsNativeBytes( + v: *mut PyObject, + buffer: *mut c_void, + n_bytes: Py_ssize_t, + flags: c_int, + ) -> Py_ssize_t; + + #[cfg(Py_3_13)] + pub fn PyLong_FromNativeBytes( + buffer: *const c_void, + n_bytes: size_t, + flags: c_int, + ) -> *mut PyObject; + + #[cfg(Py_3_13)] + pub fn PyLong_FromUnsignedNativeBytes( + buffer: *const c_void, + n_bytes: size_t, + flags: c_int, + ) -> *mut PyObject; + + // skipped PyUnstable_Long_IsCompact + // skipped PyUnstable_Long_CompactValue + + #[cfg_attr(PyPy, link_name = "_PyPyLong_FromByteArray")] + pub fn _PyLong_FromByteArray( + bytes: *const c_uchar, + n: size_t, + little_endian: c_int, + is_signed: c_int, + ) -> *mut PyObject; + + #[cfg_attr(PyPy, link_name = "_PyPyLong_AsByteArrayO")] + pub fn _PyLong_AsByteArray( + v: *mut PyLongObject, + bytes: *mut c_uchar, + n: size_t, + little_endian: c_int, + is_signed: c_int, + ) -> c_int; + + // skipped _PyLong_GCD +} diff --git a/pyo3-ffi/src/cpython/mod.rs b/pyo3-ffi/src/cpython/mod.rs index 1ab0e3c893f..1710dbc4122 100644 --- a/pyo3-ffi/src/cpython/mod.rs +++ b/pyo3-ffi/src/cpython/mod.rs @@ -18,6 +18,7 @@ pub(crate) mod import; pub(crate) mod initconfig; // skipped interpreteridobject.h pub(crate) mod listobject; +pub(crate) mod longobject; #[cfg(all(Py_3_9, not(PyPy)))] pub(crate) mod methodobject; pub(crate) mod object; @@ -53,6 +54,7 @@ pub use self::import::*; #[cfg(all(Py_3_8, not(PyPy)))] pub use self::initconfig::*; pub use self::listobject::*; +pub use self::longobject::*; #[cfg(all(Py_3_9, not(PyPy)))] pub use self::methodobject::*; pub use self::object::*; diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 0f1778f6a3d..b4f6ce5a878 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -296,6 +296,8 @@ pub struct _specialization_cache { pub getitem: *mut PyObject, #[cfg(Py_3_12)] pub getitem_version: u32, + #[cfg(Py_3_13)] + pub init: *mut PyObject, } #[repr(C)] diff --git a/pyo3-ffi/src/longobject.rs b/pyo3-ffi/src/longobject.rs index 55ea8fa1462..35a2bc1b0ff 100644 --- a/pyo3-ffi/src/longobject.rs +++ b/pyo3-ffi/src/longobject.rs @@ -1,8 +1,6 @@ use crate::object::*; use crate::pyport::Py_ssize_t; use libc::size_t; -#[cfg(not(Py_LIMITED_API))] -use std::os::raw::c_uchar; use std::os::raw::{c_char, c_double, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void}; use std::ptr::addr_of_mut; @@ -90,34 +88,12 @@ extern "C" { arg3: c_int, ) -> *mut PyObject; } -// skipped non-limited PyLong_FromUnicodeObject -// skipped non-limited _PyLong_FromBytes #[cfg(not(Py_LIMITED_API))] extern "C" { - // skipped non-limited _PyLong_Sign - #[cfg_attr(PyPy, link_name = "_PyPyLong_NumBits")] + #[cfg(not(Py_3_13))] pub fn _PyLong_NumBits(obj: *mut PyObject) -> size_t; - - // skipped _PyLong_DivmodNear - - #[cfg_attr(PyPy, link_name = "_PyPyLong_FromByteArray")] - pub fn _PyLong_FromByteArray( - bytes: *const c_uchar, - n: size_t, - little_endian: c_int, - is_signed: c_int, - ) -> *mut PyObject; - - #[cfg_attr(PyPy, link_name = "_PyPyLong_AsByteArrayO")] - pub fn _PyLong_AsByteArray( - v: *mut PyLongObject, - bytes: *mut c_uchar, - n: size_t, - little_endian: c_int, - is_signed: c_int, - ) -> c_int; } // skipped non-limited _PyLong_Format @@ -130,6 +106,5 @@ extern "C" { pub fn PyOS_strtol(arg1: *const c_char, arg2: *mut *mut c_char, arg3: c_int) -> c_long; } -// skipped non-limited _PyLong_GCD // skipped non-limited _PyLong_Rshift // skipped non-limited _PyLong_Lshift diff --git a/pytests/noxfile.py b/pytests/noxfile.py index 7c681ab1aa8..af2eb0d3a75 100644 --- a/pytests/noxfile.py +++ b/pytests/noxfile.py @@ -9,11 +9,16 @@ def test(session: nox.Session): session.env["MATURIN_PEP517_ARGS"] = "--profile=dev" session.run_always("python", "-m", "pip", "install", "-v", ".[dev]") - try: - session.install("--only-binary=numpy", "numpy>=1.16") - except CommandFailed: - # No binary wheel for numpy available on this platform - pass + + def try_install_binary(package: str, constraint: str): + try: + session.install(f"--only-binary={package}", f"{package}{constraint}") + except CommandFailed: + # No binary wheel available on this platform + pass + + try_install_binary("numpy", ">=1.16") + try_install_binary("gevent", ">=22.10.2") ignored_paths = [] if sys.version_info < (3, 10): # Match syntax is only available in Python >= 3.10 diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index aace57dd4d4..5f78a573124 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -20,7 +20,6 @@ classifiers = [ [project.optional-dependencies] dev = [ - "gevent>=22.10.2; implementation_name == 'cpython'", "hypothesis>=3.55", "pytest-asyncio>=0.21", "pytest-benchmark>=3.4", diff --git a/pytests/tests/test_misc.py b/pytests/tests/test_misc.py index de75f1c8a80..88af735e861 100644 --- a/pytests/tests/test_misc.py +++ b/pytests/tests/test_misc.py @@ -5,6 +5,11 @@ import pyo3_pytests.misc import pytest +if sys.version_info >= (3, 13): + subinterpreters = pytest.importorskip("subinterpreters") +else: + subinterpreters = pytest.importorskip("_xxsubinterpreters") + def test_issue_219(): # Should not deadlock @@ -31,23 +36,19 @@ def test_multiple_imports_same_interpreter_ok(): reason="PyPy and GraalPy do not support subinterpreters", ) def test_import_in_subinterpreter_forbidden(): - import _xxsubinterpreters - if sys.version_info < (3, 12): expected_error = "PyO3 modules do not yet support subinterpreters, see https://github.com/PyO3/pyo3/issues/576" else: expected_error = "module pyo3_pytests.pyo3_pytests does not support loading in subinterpreters" - sub_interpreter = _xxsubinterpreters.create() + sub_interpreter = subinterpreters.create() with pytest.raises( - _xxsubinterpreters.RunFailedError, + subinterpreters.RunFailedError, match=expected_error, ): - _xxsubinterpreters.run_string( - sub_interpreter, "import pyo3_pytests.pyo3_pytests" - ) + subinterpreters.run_string(sub_interpreter, "import pyo3_pytests.pyo3_pytests") - _xxsubinterpreters.destroy(sub_interpreter) + subinterpreters.destroy(sub_interpreter) def test_type_full_name_includes_module(): diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 743df8a9923..196ae28e788 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -47,6 +47,8 @@ //! assert n + 1 == value //! ``` +#[cfg(not(Py_LIMITED_API))] +use crate::ffi_ptr_ext::FfiPtrExt; #[cfg(Py_LIMITED_API)] use crate::types::{bytes::PyBytesMethods, PyBytes}; use crate::{ @@ -63,20 +65,47 @@ use num_bigint::Sign; // for identical functionality between BigInt and BigUint macro_rules! bigint_conversion { - ($rust_ty: ty, $is_signed: expr, $to_bytes: path) => { + ($rust_ty: ty, $is_signed: literal, $to_bytes: path) => { #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] impl ToPyObject for $rust_ty { #[cfg(not(Py_LIMITED_API))] fn to_object(&self, py: Python<'_>) -> PyObject { let bytes = $to_bytes(self); - unsafe { - let obj = ffi::_PyLong_FromByteArray( - bytes.as_ptr().cast(), - bytes.len(), - 1, - $is_signed, - ); - PyObject::from_owned_ptr(py, obj) + #[cfg(not(Py_3_13))] + { + unsafe { + ffi::_PyLong_FromByteArray( + bytes.as_ptr().cast(), + bytes.len(), + 1, + $is_signed.into(), + ) + .assume_owned(py) + .unbind() + } + } + #[cfg(Py_3_13)] + { + if $is_signed { + unsafe { + ffi::PyLong_FromNativeBytes( + bytes.as_ptr().cast(), + bytes.len(), + ffi::Py_ASNATIVEBYTES_LITTLE_ENDIAN, + ) + .assume_owned(py) + } + } else { + unsafe { + ffi::PyLong_FromUnsignedNativeBytes( + bytes.as_ptr().cast(), + bytes.len(), + ffi::Py_ASNATIVEBYTES_LITTLE_ENDIAN, + ) + .assume_owned(py) + } + } + .unbind() } } @@ -84,7 +113,7 @@ macro_rules! bigint_conversion { fn to_object(&self, py: Python<'_>) -> PyObject { let bytes = $to_bytes(self); let bytes_obj = PyBytes::new_bound(py, &bytes); - let kwargs = if $is_signed > 0 { + let kwargs = if $is_signed { let kwargs = crate::types::PyDict::new_bound(py); kwargs.set_item(crate::intern!(py, "signed"), true).unwrap(); Some(kwargs) @@ -107,8 +136,8 @@ macro_rules! bigint_conversion { }; } -bigint_conversion!(BigUint, 0, BigUint::to_bytes_le); -bigint_conversion!(BigInt, 1, BigInt::to_signed_bytes_le); +bigint_conversion!(BigUint, false, BigUint::to_bytes_le); +bigint_conversion!(BigInt, true, BigInt::to_signed_bytes_le); #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] impl<'py> FromPyObject<'py> for BigInt { @@ -122,13 +151,9 @@ impl<'py> FromPyObject<'py> for BigInt { num_owned = unsafe { Py::from_owned_ptr_or_err(py, ffi::PyNumber_Index(ob.as_ptr()))? }; num_owned.bind(py) }; - let n_bits = int_n_bits(num)?; - if n_bits == 0 { - return Ok(BigInt::from(0isize)); - } #[cfg(not(Py_LIMITED_API))] { - let mut buffer = int_to_u32_vec(num, (n_bits + 32) / 32, true)?; + let mut buffer = int_to_u32_vec::(num)?; let sign = if buffer.last().copied().map_or(false, |last| last >> 31 != 0) { // BigInt::new takes an unsigned array, so need to convert from two's complement // flip all bits, 'subtract' 1 (by adding one to the unsigned array) @@ -152,6 +177,10 @@ impl<'py> FromPyObject<'py> for BigInt { } #[cfg(Py_LIMITED_API)] { + let n_bits = int_n_bits(num)?; + if n_bits == 0 { + return Ok(BigInt::from(0isize)); + } let bytes = int_to_py_bytes(num, (n_bits + 8) / 8, true)?; Ok(BigInt::from_signed_bytes_le(bytes.as_bytes())) } @@ -170,31 +199,37 @@ impl<'py> FromPyObject<'py> for BigUint { num_owned = unsafe { Py::from_owned_ptr_or_err(py, ffi::PyNumber_Index(ob.as_ptr()))? }; num_owned.bind(py) }; - let n_bits = int_n_bits(num)?; - if n_bits == 0 { - return Ok(BigUint::from(0usize)); - } #[cfg(not(Py_LIMITED_API))] { - let buffer = int_to_u32_vec(num, (n_bits + 31) / 32, false)?; + let buffer = int_to_u32_vec::(num)?; Ok(BigUint::new(buffer)) } #[cfg(Py_LIMITED_API)] { + let n_bits = int_n_bits(num)?; + if n_bits == 0 { + return Ok(BigUint::from(0usize)); + } let bytes = int_to_py_bytes(num, (n_bits + 7) / 8, false)?; Ok(BigUint::from_bytes_le(bytes.as_bytes())) } } } -#[cfg(not(Py_LIMITED_API))] +#[cfg(not(any(Py_LIMITED_API, Py_3_13)))] #[inline] -fn int_to_u32_vec( - long: &Bound<'_, PyLong>, - n_digits: usize, - is_signed: bool, -) -> PyResult> { - let mut buffer = Vec::with_capacity(n_digits); +fn int_to_u32_vec(long: &Bound<'_, PyLong>) -> PyResult> { + let mut buffer = Vec::new(); + let n_bits = int_n_bits(long)?; + if n_bits == 0 { + return Ok(buffer); + } + let n_digits = if SIGNED { + (n_bits + 32) / 32 + } else { + (n_bits + 31) / 32 + }; + buffer.reserve_exact(n_digits); unsafe { crate::err::error_on_minusone( long.py(), @@ -203,7 +238,7 @@ fn int_to_u32_vec( buffer.as_mut_ptr() as *mut u8, n_digits * 4, 1, - is_signed.into(), + SIGNED.into(), ), )?; buffer.set_len(n_digits) @@ -215,6 +250,44 @@ fn int_to_u32_vec( Ok(buffer) } +#[cfg(all(not(Py_LIMITED_API), Py_3_13))] +#[inline] +fn int_to_u32_vec(long: &Bound<'_, PyLong>) -> PyResult> { + let mut buffer = Vec::new(); + let mut flags = ffi::Py_ASNATIVEBYTES_LITTLE_ENDIAN; + if !SIGNED { + flags |= ffi::Py_ASNATIVEBYTES_UNSIGNED_BUFFER | ffi::Py_ASNATIVEBYTES_REJECT_NEGATIVE; + } + let n_bytes = + unsafe { ffi::PyLong_AsNativeBytes(long.as_ptr().cast(), std::ptr::null_mut(), 0, flags) }; + let n_bytes_unsigned: usize = n_bytes + .try_into() + .map_err(|_| crate::PyErr::fetch(long.py()))?; + if n_bytes == 0 { + return Ok(buffer); + } + // TODO: use div_ceil when MSRV >= 1.73 + let n_digits = { + let adjust = if n_bytes % 4 == 0 { 0 } else { 1 }; + (n_bytes_unsigned / 4) + adjust + }; + buffer.reserve_exact(n_digits); + unsafe { + ffi::PyLong_AsNativeBytes( + long.as_ptr().cast(), + buffer.as_mut_ptr().cast(), + (n_digits * 4).try_into().unwrap(), + flags, + ); + buffer.set_len(n_digits); + }; + buffer + .iter_mut() + .for_each(|chunk| *chunk = u32::from_le(*chunk)); + + Ok(buffer) +} + #[cfg(Py_LIMITED_API)] fn int_to_py_bytes<'py>( long: &Bound<'py, PyLong>, @@ -239,6 +312,7 @@ fn int_to_py_bytes<'py>( } #[inline] +#[cfg(any(not(Py_3_13), Py_LIMITED_API))] fn int_n_bits(long: &Bound<'_, PyLong>) -> PyResult { let py = long.py(); #[cfg(not(Py_LIMITED_API))] diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index e2072d210e0..effe7c7c062 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -1,3 +1,4 @@ +use crate::ffi_ptr_ext::FfiPtrExt; #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::types::any::PyAnyMethods; @@ -63,14 +64,8 @@ macro_rules! extract_int { err_if_invalid_value($obj.py(), $error_val, unsafe { $pylong_as(long.as_ptr()) }) } else { unsafe { - let num = ffi::PyNumber_Index($obj.as_ptr()); - if num.is_null() { - Err(PyErr::fetch($obj.py())) - } else { - let result = err_if_invalid_value($obj.py(), $error_val, $pylong_as(num)); - ffi::Py_DECREF(num); - result - } + let num = ffi::PyNumber_Index($obj.as_ptr()).assume_owned_or_err($obj.py())?; + err_if_invalid_value($obj.py(), $error_val, $pylong_as(num.as_ptr())) } } }; @@ -181,7 +176,7 @@ mod fast_128bit_int_conversion { // for 128bit Integers macro_rules! int_convert_128 { - ($rust_type: ty, $is_signed: expr) => { + ($rust_type: ty, $is_signed: literal) => { impl ToPyObject for $rust_type { #[inline] fn to_object(&self, py: Python<'_>) -> PyObject { @@ -190,18 +185,44 @@ mod fast_128bit_int_conversion { } impl IntoPy for $rust_type { fn into_py(self, py: Python<'_>) -> PyObject { - // Always use little endian - let bytes = self.to_le_bytes(); - unsafe { - PyObject::from_owned_ptr( - py, + #[cfg(not(Py_3_13))] + { + let bytes = self.to_le_bytes(); + unsafe { ffi::_PyLong_FromByteArray( - bytes.as_ptr() as *const std::os::raw::c_uchar, + bytes.as_ptr().cast(), bytes.len(), 1, - $is_signed, - ), - ) + $is_signed.into(), + ) + .assume_owned(py) + .unbind() + } + } + #[cfg(Py_3_13)] + { + let bytes = self.to_ne_bytes(); + + if $is_signed { + unsafe { + ffi::PyLong_FromNativeBytes( + bytes.as_ptr().cast(), + bytes.len(), + ffi::Py_ASNATIVEBYTES_NATIVE_ENDIAN, + ) + .assume_owned(py) + } + } else { + unsafe { + ffi::PyLong_FromUnsignedNativeBytes( + bytes.as_ptr().cast(), + bytes.len(), + ffi::Py_ASNATIVEBYTES_NATIVE_ENDIAN, + ) + .assume_owned(py) + } + } + .unbind() } } @@ -213,20 +234,46 @@ mod fast_128bit_int_conversion { impl FromPyObject<'_> for $rust_type { fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult<$rust_type> { - let num = unsafe { - PyObject::from_owned_ptr_or_err(ob.py(), ffi::PyNumber_Index(ob.as_ptr()))? - }; - let mut buffer = [0; std::mem::size_of::<$rust_type>()]; + let num = + unsafe { ffi::PyNumber_Index(ob.as_ptr()).assume_owned_or_err(ob.py())? }; + let mut buffer = [0u8; std::mem::size_of::<$rust_type>()]; + #[cfg(not(Py_3_13))] crate::err::error_on_minusone(ob.py(), unsafe { ffi::_PyLong_AsByteArray( num.as_ptr() as *mut ffi::PyLongObject, buffer.as_mut_ptr(), buffer.len(), 1, - $is_signed, + $is_signed.into(), ) })?; - Ok(<$rust_type>::from_le_bytes(buffer)) + #[cfg(Py_3_13)] + { + let mut flags = ffi::Py_ASNATIVEBYTES_NATIVE_ENDIAN; + if !$is_signed { + flags |= ffi::Py_ASNATIVEBYTES_UNSIGNED_BUFFER + | ffi::Py_ASNATIVEBYTES_REJECT_NEGATIVE; + } + let actual_size: usize = unsafe { + ffi::PyLong_AsNativeBytes( + num.as_ptr(), + buffer.as_mut_ptr().cast(), + buffer + .len() + .try_into() + .expect("length of buffer fits in Py_ssize_t"), + flags, + ) + } + .try_into() + .map_err(|_| PyErr::fetch(ob.py()))?; + if actual_size as usize > buffer.len() { + return Err(crate::exceptions::PyOverflowError::new_err( + "Python int larger than 128 bits", + )); + } + } + Ok(<$rust_type>::from_ne_bytes(buffer)) } #[cfg(feature = "experimental-inspect")] @@ -237,8 +284,8 @@ mod fast_128bit_int_conversion { }; } - int_convert_128!(i128, 1); - int_convert_128!(u128, 0); + int_convert_128!(i128, true); + int_convert_128!(u128, false); } // For ABI3 we implement the conversion manually.