Rename PyClassShell with PyCell and do mutability checking

guide/src/class.md

@@ -22,6 +22,7 @@ Specifically, the following implementation is generated:
 
 ```rust
 use pyo3::prelude::*;
+use pyo3::types::PyAny;
 
 /// Class for demonstration
 struct MyClass {
@@ -33,9 +34,10 @@ impl pyo3::pyclass::PyClassAlloc for MyClass {}
 
 unsafe impl pyo3::PyTypeInfo for MyClass {
     type Type = MyClass;
-    type BaseType = pyo3::types::PyAny;
-    type ConcreteLayout = pyo3::PyClassShell<Self>;
-    type Initializer = pyo3::PyClassInitializer<Self>;
+    type BaseType = PyAny;
+    type BaseLayout = pyo3::pycell::PyCellBase<PyAny>;
+    type Layout = PyCell<Self>;
+    type Initializer = PyClassInitializer<Self>;
 
     const NAME: &'static str = "MyClass";
     const MODULE: Option<&'static str> = None;
@@ -53,6 +55,7 @@ unsafe impl pyo3::PyTypeInfo for MyClass {
 impl pyo3::pyclass::PyClass for MyClass {
     type Dict = pyo3::pyclass_slots::PyClassDummySlot;
     type WeakRef = pyo3::pyclass_slots::PyClassDummySlot;
+    type BaseNativeType = PyAny;
 }
 
 impl pyo3::IntoPy<PyObject> for MyClass {
@@ -105,47 +108,58 @@ fn mymodule(_py: Python, m: &PyModule) -> PyResult<()> {
 }
 ```
 
-## Get Python objects from `pyclass`
-You sometimes need to convert your `pyclass` into a Python object in Rust code (e.g., for testing it).
+## PyCell and interior mutability
+You sometimes need to convert your `pyclass` into a Python object and access it
+from Rust code (e.g., for testing it).
+`PyCell` is our primary interface for that.
 
-For getting *GIL-bounded* (i.e., with `'py` lifetime) references of `pyclass`,
-you can use `PyClassShell<T>`.
-Or you can use `Py<T>` directly, for *not-GIL-bounded* references.
+`PyCell<T: PyClass>` is always allocated in the Python heap, so we don't have the ownership of it.
+We can get `&PyCell<T>`, not `PyCell<T>`.
 
-### `PyClassShell`
-`PyClassShell` represents the actual layout of `pyclass` on the Python heap.
+Thus, to mutate data behind `&PyCell` safely, we employ the
+[Interior Mutability Pattern](https://doc.rust-lang.org/book/ch15-05-interior-mutability.html)
+like [std::cell::RefCell](https://doc.rust-lang.org/std/cell/struct.RefCell.html).
 
-If you want to instantiate `pyclass` in Python and get the reference,
-you can use `PyClassShell::new_ref` or `PyClassShell::new_mut`.
+Users who are familiar with `RefCell` can use `PyCell` just like `RefCell`.
+
+For users who doesn't know `RefCell` well, we repeat the Rust's borrowing rule here:
+- At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.
+- References must always be valid.
+`PyCell` ensures these borrowing rules by managing a reference counter.
+
+TODO: link to the API document
 
 ```rust
 # use pyo3::prelude::*;
 # use pyo3::types::PyDict;
-# use pyo3::PyClassShell;
+# use pyo3::PyCell;
 #[pyclass]
 struct MyClass {
+   #[pyo3(get)]
    num: i32,
    debug: bool,
 }
 let gil = Python::acquire_gil();
 let py = gil.python();
-let obj = PyClassShell::new_ref(py, MyClass { num: 3, debug: true }).unwrap();
-// You can use deref
-assert_eq!(obj.num, 3);
-let dict = PyDict::new(py);
-// You can treat a `&PyClassShell` as a normal Python object
-dict.set_item("obj", obj).unwrap();
-
-// return &mut PyClassShell<MyClass>
-let obj = PyClassShell::new_mut(py, MyClass { num: 3, debug: true }).unwrap();
-obj.num = 5;
+let obj = PyCell::new(py, MyClass { num: 3, debug: true }).unwrap();
+{
+    let obj_ref = obj.borrow(); // Get PyRef
+    assert_eq!(obj_ref.num, 3);
+    // You cannot get PyRefMut unless all PyRefs are dropped
+    assert!(obj.try_borrow_mut().is_err());
+}
+{
+    let mut obj_mut = obj.borrow_mut(); // Get PyRefMut
+    obj_mut.num = 5;
+    // You cannot get any other refs until the PyRefMut is dropped
+    assert!(obj.try_borrow().is_err());
+}
+// You can convert `&PyCell` to Python object
+pyo3::py_run!(py, obj, "assert obj.num == 5")
 ```
 
-### `Py`
-
-`Py` is an object wrapper which stores an object longer than the GIL lifetime.
-
-You can use it to avoid lifetime problems.
+`&PyCell<T>` is bouded by the same lifetime as `GILGuard`.
+To avoid this you can use `Py<T>`, which stores an object longer than the GIL lifetime.
 ```rust
 # use pyo3::prelude::*;
 #[pyclass]
@@ -159,7 +173,9 @@ fn return_myclass() -> Py<MyClass> {
 }
 let gil = Python::acquire_gil();
 let obj = return_myclass();
-assert_eq!(obj.as_ref(gil.python()).num, 1);
+let cell = obj.as_ref(gil.python()); // AsPyRef::as_ref returns &PyCell
+let obj_ref = cell.borrow(); // Get PyRef<T>
+assert_eq!(obj_ref.num, 1);
 ```
 
 ## Customizing the class
@@ -228,9 +244,12 @@ baseclass of `T`.
 But for more deeply nested inheritance, you have to return `PyClassInitializer<T>`
 explicitly.
 
+To get a parent class from child, use `PyRef<T>` instead of `&self`,
+or `PyRefMut<T>` instead of `&mut self`.
+
 ```rust
 # use pyo3::prelude::*;
-use pyo3::PyClassShell;
+use pyo3::PyCell;
 
 #[pyclass]
 struct BaseClass {
@@ -261,8 +280,9 @@ impl SubClass {
        (SubClass{ val2: 15}, BaseClass::new())
    }
 
-   fn method2(self_: &PyClassShell<Self>) -> PyResult<usize> {
-      self_.get_super().method().map(|x| x * self_.val2)
+   fn method2(self_: PyRef<Self>) -> PyResult<usize> {
+       let super_ = self_.as_super();  // Get &BaseClass
+       super_.method().map(|x| x * self_.val2)
    }
 }
 
@@ -279,29 +299,30 @@ impl SubSubClass {
            .add_subclass(SubSubClass{val3: 20})
    }
 
-   fn method3(self_: &PyClassShell<Self>) -> PyResult<usize> {
-      let super_ = self_.get_super();
-      SubClass::method2(super_).map(|x| x * self_.val3)
+   fn method3(self_: PyRef<Self>) -> PyResult<usize> {
+      let v = self_.val3;
+      let super_ = self_.into_super();  // Get PyRef<SubClass>
+      SubClass::method2(super_).map(|x| x * v)
    }
 }
 
 
 # let gil = Python::acquire_gil();
 # let py = gil.python();
-# let subsub = pyo3::PyClassShell::new_ref(py, SubSubClass::new()).unwrap();
+# let subsub = pyo3::PyCell::new(py, SubSubClass::new()).unwrap();
 # pyo3::py_run!(py, subsub, "assert subsub.method3() == 3000")
 ```
 
 To access the super class, you can use either of these two ways:
-- Use `self_: &PyClassShell<Self>` instead of `self`, and call `get_super()`
+- Use `self_: &PyCell<Self>` instead of `self`, and call `get_super()`
 - `ObjectProtocol::get_base`
-We recommend `PyClassShell` here, since it makes the context much clearer.
+We recommend `PyCell` here, since it makes the context much clearer.
 
 
 If `SubClass` does not provide a baseclass initialization, the compilation fails.
 ```compile_fail
 # use pyo3::prelude::*;
-use pyo3::PyClassShell;
+use pyo3::PyCell;
 
 #[pyclass]
 struct BaseClass {
@@ -761,16 +782,16 @@ struct GCTracked {} // Fails because it does not implement PyGCProtocol
 Iterators can be defined using the
 [`PyIterProtocol`](https://docs.rs/pyo3/latest/pyo3/class/iter/trait.PyIterProtocol.html) trait.
 It includes two methods `__iter__` and `__next__`:
-  * `fn __iter__(slf: &mut PyClassShell<Self>) -> PyResult<impl IntoPy<PyObject>>`
-  * `fn __next__(slf: &mut PyClassShell<Self>) -> PyResult<Option<impl IntoPy<PyObject>>>`
+  * `fn __iter__(slf: PyRefMut<Self>) -> PyResult<impl IntoPy<PyObject>>`
+  * `fn __next__(slf: PyRefMut<Self>) -> PyResult<Option<impl IntoPy<PyObject>>>`
 
   Returning `Ok(None)` from `__next__` indicates that that there are no further items.
 
 Example:
 
 ```rust
 use pyo3::prelude::*;
-use pyo3::{PyIterProtocol, PyClassShell};
+use pyo3::{PyIterProtocol, PyCell};
 
 #[pyclass]
 struct MyIterator {
@@ -779,10 +800,10 @@ struct MyIterator {
 
 #[pyproto]
 impl PyIterProtocol for MyIterator {
-    fn __iter__(slf: &mut PyClassShell<Self>) -> PyResult<Py<MyIterator>> {
+    fn __iter__(mut slf: PyRefMut<Self>) -> PyResult<Py<MyIterator>> {
         Ok(slf.into())
     }
-    fn __next__(slf: &mut PyClassShell<Self>) -> PyResult<Option<PyObject>> {
+    fn __next__(mut slf: PyRefMut<Self>) -> PyResult<Option<PyObject>> {
         Ok(slf.iter.next())
     }
 }

guide/src/python_from_rust.md

@@ -36,7 +36,7 @@ your Python extensions quickly.
 
 ```rust
 use pyo3::prelude::*;
-use pyo3::{PyClassShell, PyObjectProtocol, py_run};
+use pyo3::{PyCell, PyObjectProtocol, py_run};
 #  fn main() {
 #[pyclass]
 struct UserData {
@@ -61,7 +61,7 @@ let userdata = UserData {
     id: 34,
     name: "Yu".to_string(),
 };
-let userdata = PyClassShell::new_ref(py, userdata).unwrap();
+let userdata = PyCell::new(py, userdata).unwrap();
 let userdata_as_tuple = (34, "Yu");
 py_run!(py, userdata userdata_as_tuple, r#"
 assert repr(userdata) == "User Yu(id: 34)"

pyo3-derive-backend/src/method.rs

@@ -29,12 +29,16 @@ pub enum FnType {
     FnCall,
     FnClass,
     FnStatic,
-    PySelf(syn::TypeReference),
+    // self_: &PyCell<Self>,
+    PySelfRef(syn::TypeReference),
+    // self_: PyRef<Self> or PyRefMut<Self>
+    PySelfPath(syn::TypePath),
 }
 
 #[derive(Clone, PartialEq, Debug)]
 pub struct FnSpec<'a> {
     pub tp: FnType,
+    pub self_: Option<bool>,
     // Rust function name
     pub name: &'a syn::Ident,
     // Wrapped python name. This should not have any leading r#.
@@ -54,6 +58,14 @@ pub fn get_return_info(output: &syn::ReturnType) -> syn::Type {
 }
 
 impl<'a> FnSpec<'a> {
+    /// Generate the code for borrowing self
+    pub(crate) fn borrow_self(&self) -> TokenStream {
+        let is_mut = self
+            .self_
+            .expect("impl_borrow_self is called for non-self fn");
+        crate::utils::borrow_self(is_mut, true)
+    }
+
     /// Parser function signature and function attributes
     pub fn parse(
         sig: &'a syn::Signature,
@@ -67,19 +79,19 @@ impl<'a> FnSpec<'a> {
             mut python_name,
         } = parse_method_attributes(meth_attrs, allow_custom_name)?;
 
-        let mut has_self = false;
+        let mut self_ = None;
         let mut arguments = Vec::new();
         for input in sig.inputs.iter() {
             match input {
-                syn::FnArg::Receiver(_) => {
-                    has_self = true;
+                syn::FnArg::Receiver(recv) => {
+                    self_ = Some(recv.mutability.is_some());
                 }
                 syn::FnArg::Typed(syn::PatType {
                     ref pat, ref ty, ..
                 }) => {
                     // skip first argument (cls)
-                    if fn_type == FnType::FnClass && !has_self {
-                        has_self = true;
+                    if fn_type == FnType::FnClass && self_.is_none() {
+                        self_ = Some(false);
                         continue;
                     }
 
@@ -114,18 +126,18 @@ impl<'a> FnSpec<'a> {
 
         let ty = get_return_info(&sig.output);
 
-        if fn_type == FnType::Fn && !has_self {
+        if fn_type == FnType::Fn && self_.is_none() {
             if arguments.is_empty() {
                 return Err(syn::Error::new_spanned(
                     name,
                     "Static method needs #[staticmethod] attribute",
                 ));
             }
-            let tp = match arguments.remove(0).ty {
-                syn::Type::Reference(r) => replace_self(r)?,
+            fn_type = match arguments.remove(0).ty {
+                syn::Type::Reference(r) => FnType::PySelfRef(replace_self_in_ref(r)?),
+                syn::Type::Path(p) => FnType::PySelfPath(replace_self_in_path(p)),
                 x => return Err(syn::Error::new_spanned(x, "Invalid type as custom self")),
             };
-            fn_type = FnType::PySelf(tp);
         }
 
         // "Tweak" getter / setter names: strip off set_ and get_ if needed
@@ -158,9 +170,11 @@ impl<'a> FnSpec<'a> {
         };
 
         let text_signature = match &fn_type {
-            FnType::Fn | FnType::PySelf(_) | FnType::FnClass | FnType::FnStatic => {
-                utils::parse_text_signature_attrs(&mut *meth_attrs, name)?
-            }
+            FnType::Fn
+            | FnType::PySelfRef(_)
+            | FnType::PySelfPath(_)
+            | FnType::FnClass
+            | FnType::FnStatic => utils::parse_text_signature_attrs(&mut *meth_attrs, name)?,
             FnType::FnNew => parse_erroneous_text_signature(
                 "text_signature not allowed on __new__; if you want to add a signature on \
                  __new__, put it on the struct definition instead",
@@ -174,6 +188,7 @@ impl<'a> FnSpec<'a> {
 
         Ok(FnSpec {
             tp: fn_type,
+            self_,
             name,
             python_name,
             attrs: fn_attrs,
@@ -514,17 +529,24 @@ fn parse_method_name_attribute(
 }
 
 // Replace &A<Self> with &A<_>
-fn replace_self(refn: &syn::TypeReference) -> syn::Result<syn::TypeReference> {
-    fn infer(span: proc_macro2::Span) -> syn::GenericArgument {
-        syn::GenericArgument::Type(syn::Type::Infer(syn::TypeInfer {
-            underscore_token: syn::token::Underscore { spans: [span] },
-        }))
-    }
+fn replace_self_in_ref(refn: &syn::TypeReference) -> syn::Result<syn::TypeReference> {
     let mut res = refn.to_owned();
     let tp = match &mut *res.elem {
         syn::Type::Path(p) => p,
         _ => return Err(syn::Error::new_spanned(refn, "unsupported argument")),
     };
+    replace_self_impl(tp);
+    res.lifetime = None;
+    Ok(res)
+}
+
+fn replace_self_in_path(tp: &syn::TypePath) -> syn::TypePath {
+    let mut res = tp.to_owned();
+    replace_self_impl(&mut res);
+    res
+}
+
+fn replace_self_impl(tp: &mut syn::TypePath) {
     for seg in &mut tp.path.segments {
         if let syn::PathArguments::AngleBracketed(ref mut g) = seg.arguments {
             let mut args = syn::punctuated::Punctuated::new();
@@ -546,6 +568,9 @@ fn replace_self(refn: &syn::TypeReference) -> syn::Result<syn::TypeReference> {
             g.args = args;
         }
     }
-    res.lifetime = None;
-    Ok(res)
+    fn infer(span: proc_macro2::Span) -> syn::GenericArgument {
+        syn::GenericArgument::Type(syn::Type::Infer(syn::TypeInfer {
+            underscore_token: syn::token::Underscore { spans: [span] },
+        }))
+    }
 }