Clean up python bindings generator, and add support for sequences.

This is a significant refactor of the python bindings generator, with
the aim of making the generated code a bit cleaner and of better hiding
many of our implementation details from the public API. To help prove
out the approach, it adds previously-missing support for sequence types.

The key idea here is that, for any given `ComponentInterface`, we can
finitely enumerate all the possible types used in that interface
(including recursive types like sequences) and can give each such type
a unique name. This lets us define per-type helper methods on the internal
helper objects like `RustBuffer`, rather than putting these as "hidden"
methods on the public classes.

For example, for each type that lowers into a `RustBuffer`, there is
a corresponding `RustBuffer.allocFrom{{ type_name }}` classmethod for
lowering it and a `RustBuffer.consumeInto{{ type_name }}` for lifting it.

If you squint, this is a little bit like defining private traits and then
emitting a monomorphised implementation for each concrete type, except
within the constraints of python's much looser type system.

examples/rondpoint/tests/bindings/test_rondpoint.kts

@@ -77,7 +77,7 @@ fun <T> List<T>.affirmEnchaine(
     }
 }
 
-// Test the effigacy of the string transport from rust. If this fails, but everything else 
+// Test the efficacy of the string transport from rust. If this fails, but everything else 
 // works, then things are very weird.
 val wellKnown = st.wellKnownString("kotlin")
 assert("uniffi 💚 kotlin!" == wellKnown) { "wellKnownString 'uniffi 💚 kotlin!' == '$wellKnown'" }
@@ -112,4 +112,4 @@ listOf(0.0F, 1.0F, -1.0F, Float.MIN_VALUE, Float.MAX_VALUE).affirmEnchaine(st::t
 
 // Doubles
 // MIN_VALUE is 4.9E-324. Accuracy and formatting get weird at small sizes.
-listOf(0.0, 1.0, -1.0, Double.MIN_VALUE, Double.MAX_VALUE).affirmEnchaine(st::toStringDouble)  { s, n -> s.toDouble() == n }
+listOf(0.0, 1.0, -1.0, Double.MIN_VALUE, Double.MAX_VALUE).affirmEnchaine(st::toStringDouble)  { s, n -> s.toDouble() == n }

examples/rondpoint/tests/bindings/test_rondpoint.py

@@ -0,0 +1,125 @@
+import ctypes
+from rondpoint import *
+
+dico = Dictionnaire(Enumeration.DEUX, True, 0, 123456789)
+copyDico = copie_dictionnaire(dico)
+assert dico == copyDico
+
+assert copie_enumeration(Enumeration.DEUX) == Enumeration.DEUX
+assert copie_enumerations([Enumeration.UN, Enumeration.DEUX]) == [Enumeration.UN, Enumeration.DEUX]
+assert copie_carte({"1": Enumeration.UN, "2": Enumeration.DEUX}) == {"1": Enumeration.UN, "2": Enumeration.DEUX}
+
+assert switcheroo(False) is True
+
+# Test the roundtrip across the FFI.
+# This shows that the values we send come back in exactly the same state as we sent them.
+# i.e. it shows that lowering from python and lifting into rust is symmetrical with
+#      lowering from rust and lifting into python.
+rt = Retourneur()
+
+def affirmAllerRetour(vals, identique):
+  for v in vals:
+    id_v = identique(v)
+    assert id_v == v, f"Round-trip failure: {v} => {id_v}"
+
+MIN_I8 = -1 * 2**7
+MAX_I8 = 2**7 - 1
+MIN_I16 = -1 * 2**15
+MAX_I16 = 2**15 - 1
+MIN_I32 = -1 * 2**31
+MAX_I32 = 2**31 - 1
+MIN_I64 = -1 * 2**31
+MAX_I64 = 2**31 - 1
+
+# Python floats are always doubles, so won't round-trip through f32 correctly.
+# This truncates them appropriately.
+F32_ONE_THIRD = ctypes.c_float(1.0 / 3).value
+
+# Booleans
+affirmAllerRetour([True, False], rt.identique_boolean)
+
+# Bytes.
+affirmAllerRetour([MIN_I8, -1, 0, 1, MAX_I8], rt.identique_i8)
+affirmAllerRetour([0x00, 0x12, 0xFF], rt.identique_u8)
+
+# Shorts
+affirmAllerRetour([MIN_I16, -1, 0, 1, MAX_I16], rt.identique_i16)
+affirmAllerRetour([0x0000, 0x1234, 0xFFFF], rt.identique_u16)
+
+# Ints
+affirmAllerRetour([MIN_I32, -1, 0, 1, MAX_I32], rt.identique_i32)
+affirmAllerRetour([0x00000000, 0x12345678, 0xFFFFFFFF], rt.identique_u32)
+
+# Longs
+affirmAllerRetour([MIN_I64, -1, 0, 1, MAX_I64], rt.identique_i64)
+affirmAllerRetour([0x0000000000000000, 0x1234567890ABCDEF, 0xFFFFFFFFFFFFFFFF], rt.identique_u64)
+
+# Floats
+# XXX TODO: MIN_FLOAT/MAX_FLOAT
+affirmAllerRetour([0.0, 0.5, 0.25, 1.0, F32_ONE_THIRD], rt.identique_float)
+
+# Doubles
+# XXX TODO: MIN_DOUBLE/MAX_DOUBLE
+affirmAllerRetour([0.0, 0.5, 0.25, 1.0, 1.0 / 3], rt.identique_double)
+
+# Strings
+affirmAllerRetour(["", "abc", "été", "ښي لاس ته لوستلو لوستل", "😻emoji 👨‍👧‍👦multi-emoji, 🇨🇭a flag, a canal, panama"], rt.identique_string)
+
+# Test one way across the FFI.
+#
+# We send one representation of a value to lib.rs, and it transforms it into another, a string.
+# lib.rs sends the string back, and then we compare here in python.
+#
+# This shows that the values are transformed into strings the same way in both python and rust.
+# i.e. if we assume that the string return works (we test this assumption elsewhere)
+#      we show that lowering from python and lifting into rust has values that both python and rust
+#      both stringify in the same way. i.e. the same values.
+#
+# If we roundtripping proves the symmetry of our lowering/lifting from here to rust, and lowering/lifting from rust to here,
+# and this convinces us that lowering/lifting from here to rust is correct, then
+# together, we've shown the correctness of the return leg.
+st = Stringifier()
+
+def affirmEnchaine(vals, toString, rustyStringify=lambda v: str(v).lower()):
+    for v in vals:
+      str_v = toString(v)
+      assert rustyStringify(v) == str_v, f"String compare error {v} => {str_v}"
+
+# Test the efficacy of the string transport from rust. If this fails, but everything else
+# works, then things are very weird.
+wellKnown = st.well_known_string("python")
+assert "uniffi 💚 python!" == wellKnown
+
+# Booleans
+affirmEnchaine([True, False], st.to_string_boolean)
+
+# Bytes.
+affirmEnchaine([MIN_I8, -1, 0, 1, MAX_I8], st.to_string_i8)
+affirmEnchaine([0x00, 0x12, 0xFF], st.to_string_u8)
+
+# Shorts
+affirmEnchaine([MIN_I16, -1, 0, 1, MAX_I16], st.to_string_i16)
+affirmEnchaine([0x0000, 0x1234, 0xFFFF], st.to_string_u16)
+
+# Ints
+affirmEnchaine([MIN_I32, -1, 0, 1, MAX_I32], st.to_string_i32)
+affirmEnchaine([0x00000000, 0x12345678, 0xFFFFFFFF], st.to_string_u32)
+
+# Longs
+affirmEnchaine([MIN_I64, -1, 0, 1, MAX_I64], st.to_string_i64)
+affirmEnchaine([0x0000000000000000, 0x1234567890ABCDEF, 0xFFFFFFFFFFFFFFFF], st.to_string_u64)
+
+# Floats
+def rustyFloatToStr(v):
+    """Rust doesn't include the decimal part of whole enumber floats when stringifying."""
+    if int(v) == v:
+        return str(int(v))
+    return str(v)
+
+# XXX TODO: MIN_FLOAT/MAX_FLOAT
+affirmEnchaine([0.0, 0.5, 0.25, 1.0], st.to_string_float, rustyFloatToStr)
+assert st.to_string_float(F32_ONE_THIRD) == "0.33333334" # annoyingly different string repr
+
+# Doubles
+# XXX TODO: MIN_DOUBLE/MAX_DOUBLE
+affirmEnchaine([0.0, 0.5, 0.25, 1.0, 1.0 / 3], st.to_string_double, rustyFloatToStr)

examples/rondpoint/tests/bindings/test_rondpoint.swift

@@ -60,7 +60,7 @@ let rt = Retourneur()
 // together, we've shown the correctness of the return leg.
 let st = Stringifier()
 
-// Test the effigacy of the string transport from rust. If this fails, but everything else 
+// Test the efficacy of the string transport from rust. If this fails, but everything else 
 // works, then things are very weird.
 let wellKnown = st.wellKnownString(value: "swift")
 assert("uniffi 💚 swift!" == wellKnown, "wellKnownString 'uniffi 💚 swift!' == '\(wellKnown)'")

examples/rondpoint/tests/test_generated_bindings.rs

@@ -3,6 +3,6 @@ uniffi_macros::build_foreign_language_testcases!(
     [
         "tests/bindings/test_rondpoint.kts",
         "tests/bindings/test_rondpoint.swift",
-        // "tests/bindings/test_rondpoint.py",
+        "tests/bindings/test_rondpoint.py",
     ]
 );

uniffi_bindgen/src/bindings/python/gen_python.rs

@@ -52,13 +52,9 @@ mod filters {
             FFIType::Float32 => "ctypes.c_float".to_string(),
             FFIType::Float64 => "ctypes.c_double".to_string(),
             FFIType::RustBuffer => "RustBuffer".to_string(),
-            FFIType::RustError => "RustError".to_string(),
-            // We use a c_void_p instead of a c_char_p since python seems to
-            // create it's own string if we use c_char_p, and that prevents us
-            // from freeing. I could be wrong, but that's what I got from this:
-            // https://stackoverflow.com/questions/13445568/python-ctypes-how-to-free-memory-getting-invalid-pointer-error
-            FFIType::RustString => "ctypes.c_void_p".to_string(),
-            FFIType::ForeignStringRef => "ctypes.c_void_p".to_string(),
+            FFIType::RustError => "POINTER(RustError)".to_string(),
+            FFIType::RustString => "RustString".to_string(),
+            FFIType::ForeignStringRef => "ctypes.c_char_p".to_string(),
         })
     }
 
@@ -87,20 +83,16 @@ mod filters {
             | Type::Int32
             | Type::UInt32
             | Type::Int64
-            | Type::UInt64
-            | Type::Float32
-            | Type::Float64
-            | Type::String
-            | Type::Boolean
-            | Type::Object(_)
-            | Type::Error(_) => format!("{} = {}", nm, nm),
-            Type::Enum(type_name) => format!("{} = {}({})", nm, type_name, nm),
-            Type::Record(type_name) => format!("{} = {}._coerce({})", nm, type_name, nm),
+            | Type::UInt64 => format!("int({})", nm), // TODO: check max/min value
+            Type::Float32 | Type::Float64 => format!("float({})", nm),
+            Type::Boolean => format!("bool({})", nm),
+            Type::String | Type::Object(_) | Type::Error(_) | Type::Record(_) => nm.to_string(),
+            Type::Enum(name) => format!("{}({})", class_name_py(name)?, nm),
             Type::Optional(t) => format!("(None if {} is None else {})", nm, coerce_py(nm, t)?),
-            Type::Sequence(t) => format!("({} for x in {})", coerce_py(&"x", t)?, nm), // TODO: name hygiene,
+            Type::Sequence(t) => format!("list({} for x in {})", coerce_py(&"x", t)?, nm),
             Type::Map(t) => format!(
-                "({}:{} for (k, v) in {}.items())",
-                coerce_py(&"k", t)?,
+                "dict(({},{}) for (k, v) in {}.items())",
+                coerce_py(&"k", &Type::String)?,
                 coerce_py(&"v", t)?,
                 nm
             ),
@@ -109,78 +101,70 @@ mod filters {
 
     pub fn lower_py(nm: &dyn fmt::Display, type_: &Type) -> Result<String, askama::Error> {
         Ok(match type_ {
-            Type::UInt32 | Type::UInt64 | Type::Float32 | Type::Float64 | Type::Boolean => {
-                nm.to_string()
-            }
-            Type::Enum(_) => format!("{}.value", nm),
-            Type::String => format!("{}.encode('utf-8')", nm),
-            Type::Record(type_name) => format!("{}._lower({})", type_name, nm),
-            Type::Optional(_type) => format!(
-                "lowerOptional({}, lambda buf, v: {})",
-                nm,
-                lower_into_py(&"buf", &"v", type_)?
+            Type::Int8
+            | Type::UInt8
+            | Type::Int16
+            | Type::UInt16
+            | Type::Int32
+            | Type::UInt32
+            | Type::Int64
+            | Type::UInt64
+            | Type::Float32
+            | Type::Float64 => nm.to_string(),
+            Type::Boolean => format!("(1 if {} else 0)", nm),
+            Type::String => format!("RustString.allocFromString({})", nm),
+            Type::Enum(_) => format!("({}.value)", nm),
+            Type::Object(_) => format!("({}._uniffi_handle)", nm),
+            Type::Error(_) => panic!("No support for lowering errors, yet"),
+            Type::Record(_) | Type::Optional(_) | Type::Sequence(_) | Type::Map(_) => format!(
+                "RustBuffer.allocFrom{}({})",
+                class_name_py(&type_.unique_name())?,
+                nm
             ),
-            _ => panic!("[TODO: lower_py({:?})]", type_),
-        })
-    }
-
-    pub fn lowers_into_size_py(
-        nm: &dyn fmt::Display,
-        type_: &Type,
-    ) -> Result<String, askama::Error> {
-        let nm = var_name_py(nm)?;
-        Ok(match type_ {
-            Type::UInt32 => "4".to_string(),
-            Type::Float64 => "8".to_string(),
-            Type::String => format!("4 + len({}.encode('utf-8'))", nm),
-            Type::Record(type_name) => format!("{}._lowersIntoSize({})", type_name, nm),
-            _ => panic!("[TODO: lowers_into_size_py({:?})]", type_),
-        })
-    }
-    pub fn lower_into_py(
-        nm: &dyn fmt::Display,
-        target: &dyn fmt::Display,
-        type_: &Type,
-    ) -> Result<String, askama::Error> {
-        let nm = var_name_py(nm)?;
-        Ok(match type_ {
-            Type::Float64 => format!("{}.putDouble({})", target, nm),
-            Type::UInt32 => format!("{}.putInt({})", target, nm),
-            Type::String => format!("{}.putString({})", target, nm),
-            Type::Record(type_name) => format!("{}._lowerInto({}, {})", type_name, nm, target),
-            _ => panic!("[TODO: lower_into_py({:?})]", type_),
         })
     }
 
     pub fn lift_py(nm: &dyn fmt::Display, type_: &Type) -> Result<String, askama::Error> {
         Ok(match type_ {
-            Type::UInt32 | Type::UInt64 | Type::Float32 | Type::Float64 | Type::Boolean => {
-                format!("{}", nm)
-            }
-            Type::Enum(type_name) => format!("{}({})", type_name, nm),
-            Type::String => format!("liftString({})", nm),
-            Type::Record(type_name) => format!("{}._lift({})", type_name, nm),
-            Type::Optional(type_) => format!(
-                "liftOptional({}, lambda buf: {})",
-                nm,
-                lift_from_py(&"buf", type_)?
-            ),
-            Type::Sequence(type_) => format!(
-                "liftSequence({}, lambda buf: {})",
+            Type::Int8
+            | Type::UInt8
+            | Type::Int16
+            | Type::UInt16
+            | Type::Int32
+            | Type::UInt32
+            | Type::Int64
+            | Type::UInt64 => format!("int({})", nm),
+            Type::Float32 | Type::Float64 => format!("float({})", nm),
+            Type::Boolean => format!("(True if {} else False)", nm),
+            Type::String => format!("{}.consumeIntoString()", nm),
+            Type::Enum(name) => format!("{}({})", class_name_py(name)?, nm),
+            Type::Object(_) => panic!("No support for lifting objects, yet"),
+            Type::Error(_) => panic!("No support for lowering errors, yet"),
+            Type::Record(_) | Type::Optional(_) | Type::Sequence(_) | Type::Map(_) => format!(
+                "{}.consumeInto{}()",
                 nm,
-                lift_from_py(&"buf", type_)?
+                class_name_py(&type_.unique_name())?
             ),
-            _ => panic!("[TODO: lift_py({:?})]", type_),
         })
     }
 
-    pub fn lift_from_py(nm: &dyn fmt::Display, type_: &Type) -> Result<String, askama::Error> {
+    pub fn calculate_write_size(
+        nm: &dyn fmt::Display,
+        type_: &Type,
+    ) -> Result<String, askama::Error> {
         Ok(match type_ {
-            Type::UInt32 => format!("{}.getInt()", nm),
-            Type::Float64 => format!("{}.getDouble()", nm),
-            Type::Record(type_name) => format!("{}._liftFrom({})", type_name, nm),
-            Type::String => format!("{}.getString()", nm),
-            _ => panic!("[TODO: lift_from_py({:?})]", type_),
+            Type::Int8 | Type::UInt8 | Type::Boolean => "1".into(),
+            Type::Int16 | Type::UInt16 => "2".into(),
+            Type::Int32 | Type::UInt32 | Type::Float32 | Type::Enum(_) => "4".into(),
+            Type::Int64 | Type::UInt64 | Type::Float64 => "8".into(),
+            Type::String => format!("4 + len({}.encode('utf-8'))", nm),
+            Type::Object(_) => panic!("No support for writing objects, yet"),
+            Type::Error(_) => panic!("No support for writing errors, yet"),
+            Type::Record(_) | Type::Optional(_) | Type::Sequence(_) | Type::Map(_) => format!(
+                "RustBuffer.calculateWriteSizeOf{}({})",
+                class_name_py(&type_.unique_name())?,
+                nm
+            ),
         })
     }
 }

uniffi_bindgen/src/bindings/python/templates/EnumTemplate.py

@@ -1,4 +1,4 @@
-class {{ e.name() }}(enum.Enum):
+class {{ e.name()|class_name_py }}(enum.Enum):
     {% for variant in e.variants() -%}
     {{ variant|enum_name_py }} = {{ loop.index }}
-    {% endfor -%}
+    {% endfor %}

uniffi_bindgen/src/bindings/python/templates/ObjectTemplate.py

@@ -3,24 +3,28 @@ class {{ obj.name()|class_name_py }}(object):
     {%- for cons in obj.constructors() %}
     def __init__(self, {% call py::arg_list_decl(cons) -%}):
         {%- call py::coerce_args_extra_indent(cons) %}
-        self._handle = {% call py::to_ffi_call(cons) %}
+        self._uniffi_handle = {% call py::to_ffi_call(cons) %}
     {%- endfor %}
 
     def __del__(self):
-        _UniFFILib.{{ obj.ffi_object_free().name() }}(self._handle)
+        # XXX TODO: if we have multiple instances of the python-side class
+        # that somehow share the same underlying handle, then the rust side
+        # of *all* of them will be poisoned whenever the first one is GC'd.
+        # Maybe we need a python-side map of handles to instances?
+        _UniFFILib.{{ obj.ffi_object_free().name() }}(self._uniffi_handle)
 
     {% for meth in obj.methods() -%}
     {%- match meth.return_type() -%}
 
     {%- when Some with (return_type) -%}
     def {{ meth.name()|fn_name_py }}(self, {% call py::arg_list_decl(meth) %}):
         {%- call py::coerce_args_extra_indent(meth) %}
-        _retval = {% call py::to_ffi_call_with_prefix("self._handle", meth) %}
+        _retval = {% call py::to_ffi_call_with_prefix("self._uniffi_handle", meth) %}
         return {{ "_retval"|lift_py(return_type) }}
-    
+
     {%- when None -%}
     def {{ meth.name()|fn_name_py }}(self, {% call py::arg_list_decl(meth) %}):
         {%- call py::coerce_args_extra_indent(meth) %}
-        {% call py::to_ffi_call_with_prefix("self._handle", meth) %}
+        {% call py::to_ffi_call_with_prefix("self._uniffi_handle", meth) %}
     {% endmatch %}
     {% endfor %}