From 4d662d5526e986deb7aab3c33f2c88ef5935d9a9 Mon Sep 17 00:00:00 2001 From: Ryan Kelly Date: Sat, 8 Aug 2020 15:22:09 +1000 Subject: [PATCH] Clean up python bindings generator, add support for remaining data types. This is a significant refactor of the python bindings generator, with the aim of making the generated code a bit cleaner and of better hiding many of our implementation details from the public API. To help prove out the approach, it adds previously-missing support for data types like sequences and maps. We use the new `TypeUniverse` structure to iterate over all types used in the interface, and generate various internal helper functions as methods on our utility classes. This keeps them out of the public API as seen by consumers. For example, for each type that lowers into a `RustBuffer`, there is a corresponding `RustBuffer.allocFrom{{ type_name }}` staticmethod for lowering it and a `RustBuffer.consumeInto{{ type_name }}` for lifting it. --- .../tests/bindings/test_rondpoint.kts | 5 - .../tests/bindings/test_rondpoint.py | 134 +++++++++++++ .../tests/test_generated_bindings.rs | 2 +- .../src/bindings/python/gen_python.rs | 105 ++++------ .../bindings/python/templates/EnumTemplate.py | 4 +- .../python/templates/ObjectTemplate.py | 2 +- .../python/templates/RecordTemplate.py | 38 +--- .../python/templates/RustBufferBuilder.py | 166 ++++++++++++++++ .../python/templates/RustBufferHelper.py | 111 ----------- .../python/templates/RustBufferStream.py | 179 ++++++++++++++++++ .../python/templates/RustBufferTemplate.py | 113 ++++++++++- .../src/bindings/python/templates/macros.py | 32 ++-- .../src/bindings/python/templates/wrapper.py | 4 +- uniffi_bindgen/src/interface/mod.rs | 47 +++++ 14 files changed, 696 insertions(+), 246 deletions(-) create mode 100644 examples/rondpoint/tests/bindings/test_rondpoint.py create mode 100644 uniffi_bindgen/src/bindings/python/templates/RustBufferBuilder.py delete mode 100644 uniffi_bindgen/src/bindings/python/templates/RustBufferHelper.py create mode 100644 uniffi_bindgen/src/bindings/python/templates/RustBufferStream.py diff --git a/examples/rondpoint/tests/bindings/test_rondpoint.kts b/examples/rondpoint/tests/bindings/test_rondpoint.kts index 4e6db7d868..435ad014c5 100644 --- a/examples/rondpoint/tests/bindings/test_rondpoint.kts +++ b/examples/rondpoint/tests/bindings/test_rondpoint.kts @@ -88,11 +88,6 @@ fun List.affirmEnchaine( val wellKnown = st.wellKnownString("kotlin") assert("uniffi ๐Ÿ’š kotlin!" == wellKnown) { "wellKnownString 'uniffi ๐Ÿ’š kotlin!' == '$wellKnown'" } -// NB. Numbers are all signed in kotlin. This makes roundtripping of unsigned numbers tricky to show. -// Uniffi does not generate unsigned types for kotlin, but the work tracked is -// in https://github.com/mozilla/uniffi-rs/issues/249. Tests using unsigned types are -// commented out for now. - // Booleans listOf(true, false).affirmEnchaine(st::toStringBoolean) diff --git a/examples/rondpoint/tests/bindings/test_rondpoint.py b/examples/rondpoint/tests/bindings/test_rondpoint.py new file mode 100644 index 0000000000..86abb17b09 --- /dev/null +++ b/examples/rondpoint/tests/bindings/test_rondpoint.py @@ -0,0 +1,134 @@ +import sys +import ctypes +from rondpoint import * + +dico = Dictionnaire(Enumeration.DEUX, True, 0, 123456789) +copyDico = copie_dictionnaire(dico) +assert dico == copyDico + +assert copie_enumeration(Enumeration.DEUX) == Enumeration.DEUX +assert copie_enumerations([Enumeration.UN, Enumeration.DEUX]) == [Enumeration.UN, Enumeration.DEUX] +assert copie_carte({"1": Enumeration.UN, "2": Enumeration.DEUX}) == {"1": Enumeration.UN, "2": Enumeration.DEUX} + +assert switcheroo(False) is True + +# Test the roundtrip across the FFI. +# This shows that the values we send come back in exactly the same state as we sent them. +# i.e. it shows that lowering from python and lifting into rust is symmetrical with +# lowering from rust and lifting into python. +rt = Retourneur() + +def affirmAllerRetour(vals, identique): + for v in vals: + id_v = identique(v) + assert id_v == v, f"Round-trip failure: {v} => {id_v}" + +MIN_I8 = -1 * 2**7 +MAX_I8 = 2**7 - 1 +MIN_I16 = -1 * 2**15 +MAX_I16 = 2**15 - 1 +MIN_I32 = -1 * 2**31 +MAX_I32 = 2**31 - 1 +MIN_I64 = -1 * 2**31 +MAX_I64 = 2**31 - 1 + +# Python floats are always doubles, so won't round-trip through f32 correctly. +# This truncates them appropriately. +F32_ONE_THIRD = ctypes.c_float(1.0 / 3).value + +# Booleans +affirmAllerRetour([True, False], rt.identique_boolean) + +# Bytes. +affirmAllerRetour([MIN_I8, -1, 0, 1, MAX_I8], rt.identique_i8) +affirmAllerRetour([0x00, 0x12, 0xFF], rt.identique_u8) + +# Shorts +affirmAllerRetour([MIN_I16, -1, 0, 1, MAX_I16], rt.identique_i16) +affirmAllerRetour([0x0000, 0x1234, 0xFFFF], rt.identique_u16) + +# Ints +affirmAllerRetour([MIN_I32, -1, 0, 1, MAX_I32], rt.identique_i32) +affirmAllerRetour([0x00000000, 0x12345678, 0xFFFFFFFF], rt.identique_u32) + +# Longs +affirmAllerRetour([MIN_I64, -1, 0, 1, MAX_I64], rt.identique_i64) +affirmAllerRetour([0x0000000000000000, 0x1234567890ABCDEF, 0xFFFFFFFFFFFFFFFF], rt.identique_u64) + +# Floats +affirmAllerRetour([0.0, 0.5, 0.25, 1.0, F32_ONE_THIRD], rt.identique_float) + +# Doubles +affirmAllerRetour( + [0.0, 0.5, 0.25, 1.0, 1.0 / 3, sys.float_info.max, sys.float_info.min], + rt.identique_double +) + +# Strings +affirmAllerRetour( + ["", "abc", "รฉtรฉ", "ฺšูŠ ู„ุงุณ ุชู‡ ู„ูˆุณุชู„ูˆ ู„ูˆุณุชู„", "๐Ÿ˜ปemoji ๐Ÿ‘จโ€๐Ÿ‘งโ€๐Ÿ‘ฆmulti-emoji, ๐Ÿ‡จ๐Ÿ‡ญa flag, a canal, panama"], + rt.identique_string +) + +# Test one way across the FFI. +# +# We send one representation of a value to lib.rs, and it transforms it into another, a string. +# lib.rs sends the string back, and then we compare here in python. +# +# This shows that the values are transformed into strings the same way in both python and rust. +# i.e. if we assume that the string return works (we test this assumption elsewhere) +# we show that lowering from python and lifting into rust has values that both python and rust +# both stringify in the same way. i.e. the same values. +# +# If we roundtripping proves the symmetry of our lowering/lifting from here to rust, and lowering/lifting from rust to here, +# and this convinces us that lowering/lifting from here to rust is correct, then +# together, we've shown the correctness of the return leg. +st = Stringifier() + +def affirmEnchaine(vals, toString, rustyStringify=lambda v: str(v).lower()): + for v in vals: + str_v = toString(v) + assert rustyStringify(v) == str_v, f"String compare error {v} => {str_v}" + +# Test the efficacy of the string transport from rust. If this fails, but everything else +# works, then things are very weird. +wellKnown = st.well_known_string("python") +assert "uniffi ๐Ÿ’š python!" == wellKnown + +# Booleans +affirmEnchaine([True, False], st.to_string_boolean) + +# Bytes. +affirmEnchaine([MIN_I8, -1, 0, 1, MAX_I8], st.to_string_i8) +affirmEnchaine([0x00, 0x12, 0xFF], st.to_string_u8) + +# Shorts +affirmEnchaine([MIN_I16, -1, 0, 1, MAX_I16], st.to_string_i16) +affirmEnchaine([0x0000, 0x1234, 0xFFFF], st.to_string_u16) + +# Ints +affirmEnchaine([MIN_I32, -1, 0, 1, MAX_I32], st.to_string_i32) +affirmEnchaine([0x00000000, 0x12345678, 0xFFFFFFFF], st.to_string_u32) + +# Longs +affirmEnchaine([MIN_I64, -1, 0, 1, MAX_I64], st.to_string_i64) +affirmEnchaine([0x0000000000000000, 0x1234567890ABCDEF, 0xFFFFFFFFFFFFFFFF], st.to_string_u64) + +# Floats +def rustyFloatToStr(v): + """Stringify a float in the same way that rust seems to.""" + # Rust doesn't include the decimal part of whole enumber floats when stringifying. + if int(v) == v: + return str(int(v)) + return str(v) + +affirmEnchaine([0.0, 0.5, 0.25, 1.0], st.to_string_float, rustyFloatToStr) +assert st.to_string_float(F32_ONE_THIRD) == "0.33333334" # annoyingly different string repr + +# Doubles +# TODO: float_info.max/float_info.min don't stringify-roundtrip properly yet, TBD. +affirmEnchaine( + [0.0, 0.5, 0.25, 1.0, 1.0 / 3], + st.to_string_double, + rustyFloatToStr, +) diff --git a/examples/rondpoint/tests/test_generated_bindings.rs b/examples/rondpoint/tests/test_generated_bindings.rs index 7367b5a3f8..c5266a58e2 100644 --- a/examples/rondpoint/tests/test_generated_bindings.rs +++ b/examples/rondpoint/tests/test_generated_bindings.rs @@ -3,6 +3,6 @@ uniffi_macros::build_foreign_language_testcases!( [ "tests/bindings/test_rondpoint.kts", "tests/bindings/test_rondpoint.swift", - // "tests/bindings/test_rondpoint.py", + "tests/bindings/test_rondpoint.py", ] ); diff --git a/uniffi_bindgen/src/bindings/python/gen_python.rs b/uniffi_bindgen/src/bindings/python/gen_python.rs index 0e0c41989f..628ea816a9 100644 --- a/uniffi_bindgen/src/bindings/python/gen_python.rs +++ b/uniffi_bindgen/src/bindings/python/gen_python.rs @@ -53,7 +53,7 @@ mod filters { FFIType::Float64 => "ctypes.c_double".to_string(), FFIType::RustCString => "ctypes.c_voidp".to_string(), FFIType::RustBuffer => "RustBuffer".to_string(), - FFIType::RustError => "RustError".to_string(), + FFIType::RustError => "ctypes.POINTER(RustError)".to_string(), FFIType::ForeignBytes => "ForeignBytes".to_string(), }) } @@ -83,20 +83,16 @@ mod filters { | Type::Int32 | Type::UInt32 | Type::Int64 - | Type::UInt64 - | Type::Float32 - | Type::Float64 - | Type::String - | Type::Boolean - | Type::Object(_) - | Type::Error(_) => format!("{} = {}", nm, nm), - Type::Enum(type_name) => format!("{} = {}({})", nm, type_name, nm), - Type::Record(type_name) => format!("{} = {}._coerce({})", nm, type_name, nm), + | Type::UInt64 => format!("int({})", nm), // TODO: check max/min value + Type::Float32 | Type::Float64 => format!("float({})", nm), + Type::Boolean => format!("bool({})", nm), + Type::String | Type::Object(_) | Type::Error(_) | Type::Record(_) => nm.to_string(), + Type::Enum(name) => format!("{}({})", class_name_py(name)?, nm), Type::Optional(t) => format!("(None if {} is None else {})", nm, coerce_py(nm, t)?), - Type::Sequence(t) => format!("({} for x in {})", coerce_py(&"x", t)?, nm), // TODO: name hygiene, + Type::Sequence(t) => format!("list({} for x in {})", coerce_py(&"x", t)?, nm), Type::Map(t) => format!( - "({}:{} for (k, v) in {}.items())", - coerce_py(&"k", t)?, + "dict(({},{}) for (k, v) in {}.items())", + coerce_py(&"k", &Type::String)?, coerce_py(&"v", t)?, nm ), @@ -106,80 +102,49 @@ mod filters { pub fn lower_py(nm: &dyn fmt::Display, type_: &Type) -> Result { Ok(match type_ { Type::Int8 - | Type::Int16 - | Type::Int32 - | Type::Int64 | Type::UInt8 + | Type::Int16 | Type::UInt16 + | Type::Int32 | Type::UInt32 + | Type::Int64 | Type::UInt64 | Type::Float32 - | Type::Float64 - | Type::Boolean => nm.to_string(), - Type::Enum(_) => format!("{}.value", nm), - Type::String => format!("{}.encode('utf-8')", nm), - Type::Record(type_name) => format!("{}._lower({})", type_name, nm), - Type::Optional(_type) => format!( - "lowerOptional({}, lambda buf, v: {})", - nm, - lower_into_py(&"buf", &"v", type_)? + | Type::Float64 => nm.to_string(), + Type::Boolean => format!("(1 if {} else 0)", nm), + Type::String => format!("RustBuffer.allocFromString({})", nm), + Type::Enum(_) => format!("({}.value)", nm), + Type::Object(_) => format!("({}._handle)", nm), + Type::Error(_) => panic!("No support for lowering errors, yet"), + Type::Record(_) | Type::Optional(_) | Type::Sequence(_) | Type::Map(_) => format!( + "RustBuffer.allocFrom{}({})", + class_name_py(&type_.canonical_name())?, + nm ), - _ => panic!("[TODO: lower_py({:?})]", type_), - }) - } - - pub fn lower_into_py( - nm: &dyn fmt::Display, - target: &dyn fmt::Display, - type_: &Type, - ) -> Result { - let nm = var_name_py(nm)?; - Ok(match type_ { - Type::Float64 => format!("{}.putDouble({})", target, nm), - Type::UInt32 => format!("{}.putInt({})", target, nm), - Type::String => format!("{}.putString({})", target, nm), - Type::Record(type_name) => format!("{}._lowerInto({}, {})", type_name, nm, target), - _ => panic!("[TODO: lower_into_py({:?})]", type_), }) } pub fn lift_py(nm: &dyn fmt::Display, type_: &Type) -> Result { Ok(match type_ { Type::Int8 - | Type::Int16 - | Type::Int32 - | Type::Int64 | Type::UInt8 + | Type::Int16 | Type::UInt16 + | Type::Int32 | Type::UInt32 - | Type::UInt64 - | Type::Float32 - | Type::Float64 - | Type::Boolean => format!("{}", nm), - Type::Enum(type_name) => format!("{}({})", type_name, nm), - Type::String => format!("liftString({})", nm), - Type::Record(type_name) => format!("{}._lift({})", type_name, nm), - Type::Optional(type_) => format!( - "liftOptional({}, lambda buf: {})", - nm, - lift_from_py(&"buf", type_)? - ), - Type::Sequence(type_) => format!( - "liftSequence({}, lambda buf: {})", + | Type::Int64 + | Type::UInt64 => format!("int({})", nm), + Type::Float32 | Type::Float64 => format!("float({})", nm), + Type::Boolean => format!("(True if {} else False)", nm), + Type::String => format!("{}.consumeIntoString()", nm), + Type::Enum(name) => format!("{}({})", class_name_py(name)?, nm), + Type::Object(_) => panic!("No support for lifting objects, yet"), + Type::Error(_) => panic!("No support for lowering errors, yet"), + Type::Record(_) | Type::Optional(_) | Type::Sequence(_) | Type::Map(_) => format!( + "{}.consumeInto{}()", nm, - lift_from_py(&"buf", type_)? + class_name_py(&type_.canonical_name())? ), - _ => panic!("[TODO: lift_py({:?})]", type_), - }) - } - - pub fn lift_from_py(nm: &dyn fmt::Display, type_: &Type) -> Result { - Ok(match type_ { - Type::UInt32 => format!("{}.getInt()", nm), - Type::Float64 => format!("{}.getDouble()", nm), - Type::Record(type_name) => format!("{}._liftFrom({})", type_name, nm), - Type::String => format!("{}.getString()", nm), - _ => panic!("[TODO: lift_from_py({:?})]", type_), }) } } diff --git a/uniffi_bindgen/src/bindings/python/templates/EnumTemplate.py b/uniffi_bindgen/src/bindings/python/templates/EnumTemplate.py index 4459fc822a..73d56578f0 100644 --- a/uniffi_bindgen/src/bindings/python/templates/EnumTemplate.py +++ b/uniffi_bindgen/src/bindings/python/templates/EnumTemplate.py @@ -1,4 +1,4 @@ -class {{ e.name() }}(enum.Enum): +class {{ e.name()|class_name_py }}(enum.Enum): {% for variant in e.variants() -%} {{ variant|enum_name_py }} = {{ loop.index }} - {% endfor -%} \ No newline at end of file + {% endfor %} diff --git a/uniffi_bindgen/src/bindings/python/templates/ObjectTemplate.py b/uniffi_bindgen/src/bindings/python/templates/ObjectTemplate.py index 59f7290b57..930e670635 100644 --- a/uniffi_bindgen/src/bindings/python/templates/ObjectTemplate.py +++ b/uniffi_bindgen/src/bindings/python/templates/ObjectTemplate.py @@ -21,7 +21,7 @@ def {{ meth.name()|fn_name_py }}(self, {% call py::arg_list_decl(meth) %}): {%- call py::coerce_args_extra_indent(meth) %} _retval = {% call py::to_ffi_call_with_prefix("self._handle", meth) %} return {{ "_retval"|lift_py(return_type) }} - + {%- when None -%} def {{ meth.name()|fn_name_py }}(self, {% call py::arg_list_decl(meth) %}): {%- call py::coerce_args_extra_indent(meth) %} diff --git a/uniffi_bindgen/src/bindings/python/templates/RecordTemplate.py b/uniffi_bindgen/src/bindings/python/templates/RecordTemplate.py index 13620d91a2..e399139454 100644 --- a/uniffi_bindgen/src/bindings/python/templates/RecordTemplate.py +++ b/uniffi_bindgen/src/bindings/python/templates/RecordTemplate.py @@ -1,11 +1,11 @@ -class {{ rec.name() }}(object): +class {{ rec.name()|class_name_py }}(object): def __init__(self,{% for field in rec.fields() %}{{ field.name()|var_name_py }}{% if loop.last %}{% else %}, {% endif %}{% endfor %}): {%- for field in rec.fields() %} self.{{ field.name()|var_name_py }} = {{ field.name()|var_name_py }} {%- endfor %} def __str__(self): - return "{{ rec.name() }}({% for field in rec.fields() %}{{ field.name() }}={}{% if loop.last %}{% else %}, {% endif %}{% endfor %})".format({% for field in rec.fields() %}self.{{ field.name() }}{% if loop.last %}{% else %}, {% endif %}{% endfor %}) + return "{{ rec.name()|class_name_py }}({% for field in rec.fields() %}{{ field.name() }}={}{% if loop.last %}{% else %}, {% endif %}{% endfor %})".format({% for field in rec.fields() %}self.{{ field.name() }}{% if loop.last %}{% else %}, {% endif %}{% endfor %}) def __eq__(self, other): {%- for field in rec.fields() %} @@ -13,37 +13,3 @@ def __eq__(self, other): return False return True {%- endfor %} - - @classmethod - def _coerce(cls, v): - # TODO: maybe we could do a bit of duck-typing here, details TBD - assert isinstance(v, {{ rec.name() }}) - return v - - @classmethod - def _lift(cls, rbuf): - return cls._liftFrom(RustBufferStream(rbuf)) - - @classmethod - def _liftFrom(cls, buf): - return cls( - {%- for field in rec.fields() %} - {{ "buf"|lift_from_py(field.type_()) }}{% if loop.last %}{% else %},{% endif %} - {%- endfor %} - ) - - @classmethod - def _lower(cls, v): - buf = RustBufferBuilder() - try: - cls._lowerInto(v, buf) - return buf.finalize() - except Exception: - buf.discard() - raise - - @classmethod - def _lowerInto(cls, v, buf): - {%- for field in rec.fields() %} - {{ "(v.{})"|format(field.name())|lower_into_py("buf", field.type_()) }} - {%- endfor %} diff --git a/uniffi_bindgen/src/bindings/python/templates/RustBufferBuilder.py b/uniffi_bindgen/src/bindings/python/templates/RustBufferBuilder.py new file mode 100644 index 0000000000..166b67ae03 --- /dev/null +++ b/uniffi_bindgen/src/bindings/python/templates/RustBufferBuilder.py @@ -0,0 +1,166 @@ + +class RustBufferBuilder(object): + """Helper for structured writing of values into a RustBuffer.""" + + def __init__(self): + self.rbuf = RustBuffer.alloc(16) + self.rbuf.len = 0 + + def finalize(self): + rbuf = self.rbuf + self.rbuf = None + return rbuf + + def discard(self): + if self.rbuf is not None: + rbuf = self.finalize() + rbuf.free() + + @contextlib.contextmanager + def _reserve(self, numBytes): + if self.rbuf.len + numBytes > self.rbuf.capacity: + self.rbuf = RustBuffer.reserve(self.rbuf, numBytes) + yield None + self.rbuf.len += numBytes + + def _pack_into(self, size, format, value): + with self._reserve(size): + # XXX TODO: I feel like I should be able to use `struct.pack_into` here but can't figure it out. + for i, byte in enumerate(struct.pack(format, value)): + self.rbuf.data[self.rbuf.len + i] = byte + + def write(self, value): + with self._reserve(len(value)): + for i, byte in enumerate(value): + self.rbuf.data[self.rbuf.len + i] = byte + + # For every type used in the interface, we provide helper methods for conveniently + # writing values of that type in a buffer. Putting them on this internal helper object + # (rather than, say, as methods on the public classes) makes it easier for us to hide + # these implementation details from consumers, in the face of python's free-for-all + # type system. + + {%- for typ in ci.iter_types() -%} + {%- let canonical_type_name = typ.canonical_name()|class_name_py -%} + {%- match typ -%} + + {% when Type::Int8 -%} + + def writeI8(self, v): + self._pack_into(1, ">b", v) + + {% when Type::UInt8 -%} + + def writeU8(self, v): + self._pack_into(1, ">B", v) + + {% when Type::Int16 -%} + + def writeI16(self, v): + self._pack_into(2, ">h", v) + + {% when Type::UInt16 -%} + + def writeU16(self, v): + self._pack_into(1, ">H", v) + + {% when Type::Int32 -%} + + def writeI32(self, v): + self._pack_into(4, ">i", v) + + {% when Type::UInt32 -%} + + def writeU32(self, v): + self._pack_into(4, ">I", v) + + {% when Type::Int64 -%} + + def writeI64(self, v): + self._pack_into(8, ">q", v) + + {% when Type::UInt64 -%} + + def writeU64(self, v): + self._pack_into(8, ">Q", v) + + {% when Type::Float32 -%} + + def writeF32(self, v): + self._pack_into(4, ">f", v) + + {% when Type::Float64 -%} + + def writeF64(self, v): + self._pack_into(8, ">d", v) + + {% when Type::Boolean -%} + + def writeBool(self, v): + self._pack_into(1, ">b", 0 if v else 1) + + {% when Type::String -%} + + def writeString(self, v): + utf8Bytes = v.encode("utf-8") + self._pack_into(4, ">i", len(utf8Bytes)) + self.write(utf8Bytes) + + {% when Type::Object with (object_name) -%} + # The Object type {{ object_name }}. + # Objects cannot currently be serialized, but we can produce a helpful error. + + def write{{ canonical_type_name }}(self): + raise InternalError("RustBufferStream.write() not implemented yet for {{ canonical_type_name }}") + + {% when Type::Error with (error_name) -%} + # The Error type {{ error_name }}. + # Errors cannot currently be serialized, but we can produce a helpful error. + + def write{{ canonical_type_name }}(self): + raise InternalError("RustBufferStream.write() not implemented yet for {{ canonical_type_name }}") + + {% when Type::Enum with (enum_name) -%} + # The Enum type {{ enum_name }}. + + def write{{ canonical_type_name }}(self, v): + self._pack_into(4, ">i", v.value) + + {% when Type::Record with (record_name) -%} + {%- let rec = ci.get_record_definition(record_name).unwrap() -%} + # The Record type {{ record_name }}. + + def write{{ canonical_type_name }}(self, v): + {%- for field in rec.fields() %} + self.write{{ field.type_().canonical_name()|class_name_py }}(v.{{ field.name() }}) + {%- endfor %} + + {% when Type::Optional with (inner_type) -%} + # The Optional type for {{ inner_type.canonical_name() }}. + + def write{{ canonical_type_name }}(self, v): + if v is None: + self._pack_into(1, ">b", 0) + else: + self._pack_into(1, ">b", 1) + self.write{{ inner_type.canonical_name()|class_name_py }}(v) + + {% when Type::Sequence with (inner_type) -%} + # The Sequence type for {{ inner_type.canonical_name() }}. + + def write{{ canonical_type_name }}(self, items): + self._pack_into(4, ">i", len(items)) + for item in items: + self.write{{ inner_type.canonical_name()|class_name_py }}(item) + + {% when Type::Map with (inner_type) -%} + # The Map type for {{ inner_type.canonical_name() }}. + + def write{{ canonical_type_name }}(self, items): + self._pack_into(4, ">i", len(items)) + for (k, v) in items.items(): + self.writeString(k) + self.write{{ inner_type.canonical_name()|class_name_py }}(v) + + {%- endmatch -%} + {%- endfor %} \ No newline at end of file diff --git a/uniffi_bindgen/src/bindings/python/templates/RustBufferHelper.py b/uniffi_bindgen/src/bindings/python/templates/RustBufferHelper.py deleted file mode 100644 index 9b2bf49239..0000000000 --- a/uniffi_bindgen/src/bindings/python/templates/RustBufferHelper.py +++ /dev/null @@ -1,111 +0,0 @@ -# Helpers for lifting/lowering primitive data types from/to a bytebuffer. - -class RustBufferStream(object): - """Helper for structured reading of values for a RustBuffer.""" - - def __init__(self, rbuf): - self.rbuf = rbuf - self.offset = 0 - - @contextlib.contextmanager - def checked_access(self, numBytes): - if self.offset + numBytes > self.rbuf.len: - raise RuntimeError("access past end of rust buffer") - yield None - self.offset += numBytes - - def _unpack_from(self, size, format): - if self.offset + size > self.rbuf.len: - raise RuntimeError("read past end of rust buffer") - value = struct.unpack(format, self.rbuf.data[self.offset:self.offset+size])[0] - self.offset += size - return value - - def getByte(self): - return self._unpack_from(1, ">c") - - def getDouble(self): - return self._unpack_from(8, ">d") - - def getInt(self): - return self._unpack_from(4, ">I") - - def getLong(self): - return self._unpack_from(8, ">Q") - - def getString(self): - numBytes = self.getInt() - return self._unpack_from(numBytes, ">{}s".format(numBytes)).decode('utf-8') - - -class RustBufferBuilder(object): - """Helper for structured writing of values into a RustBuffer.""" - - def __init__(self): - self.rbuf = RustBuffer.alloc(16) - self.rbuf.len = 0 - - def finalize(self): - rbuf = self.rbuf - self.rbuf = None - return rbuf - - def discard(self): - rbuf = self.finalize() - rbuf.free() - - @contextlib.contextmanager - def _reserve(self, numBytes): - if self.rbuf.len + numBytes > self.rbuf.capacity: - self.rbuf = RustBuffer.reserve(self.rbuf, numBytes) - yield None - self.rbuf.len += numBytes - - def _pack_into(self, size, format, value): - with self._reserve(size): - # XXX TODO: I feel like I should be able to use `struct.pack_into` here but can't figure it out. - for i, byte in enumerate(struct.pack(format, value)): - self.rbuf.data[self.rbuf.len + i] = byte - - def putByte(self, v): - self._pack_into(1, ">c", v) - - def putDouble(self, v): - self._pack_into(8, ">d", v) - - def putInt(self, v): - self._pack_into(4, ">I", v) - - def putLong(self, v): - self._pack_into(8, ">Q", v) - - def putString(self, v): - valueBytes = v.encode('utf-8') - numBytes = len(valueBytes) - self.putInt(numBytes) - self._pack_into(numBytes, ">{}s".format(numBytes), valueBytes) - - -def liftSequence(rbuf, liftFrom): - return liftFromSequence(RustBufferStream(rbuf), liftFrom) - -def liftFromSequence(buf, liftFrom): - seq_len = buf.getInt() - seq = [] - for i in range(0, seq_len): - seq.append(listFrom(buf)) - return seq - -def liftOptional(rbuf, liftFrom): - return liftFromOptional(RustBufferStream(rbuf), liftFrom) - -def liftFromOptional(buf, liftFrom): - if buf.getByte() == b"\x00": - return None - return liftFrom(buf) - -def liftString(cPtr): - # TODO: update strings to lift from a `RustBuffer`. - # There's currently no test coverage for this, so it can come in a separate PR - # that cleans up a bunch of python lifting/lowering stuff. - raise NotImplementedError diff --git a/uniffi_bindgen/src/bindings/python/templates/RustBufferStream.py b/uniffi_bindgen/src/bindings/python/templates/RustBufferStream.py new file mode 100644 index 0000000000..428af8fa62 --- /dev/null +++ b/uniffi_bindgen/src/bindings/python/templates/RustBufferStream.py @@ -0,0 +1,179 @@ + +class RustBufferStream(object): + """Helper for structured reading of values from a RustBuffer.""" + + def __init__(self, rbuf): + self.rbuf = rbuf + self.offset = 0 + + def remaining(self): + return self.rbuf.len - self.offset + + def _unpack_from(self, size, format): + if self.offset + size > self.rbuf.len: + raise InternalError("read past end of rust buffer") + value = struct.unpack(format, self.rbuf.data[self.offset:self.offset+size])[0] + self.offset += size + return value + + def read(self, size): + if self.offset + size > self.rbuf.len: + raise InternalError("read past end of rust buffer") + data = self.rbuf.data[self.offset:self.offset+size] + self.offset += size + return data + + # For every type used in the interface, we provide helper methods for conveniently + # reading that type in a buffer. Putting them on this internal helper object (rather + # than, say, as methods on the public classes) makes it easier for us to hide these + # implementation details from consumers, in the face of python's free-for-all type + # system. + + {%- for typ in ci.iter_types() -%} + {%- let canonical_type_name = typ.canonical_name()|class_name_py -%} + {%- match typ -%} + + {% when Type::Int8 -%} + + def readI8(self): + return self._unpack_from(1, ">b") + + {% when Type::UInt8 -%} + + def readU8(self): + return self._unpack_from(1, ">B") + + {% when Type::Int16 -%} + + def readI16(self): + return self._unpack_from(2, ">h") + + {% when Type::UInt16 -%} + + def readU16(self): + return self._unpack_from(1, ">H") + + {% when Type::Int32 -%} + + def readI32(self): + return self._unpack_from(4, ">i") + + {% when Type::UInt32 -%} + + def readU32(self): + return self._unpack_from(4, ">I") + + {% when Type::Int64 -%} + + def readI64(self): + return self._unpack_from(8, ">q") + + {% when Type::UInt64 -%} + + def readU64(self): + return self._unpack_from(8, ">Q") + + {% when Type::Float32 -%} + + def readF32(self): + v = self._unpack_from(4, ">f") + return v + + {% when Type::Float64 -%} + + def readF64(self): + return self._unpack_from(8, ">d") + + {% when Type::Boolean -%} + + def readBool(self): + v = self._unpack_from(1, ">b") + if v == 0: + return False + if v == 1: + return True + raise InternalError("Unexpected byte for Boolean type") + + {% when Type::String -%} + + def readString(self): + size = self._unpack_from(4, ">i") + if size < 0: + raise InternalError("Unexpected negative string length") + utf8Bytes = self.read(size) + return utf8Bytes.decode("utf-8") + + {% when Type::Object with (object_name) -%} + # The Object type {{ object_name }}. + # Objects cannot currently be serialized, but we can produce a helpful error. + + def read{{ canonical_type_name }}(self): + raise InternalError("RustBufferStream.read not implemented yet for {{ canonical_type_name }}") + + {% when Type::Error with (error_name) -%} + # The Error type {{ error_name }}. + # Errors cannot currently be serialized, but we can produce a helpful error. + + def read{{ canonical_type_name }}(self): + raise InternalError("RustBufferStream.read not implemented yet for {{ canonical_type_name }}") + + {% when Type::Enum with (enum_name) -%} + # The Enum type {{ enum_name }}. + + def read{{ canonical_type_name }}(self): + return {{ enum_name|class_name_py }}( + self._unpack_from(4, ">i") + ) + + {% when Type::Record with (record_name) -%} + {%- let rec = ci.get_record_definition(record_name).unwrap() -%} + # The Record type {{ record_name }}. + + def read{{ canonical_type_name }}(self): + return {{ rec.name()|class_name_py }}( + {%- for field in rec.fields() %} + self.read{{ field.type_().canonical_name()|class_name_py }}(){% if loop.last %}{% else %},{% endif %} + {%- endfor %} + ) + + {% when Type::Optional with (inner_type) -%} + # The Optional type for {{ inner_type.canonical_name() }}. + + def read{{ canonical_type_name }}(self): + flag = self._unpack_from(1, ">b") + if flag == 0: + return None + elif flag == 1: + return self.read{{ inner_type.canonical_name()|class_name_py }}() + else: + raise InternalError("Unexpected flag byte for {{ canonical_type_name }}") + + {% when Type::Sequence with (inner_type) -%} + # The Sequence type for {{ inner_type.canonical_name() }}. + + def read{{ canonical_type_name }}(self): + count = self._unpack_from(4, ">i") + if count < 0: + raise InternalError("Unexpected negative sequence length") + items = [] + while count > 0: + items.append(self.read{{ inner_type.canonical_name()|class_name_py }}()) + count -= 1 + return items + + {% when Type::Map with (inner_type) -%} + # The Map type for {{ inner_type.canonical_name() }}. + + def read{{ canonical_type_name }}(self): + count = self._unpack_from(4, ">i") + if count < 0: + raise InternalError("Unexpected negative map size") + items = {} + while count > 0: + key = self.readString() + items[key] = self.read{{ inner_type.canonical_name()|class_name_py }}() + count -= 1 + return items + + {%- endmatch -%} + {%- endfor %} diff --git a/uniffi_bindgen/src/bindings/python/templates/RustBufferTemplate.py b/uniffi_bindgen/src/bindings/python/templates/RustBufferTemplate.py index 2b9d806b6c..7a603a2d2a 100644 --- a/uniffi_bindgen/src/bindings/python/templates/RustBufferTemplate.py +++ b/uniffi_bindgen/src/bindings/python/templates/RustBufferTemplate.py @@ -1,6 +1,3 @@ -# This is a helper for safely working with byte buffers returned from the Rust code. -# It's basically a wrapper around a length and a data pointer, corresponding to the -# `ffi_support::ByteBuffer` struct on the rust side. class RustBuffer(ctypes.Structure): _fields_ = [ @@ -27,6 +24,116 @@ def __str__(self): self.data[0:self.len] ) + @contextlib.contextmanager + def allocWithBuilder(): + """Context-manger to allocate a buffer using a RustBufferBuilder. + + The allocated buffer will be automatically freed if an error occurs, ensuring that + we don't accidentally leak it. + """ + builder = RustBufferBuilder() + try: + yield builder + except: + builder.discard() + raise + + @contextlib.contextmanager + def consumeWithStream(self): + """Context-manager to consume a buffer using a RustBufferStream. + + The RustBuffer will be freed once the context-manager exits, ensuring that we don't + leak it even if an error occurs. + """ + try: + s = RustBufferStream(self) + yield s + if s.remaining() != 0: + raise RuntimeError("junk data left in buffer after consuming") + finally: + self.free() + + # For every type that lowers into a RustBuffer, we provide helper methods for + # conveniently doing the lifting and lowering. Putting them on this internal + # helper object (rather than, say, as methods on the public classes) makes it + # easier for us to hide these implementation details from consumers, in the face + # of python's free-for-all type system. + + {%- for typ in ci.iter_types() -%} + {%- let canonical_type_name = typ.canonical_name() -%} + {%- match typ -%} + + {% when Type::String -%} + # The primitive String type. + + @staticmethod + def allocFromString(value): + with RustBuffer.allocWithBuilder() as builder: + builder.write(value.encode("utf-8")) + return builder.finalize() + + def consumeIntoString(self): + with self.consumeWithStream() as stream: + return stream.read(stream.remaining()).decode("utf-8") + + {% when Type::Record with (record_name) -%} + {%- let rec = ci.get_record_definition(record_name).unwrap() -%} + # The Record type {{ record_name }}. + + @staticmethod + def allocFrom{{ canonical_type_name }}(v): + with RustBuffer.allocWithBuilder() as builder: + builder.write{{ canonical_type_name }}(v) + return builder.finalize() + + def consumeInto{{ canonical_type_name }}(self): + with self.consumeWithStream() as stream: + return stream.read{{ canonical_type_name }}() + + {% when Type::Optional with (inner_type) -%} + # The Optional type for {{ inner_type.canonical_name() }}. + + @staticmethod + def allocFrom{{ canonical_type_name }}(v): + with RustBuffer.allocWithBuilder() as builder: + builder.write{{ canonical_type_name }}(v) + return builder.finalize() + + def consumeInto{{ canonical_type_name }}(self): + with self.consumeWithStream() as stream: + return stream.read{{ canonical_type_name }}() + + {% when Type::Sequence with (inner_type) -%} + # The Sequence type for {{ inner_type.canonical_name() }}. + + @staticmethod + def allocFrom{{ canonical_type_name }}(v): + with RustBuffer.allocWithBuilder() as builder: + builder.write{{ canonical_type_name }}(v) + return builder.finalize() + + def consumeInto{{ canonical_type_name }}(self): + with self.consumeWithStream() as stream: + return stream.read{{ canonical_type_name }}() + + {% when Type::Map with (inner_type) -%} + # The Map type for {{ inner_type.canonical_name() }}. + + @staticmethod + def allocFrom{{ canonical_type_name }}(v): + with RustBuffer.allocWithBuilder() as builder: + builder.write{{ canonical_type_name }}(v) + return builder.finalize() + + def consumeInto{{ canonical_type_name }}(self): + with self.consumeWithStream() as stream: + return stream.read{{ canonical_type_name }}() + + {%- else -%} + {#- No code emitted for types that don't lower into a RustBuffer -#} + {%- endmatch -%} + {%- endfor %} + class ForeignBytes(ctypes.Structure): _fields_ = [ diff --git a/uniffi_bindgen/src/bindings/python/templates/macros.py b/uniffi_bindgen/src/bindings/python/templates/macros.py index af2fbbeaf6..520996be65 100644 --- a/uniffi_bindgen/src/bindings/python/templates/macros.py +++ b/uniffi_bindgen/src/bindings/python/templates/macros.py @@ -5,15 +5,15 @@ #} {%- macro to_ffi_call(func) -%} - rust_call_with_error( {%- match func.throws() -%} {%- when Some with (e) -%} - {{ e|class_name_py }} + {{ e|class_name_py }}, {%- else -%} - InternalError - {%- endmatch -%}, - _UniFFILib.{{ func.ffi_func().name() }}{% if func.arguments().len() > 0 %},{% endif %}{% call _arg_list_ffi_call(func) -%} + InternalError, + {%- endmatch -%} + _UniFFILib.{{ func.ffi_func().name() }}, + {%- call _arg_list_ffi_call(func) -%} ) {%- endmacro -%} @@ -21,11 +21,13 @@ rust_call_with_error( {%- match func.throws() -%} {%- when Some with (e) -%} - {{ e|class_name_py }} + {{ e|class_name_py }}, {%- else -%} - InternalError - {%- endmatch -%}, - _UniFFILib.{{ func.ffi_func().name() }},{{- prefix }}{% if func.arguments().len() > 0 %},{% endif %}{% call _arg_list_ffi_call(func) %}) + InternalError, + {%- endmatch -%} + _UniFFILib.{{ func.ffi_func().name() }}, + {{- prefix }}, + {%- call _arg_list_ffi_call(func) -%} ) {%- endmacro -%} @@ -53,20 +55,20 @@ // Note unfiltered name but type_ffi filters. -#} {%- macro arg_list_ffi_decl(func) %} - {%- for arg in func.arguments() -%} - {{ arg.type_()|type_ffi }},{##} + {%- for arg in func.arguments() %} + {{ arg.type_()|type_ffi }}, {%- endfor %} - ctypes.POINTER(RustError) -{%- endmacro -%} + ctypes.POINTER(RustError), +{% endmacro -%} {%- macro coerce_args(func) %} {%- for arg in func.arguments() %} - {{ arg.name()|coerce_py(arg.type_()) -}} + {{ arg.name() }} = {{ arg.name()|coerce_py(arg.type_()) -}} {% endfor -%} {%- endmacro -%} {%- macro coerce_args_extra_indent(func) %} {%- for arg in func.arguments() %} - {{ arg.name()|coerce_py(arg.type_()) }} + {{ arg.name() }} = {{ arg.name()|coerce_py(arg.type_()) }} {%- endfor %} {%- endmacro -%} diff --git a/uniffi_bindgen/src/bindings/python/templates/wrapper.py b/uniffi_bindgen/src/bindings/python/templates/wrapper.py index 83073f9d62..f462604539 100644 --- a/uniffi_bindgen/src/bindings/python/templates/wrapper.py +++ b/uniffi_bindgen/src/bindings/python/templates/wrapper.py @@ -20,8 +20,8 @@ import contextlib {% include "RustBufferTemplate.py" %} - -{% include "RustBufferHelper.py" %} +{% include "RustBufferStream.py" %} +{% include "RustBufferBuilder.py" %} # Error definitions {% include "ErrorTemplate.py" %} diff --git a/uniffi_bindgen/src/interface/mod.rs b/uniffi_bindgen/src/interface/mod.rs index fa194eb781..8c3c789fb0 100644 --- a/uniffi_bindgen/src/interface/mod.rs +++ b/uniffi_bindgen/src/interface/mod.rs @@ -100,30 +100,69 @@ impl<'ci> ComponentInterface { Ok(ci) } + /// The string namespace within which this API should be presented to the caller. + /// + /// This string would typically be used to prefix function names in the FFI, to build + /// a package or module name for the foreign language, etc. pub fn namespace(&self) -> &str { self.namespace.as_str() } + /// List the definitions for every Enum type in the interface. pub fn iter_enum_definitions(&self) -> Vec { self.enums.to_vec() } + /// Get an Enum definition by name, or None if no such Enum is defined. + pub fn get_enum_definition(&self, name: &str) -> Option<&Enum> { + // TODO: probably we could store these internally in a HashMap to make this easier? + self.enums.iter().find(|e| e.name == name) + } + + /// List the definitions for every Record type in the interface. pub fn iter_record_definitions(&self) -> Vec { self.records.to_vec() } + /// Get a Record definition by name, or None if no such Record is defined. + pub fn get_record_definition(&self, name: &str) -> Option<&Record> { + // TODO: probably we could store these internally in a HashMap to make this easier? + self.records.iter().find(|r| r.name == name) + } + + /// List the definitions for every Function in the interface. pub fn iter_function_definitions(&self) -> Vec { self.functions.to_vec() } + /// Get a Function definition by name, or None if no such Function is defined. + pub fn get_function_definition(&self, name: &str) -> Option<&Function> { + // TODO: probably we could store these internally in a HashMap to make this easier? + self.functions.iter().find(|f| f.name == name) + } + + /// List the definitions for every Object type in the interface. pub fn iter_object_definitions(&self) -> Vec { self.objects.to_vec() } + /// Get an Object definition by name, or None if no such Object is defined. + pub fn get_object_definition(&self, name: &str) -> Option<&Object> { + // TODO: probably we could store these internally in a HashMap to make this easier? + self.objects.iter().find(|o| o.name == name) + } + + /// List the definitions for every Error type in the interface. pub fn iter_error_definitions(&self) -> Vec { self.errors.to_vec() } + /// Get an Error definition by name, or None if no such Error is defined. + pub fn get_error_definition(&self, name: &str) -> Option<&Error> { + // TODO: probably we could store these internally in a HashMap to make this easier? + self.errors.iter().find(|e| e.name == name) + } + pub fn iter_types(&self) -> Vec { self.types.iter_known_types().collect() } @@ -256,6 +295,9 @@ impl<'ci> ComponentInterface { } } + /// List the definitions of all FFI functions in the interface. + /// + /// The set of FFI functions is derived automatically from the set of higher-level types. pub fn iter_ffi_function_definitions(&self) -> Vec { self.objects .iter() @@ -329,6 +371,10 @@ impl<'ci> ComponentInterface { Ok(()) } + /// Automatically derive the low-level FFI functions from the high-level types in the interface. + /// + /// This should only be called after the high-level types have been completed defined, otherwise + /// the resulting set will be missing some entries. fn derive_ffi_funcs(&mut self) -> Result<()> { let ci_prefix = self.ffi_namespace().to_string(); for func in self.functions.iter_mut() { @@ -341,6 +387,7 @@ impl<'ci> ComponentInterface { } } +/// Convenience implementation for parsing a `ComponentInterface` from a string. impl FromStr for ComponentInterface { type Err = anyhow::Error; fn from_str(s: &str) -> Result {