Implement compatibility with bytes crate (#134)

* Implement compatibility with bytes crate

* Fix corruption of embedded Uint8Array arrays

* Update CHANGELOG

CHANGELOG.md

@@ -3,6 +3,10 @@
 ## main
 
 - Add support for arrays of primitives (#94).
+- Added the `bytes-compat` feature for compatibility with the `bytes` crate.
+- Fixed an issue where embedded `Uint8Array`s that were returned to the
+  TypeScript runtime from `Bytes` or `ByteBuf` types in the Rust plugin could
+  end up being corrupted.
 
 ## 2.0.1
 

README.md

@@ -147,6 +147,7 @@ keep their TypeScript types in sync with the protocol.
 The `fp-bindgen` crate supports optional Cargo features for compatibility with some common types
 from the crate ecosystem:
 
+- `bytes-compat`: Enables compatibility with the `Bytes` type from the `bytes` crate.
 - `http-compat`: Enables compatibility with types from the `http` crate.
 - `serde-bytes-compat`: Enables compatibility with `serde_bytes`'s `ByteBuf` type (the `Bytes` type
   is a reference type, which `fp-bindgen` doesn't support in general).
@@ -171,7 +172,7 @@ Currently, we support the following binding types:
 
 - `BindingsType::RustPlugin`: Generates bindings for a Rust plugin.
 - `BindingsType::RustWasmerRuntime`: Generates runtime bindings for use with Wasmer.
-- `BindingsType::TsRuntime`: Generates bindings for a TypeScript runtime.
+- `BindingsType::TsRuntimeWithExtendedConfig`: Generates bindings for a TypeScript runtime.
 
 Note that some binding types take an additional config argument.
 

examples/example-deno-runtime/tests.ts

@@ -96,8 +96,12 @@ const imports: Imports = {
     };
   },
 
-  importGetBytes: (): Result<ArrayBuffer, string> => {
-    return { Ok: new TextEncoder().encode("Hello, world!") };
+  importGetBytes: (): Result<Uint8Array, string> => {
+    return { Ok: new TextEncoder().encode("hello") };
+  },
+
+  importGetSerdeBytes: (): Result<ArrayBuffer, string> => {
+    return { Ok: new TextEncoder().encode("hello") };
   },
 
   importMultiplePrimitives: (arg1: number, arg2: string): bigint => {
@@ -486,3 +490,25 @@ Deno.test("fetch async data", async () => {
     Ok: JSON.stringify({ "status": "confirmed" })
   });
 });
+
+Deno.test("bytes", async () => {
+  const { exportGetBytes, exportGetSerdeBytes } = await loadExamplePlugin();
+  assert(exportGetBytes);
+  assert(exportGetSerdeBytes);
+
+  const encoder = new TextEncoder();
+  assertEquals(unwrap(exportGetBytes()), encoder.encode("hello, world"));
+  assertEquals(unwrap(exportGetSerdeBytes()), encoder.encode("hello, world"));
+});
+
+function isOk<T, E>(result: Result<T, E>): result is { Ok: T } {
+  return "Ok" in result;
+}
+
+function unwrap<T, E>(result: Result<T, E>): T {
+  if (!isOk(result)) {
+    throw result.Err;
+  }
+
+  return result.Ok;
+}

examples/example-plugin/Cargo.toml

@@ -8,6 +8,7 @@ version = "0.1.0"
 crate-type = ["cdylib"]
 
 [dependencies]
+bytes = "1"
 example-bindings = {path = "../example-protocol/bindings/rust-plugin"}
 http = {version = "0.2"}
 once_cell = {version = "1.10"}

examples/example-plugin/src/lib.rs

@@ -1,3 +1,4 @@
+use bytes::{Bytes, BytesMut};
 use ::http::{Method, Uri};
 use example_bindings::*;
 use serde_bytes::ByteBuf;
@@ -360,6 +361,26 @@ async fn fetch_data(r#type: String) -> Result<String, String> {
     }
 }
 
+#[fp_export_impl(example_bindings)]
+fn export_get_bytes() -> Result<Bytes, String> {
+    import_get_bytes().map(|bytes| {
+        let mut new_bytes = BytesMut::with_capacity(bytes.len() + 7);
+        new_bytes.extend_from_slice(&bytes);
+        new_bytes.extend_from_slice(b", world");
+        new_bytes.freeze()
+    })
+}
+
+#[fp_export_impl(example_bindings)]
+fn export_get_serde_bytes() -> Result<ByteBuf, String> {
+    import_get_serde_bytes().map(|bytes| {
+        let mut new_bytes = ByteBuf::with_capacity(bytes.len() + 7);
+        new_bytes.extend_from_slice(&bytes);
+        new_bytes.extend_from_slice(b", world");
+        new_bytes
+    })
+}
+
 #[fp_export_impl(example_bindings)]
 fn init() {
     init_panic_hook();

examples/example-protocol/Cargo.toml

@@ -5,7 +5,9 @@ name = "example-protocol"
 version = "0.1.0"
 
 [dependencies]
+bytes = {version = "1", features = ["serde"]}
 fp-bindgen = {path = "../../fp-bindgen", features = [
+  "bytes-compat",
   "http-compat",
   "serde-bytes-compat",
   "time-compat",

examples/example-protocol/src/assets/rust_plugin_test/expected_Cargo.toml

@@ -5,6 +5,7 @@ authors = ["Fiberplane <info@fiberplane.com>"]
 edition = "2018"
 
 [dependencies]
+bytes = { version = "1", features = ["serde"] }
 fp-bindgen-support = { path = "../../../../fp-bindgen-support", version = "2.0.1", features = ["async", "guest", "http"] }
 http = { version = "0.2" }
 once_cell = { version = "1.4" }

examples/example-protocol/src/assets/rust_plugin_test/expected_export.rs

@@ -49,7 +49,10 @@ pub fn export_fp_untagged(arg: FpUntagged) -> FpUntagged;
 pub fn export_generics(arg: StructWithGenerics<u64>) -> StructWithGenerics<u64>;
 
 #[fp_bindgen_support::fp_export_signature]
-pub fn export_get_bytes() -> Result<serde_bytes::ByteBuf, String>;
+pub fn export_get_bytes() -> Result<bytes::Bytes, String>;
+
+#[fp_bindgen_support::fp_export_signature]
+pub fn export_get_serde_bytes() -> Result<serde_bytes::ByteBuf, String>;
 
 #[fp_bindgen_support::fp_export_signature]
 pub fn export_multiple_primitives(arg1: i8, arg2: String) -> i64;

examples/example-protocol/src/assets/rust_plugin_test/expected_import.rs

@@ -49,7 +49,10 @@ pub fn import_fp_untagged(arg: FpUntagged) -> FpUntagged;
 pub fn import_generics(arg: StructWithGenerics<u64>) -> StructWithGenerics<u64>;
 
 #[fp_bindgen_support::fp_import_signature]
-pub fn import_get_bytes() -> Result<serde_bytes::ByteBuf, String>;
+pub fn import_get_bytes() -> Result<bytes::Bytes, String>;
+
+#[fp_bindgen_support::fp_import_signature]
+pub fn import_get_serde_bytes() -> Result<serde_bytes::ByteBuf, String>;
 
 #[fp_bindgen_support::fp_import_signature]
 pub fn import_multiple_primitives(arg1: i8, arg2: String) -> i64;

examples/example-protocol/src/assets/rust_wasmer_runtime_test/expected_bindings.rs

@@ -442,9 +442,7 @@ impl Runtime {
         Ok(result)
     }
 
-    pub fn export_get_bytes(
-        &self,
-    ) -> Result<Result<serde_bytes::ByteBuf, String>, InvocationError> {
+    pub fn export_get_bytes(&self) -> Result<Result<bytes::Bytes, String>, InvocationError> {
         let result = self.export_get_bytes_raw();
         let result = result.map(|ref data| deserialize_from_slice(data));
         result
@@ -465,6 +463,29 @@ impl Runtime {
         Ok(result)
     }
 
+    pub fn export_get_serde_bytes(
+        &self,
+    ) -> Result<Result<serde_bytes::ByteBuf, String>, InvocationError> {
+        let result = self.export_get_serde_bytes_raw();
+        let result = result.map(|ref data| deserialize_from_slice(data));
+        result
+    }
+    pub fn export_get_serde_bytes_raw(&self) -> Result<Vec<u8>, InvocationError> {
+        let mut env = RuntimeInstanceData::default();
+        let import_object = create_import_object(self.module.store(), &env);
+        let instance = Instance::new(&self.module, &import_object).unwrap();
+        env.init_with_instance(&instance).unwrap();
+        let function = instance
+            .exports
+            .get_native_function::<(), FatPtr>("__fp_gen_export_get_serde_bytes")
+            .map_err(|_| {
+                InvocationError::FunctionNotExported("__fp_gen_export_get_serde_bytes".to_owned())
+            })?;
+        let result = function.call()?;
+        let result = import_from_guest_raw(&env, result);
+        Ok(result)
+    }
+
     pub fn export_multiple_primitives(
         &self,
         arg1: i8,
@@ -1061,6 +1082,7 @@ fn create_import_object(store: &Store, env: &RuntimeInstanceData) -> ImportObjec
             "__fp_gen_import_fp_untagged" => Function::new_native_with_env(store, env.clone(), _import_fp_untagged),
             "__fp_gen_import_generics" => Function::new_native_with_env(store, env.clone(), _import_generics),
             "__fp_gen_import_get_bytes" => Function::new_native_with_env(store, env.clone(), _import_get_bytes),
+            "__fp_gen_import_get_serde_bytes" => Function::new_native_with_env(store, env.clone(), _import_get_serde_bytes),
             "__fp_gen_import_multiple_primitives" => Function::new_native_with_env(store, env.clone(), _import_multiple_primitives),
             "__fp_gen_import_primitive_bool" => Function::new_native_with_env(store, env.clone(), _import_primitive_bool),
             "__fp_gen_import_primitive_f32" => Function::new_native_with_env(store, env.clone(), _import_primitive_f32),
@@ -1190,6 +1212,11 @@ pub fn _import_get_bytes(env: &RuntimeInstanceData) -> FatPtr {
     export_to_guest(env, &result)
 }
 
+pub fn _import_get_serde_bytes(env: &RuntimeInstanceData) -> FatPtr {
+    let result = super::import_get_serde_bytes();
+    export_to_guest(env, &result)
+}
+
 pub fn _import_multiple_primitives(
     env: &RuntimeInstanceData,
     arg1: <i8 as WasmAbi>::AbiType,

examples/example-protocol/src/assets/ts_runtime_test/expected_index.ts

@@ -28,7 +28,8 @@ export type Imports = {
     importFpStruct: (arg: types.FpPropertyRenaming) => types.FpPropertyRenaming;
     importFpUntagged: (arg: types.FpUntagged) => types.FpUntagged;
     importGenerics: (arg: types.StructWithGenerics<number>) => types.StructWithGenerics<number>;
-    importGetBytes: () => types.Result<ArrayBuffer, string>;
+    importGetBytes: () => types.Result<Uint8Array, string>;
+    importGetSerdeBytes: () => types.Result<ArrayBuffer, string>;
     importMultiplePrimitives: (arg1: number, arg2: string) => bigint;
     importPrimitiveBool: (arg: boolean) => boolean;
     importPrimitiveF32: (arg: number) => number;
@@ -73,7 +74,8 @@ export type Exports = {
     exportFpStruct?: (arg: types.FpPropertyRenaming) => types.FpPropertyRenaming;
     exportFpUntagged?: (arg: types.FpUntagged) => types.FpUntagged;
     exportGenerics?: (arg: types.StructWithGenerics<number>) => types.StructWithGenerics<number>;
-    exportGetBytes?: () => types.Result<ArrayBuffer, string>;
+    exportGetBytes?: () => types.Result<Uint8Array, string>;
+    exportGetSerdeBytes?: () => types.Result<ArrayBuffer, string>;
     exportMultiplePrimitives?: (arg1: number, arg2: string) => bigint;
     exportPrimitiveBool?: (arg: boolean) => boolean;
     exportPrimitiveF32?: (arg: number) => number;
@@ -115,6 +117,7 @@ export type Exports = {
     exportFpUntaggedRaw?: (arg: Uint8Array) => Uint8Array;
     exportGenericsRaw?: (arg: Uint8Array) => Uint8Array;
     exportGetBytesRaw?: () => Uint8Array;
+    exportGetSerdeBytesRaw?: () => Uint8Array;
     exportMultiplePrimitivesRaw?: (arg1: number, arg2: Uint8Array) => bigint;
     exportPrimitiveBoolRaw?: (arg: boolean) => boolean;
     exportPrimitiveI16Raw?: (arg: number) => number;
@@ -185,8 +188,13 @@ export async function createRuntime(
     function parseObject<T>(fatPtr: FatPtr): T {
         const [ptr, len] = fromFatPtr(fatPtr);
         const buffer = new Uint8Array(memory.buffer, ptr, len);
-        const object = decode(buffer) as unknown as T;
+        // Without creating a copy of the memory, we risk corruption of any
+        // embedded `Uint8Array` objects returned from `decode()` after `free()`
+        // has been called :(
+        const copy = new Uint8Array(len);
+        copy.set(buffer);
         free(fatPtr);
+        const object = decode(copy) as unknown as T;
         return object;
     }
 
@@ -310,6 +318,9 @@ export async function createRuntime(
             __fp_gen_import_get_bytes: (): FatPtr => {
                 return serializeObject(importFunctions.importGetBytes());
             },
+            __fp_gen_import_get_serde_bytes: (): FatPtr => {
+                return serializeObject(importFunctions.importGetSerdeBytes());
+            },
             __fp_gen_import_multiple_primitives: (arg1: number, arg2_ptr: FatPtr): bigint => {
                 const arg2 = parseObject<string>(arg2_ptr);
                 return interpretBigSign(importFunctions.importMultiplePrimitives(arg1, arg2), 9223372036854775808n);
@@ -573,6 +584,12 @@ export async function createRuntime(
             const export_fn = instance.exports.__fp_gen_export_get_bytes as any;
             if (!export_fn) return;
 
+            return () => parseObject<types.Result<Uint8Array, string>>(export_fn());
+        })(),
+        exportGetSerdeBytes: (() => {
+            const export_fn = instance.exports.__fp_gen_export_get_serde_bytes as any;
+            if (!export_fn) return;
+
             return () => parseObject<types.Result<ArrayBuffer, string>>(export_fn());
         })(),
         exportMultiplePrimitives: (() => {
@@ -862,6 +879,12 @@ export async function createRuntime(
 
             return () => importFromMemory(export_fn());
         })(),
+        exportGetSerdeBytesRaw: (() => {
+            const export_fn = instance.exports.__fp_gen_export_get_serde_bytes as any;
+            if (!export_fn) return;
+
+            return () => importFromMemory(export_fn());
+        })(),
         exportMultiplePrimitivesRaw: (() => {
             const export_fn = instance.exports.__fp_gen_export_multiple_primitives as any;
             if (!export_fn) return;

examples/example-protocol/src/main.rs

@@ -1,5 +1,6 @@
 #![allow(dead_code)]
 
+use bytes::Bytes;
 use fp_bindgen::{prelude::*, types::CargoDependency};
 use once_cell::sync::Lazy;
 use redux_example::{ReduxAction, StateUpdate};
@@ -91,7 +92,8 @@ fp_import! {
     fn import_explicit_bound_point(arg: ExplicitBoundPoint<u64>);
 
     // Custom type in a generic position.
-    fn import_get_bytes() -> Result<ByteBuf, String>;
+    fn import_get_bytes() -> Result<Bytes, String>;
+    fn import_get_serde_bytes() -> Result<ByteBuf, String>;
 
     // Passing custom types with property/variant renaming.
     //
@@ -175,7 +177,8 @@ fp_export! {
     fn export_generics(arg: StructWithGenerics<u64>) -> StructWithGenerics<u64>;
 
     // Custom type in a generic position.
-    fn export_get_bytes() -> Result<ByteBuf, String>;
+    fn export_get_bytes() -> Result<Bytes, String>;
+    fn export_get_serde_bytes() -> Result<ByteBuf, String>;
 
     // Passing custom types with property/variant renaming.
     //

examples/example-rust-runtime/Cargo.toml

@@ -6,6 +6,7 @@ version = "0.1.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+bytes = "1"
 redux-example = { path = "../redux-example" }
 fp-bindgen-support = { path = "../../fp-bindgen-support", features = [
   "async",

examples/example-rust-runtime/src/spec/mod.rs

@@ -1,10 +1,8 @@
 pub mod bindings;
 pub mod types;
 
-use std::collections::HashMap;
-
+use bytes::Bytes;
 use serde_bytes::ByteBuf;
-use time::OffsetDateTime;
 use types::*;
 
 fn import_void_function() {}
@@ -98,8 +96,11 @@ fn import_generics(arg: StructWithGenerics<u64>) -> StructWithGenerics<u64> {
     todo!()
 }
 
-fn import_get_bytes() -> Result<serde_bytes::ByteBuf, String> {
-    todo!()
+fn import_get_bytes() -> Result<Bytes, String> {
+    Ok(Bytes::from("hello"))
+}
+fn import_get_serde_bytes() -> Result<ByteBuf, String> {
+    Ok(ByteBuf::from("hello"))
 }
 
 fn import_fp_struct(arg: FpPropertyRenaming) -> FpPropertyRenaming {

examples/example-rust-runtime/src/test.rs

@@ -1,10 +1,10 @@
-use std::collections::BTreeMap;
-
+use crate::spec::types::*;
 use anyhow::Result;
+use bytes::Bytes;
+use serde_bytes::ByteBuf;
+use std::collections::BTreeMap;
 use time::{macros::datetime, OffsetDateTime};
 
-use crate::spec::types::*;
-
 const WASM_BYTES: &'static [u8] =
     include_bytes!("../../example-plugin/target/wasm32-unknown-unknown/debug/example_plugin.wasm");
 
@@ -250,9 +250,16 @@ async fn fetch_async_data() -> Result<()> {
 
     let response = rt.fetch_data("sign-up".to_string()).await?;
 
-    assert_eq!(
-        response,
-        Ok(r#"status: "confirmed"#.to_string())
-    );
+    assert_eq!(response, Ok(r#"status: "confirmed"#.to_string()));
+    Ok(())
+}
+
+#[test]
+fn bytes() -> Result<()> {
+    let rt = crate::spec::bindings::Runtime::new(WASM_BYTES)?;
+
+    assert_eq!(rt.export_get_bytes()?, Ok(Bytes::from("hello, world")));
+    assert_eq!(rt.export_get_serde_bytes()?, Ok(ByteBuf::from("hello, world")));
+
     Ok(())
 }