From 1b62f0c97220fb5bb53fa886a3472470159762bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 17 Sep 2019 14:10:52 +0200 Subject: [PATCH 01/76] Adds first version of traits for generating the host functions --- Cargo.lock | 9 ++ Cargo.toml | 1 + core/executor/src/wasm_executor.rs | 25 +--- core/runtime-interface/Cargo.toml | 14 ++ core/runtime-interface/src/lib.rs | 207 +++++++++++++++++++++++++++++ core/sr-io/src/lib.rs | 2 +- core/sr-std/with_std.rs | 1 + core/sr-std/without_std.rs | 1 + core/wasm-interface/src/lib.rs | 44 ++++++ 9 files changed, 279 insertions(+), 25 deletions(-) create mode 100644 core/runtime-interface/Cargo.toml create mode 100644 core/runtime-interface/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 68a2e1bf41de8..44b84c4d94a55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5217,6 +5217,15 @@ dependencies = [ "sr-primitives 2.0.0", ] +[[package]] +name = "substrate-runtime-interface" +version = "2.0.0" +dependencies = [ + "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sr-std 2.0.0", + "substrate-wasm-interface 2.0.0", +] + [[package]] name = "substrate-runtime-test" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index bc9fd128ff145..2cadde762b6b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ members = [ "core/rpc", "core/rpc/primitives", "core/rpc-servers", + "core/runtime-interface", "core/serializer", "core/service", "core/service/test", diff --git a/core/executor/src/wasm_executor.rs b/core/executor/src/wasm_executor.rs index cc7d5d9344ddb..c37a67a02c927 100644 --- a/core/executor/src/wasm_executor.rs +++ b/core/executor/src/wasm_executor.rs @@ -40,7 +40,7 @@ use crate::allocator; use log::trace; use wasm_interface::{ FunctionContext, HostFunctions, Pointer, WordSize, Sandbox, MemoryId, PointerType, - Result as WResult, + Result as WResult, ReadPrimitive, WritePrimitive, }; #[cfg(feature="wasm-extern-trace")] @@ -239,29 +239,6 @@ impl Sandbox for FunctionExecutor { } } -trait WritePrimitive { - fn write_primitive(&mut self, ptr: Pointer, t: T) -> WResult<()>; -} - -impl WritePrimitive for &mut dyn FunctionContext { - fn write_primitive(&mut self, ptr: Pointer, t: u32) -> WResult<()> { - let r = t.to_le_bytes(); - self.write_memory(ptr.cast(), &r) - } -} - -trait ReadPrimitive { - fn read_primitive(&self, offset: Pointer) -> WResult; -} - -impl ReadPrimitive for &mut dyn FunctionContext { - fn read_primitive(&self, ptr: Pointer) -> WResult { - let mut r = [0u8; 4]; - self.read_memory_into(ptr.cast(), &mut r)?; - Ok(u32::from_le_bytes(r)) - } -} - fn deadline_to_timestamp(deadline: u64) -> Option { if deadline == 0 { None diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml new file mode 100644 index 0000000000000..da78b00f8cdbc --- /dev/null +++ b/core/runtime-interface/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "substrate-runtime-interface" +version = "2.0.0" +authors = ["Parity Technologies "] +edition = "2018" + +[dependencies] +wasm-interface = { package = "substrate-wasm-interface", path = "../wasm-interface", optional = true } +rstd = { package = "sr-std", path = "../sr-std", default-features = false } +codec = { package = "parity-scale-codec", version = "1.0.5", default-features = false } + +[features] +default = [ "std" ] +std = [ "wasm-interface", "rstd/std", "codec/std" ] diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs new file mode 100644 index 0000000000000..17a28a291429b --- /dev/null +++ b/core/runtime-interface/src/lib.rs @@ -0,0 +1,207 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Traits and macros for creating interfaces between the runtime and the node. + +use rstd::{any::TypeId, borrow::Cow, mem}; + +use wasm_interface::{FunctionContext, IntoValue, TryFromValue, Pointer, Result}; + +use codec::{Encode, Decode}; + +pub trait AsFFIArg { + /// The owned rust type that converts into `Self::FFIType`. + type RTOwned: IntoFFIArg; + /// The borrowed rust type that converts into `Self::FFIType`. + type RTBorrowed: ?Sized + IntoFFIArg; + type FFIType; + + fn as_ffi_arg<'a>(&'a self) -> FFIArg<'a, Self::FFIType, Self::RTOwned, Self::RTBorrowed>; +} + +pub trait FromFFIArg: Sized { + fn from_ffi_arg(arg: T) -> Self; +} + +pub trait FromWasmFFIArg: Sized { + /// The `Self` instance returned by `from_wasm_ffi_arg`. + /// + /// For types that are do not implement `Sized` we can not return `Self`. So, we need to use a + /// wrapper type that stores `Self`. + type SelfInstance; + + fn from_wasm_ffi_arg(context: &mut dyn FunctionContext, arg: T) -> Result; +} + +pub trait IntoFFIArg { + fn into_ffi_arg(&self) -> T; +} + +pub trait IntoWasmFFIArg { + type SelfInstance; + + fn into_wasm_ffi_arg(self_instance: Self::SelfInstance, context: &mut dyn FunctionContext) -> Result; +} + +pub enum FFIArg<'a, T, O, R: ?Sized = O> where O: IntoFFIArg, R: IntoFFIArg { + Ref(&'a R, std::marker::PhantomData), + Owned(O), +} + +impl<'a, T, O: IntoFFIArg, R: ?Sized + IntoFFIArg> FFIArg<'a, T, O, R> { + pub fn from_owned(o: O) -> Self { + Self::Owned(o) + } + + pub fn from_ref(r: &'a R) -> Self { + Self::Ref(r, Default::default()) + } + + pub fn into_ffi_arg(&self) -> T { + match self { + Self::Ref(data, _) => data.into_ffi_arg(), + Self::Owned(ref data) => data.into_ffi_arg(), + } + } +} + +impl AsFFIArg for u32 { + type RTOwned = u32; + type RTBorrowed = u32; + type FFIType = u32; + + fn as_ffi_arg<'a>(&'a self) -> FFIArg<'a, u32, u32> { + FFIArg::from_owned(*self) + } +} + +impl IntoFFIArg for u32 { + fn into_ffi_arg(&self) -> u32 { + self.to_le() + } +} + +impl FromFFIArg for u32 { + fn from_ffi_arg(arg: u32) -> u32 { + u32::from_le(arg) + } +} + +impl> IntoFFIArg for T { + fn into_ffi_arg(&self) -> u64 { + let data = self.as_ref(); + + let ptr_address = data.as_ptr() as u64; + + ((data.len() as u64) | ptr_address << 32).to_le() + } +} + +impl AsFFIArg for &[T] { + type RTOwned = Vec; + type RTBorrowed = [u8]; + type FFIType = u64; + + fn as_ffi_arg<'a>(&'a self) -> FFIArg<'a, u64, Vec, [u8]> { + if TypeId::of::() == TypeId::of::() { + let transmuted = unsafe { mem::transmute::<&[T], &[u8]>(self) }; + FFIArg::from_ref(transmuted) + } else { + FFIArg::from_owned(self.encode()) + } + } +} + +impl AsFFIArg for Vec { + type RTOwned = Vec; + type RTBorrowed = [u8]; + type FFIType = u64; + + fn as_ffi_arg<'a>(&'a self) -> FFIArg<'a, u64, Vec, [u8]> { + if TypeId::of::() == TypeId::of::() { + let transmuted = unsafe { mem::transmute::<&[T], &[u8]>(&self[..]) }; + FFIArg::from_ref(transmuted) + } else { + FFIArg::from_owned(self.encode()) + } + } +} + +impl FromFFIArg for Vec { + fn from_ffi_arg(arg: u64) -> Vec { + let arg = u64::from_le(arg); + let len: usize = (arg & (!0u32 as u64)) as usize; + let ptr: usize = (arg >> 32) as usize; + + if TypeId::of::() == TypeId::of::() { + unsafe { std::mem::transmute(Vec::from_raw_parts(ptr as *mut u8, len, len)) } + } else { + let slice = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; + Self::decode(&mut &slice[..]).expect("Host to Wasm values are encoded correctly; qed") + } + } +} + +impl IntoWasmFFIArg for Vec { + type SelfInstance = Vec; + + fn into_wasm_ffi_arg(self_instance: Vec, context: &mut dyn FunctionContext) -> Result { + <&[T] as IntoWasmFFIArg>::into_wasm_ffi_arg(self_instance, context) + } +} + +impl FromWasmFFIArg for Vec { + type SelfInstance = Vec; + + fn from_wasm_ffi_arg(context: &mut dyn FunctionContext, arg: u64) -> Result> { + <&[T] as FromWasmFFIArg>::from_wasm_ffi_arg(context, arg) + } +} + +impl IntoWasmFFIArg for &[T] { + type SelfInstance = Vec; + + fn into_wasm_ffi_arg(self_instance: Vec, context: &mut dyn FunctionContext) -> Result { + let vec: Cow<'_, [u8]> = if TypeId::of::() == TypeId::of::() { + unsafe { Cow::Borrowed(std::mem::transmute(&self_instance[..])) } + } else { + Cow::Owned(self_instance.encode()) + }; + + let ptr = context.allocate_memory(vec.as_ref().len() as u32)?; + context.write_memory(ptr, &vec)?; + + Ok((vec.len() as u64) | u64::from(ptr) << 32) + } +} + +impl FromWasmFFIArg for &[T] { + type SelfInstance = Vec; + + fn from_wasm_ffi_arg(context: &mut dyn FunctionContext, arg: u64) -> Result> { + let arg = u64::from_le(arg); + let len = (arg & (!0u32 as u64)) as u32; + let ptr = (arg >> 32) as u32; + + let vec = context.read_memory(Pointer::new(ptr), len)?; + + if TypeId::of::() == TypeId::of::() { + Ok(unsafe { mem::transmute(vec) }) + } else { + Ok(Vec::::decode(&mut &vec[..]).expect("Wasm to Host values are encoded correctly; qed")) + } + } +} diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index aee9e23909b22..f763bebb4cb39 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -216,7 +216,7 @@ export_api! { export_api! { pub(crate) trait HashingApi { /// Conduct a 256-bit Keccak hash. - fn keccak_256(data: &[u8]) -> [u8; 32] ; + fn keccak_256(data: &[u8]) -> [u8; 32]; /// Conduct a 128-bit Blake2 hash. fn blake2_128(data: &[u8]) -> [u8; 16]; diff --git a/core/sr-std/with_std.rs b/core/sr-std/with_std.rs index 0d5ee04ed51ab..bbb932022bdaa 100644 --- a/core/sr-std/with_std.rs +++ b/core/sr-std/with_std.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +pub use std::any; pub use std::borrow; pub use std::boxed; pub use std::cell; diff --git a/core/sr-std/without_std.rs b/core/sr-std/without_std.rs index ed9eea6c31b74..aadd7d19ec8a7 100755 --- a/core/sr-std/without_std.rs +++ b/core/sr-std/without_std.rs @@ -49,6 +49,7 @@ mod __impl { pub use alloc::boxed; pub use alloc::rc; pub use alloc::vec; +pub use core::any; pub use core::cell; pub use core::clone; pub use core::cmp; diff --git a/core/wasm-interface/src/lib.rs b/core/wasm-interface/src/lib.rs index b3cbde556eea0..accbc52d03c13 100644 --- a/core/wasm-interface/src/lib.rs +++ b/core/wasm-interface/src/lib.rs @@ -119,6 +119,12 @@ impl From> for u32 { } } +impl From> for u64 { + fn from(ptr: Pointer) -> Self { + u64::from(ptr.ptr) + } +} + impl From> for usize { fn from(ptr: Pointer) -> Self { ptr.ptr as _ @@ -305,6 +311,44 @@ impl_into_and_from_value! { i64, I64, } +pub trait WritePrimitive { + fn write_primitive(&mut self, ptr: Pointer, t: T) -> Result<()>; +} + +impl WritePrimitive for &mut dyn FunctionContext { + fn write_primitive(&mut self, ptr: Pointer, t: u32) -> Result<()> { + let r = t.to_le_bytes(); + self.write_memory(ptr.cast(), &r) + } +} + +impl WritePrimitive for &mut dyn FunctionContext { + fn write_primitive(&mut self, ptr: Pointer, t: u64) -> Result<()> { + let r = t.to_le_bytes(); + self.write_memory(ptr.cast(), &r) + } +} + +pub trait ReadPrimitive { + fn read_primitive(&self, offset: Pointer) -> Result; +} + +impl ReadPrimitive for &mut dyn FunctionContext { + fn read_primitive(&self, ptr: Pointer) -> Result { + let mut r = [0u8; 4]; + self.read_memory_into(ptr.cast(), &mut r)?; + Ok(u32::from_le_bytes(r)) + } +} + +impl ReadPrimitive for &mut dyn FunctionContext { + fn read_primitive(&self, ptr: Pointer) -> Result { + let mut r = [0u8; 8]; + self.read_memory_into(ptr.cast(), &mut r)?; + Ok(u64::from_le_bytes(r)) + } +} + #[cfg(test)] mod tests { use super::*; From 63fd2ce9bc3e150d131cf3b25156e5b1d704d0f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 19 Sep 2019 20:58:57 +0200 Subject: [PATCH 02/76] First steps of the procedural macro --- Cargo.lock | 23 +++ Cargo.toml | 1 + core/runtime-interface/Cargo.toml | 4 +- core/runtime-interface/proc-macro/Cargo.toml | 15 ++ .../proc-macro/src/bare_function_interface.rs | 174 ++++++++++++++++++ core/runtime-interface/proc-macro/src/lib.rs | 65 +++++++ .../proc-macro/src/trait_decl_impl.rs | 116 ++++++++++++ .../runtime-interface/proc-macro/src/utils.rs | 62 +++++++ core/runtime-interface/src/lib.rs | 24 +++ 9 files changed, 483 insertions(+), 1 deletion(-) create mode 100644 core/runtime-interface/proc-macro/Cargo.toml create mode 100644 core/runtime-interface/proc-macro/src/bare_function_interface.rs create mode 100644 core/runtime-interface/proc-macro/src/lib.rs create mode 100644 core/runtime-interface/proc-macro/src/trait_decl_impl.rs create mode 100644 core/runtime-interface/proc-macro/src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 44b84c4d94a55..99e1282d62c7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,14 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "adler32" version = "1.0.3" @@ -5223,9 +5232,22 @@ version = "2.0.0" dependencies = [ "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", + "substrate-primitives 2.0.0", + "substrate-runtime-interface-proc-macro 2.0.0", "substrate-wasm-interface 2.0.0", ] +[[package]] +name = "substrate-runtime-interface-proc-macro" +version = "2.0.0" +dependencies = [ + "Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "substrate-runtime-test" version = "2.0.0" @@ -6475,6 +6497,7 @@ dependencies = [ ] [metadata] +"checksum Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)" = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" "checksum adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c" "checksum aes-ctr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee" "checksum aes-soft 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d" diff --git a/Cargo.toml b/Cargo.toml index 2cadde762b6b0..d93481769e1de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ members = [ "core/rpc/primitives", "core/rpc-servers", "core/runtime-interface", + "core/runtime-interface/proc-macro", "core/serializer", "core/service", "core/service/test", diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index da78b00f8cdbc..a03129473ea11 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -7,8 +7,10 @@ edition = "2018" [dependencies] wasm-interface = { package = "substrate-wasm-interface", path = "../wasm-interface", optional = true } rstd = { package = "sr-std", path = "../sr-std", default-features = false } +primitives = { package = "substrate-primitives", path = "../primitives", default-features = false } +proc-macro = { package = "substrate-runtime-interface-proc-macro", path = "proc-macro" } codec = { package = "parity-scale-codec", version = "1.0.5", default-features = false } [features] default = [ "std" ] -std = [ "wasm-interface", "rstd/std", "codec/std" ] +std = [ "wasm-interface", "rstd/std", "codec/std", "primitives/std" ] diff --git a/core/runtime-interface/proc-macro/Cargo.toml b/core/runtime-interface/proc-macro/Cargo.toml new file mode 100644 index 0000000000000..2d4c4bf1148ba --- /dev/null +++ b/core/runtime-interface/proc-macro/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "substrate-runtime-interface-proc-macro" +version = "2.0.0" +authors = ["Parity Technologies "] +edition = "2018" + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.5", features = [ "full", "visit", "fold", "extra-traits" ] } +quote = "1.0.2" +proc-macro2 = "1.0.3" +Inflector = "0.11.4" +proc-macro-crate = "0.1.4" diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs new file mode 100644 index 0000000000000..c5a7f4acf1f30 --- /dev/null +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -0,0 +1,174 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Generates the bare function interface for a given trait definition. + +use crate::utils::{generate_crate_access, create_host_function_ident}; + +use syn::{ + Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Pat, PatType, Type, Result, ReturnType, + TraitItem, +}; + +use proc_macro2::{TokenStream, Span}; + +use quote::quote; + +/// Generate the bare-function interface. +pub fn generate(trait_def: &ItemTrait) -> Result { + let trait_name = &trait_def.ident; + trait_def + .items + .iter() + .filter_map(|i| match i { + TraitItem::Method(ref method) => Some(method), + _ => None, + }) + .try_fold(TokenStream::new(), |mut t, m| { + t.extend(function_for_method(trait_name, m)?); + Ok(t) + }) +} + +/// Generates the bare function implementation for the given method. +fn function_for_method(trait_name: &Ident, method: &TraitItemMethod) -> Result { + let std_impl = function_std_impl(trait_name, method)?; + let no_std_impl = function_no_std_impl(method)?; + let function_impl_name = create_function_impl_ident(&method.sig.ident); + let function_name = &method.sig.ident; + let args = get_function_arguments(&method.sig); + let arg_names = get_function_argument_names(&method.sig); + let return_value = &method.sig.output; + + Ok( + quote! { + pub fn #function_name( #( #args, )* ) #return_value { + #std_impl + + #no_std_impl + + #function_impl_name( #( #arg_names, )* ) + } + } + ) +} + +/// Generates the bare function implementation for `cfg(not(feature = "std"))`. +fn function_no_std_impl(method: &TraitItemMethod) -> Result { + let function_name = create_function_impl_ident(&method.sig.ident); + let host_function_name = create_host_function_ident(&method.sig.ident); + let args = get_function_arguments(&method.sig); + let arg_names = get_function_argument_names(&method.sig); + let arg_names2 = get_function_argument_names(&method.sig); + let arg_types = get_function_argument_types_without_ref(&method.sig); + let crate_ = generate_crate_access(); + let return_value = &method.sig.output; + let convert_return_value = match return_value { + ReturnType::Default => quote!(), + ReturnType::Type(_, ref ty) => quote! { + <#ty as #crate_::FromFFIArg<_>>::from_ffi_arg(result) + } + }; + + Ok( + quote! { + #[cfg(not(feature = "std"))] + fn #function_name( #( #args, )* ) #return_value { + use #crate_::IntoFFIArg as _; + + // Generate all ffi arg wrappers. + #( + let #arg_names = <#arg_types as #crate_::AsFFIArg>::as_ffi_arg(&#arg_names); + )* + + // Call the host function + let result = unsafe { #host_function_name( #( #arg_names2.into_ffi_arg() )* ) }; + + #convert_return_value + } + } + ) +} + +/// Generates the bare function implementation for `cfg(feature = "std")`. +fn function_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result { + let method_name = &method.sig.ident; + let function_name = create_function_impl_ident(&method.sig.ident); + let args = get_function_arguments(&method.sig); + let arg_names = get_function_argument_names(&method.sig); + let crate_ = generate_crate_access(); + let return_value = &method.sig.output; + + let call_to_trait = if takes_self_argument(&method.sig) { + quote! { + #crate_::with_externalities(|ext| #trait_name::#method_name(&ext, #( #arg_names, )*)) + } + } else { + quote! { + <&mut dyn #crate_::Externalities<#crate_::Blake2Hasher> as #trait_name>::#method_name( + #( #arg_names, )* + ) + } + }; + + Ok( + quote! { + #[cfg(feature = "std")] + fn #function_name( #( #args, )* ) #return_value { + #call_to_trait + } + } + ) +} + +/// Create the function identifier for the internal implementation function. +fn create_function_impl_ident(method_name: &Ident) -> Ident { + Ident::new(&format!("{}_impl", method_name), Span::call_site()) +} + +/// Returns the function arguments of the given `Signature`, minus any `self` arguments. +fn get_function_arguments<'a>(sig: &'a Signature) -> impl Iterator { + sig.inputs + .iter() + .filter_map(|a| match a { + FnArg::Receiver(_) => None, + FnArg::Typed(pat_type) => Some(pat_type), + }) +} + +/// Returns the function argument names of the given `Signature`, minus any `self`. +fn get_function_argument_names<'a>(sig: &'a Signature) -> impl Iterator> { + get_function_arguments(sig).map(|pt| &pt.pat) +} + +/// Returns the function argument types, minus any `Self` type. If any of the arguments is a reference, +/// the underlying type without the ref is returned. +fn get_function_argument_types_without_ref<'a>(sig: &'a Signature) -> impl Iterator> { + get_function_arguments(sig) + .map(|pt| &pt.ty) + .map(|ty| match &**ty { + Type::Reference(type_ref) => &type_ref.elem, + _ => ty, + }) +} + +/// Returns if the given `Signature` takes a `self` argument. +fn takes_self_argument(sig: &Signature) -> bool { + match sig.inputs.first() { + Some(FnArg::Receiver(_)) => true, + _ => false, + } +} diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs new file mode 100644 index 0000000000000..da60d8fd03339 --- /dev/null +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -0,0 +1,65 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Procedural macros for generating runtime interfaces. + +extern crate proc_macro; + +use proc_macro2::{Span, TokenStream}; + +use syn::{parse_macro_input, Ident, ItemTrait, Result}; + +use quote::quote; + +use inflector::Inflector; + +use utils::generate_runtime_interface_include; + +mod bare_function_interface; +mod trait_decl_impl; +mod utils; + +#[proc_macro_attribute] +pub fn runtime_interface( + _: proc_macro::TokenStream, + input: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + let trait_def = parse_macro_input!(input as ItemTrait); + + runtime_interface_impl(trait_def).unwrap_or_else(|e| e.to_compile_error()).into() +} + +fn runtime_interface_impl(trait_def: ItemTrait) -> Result { + let bare_functions = bare_function_interface::generate(&trait_def)?; + let crate_include = generate_runtime_interface_include(); + let mod_name = Ident::new(&trait_def.ident.to_string().to_snake_case(), Span::call_site()); + let trait_decl_impl = trait_decl_impl::process(&trait_def)?; + + let res = quote! { + pub mod #mod_name { + use super::*; + #crate_include + + #bare_functions + + #trait_decl_impl + } + }; + + println!("{}", res); + + Ok(res) +} diff --git a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs new file mode 100644 index 0000000000000..bdcdee2472dc4 --- /dev/null +++ b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs @@ -0,0 +1,116 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Checks the trait declaration, folds and implements it. + +use crate::utils::generate_crate_access; + +use syn::{ + ItemTrait, TraitItemMethod, Result, TraitItem, Error, fold::{self, Fold}, spanned::Spanned, + Visibility, +}; + +use proc_macro2::TokenStream; + +use quote::quote; + +/// Process the given trait definition, by checking that the definition is valid, fold and +/// implement it. +pub fn process(trait_def: &ItemTrait) -> Result { + let impl_trait = impl_trait_for_externalities(trait_def)?; + let essential_trait_def = ToEssentialTraitDef::convert(trait_def.clone())?; + + Ok( + quote! { + #impl_trait + + #essential_trait_def + } + ) +} + +/// Converts the given trait definition into the essential trait definition without method +/// default implementations and visibility set to inherited. +struct ToEssentialTraitDef { + /// All errors found while doing the conversion. + errors: Vec, +} + +impl ToEssentialTraitDef { + /// Convert the given trait definition to the essential trait definition. + fn convert(trait_def: ItemTrait) -> Result { + let mut folder = ToEssentialTraitDef { + errors: Vec::new(), + }; + + let res = fold::fold_item_trait(&mut folder, trait_def); + + if let Some(first_error) = folder.errors.pop() { + Err( + folder.errors.into_iter().fold(first_error, |mut o, n| { + o.combine(n); + o + }) + ) + } else { + Ok(res) + } + } + + fn push_error(&mut self, span: &S, msg: &str) { + self.errors.push(Error::new(span.span(), msg)); + } +} + +impl Fold for ToEssentialTraitDef { + fn fold_trait_item_method(&mut self, mut method: TraitItemMethod) -> TraitItemMethod { + if method.default.take().is_none() { + self.push_error(&method, "Methods need to have an implementation"); + } + + method + } + + fn fold_item_trait(&mut self, mut trait_def: ItemTrait) -> ItemTrait { + if let Some(first_param) = trait_def.generics.params.first() { + self.push_error(first_param, "Generic parameters are not supported"); + } + + trait_def.vis = Visibility::Inherited; + trait_def + } +} + +/// Implements the given trait definition for `dyn Externalities`. +fn impl_trait_for_externalities(trait_def: &ItemTrait) -> Result { + let trait_ = &trait_def.ident; + let crate_ = generate_crate_access(); + let methods = trait_def + .items + .iter() + .filter_map(|i| match i { + TraitItem::Method(ref method) => Some(method), + _ => None, + }); + + Ok( + quote! { + impl #trait_ for &mut dyn #crate_::Externalities<#crate_::Blake2Hasher> { + #( #methods )* + } + } + ) +} diff --git a/core/runtime-interface/proc-macro/src/utils.rs b/core/runtime-interface/proc-macro/src/utils.rs new file mode 100644 index 0000000000000..6c669b78aa077 --- /dev/null +++ b/core/runtime-interface/proc-macro/src/utils.rs @@ -0,0 +1,62 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see + +//! Util function used by this crate. + +use proc_macro2::{TokenStream, Span}; + +use syn::{Ident, Error}; + +use proc_macro_crate::crate_name; + +use std::env; + +use quote::quote; + +/// Generates the include for the runtime-interface crate. +pub fn generate_runtime_interface_include() -> TokenStream { + if env::var("CARGO_PKG_NAME").unwrap() == "substrate-runtime-interface" { + TokenStream::new() + } else { + match crate_name("substrate-runtime-interface") { + Ok(crate_name) => { + let crate_name = Ident::new(&crate_name, Span::call_site()); + quote!( + #[doc(hidden)] + extern crate #crate_name as proc_macro_runtime_interface; + ) + }, + Err(e) => { + let err = Error::new(Span::call_site(), &e).to_compile_error(); + quote!( #err ) + } + } + } +} + +/// Generates the access to the `substrate-runtime-interface` crate. +pub fn generate_crate_access() -> TokenStream { + if env::var("CARGO_PKG_NAME").unwrap() == "substrate-runtime-interface" { + quote!( crate ) + } else { + quote!( proc_macro_runtime_interface ) + } +} + +/// Create the host function identifier for the given function name. +pub fn create_host_function_ident(name: &Ident) -> Ident { + Ident::new(&format!("ext_{}", name), Span::call_site()) +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 17a28a291429b..3fbc667cebec3 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -22,6 +22,14 @@ use wasm_interface::{FunctionContext, IntoValue, TryFromValue, Pointer, Result}; use codec::{Encode, Decode}; +#[doc(hidden)] +pub use primitives::{Blake2Hasher, traits::Externalities}; + +pub use proc_macro::runtime_interface; + +pub fn with_externalities) -> R, R>(f: F) -> R { + unimplemented!() +} pub trait AsFFIArg { /// The owned rust type that converts into `Self::FFIType`. type RTOwned: IntoFFIArg; @@ -205,3 +213,19 @@ impl FromWasmFFIArg for &[T] { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[runtime_interface] + trait Test { + fn test() { + + } + + fn test_with_self(&self) { + + } + } +} From aecba72996cbfeb341bb0d3a46cf1b469fc81e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 19 Sep 2019 23:18:11 +0200 Subject: [PATCH 03/76] Implements generation of the host extern functions --- .../proc-macro/src/bare_function_interface.rs | 38 ++-------- .../proc-macro/src/host_function_interface.rs | 72 +++++++++++++++++++ core/runtime-interface/proc-macro/src/lib.rs | 4 ++ .../proc-macro/src/trait_decl_impl.rs | 10 ++- .../runtime-interface/proc-macro/src/utils.rs | 30 +++++++- core/runtime-interface/src/lib.rs | 11 +-- 6 files changed, 127 insertions(+), 38 deletions(-) create mode 100644 core/runtime-interface/proc-macro/src/host_function_interface.rs diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index c5a7f4acf1f30..84bfe9f07e0c2 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -16,13 +16,13 @@ //! Generates the bare function interface for a given trait definition. -use crate::utils::{generate_crate_access, create_host_function_ident}; - -use syn::{ - Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Pat, PatType, Type, Result, ReturnType, - TraitItem, +use crate::utils::{ + generate_crate_access, create_host_function_ident, get_function_arguments, + get_function_argument_names, get_function_argument_types_without_ref, }; +use syn::{Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Result, ReturnType, TraitItem}; + use proc_macro2::{TokenStream, Span}; use quote::quote; @@ -114,7 +114,7 @@ fn function_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result Ident { Ident::new(&format!("{}_impl", method_name), Span::call_site()) } -/// Returns the function arguments of the given `Signature`, minus any `self` arguments. -fn get_function_arguments<'a>(sig: &'a Signature) -> impl Iterator { - sig.inputs - .iter() - .filter_map(|a| match a { - FnArg::Receiver(_) => None, - FnArg::Typed(pat_type) => Some(pat_type), - }) -} - -/// Returns the function argument names of the given `Signature`, minus any `self`. -fn get_function_argument_names<'a>(sig: &'a Signature) -> impl Iterator> { - get_function_arguments(sig).map(|pt| &pt.pat) -} - -/// Returns the function argument types, minus any `Self` type. If any of the arguments is a reference, -/// the underlying type without the ref is returned. -fn get_function_argument_types_without_ref<'a>(sig: &'a Signature) -> impl Iterator> { - get_function_arguments(sig) - .map(|pt| &pt.ty) - .map(|ty| match &**ty { - Type::Reference(type_ref) => &type_ref.elem, - _ => ty, - }) -} - /// Returns if the given `Signature` takes a `self` argument. fn takes_self_argument(sig: &Signature) -> bool { match sig.inputs.first() { diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs new file mode 100644 index 0000000000000..90501ea6b32b1 --- /dev/null +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -0,0 +1,72 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Generates the extern host function for wasm and the host functions interface for native. + +use crate::utils::{ + generate_crate_access, create_host_function_ident, get_function_argument_names, + get_function_argument_types_without_ref, +}; + +use syn::{ + ItemTrait, TraitItemMethod, Result, ReturnType, + TraitItem +}; + +use proc_macro2::TokenStream; + +use quote::quote; + +/// Generate the interface. +pub fn generate(trait_def: &ItemTrait) -> Result { + trait_def + .items + .iter() + .filter_map(|i| match i { + TraitItem::Method(ref method) => Some(method), + _ => None, + }) + .try_fold(TokenStream::new(), |mut t, m| { + t.extend(generate_extern_host_function(m)?); + Ok(t) + }) +} + +/// Generate the extern host function for the given method. +fn generate_extern_host_function(method: &TraitItemMethod) -> Result { + let crate_ = generate_crate_access(); + let arg_types = get_function_argument_types_without_ref(&method.sig); + let arg_names = get_function_argument_names(&method.sig); + let function = create_host_function_ident(&method.sig.ident); + + let output = match method.sig.output { + ReturnType::Default => quote!(), + ReturnType::Type(_, ref ty) => quote! { + -> <#ty as #crate_::AsFFIArg>::FFIType + } + }; + + Ok( + quote! { + #[cfg(not(feature = "std"))] + extern "C" { + pub fn #function ( + #( #arg_names: <#arg_types as #crate_::AsFFIArg>::FFIType ),* + ) #output; + } + } + ) +} diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index da60d8fd03339..5dac0ecd79333 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -29,6 +29,7 @@ use inflector::Inflector; use utils::generate_runtime_interface_include; mod bare_function_interface; +mod host_function_interface; mod trait_decl_impl; mod utils; @@ -47,6 +48,7 @@ fn runtime_interface_impl(trait_def: ItemTrait) -> Result { let crate_include = generate_runtime_interface_include(); let mod_name = Ident::new(&trait_def.ident.to_string().to_snake_case(), Span::call_site()); let trait_decl_impl = trait_decl_impl::process(&trait_def)?; + let host_functions = host_function_interface::generate(&trait_def)?; let res = quote! { pub mod #mod_name { @@ -56,6 +58,8 @@ fn runtime_interface_impl(trait_def: ItemTrait) -> Result { #bare_functions #trait_decl_impl + + #host_functions } }; diff --git a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs index bdcdee2472dc4..37c9f3de258c9 100644 --- a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs +++ b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs @@ -20,7 +20,7 @@ use crate::utils::generate_crate_access; use syn::{ ItemTrait, TraitItemMethod, Result, TraitItem, Error, fold::{self, Fold}, spanned::Spanned, - Visibility, + Visibility, Receiver }; use proc_macro2::TokenStream; @@ -92,6 +92,14 @@ impl Fold for ToEssentialTraitDef { trait_def.vis = Visibility::Inherited; trait_def } + + fn fold_receiver(&mut self, mut receiver: Receiver) -> Receiver { + if receiver.reference.is_none() { + self.push_error(&receiver, "Taking `Self` by value is not allowed"); + } + + receiver + } } /// Implements the given trait definition for `dyn Externalities`. diff --git a/core/runtime-interface/proc-macro/src/utils.rs b/core/runtime-interface/proc-macro/src/utils.rs index 6c669b78aa077..b8ad163da1dbe 100644 --- a/core/runtime-interface/proc-macro/src/utils.rs +++ b/core/runtime-interface/proc-macro/src/utils.rs @@ -18,7 +18,7 @@ use proc_macro2::{TokenStream, Span}; -use syn::{Ident, Error}; +use syn::{Ident, Error, Signature, Pat, PatType, FnArg, Type}; use proc_macro_crate::crate_name; @@ -60,3 +60,31 @@ pub fn generate_crate_access() -> TokenStream { pub fn create_host_function_ident(name: &Ident) -> Ident { Ident::new(&format!("ext_{}", name), Span::call_site()) } + +/// Returns the function arguments of the given `Signature`, minus any `self` arguments. +pub fn get_function_arguments<'a>(sig: &'a Signature) -> impl Iterator { + sig.inputs + .iter() + .filter_map(|a| match a { + FnArg::Receiver(_) => None, + FnArg::Typed(pat_type) => Some(pat_type), + }) +} + +/// Returns the function argument names of the given `Signature`, minus any `self`. +pub fn get_function_argument_names<'a>(sig: &'a Signature) -> impl Iterator> { + get_function_arguments(sig).map(|pt| &pt.pat) +} + +/// Returns the function argument types, minus any `Self` type. If any of the arguments +/// is a reference, the underlying type without the ref is returned. +pub fn get_function_argument_types_without_ref<'a>( + sig: &'a Signature, +) -> impl Iterator> { + get_function_arguments(sig) + .map(|pt| &pt.ty) + .map(|ty| match &**ty { + Type::Reference(type_ref) => &type_ref.elem, + _ => ty, + }) +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 3fbc667cebec3..13b64e1a21e2a 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -118,7 +118,7 @@ impl> IntoFFIArg for T { } } -impl AsFFIArg for &[T] { +impl AsFFIArg for [T] { type RTOwned = Vec; type RTBorrowed = [u8]; type FFIType = u64; @@ -220,12 +220,15 @@ mod tests { #[runtime_interface] trait Test { - fn test() { - + fn test(lol: Vec) -> Vec { + Vec::new() } - fn test_with_self(&self) { + fn test_with_self(&mut self, data: &[u8]) { + self.clear_storage(data); + } + fn test_with_self2(&self, data: &[u8]) { } } } From f7fa0abc1d06c90bc4b38d79fad7fa1891e1b5c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 21 Sep 2019 00:47:39 +0200 Subject: [PATCH 04/76] Prefix ext host function with snake case trait name --- .../proc-macro/src/bare_function_interface.rs | 6 +++--- .../proc-macro/src/host_function_interface.rs | 12 +++++------- .../proc-macro/src/trait_decl_impl.rs | 2 +- core/runtime-interface/proc-macro/src/utils.rs | 13 +++++++++++-- core/runtime-interface/src/lib.rs | 10 ++++++++++ 5 files changed, 30 insertions(+), 13 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index 84bfe9f07e0c2..890b54db3087d 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -46,7 +46,7 @@ pub fn generate(trait_def: &ItemTrait) -> Result { /// Generates the bare function implementation for the given method. fn function_for_method(trait_name: &Ident, method: &TraitItemMethod) -> Result { let std_impl = function_std_impl(trait_name, method)?; - let no_std_impl = function_no_std_impl(method)?; + let no_std_impl = function_no_std_impl(trait_name, method)?; let function_impl_name = create_function_impl_ident(&method.sig.ident); let function_name = &method.sig.ident; let args = get_function_arguments(&method.sig); @@ -67,9 +67,9 @@ fn function_for_method(trait_name: &Ident, method: &TraitItemMethod) -> Result Result { +fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result { let function_name = create_function_impl_ident(&method.sig.ident); - let host_function_name = create_host_function_ident(&method.sig.ident); + let host_function_name = create_host_function_ident(&method.sig.ident, trait_name); let args = get_function_arguments(&method.sig); let arg_names = get_function_argument_names(&method.sig); let arg_names2 = get_function_argument_names(&method.sig); diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 90501ea6b32b1..0e982ed1aaa6a 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -21,10 +21,7 @@ use crate::utils::{ get_function_argument_types_without_ref, }; -use syn::{ - ItemTrait, TraitItemMethod, Result, ReturnType, - TraitItem -}; +use syn::{ItemTrait, TraitItemMethod, Result, ReturnType, Ident, TraitItem}; use proc_macro2::TokenStream; @@ -32,6 +29,7 @@ use quote::quote; /// Generate the interface. pub fn generate(trait_def: &ItemTrait) -> Result { + let trait_name = &trait_def.ident; trait_def .items .iter() @@ -40,17 +38,17 @@ pub fn generate(trait_def: &ItemTrait) -> Result { _ => None, }) .try_fold(TokenStream::new(), |mut t, m| { - t.extend(generate_extern_host_function(m)?); + t.extend(generate_extern_host_function(m, trait_name)?); Ok(t) }) } /// Generate the extern host function for the given method. -fn generate_extern_host_function(method: &TraitItemMethod) -> Result { +fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) -> Result { let crate_ = generate_crate_access(); let arg_types = get_function_argument_types_without_ref(&method.sig); let arg_names = get_function_argument_names(&method.sig); - let function = create_host_function_ident(&method.sig.ident); + let function = create_host_function_ident(&method.sig.ident, trait_name); let output = match method.sig.output { ReturnType::Default => quote!(), diff --git a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs index 37c9f3de258c9..b8329e990d8af 100644 --- a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs +++ b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs @@ -93,7 +93,7 @@ impl Fold for ToEssentialTraitDef { trait_def } - fn fold_receiver(&mut self, mut receiver: Receiver) -> Receiver { + fn fold_receiver(&mut self, receiver: Receiver) -> Receiver { if receiver.reference.is_none() { self.push_error(&receiver, "Taking `Self` by value is not allowed"); } diff --git a/core/runtime-interface/proc-macro/src/utils.rs b/core/runtime-interface/proc-macro/src/utils.rs index b8ad163da1dbe..b3ac2dac1420f 100644 --- a/core/runtime-interface/proc-macro/src/utils.rs +++ b/core/runtime-interface/proc-macro/src/utils.rs @@ -26,6 +26,8 @@ use std::env; use quote::quote; +use inflector::Inflector; + /// Generates the include for the runtime-interface crate. pub fn generate_runtime_interface_include() -> TokenStream { if env::var("CARGO_PKG_NAME").unwrap() == "substrate-runtime-interface" { @@ -57,8 +59,15 @@ pub fn generate_crate_access() -> TokenStream { } /// Create the host function identifier for the given function name. -pub fn create_host_function_ident(name: &Ident) -> Ident { - Ident::new(&format!("ext_{}", name), Span::call_site()) +pub fn create_host_function_ident(name: &Ident, trait_name: &Ident) -> Ident { + Ident::new( + &format!( + "ext_{}_{}", + trait_name.to_string().to_snake_case(), + name, + ), + Span::call_site(), + ) } /// Returns the function arguments of the given `Signature`, minus any `self` arguments. diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 13b64e1a21e2a..b6073908ab939 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -64,6 +64,16 @@ pub trait IntoWasmFFIArg { fn into_wasm_ffi_arg(self_instance: Self::SelfInstance, context: &mut dyn FunctionContext) -> Result; } +pub trait IntoPreAllocatedWasmFFIArg { + type SelfInstance; + + fn into_wasm_ffi_arg( + self_instance: Self::SelfInstance, + context: &mut dyn FunctionContext, + allocated: T, + ) -> Result<()>; +} + pub enum FFIArg<'a, T, O, R: ?Sized = O> where O: IntoFFIArg, R: IntoFFIArg { Ref(&'a R, std::marker::PhantomData), Owned(O), From a922101db64d8362cb5a9dacca45a38e1f3a443f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 24 Sep 2019 17:11:35 +0200 Subject: [PATCH 05/76] Implement host functions implementation on the host --- .../proc-macro/src/host_function_interface.rs | 277 +++++++++++++++++- .../runtime-interface/proc-macro/src/utils.rs | 27 +- core/runtime-interface/src/lib.rs | 43 ++- 3 files changed, 312 insertions(+), 35 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 0e982ed1aaa6a..53ecd2bbeca12 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -18,19 +18,27 @@ use crate::utils::{ generate_crate_access, create_host_function_ident, get_function_argument_names, - get_function_argument_types_without_ref, + get_function_argument_types_without_ref, get_function_argument_types_ref_and_mut, + get_function_argument_names_and_types_without_ref, }; -use syn::{ItemTrait, TraitItemMethod, Result, ReturnType, Ident, TraitItem}; +use syn::{ + ItemTrait, TraitItemMethod, Result, ReturnType, Ident, TraitItem, Pat, Error, Signature, + spanned::Spanned, +}; + +use proc_macro2::{TokenStream, Span}; -use proc_macro2::TokenStream; +use quote::{quote, ToTokens}; -use quote::quote; +use inflector::Inflector; +use std::iter::Iterator; -/// Generate the interface. +/// Generate the extern host functions for wasm and the `HostFunctions` struct that provides the +/// implementations for the host functions on the host. pub fn generate(trait_def: &ItemTrait) -> Result { let trait_name = &trait_def.ident; - trait_def + let extern_host_functions = trait_def .items .iter() .filter_map(|i| match i { @@ -39,8 +47,16 @@ pub fn generate(trait_def: &ItemTrait) -> Result { }) .try_fold(TokenStream::new(), |mut t, m| { t.extend(generate_extern_host_function(m, trait_name)?); - Ok(t) - }) + Ok::<_, Error>(t) + })?; + let host_functions_struct = generate_host_functions_struct(trait_def)?; + + Ok( + quote! { + #extern_host_functions + #host_functions_struct + } + ) } /// Generate the extern host function for the given method. @@ -68,3 +84,248 @@ fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) - } ) } + +/// Generate the `HostFunctions` struct that implements `wasm-interface::HostFunctions` to provide +/// implementations for the extern host functions. +fn generate_host_functions_struct(trait_def: &ItemTrait) -> Result { + let crate_ = generate_crate_access(); + let host_functions = trait_def + .items + .iter() + .filter_map(|i| match i { + TraitItem::Method(ref method) => Some(method), + _ => None, + }) + .map(generate_host_function_implementation) + .collect::>>()?; + + Ok( + quote! { + /// Provides implementations for the extern host functions. + #[cfg(feature = "std")] + pub struct HostFunctions; + + #[cfg(feature = "std")] + impl #crate_::wasm_interface::HostFunctions for HostFunctions { + fn functions() -> &'static [&'static dyn #crate_::wasm_interface::Function] { + &[ #( #host_functions ),* ] + } + } + } + ) +} + +/// Generates the host function struct that implements `wasm_interface::Function` and returns a static +/// reference to this struct. +/// +/// When calling from wasm into the host, we will call the `execute` function that calls the native +/// implementation of the function. +fn generate_host_function_implementation(method: &TraitItemMethod) -> Result { + let name = method.sig.ident.to_string(); + let struct_name = Ident::new(&name.to_pascal_case(), Span::call_site()); + let crate_ = generate_crate_access(); + let signature = generate_wasm_interface_signature_for_host_function(&method.sig)?; + let wasm_to_ffi_values = generate_wasm_to_ffi_values(&method.sig).collect::>>()?; + let ffi_to_host_values = generate_ffi_to_host_value(&method.sig).collect::>>()?; + let host_function_call = generate_host_function_call(&method.sig); + let into_preallocated_ffi_arg = generate_into_preallocated_ffi_arg(&method.sig)?; + let convert_return_value = generate_return_value_into_wasm_value(&method.sig); + + Ok( + quote! { + { + struct #struct_name; + + #[allow(unused)] + impl #crate_::wasm_interface::Function for #struct_name { + fn name(&self) -> &str { + #name + } + + fn signature(&self) -> #crate_::wasm_interface::Signature { + #signature + } + + fn execute( + &self, + context: &mut dyn #crate_::wasm_interface::FunctionContext, + args: &mut dyn Iterator, + ) -> std::result::Result, String> { + #( #wasm_to_ffi_values )* + #( #ffi_to_host_values )* + #host_function_call + #into_preallocated_ffi_arg + #convert_return_value + } + } + + &#struct_name as &dyn #crate_::wasm_interface::Function + } + } + ) +} + +/// Generate the `wasm_interface::Signature` for the given host function `sig`. +fn generate_wasm_interface_signature_for_host_function(sig: &Signature) -> Result { + let crate_ = generate_crate_access(); + let return_value = match &sig.output { + ReturnType::Type(_, ty) => + quote! { + Some( <<#ty as #crate_::AsFFIArg>::FFIType as #crate_::wasm_interface::IntoValue>::VALUE_TYPE ) + }, + ReturnType::Default => quote!( None ), + }; + let arg_types = get_function_argument_types_without_ref(sig) + .map(|ty| quote! { + <<#ty as #crate_::AsFFIArg>::FFIType as #crate_::wasm_interface::IntoValue>::VALUE_TYPE + }); + + Ok( + quote! { + #crate_::wasm_interface::Signature { + args: std::borrow::Cow::Borrowed(&[ #( #arg_types ),* ][..]), + return_value: #return_value, + } + } + ) +} + +/// Generate the code that converts the wasm values given to `HostFunctions::execute` into the FFI +/// values. +fn generate_wasm_to_ffi_values<'a>( + sig: &'a Signature, +) -> impl Iterator> + 'a { + let crate_ = generate_crate_access(); + let function_name = &sig.ident; + let error_message = format!( + "Number of arguments given to `{}` does not match the expected number of arguments!", + function_name, + ); + + get_function_argument_names_and_types_without_ref(sig) + .map(move |(name, ty)| { + let try_from_error = format!( + "Could not instantiate `{}` from wasm value while executing `{}`!", + name.to_token_stream(), + function_name, + ); + + let var_name = generate_ffi_value_var_name(name)?; + + Ok(quote! { + let val = args.next().ok_or_else(|| #error_message)?; + let #var_name = < + <#ty as #crate_::AsFFIArg>::FFIType as #crate_::wasm_interface::TryFromValue + >::try_from_value(val).ok_or_else(|| #try_from_error)?; + }) + }) +} + +/// Generate the code to convert the ffi values on the host to the host values using `FromWasmFFIArg`. +fn generate_ffi_to_host_value<'a>( + sig: &'a Signature, +) -> impl Iterator> + 'a { + let mut_access = get_function_argument_types_ref_and_mut(sig); + let crate_ = generate_crate_access(); + + get_function_argument_names_and_types_without_ref(sig) + .zip(mut_access.map(|v| v.and_then(|m| m.1))) + .map(move |((name, ty), mut_access)| { + let ffi_value_var_name = generate_ffi_value_var_name(name)?; + + Ok( + quote! { + let #mut_access #name = <#ty as #crate_::FromWasmFFIArg<_>>::from_wasm_ffi_arg( + context, + #ffi_value_var_name, + )?; + } + ) + }) +} + +/// Generate the code to call the host function and the ident that stores the result. +fn generate_host_function_call(sig: &Signature) -> TokenStream { + let host_function_name = &sig.ident; + let result_var_name = generate_host_function_result_var_name(&sig.ident); + let ref_and_mut = get_function_argument_types_ref_and_mut(sig).map(|ram| + ram.map(|(vr, vm)| quote!(#vr #vm)) + ); + let names = get_function_argument_names(sig); + + let var_access = names.zip(ref_and_mut).map(|(n, ref_and_mut)| { + quote!( #ref_and_mut #n ) + }); + + quote! { + let #result_var_name = #host_function_name ( #( #var_access ),* ); + } +} + +/// Generate the variable name that stores the result of the host function. +fn generate_host_function_result_var_name(name: &Ident) -> Ident { + Ident::new(&format!("{}_result", name), Span::call_site()) +} + +/// Generate the variable name that stores the FFI value. +fn generate_ffi_value_var_name(pat: &Pat) -> Result { + match pat { + Pat::Ident(pat_ident) => { + if let Some(by_ref) = pat_ident.by_ref { + Err(Error::new(by_ref.span(), "`ref` not supported!")) + } else if let Some(sub_pattern) = &pat_ident.subpat { + Err(Error::new(sub_pattern.0.span(), "Not supported!")) + } else { + Ok(Ident::new(&format!("{}_ffi_value", pat_ident.ident), Span::call_site())) + } + } + _ => Err(Error::new(pat.span(), "Not supported as variable name!")) + } +} + +/// Generate code that copies data from the host back to preallocated wasm memory. +/// +/// Any argument that is given as `&mut` is interpreted as preallocated memory and it is expected +/// that the type implements `IntoPreAllocatedWasmFFIArg`. +fn generate_into_preallocated_ffi_arg(sig: &Signature) -> Result { + let crate_ = generate_crate_access(); + let ref_and_mut = get_function_argument_types_ref_and_mut(sig).map(|ram| + ram.and_then(|(vr, vm)| vm.map(|v| (vr, v))) + ); + let names_and_types = get_function_argument_names_and_types_without_ref(sig); + + ref_and_mut.zip(names_and_types) + .filter_map(|(ram, (name, ty))| ram.map(|_| (name, ty))) + .map(|(name, ty)| { + let ffi_var_name = generate_ffi_value_var_name(name)?; + + Ok( + quote! { + <#ty as #crate_::IntoPreAllocatedWasmFFIArg<_>>::into_wasm_ffi_arg( + #name, + context, + #ffi_var_name, + )?; + } + ) + }) + .collect() +} + +/// Generate the code that converts the return value into the appropriate wasm value. +fn generate_return_value_into_wasm_value(sig: &Signature) -> TokenStream { + let crate_ = generate_crate_access(); + + match &sig.output { + ReturnType::Default => quote!( Ok(None) ), + ReturnType::Type(_, ty) => { + let result_var_name = generate_host_function_result_var_name(&sig.ident); + + quote! { + <#ty as #crate_::IntoWasmFFIArg<_>>::into_wasm_ffi_arg(#result_var_name, context) + .map(#crate_::wasm_interface::IntoValue::into_value) + .map(Some) + } + } + } +} diff --git a/core/runtime-interface/proc-macro/src/utils.rs b/core/runtime-interface/proc-macro/src/utils.rs index b3ac2dac1420f..112ec79debd34 100644 --- a/core/runtime-interface/proc-macro/src/utils.rs +++ b/core/runtime-interface/proc-macro/src/utils.rs @@ -18,7 +18,7 @@ use proc_macro2::{TokenStream, Span}; -use syn::{Ident, Error, Signature, Pat, PatType, FnArg, Type}; +use syn::{Ident, Error, Signature, Pat, PatType, FnArg, Type, token}; use proc_macro_crate::crate_name; @@ -97,3 +97,28 @@ pub fn get_function_argument_types_without_ref<'a>( _ => ty, }) } + +/// Returns the function argument names and types, minus any `self`. If any of the arguments +/// is a reference, the underlying type without the ref is returned. +pub fn get_function_argument_names_and_types_without_ref<'a>( + sig: &'a Signature, +) -> impl Iterator, &'a Box)> { + get_function_arguments(sig) + .map(|pt| match &*pt.ty { + Type::Reference(type_ref) => (&pt.pat, &type_ref.elem), + _ => (&pt.pat, &pt.ty), + }) +} + +/// Returns the `&`/`&mut` for all function argument types, minus the `self` arg. If a function +/// argument is not a reference, `None` is returned. +pub fn get_function_argument_types_ref_and_mut<'a>( + sig: &'a Signature, +) -> impl Iterator)>> { + get_function_arguments(sig) + .map(|pt| &pt.ty) + .map(|ty| match &**ty { + Type::Reference(type_ref) => Some((&type_ref.and_token, type_ref.mutability.as_ref())), + _ => None, + }) +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index b6073908ab939..b71ee43f98131 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -25,6 +25,9 @@ use codec::{Encode, Decode}; #[doc(hidden)] pub use primitives::{Blake2Hasher, traits::Externalities}; +#[doc(hidden)] +pub use wasm_interface; + pub use proc_macro::runtime_interface; pub fn with_externalities) -> R, R>(f: F) -> R { @@ -44,7 +47,7 @@ pub trait FromFFIArg: Sized { fn from_ffi_arg(arg: T) -> Self; } -pub trait FromWasmFFIArg: Sized { +pub trait FromWasmFFIArg { /// The `Self` instance returned by `from_wasm_ffi_arg`. /// /// For types that are do not implement `Sized` we can not return `Self`. So, we need to use a @@ -59,9 +62,7 @@ pub trait IntoFFIArg { } pub trait IntoWasmFFIArg { - type SelfInstance; - - fn into_wasm_ffi_arg(self_instance: Self::SelfInstance, context: &mut dyn FunctionContext) -> Result; + fn into_wasm_ffi_arg(self, context: &mut dyn FunctionContext) -> Result; } pub trait IntoPreAllocatedWasmFFIArg { @@ -174,29 +175,11 @@ impl FromFFIArg for Vec { } impl IntoWasmFFIArg for Vec { - type SelfInstance = Vec; - - fn into_wasm_ffi_arg(self_instance: Vec, context: &mut dyn FunctionContext) -> Result { - <&[T] as IntoWasmFFIArg>::into_wasm_ffi_arg(self_instance, context) - } -} - -impl FromWasmFFIArg for Vec { - type SelfInstance = Vec; - - fn from_wasm_ffi_arg(context: &mut dyn FunctionContext, arg: u64) -> Result> { - <&[T] as FromWasmFFIArg>::from_wasm_ffi_arg(context, arg) - } -} - -impl IntoWasmFFIArg for &[T] { - type SelfInstance = Vec; - - fn into_wasm_ffi_arg(self_instance: Vec, context: &mut dyn FunctionContext) -> Result { + fn into_wasm_ffi_arg(self, context: &mut dyn FunctionContext) -> Result { let vec: Cow<'_, [u8]> = if TypeId::of::() == TypeId::of::() { - unsafe { Cow::Borrowed(std::mem::transmute(&self_instance[..])) } + unsafe { Cow::Borrowed(std::mem::transmute(&self[..])) } } else { - Cow::Owned(self_instance.encode()) + Cow::Owned(self.encode()) }; let ptr = context.allocate_memory(vec.as_ref().len() as u32)?; @@ -206,7 +189,15 @@ impl IntoWasmFFIArg for &[T] { } } -impl FromWasmFFIArg for &[T] { +impl FromWasmFFIArg for Vec { + type SelfInstance = Vec; + + fn from_wasm_ffi_arg(context: &mut dyn FunctionContext, arg: u64) -> Result> { + <[T] as FromWasmFFIArg>::from_wasm_ffi_arg(context, arg) + } +} + +impl FromWasmFFIArg for [T] { type SelfInstance = Vec; fn from_wasm_ffi_arg(context: &mut dyn FunctionContext, arg: u64) -> Result> { From 156d8a09415a5ea6ee6c3d2a01c1bf49f362491d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 24 Sep 2019 18:22:09 +0200 Subject: [PATCH 06/76] Change `HostFunctions` interface --- core/executor/src/wasm_executor.rs | 65 +++++++++++-------- core/executor/src/wasm_utils.rs | 12 +++- .../proc-macro/src/host_function_interface.rs | 16 ++++- core/wasm-interface/src/lib.rs | 8 ++- 4 files changed, 68 insertions(+), 33 deletions(-) diff --git a/core/executor/src/wasm_executor.rs b/core/executor/src/wasm_executor.rs index c37a67a02c927..7e76b3c55f116 100644 --- a/core/executor/src/wasm_executor.rs +++ b/core/executor/src/wasm_executor.rs @@ -255,28 +255,35 @@ impl FunctionExecutor { -> std::result::Result { let signature = wasm_interface::Signature::from(signature); + let num_functions = SubstrateExternals::num_functions(); + let mut function_index = 0; - if let Some((index, func)) = SubstrateExternals::functions().iter() - .enumerate() - .find(|f| name == f.1.name()) - { - if signature == func.signature() { - Ok(wasmi::FuncInstance::alloc_host(signature.into(), index)) - } else { - Err(wasmi::Error::Instantiation( - format!( - "Invalid signature for function `{}` expected `{:?}`, got `{:?}`", - func.name(), - signature, - func.signature(), + while function_index < num_functions { + let function = SubstrateExternals::get_function(function_index); + + if name == function.name() { + if signature == function.signature() { + return Ok( + wasmi::FuncInstance::alloc_host(signature.into(), function_index), ) - )) + } else { + return Err(wasmi::Error::Instantiation( + format!( + "Invalid signature for function `{}` expected `{:?}`, got `{:?}`", + function.name(), + signature, + function.signature(), + ), + )) + } } - } else { - Err(wasmi::Error::Instantiation( - format!("Export {} not found", name), - )) + + function_index += 1; } + + Err(wasmi::Error::Instantiation( + format!("Export {} not found", name), + )) } } &Resolver @@ -287,17 +294,21 @@ impl wasmi::Externals for FunctionExecutor { fn invoke_index(&mut self, index: usize, args: wasmi::RuntimeArgs) -> std::result::Result, wasmi::Trap> { - let mut args = args.as_ref().iter().copied().map(Into::into); - let function = SubstrateExternals::functions().get(index).ok_or_else(|| - Error::from( - format!("Could not find host function with index: {}", index), + if index >= SubstrateExternals::num_functions() { + Err( + Error::from( + format!("Could not find host function with index: {}", index), + ).into() ) - )?; + } else { + let mut args = args.as_ref().iter().copied().map(Into::into); - function.execute(self, &mut args) - .map_err(Error::FunctionExecution) - .map_err(wasmi::Trap::from) - .map(|v| v.map(Into::into)) + SubstrateExternals::get_function(index) + .execute(self, &mut args) + .map_err(Error::FunctionExecution) + .map_err(wasmi::Trap::from) + .map(|v| v.map(Into::into)) + } } } struct SubstrateExternals; diff --git a/core/executor/src/wasm_utils.rs b/core/executor/src/wasm_utils.rs index b217350ac6fc4..1513690e87806 100644 --- a/core/executor/src/wasm_utils.rs +++ b/core/executor/src/wasm_utils.rs @@ -162,11 +162,19 @@ macro_rules! impl_wasm_host_interface { ) => ( impl $crate::wasm_interface::HostFunctions for $interface_name { #[allow(non_camel_case_types)] - fn functions() -> &'static [&'static dyn $crate::wasm_interface::Function] { + fn get_function(index: usize) -> &'static dyn $crate::wasm_interface::Function { gen_functions!( $context, $( $name( $( $names: $params ),* ) $( -> $returns )? { $( $body )* } )* - ) + )[index] + } + + #[allow(non_camel_case_types)] + fn num_functions() -> usize { + gen_functions!( + $context, + $( $name( $( $names: $params ),* ) $( -> $returns )? { $( $body )* } )* + ).len() } } ); diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 53ecd2bbeca12..296df16d49469 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -98,6 +98,14 @@ fn generate_host_functions_struct(trait_def: &ItemTrait) -> Result }) .map(generate_host_function_implementation) .collect::>>()?; + let host_functions_count = trait_def + .items + .iter() + .filter(|i| match i { + TraitItem::Method(_) => true, + _ => false, + }) + .count(); Ok( quote! { @@ -107,8 +115,12 @@ fn generate_host_functions_struct(trait_def: &ItemTrait) -> Result #[cfg(feature = "std")] impl #crate_::wasm_interface::HostFunctions for HostFunctions { - fn functions() -> &'static [&'static dyn #crate_::wasm_interface::Function] { - &[ #( #host_functions ),* ] + fn get_function(index: usize) -> &'static dyn #crate_::wasm_interface::Function { + [ #( #host_functions ),* ][index] + } + + fn num_functions() -> usize { + #host_functions_count } } } diff --git a/core/wasm-interface/src/lib.rs b/core/wasm-interface/src/lib.rs index accbc52d03c13..674f63e2eb645 100644 --- a/core/wasm-interface/src/lib.rs +++ b/core/wasm-interface/src/lib.rs @@ -261,8 +261,12 @@ pub trait Sandbox { /// Something that provides implementations for host functions. pub trait HostFunctions { - /// Returns all host functions. - fn functions() -> &'static [&'static dyn Function]; + /// Returns the function at the given index. + /// + /// Panics if the given index is invalid. + fn get_function(index: usize) -> &'static dyn Function; + /// Returns the number of host functions. + fn num_functions() -> usize; } /// Something that can be converted into a wasm compatible `Value`. From 936d0294ccd74a6d908691b71e2b70d85d93e64f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Sep 2019 11:07:48 +0200 Subject: [PATCH 07/76] Implement `HostFunctions` for tuples --- Cargo.lock | 19 ++++++++++--------- core/wasm-interface/Cargo.toml | 1 + core/wasm-interface/src/lib.rs | 23 +++++++++++++++++++++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99e1282d62c7a..3afb40b744534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1390,7 +1390,7 @@ dependencies = [ [[package]] name = "impl-trait-for-tuples" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3842,7 +3842,7 @@ dependencies = [ name = "sr-primitives" version = "2.0.0" dependencies = [ - "impl-trait-for-tuples 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-trait-for-tuples 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "integer-sqrt 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3955,7 +3955,7 @@ dependencies = [ name = "srml-authorship" version = "0.1.0" dependencies = [ - "impl-trait-for-tuples 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-trait-for-tuples 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "sr-io 2.0.0", "sr-primitives 2.0.0", @@ -4114,7 +4114,7 @@ dependencies = [ name = "srml-finality-tracker" version = "2.0.0" dependencies = [ - "impl-trait-for-tuples 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-trait-for-tuples 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)", "sr-io 2.0.0", @@ -4252,7 +4252,7 @@ dependencies = [ name = "srml-session" version = "2.0.0" dependencies = [ - "impl-trait-for-tuples 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-trait-for-tuples 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4310,7 +4310,7 @@ name = "srml-support" version = "2.0.0" dependencies = [ "bitmask 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "impl-trait-for-tuples 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-trait-for-tuples 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "once_cell 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "paste 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4376,7 +4376,7 @@ name = "srml-system" version = "2.0.0" dependencies = [ "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "impl-trait-for-tuples 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-trait-for-tuples 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4392,7 +4392,7 @@ dependencies = [ name = "srml-timestamp" version = "2.0.0" dependencies = [ - "impl-trait-for-tuples 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-trait-for-tuples 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.97 (registry+https://github.com/rust-lang/crates.io-index)", "sr-io 2.0.0", @@ -5531,6 +5531,7 @@ version = "1.0.3" name = "substrate-wasm-interface" version = "2.0.0" dependencies = [ + "impl-trait-for-tuples 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "wasmi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -6654,7 +6655,7 @@ dependencies = [ "checksum impl-codec 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "78c441b3d2b5e24b407161e76d482b7bbd29b5da357707839ac40d95152f031f" "checksum impl-serde 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5158079de9d4158e0ce1de3ae0bd7be03904efc40b3d7dd8b8c301cbf6b52b56" "checksum impl-serde 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d26be4b97d738552ea423f76c4f681012ff06c3fa36fa968656b3679f60b4a1" -"checksum impl-trait-for-tuples 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7b0df44cb13008e863c3d80788d5f4cb0f94d09b31bb0190a8ecc05151b2ac8a" +"checksum impl-trait-for-tuples 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6947b372790f8948f439bb6aaa6baabdf80be1a207a477ff072f83fb793e428f" "checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d" "checksum integer-sqrt 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ea155abb3ba6f382a75f1418988c05fe82959ed9ce727de427f9cfd425b0c903" "checksum interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "141340095b15ed7491bd3d4ced9d20cebfb826174b6bb03386381f62b01e3d77" diff --git a/core/wasm-interface/Cargo.toml b/core/wasm-interface/Cargo.toml index c388b32930ecd..39695d4695c2b 100644 --- a/core/wasm-interface/Cargo.toml +++ b/core/wasm-interface/Cargo.toml @@ -6,3 +6,4 @@ edition = "2018" [dependencies] wasmi = "0.5.0" +impl-trait-for-tuples = "0.1.2" diff --git a/core/wasm-interface/src/lib.rs b/core/wasm-interface/src/lib.rs index 674f63e2eb645..6f0145df4bf84 100644 --- a/core/wasm-interface/src/lib.rs +++ b/core/wasm-interface/src/lib.rs @@ -269,6 +269,29 @@ pub trait HostFunctions { fn num_functions() -> usize; } +#[impl_trait_for_tuples::impl_for_tuples(1, 30)] +impl HostFunctions for Tuple { + fn get_function(mut index: usize) -> &'static dyn Function { + for_tuples!( + #( + let num_functions = Tuple::num_functions(); + + if index < num_functions { + return Tuple::get_function(index) + } + + index -= num_functions; + )* + ); + + panic!("Invalid index for `get_function`: {}", index) + } + + fn num_functions() -> usize { + for_tuples!( #( Tuple::num_functions() )+* ) + } +} + /// Something that can be converted into a wasm compatible `Value`. pub trait IntoValue { /// The type of the value in wasm. From dfe833a14341e74f57a772cf94e7ab0f6cf19ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Sep 2019 13:34:31 +0200 Subject: [PATCH 08/76] Make `WasmExecutor` generic over the host functions --- core/executor/src/sandbox.rs | 20 +-- core/executor/src/wasm_executor.rs | 126 ++++++++++-------- core/executor/src/wasm_runtimes_cache.rs | 11 +- core/executor/src/wasm_utils.rs | 9 +- .../proc-macro/src/host_function_interface.rs | 4 +- core/wasm-interface/src/lib.rs | 22 ++- 6 files changed, 111 insertions(+), 81 deletions(-) diff --git a/core/executor/src/sandbox.rs b/core/executor/src/sandbox.rs index 2c38499ec8bf4..9931dec852099 100644 --- a/core/executor/src/sandbox.rs +++ b/core/executor/src/sandbox.rs @@ -598,7 +598,7 @@ mod tests { "#).unwrap(); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox", &code).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sandbox", &code).unwrap(), vec![1], ); } @@ -619,7 +619,7 @@ mod tests { "#).unwrap(); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox", &code).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sandbox", &code).unwrap(), vec![0], ); } @@ -639,7 +639,7 @@ mod tests { ) "#).unwrap(); - let res = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_exhaust_heap", &code); + let res = ::new().call(&mut ext, 8, &test_code[..], "test_exhaust_heap", &code); assert_eq!(res.is_err(), true); if let Err(err) = res { assert_eq!( @@ -685,7 +685,7 @@ mod tests { "#).unwrap(); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox", &code).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sandbox", &code).unwrap(), vec![1], ); } @@ -719,7 +719,7 @@ mod tests { "#).unwrap(); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox_args", &code).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sandbox_args", &code).unwrap(), vec![1], ); } @@ -741,7 +741,7 @@ mod tests { "#).unwrap(); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox_return_val", &code).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sandbox_return_val", &code).unwrap(), vec![1], ); } @@ -761,7 +761,7 @@ mod tests { "#).unwrap(); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox_instantiate", &code).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sandbox_instantiate", &code).unwrap(), vec![1], ); } @@ -775,7 +775,7 @@ mod tests { let code = &[0, 0, 0, 0, 1, 0, 0, 0]; assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox_instantiate", code).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sandbox_instantiate", code).unwrap(), vec![1], ); } @@ -798,7 +798,7 @@ mod tests { "#).unwrap(); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox_instantiate", &code).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sandbox_instantiate", &code).unwrap(), vec![0], ); } @@ -822,7 +822,7 @@ mod tests { "#).unwrap(); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sandbox_instantiate", &code).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sandbox_instantiate", &code).unwrap(), vec![2], ); } diff --git a/core/executor/src/wasm_executor.rs b/core/executor/src/wasm_executor.rs index 7e76b3c55f116..f722549a28506 100644 --- a/core/executor/src/wasm_executor.rs +++ b/core/executor/src/wasm_executor.rs @@ -19,7 +19,7 @@ //! This module defines and implements the wasm part of Substrate Host Interface and provides //! an interface for calling into the wasm runtime. -use std::{convert::TryFrom, str, panic}; +use std::{convert::TryFrom, str, panic, marker::PhantomData}; use tiny_keccak; use secp256k1; @@ -39,7 +39,7 @@ use crate::sandbox; use crate::allocator; use log::trace; use wasm_interface::{ - FunctionContext, HostFunctions, Pointer, WordSize, Sandbox, MemoryId, PointerType, + FunctionContext, HostFunctions, Pointer, WordSize, Sandbox, MemoryId, Result as WResult, ReadPrimitive, WritePrimitive, }; @@ -53,25 +53,27 @@ macro_rules! debug_trace { ( $( $x:tt )* ) => () } -struct FunctionExecutor { +struct FunctionExecutor { sandbox_store: sandbox::Store, heap: allocator::FreeingBumpHeapAllocator, memory: MemoryRef, table: Option, + _marker: PhantomData, } -impl FunctionExecutor { +impl FunctionExecutor { fn new(m: MemoryRef, heap_base: u32, t: Option) -> Result { Ok(FunctionExecutor { sandbox_store: sandbox::Store::new(), heap: allocator::FreeingBumpHeapAllocator::new(m.clone(), heap_base), memory: m, table: t, + _marker: Default::default(), }) } } -impl sandbox::SandboxCapabilities for FunctionExecutor { +impl sandbox::SandboxCapabilities for FunctionExecutor { fn store(&self) -> &sandbox::Store { &self.sandbox_store } @@ -92,7 +94,7 @@ impl sandbox::SandboxCapabilities for FunctionExecutor { } } -impl FunctionContext for FunctionExecutor { +impl FunctionContext for FunctionExecutor { fn read_memory_into(&self, address: Pointer, dest: &mut [u8]) -> WResult<()> { self.memory.get_into(address.into(), dest).map_err(|e| format!("{:?}", e)) } @@ -114,7 +116,7 @@ impl FunctionContext for FunctionExecutor { } } -impl Sandbox for FunctionExecutor { +impl Sandbox for FunctionExecutor { fn memory_get( &self, memory_id: MemoryId, @@ -247,19 +249,20 @@ fn deadline_to_timestamp(deadline: u64) -> Option { } } -impl FunctionExecutor { +impl FunctionExecutor { fn resolver() -> &'static dyn wasmi::ModuleImportResolver { - struct Resolver; - impl wasmi::ModuleImportResolver for Resolver { + struct Resolver(PhantomData); + impl wasmi::ModuleImportResolver for Resolver { fn resolve_func(&self, name: &str, signature: &wasmi::Signature) -> std::result::Result { let signature = wasm_interface::Signature::from(signature); - let num_functions = SubstrateExternals::num_functions(); + let num_functions = HF::num_functions(); let mut function_index = 0; while function_index < num_functions { - let function = SubstrateExternals::get_function(function_index); + let function = HF::get_function(function_index) + .expect("Any index below the number of functions is valid; qed"); if name == function.name() { if signature == function.signature() { @@ -286,29 +289,23 @@ impl FunctionExecutor { )) } } - &Resolver + &Resolver::(PhantomData::) } } -impl wasmi::Externals for FunctionExecutor { +impl wasmi::Externals for FunctionExecutor { fn invoke_index(&mut self, index: usize, args: wasmi::RuntimeArgs) -> std::result::Result, wasmi::Trap> { - if index >= SubstrateExternals::num_functions() { - Err( - Error::from( - format!("Could not find host function with index: {}", index), - ).into() - ) - } else { - let mut args = args.as_ref().iter().copied().map(Into::into); + let function = HF::get_function(index).ok_or_else(|| + Error::from(format!("Could not find host function with index: {}", index)) + )?; - SubstrateExternals::get_function(index) - .execute(self, &mut args) - .map_err(Error::FunctionExecution) - .map_err(wasmi::Trap::from) - .map(|v| v.map(Into::into)) - } + let mut args = args.as_ref().iter().copied().map(Into::into); + function.execute(self, &mut args) + .map_err(Error::FunctionExecution) + .map_err(wasmi::Trap::from) + .map(|v| v.map(Into::into)) } } struct SubstrateExternals; @@ -1372,12 +1369,12 @@ fn with_external_storage(f: F) -> std::result::Result /// /// Executes the provided code in a sandboxed wasm runtime. #[derive(Debug, Clone)] -pub struct WasmExecutor; +pub struct WasmExecutor(PhantomData); -impl WasmExecutor { +impl WasmExecutor { /// Create a new instance. pub fn new() -> Self { - WasmExecutor + WasmExecutor(PhantomData::) } /// Call a given method in the given code. @@ -1503,7 +1500,7 @@ impl WasmExecutor { .and_then(|e| e.as_table().cloned()); let heap_base = Self::get_heap_base(module_instance)?; - let mut fec = FunctionExecutor::new( + let mut fec = FunctionExecutor::<(SubstrateExternals, HF)>::new( memory.clone(), heap_base, table, @@ -1544,7 +1541,7 @@ impl WasmExecutor { let intermediate_instance = ModuleInstance::new( module, &ImportsBuilder::new() - .with_resolver("env", FunctionExecutor::resolver()) + .with_resolver("env", FunctionExecutor::<(SubstrateExternals, HF)>::resolver()) )?; // Verify that the module has the heap base global variable. @@ -1583,7 +1580,14 @@ mod tests { let mut ext = TestExternalities::default(); let test_code = WASM_BINARY; - let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_empty_return", &[]).unwrap(); + let output = ::new() + .call( + &mut ext, + 8, + &test_code[..], + "test_empty_return", + &[], + ).unwrap(); assert_eq!(output, vec![0u8; 0]); } @@ -1592,13 +1596,13 @@ mod tests { let mut ext = TestExternalities::default(); let test_code = WASM_BINARY; - let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_panic", &[]); + let output = ::new().call(&mut ext, 8, &test_code[..], "test_panic", &[]); assert!(output.is_err()); - let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_conditional_panic", &[]); + let output = ::new().call(&mut ext, 8, &test_code[..], "test_conditional_panic", &[]); assert_eq!(output.unwrap(), vec![0u8; 0]); - let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_conditional_panic", &[2]); + let output = ::new().call(&mut ext, 8, &test_code[..], "test_conditional_panic", &[2]); assert!(output.is_err()); } @@ -1608,7 +1612,14 @@ mod tests { ext.set_storage(b"foo".to_vec(), b"bar".to_vec()); let test_code = WASM_BINARY; - let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_data_in", b"Hello world").unwrap(); + let output = ::new() + .call( + &mut ext, + 8, + &test_code[..], + "test_data_in", + b"Hello world", + ).unwrap(); assert_eq!(output, b"all ok!".to_vec()); @@ -1631,7 +1642,14 @@ mod tests { let test_code = WASM_BINARY; // This will clear all entries which prefix is "ab". - let output = WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_clear_prefix", b"ab").unwrap(); + let output = ::new() + .call( + &mut ext, + 8, + &test_code[..], + "test_clear_prefix", + b"ab", + ).unwrap(); assert_eq!(output, b"all ok!".to_vec()); @@ -1648,11 +1666,11 @@ mod tests { let mut ext = TestExternalities::default(); let test_code = WASM_BINARY; assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_blake2_256", &[]).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_blake2_256", &[]).unwrap(), blake2_256(&b""[..]).encode() ); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_blake2_256", b"Hello world!").unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_blake2_256", b"Hello world!").unwrap(), blake2_256(&b"Hello world!"[..]).encode() ); } @@ -1662,11 +1680,11 @@ mod tests { let mut ext = TestExternalities::default(); let test_code = WASM_BINARY; assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_blake2_128", &[]).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_blake2_128", &[]).unwrap(), blake2_128(&b""[..]).encode() ); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_blake2_128", b"Hello world!").unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_blake2_128", b"Hello world!").unwrap(), blake2_128(&b"Hello world!"[..]).encode() ); } @@ -1676,11 +1694,11 @@ mod tests { let mut ext = TestExternalities::default(); let test_code = WASM_BINARY; assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_twox_256", &[]).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_twox_256", &[]).unwrap(), hex!("99e9d85137db46ef4bbea33613baafd56f963c64b1f3685a4eb4abd67ff6203a"), ); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_twox_256", b"Hello world!").unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_twox_256", b"Hello world!").unwrap(), hex!("b27dfd7f223f177f2a13647b533599af0c07f68bda23d96d059da2b451a35a74"), ); } @@ -1690,11 +1708,11 @@ mod tests { let mut ext = TestExternalities::default(); let test_code = WASM_BINARY; assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_twox_128", &[]).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_twox_128", &[]).unwrap(), hex!("99e9d85137db46ef4bbea33613baafd5") ); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_twox_128", b"Hello world!").unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_twox_128", b"Hello world!").unwrap(), hex!("b27dfd7f223f177f2a13647b533599af") ); } @@ -1710,7 +1728,7 @@ mod tests { calldata.extend_from_slice(sig.as_ref()); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_ed25519_verify", &calldata).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_ed25519_verify", &calldata).unwrap(), vec![1] ); @@ -1720,7 +1738,7 @@ mod tests { calldata.extend_from_slice(other_sig.as_ref()); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_ed25519_verify", &calldata).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_ed25519_verify", &calldata).unwrap(), vec![0] ); } @@ -1736,7 +1754,7 @@ mod tests { calldata.extend_from_slice(sig.as_ref()); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sr25519_verify", &calldata).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sr25519_verify", &calldata).unwrap(), vec![1] ); @@ -1746,7 +1764,7 @@ mod tests { calldata.extend_from_slice(other_sig.as_ref()); assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_sr25519_verify", &calldata).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_sr25519_verify", &calldata).unwrap(), vec![0] ); } @@ -1757,7 +1775,7 @@ mod tests { let trie_input = vec![b"zero".to_vec(), b"one".to_vec(), b"two".to_vec()]; let test_code = WASM_BINARY; assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_ordered_trie_root", &[]).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_ordered_trie_root", &[]).unwrap(), Layout::::ordered_trie_root(trie_input.iter()).as_fixed_bytes().encode() ); } @@ -1771,7 +1789,7 @@ mod tests { ext.set_offchain_externalities(offchain); let test_code = WASM_BINARY; assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_offchain_local_storage", &[]).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_offchain_local_storage", &[]).unwrap(), vec![0] ); assert_eq!(state.read().persistent_storage.get(b"", b"test"), Some(vec![])); @@ -1798,7 +1816,7 @@ mod tests { let test_code = WASM_BINARY; assert_eq!( - WasmExecutor::new().call(&mut ext, 8, &test_code[..], "test_offchain_http", &[]).unwrap(), + ::new().call(&mut ext, 8, &test_code[..], "test_offchain_http", &[]).unwrap(), vec![0] ); } diff --git a/core/executor/src/wasm_runtimes_cache.rs b/core/executor/src/wasm_runtimes_cache.rs index fb207dc18b88d..2d8ee53684d08 100644 --- a/core/executor/src/wasm_runtimes_cache.rs +++ b/core/executor/src/wasm_runtimes_cache.rs @@ -25,6 +25,7 @@ use primitives::{storage::well_known_keys, Blake2Hasher, traits::Externalities}; use runtime_version::RuntimeVersion; use std::{collections::hash_map::{Entry, HashMap}, mem, rc::Rc}; use wasmi::{Module as WasmModule, ModuleRef as WasmModuleInstanceRef, RuntimeValue}; +use wasm_interface::HostFunctions; #[derive(Debug)] enum CacheError { @@ -236,9 +237,9 @@ impl RuntimesCache { /// /// `Error::InvalidMemoryReference` is returned if no memory export with the /// identifier `memory` can be found in the runtime. - pub fn fetch_runtime>( + pub fn fetch_runtime, HF: HostFunctions>( &mut self, - wasm_executor: &WasmExecutor, + wasm_executor: &WasmExecutor, ext: &mut E, default_heap_pages: Option, ) -> Result, Error> { @@ -291,8 +292,8 @@ impl RuntimesCache { } } - fn create_wasm_instance>( - wasm_executor: &WasmExecutor, + fn create_wasm_instance, HF: HostFunctions>( + wasm_executor: &WasmExecutor, ext: &mut E, heap_pages: u64, ) -> Result, CacheError> { @@ -308,7 +309,7 @@ impl RuntimesCache { let data_segments = extract_data_segments(&code)?; // Instantiate this module. - let instance = WasmExecutor::instantiate_module(heap_pages as usize, &module) + let instance = WasmExecutor::::instantiate_module(heap_pages as usize, &module) .map_err(CacheError::Instantiation)?; // Take state snapshot before executing anything. diff --git a/core/executor/src/wasm_utils.rs b/core/executor/src/wasm_utils.rs index 1513690e87806..5172d4924ec9e 100644 --- a/core/executor/src/wasm_utils.rs +++ b/core/executor/src/wasm_utils.rs @@ -162,11 +162,13 @@ macro_rules! impl_wasm_host_interface { ) => ( impl $crate::wasm_interface::HostFunctions for $interface_name { #[allow(non_camel_case_types)] - fn get_function(index: usize) -> &'static dyn $crate::wasm_interface::Function { + fn get_function(index: usize) -> Option<&'static dyn $crate::wasm_interface::Function> { gen_functions!( $context, $( $name( $( $names: $params ),* ) $( -> $returns )? { $( $body )* } )* - )[index] + ) + .get(index) + .map(|f| *f) } #[allow(non_camel_case_types)] @@ -174,7 +176,8 @@ macro_rules! impl_wasm_host_interface { gen_functions!( $context, $( $name( $( $names: $params ),* ) $( -> $returns )? { $( $body )* } )* - ).len() + ) + .len() } } ); diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 296df16d49469..386fd0b02383f 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -115,8 +115,8 @@ fn generate_host_functions_struct(trait_def: &ItemTrait) -> Result #[cfg(feature = "std")] impl #crate_::wasm_interface::HostFunctions for HostFunctions { - fn get_function(index: usize) -> &'static dyn #crate_::wasm_interface::Function { - [ #( #host_functions ),* ][index] + fn get_function(index: usize) -> Option<&'static dyn #crate_::wasm_interface::Function> { + [ #( #host_functions ),* ].get(index).map(|f| *f) } fn num_functions() -> usize { diff --git a/core/wasm-interface/src/lib.rs b/core/wasm-interface/src/lib.rs index 6f0145df4bf84..ae0599a1e8497 100644 --- a/core/wasm-interface/src/lib.rs +++ b/core/wasm-interface/src/lib.rs @@ -260,18 +260,16 @@ pub trait Sandbox { } /// Something that provides implementations for host functions. -pub trait HostFunctions { - /// Returns the function at the given index. - /// - /// Panics if the given index is invalid. - fn get_function(index: usize) -> &'static dyn Function; +pub trait HostFunctions: 'static { + /// Returns the function at the given index or `None` if the index is invalid. + fn get_function(index: usize) -> Option<&'static dyn Function>; /// Returns the number of host functions. fn num_functions() -> usize; } #[impl_trait_for_tuples::impl_for_tuples(1, 30)] impl HostFunctions for Tuple { - fn get_function(mut index: usize) -> &'static dyn Function { + fn get_function(mut index: usize) -> Option<&'static dyn Function> { for_tuples!( #( let num_functions = Tuple::num_functions(); @@ -284,7 +282,7 @@ impl HostFunctions for Tuple { )* ); - panic!("Invalid index for `get_function`: {}", index) + None } fn num_functions() -> usize { @@ -292,6 +290,16 @@ impl HostFunctions for Tuple { } } +impl HostFunctions for () { + fn get_function(_: usize) -> Option<&'static dyn Function> { + None + } + + fn num_functions() -> usize { + 0 + } +} + /// Something that can be converted into a wasm compatible `Value`. pub trait IntoValue { /// The type of the value in wasm. From bd8ac3561863fa9ba41e483d0a2b77380fa7761e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Sep 2019 16:07:41 +0200 Subject: [PATCH 09/76] Begin to add a test and make it compile --- Cargo.lock | 11 ++++ Cargo.toml | 1 + core/runtime-interface/Cargo.toml | 6 +- .../proc-macro/src/bare_function_interface.rs | 4 +- .../proc-macro/src/trait_decl_impl.rs | 1 + core/runtime-interface/src/lib.rs | 57 ++++++++++++++----- core/runtime-interface/test-wasm/Cargo.toml | 18 ++++++ core/runtime-interface/test-wasm/build.rs | 30 ++++++++++ core/runtime-interface/test-wasm/src/lib.rs | 52 +++++++++++++++++ 9 files changed, 163 insertions(+), 17 deletions(-) create mode 100644 core/runtime-interface/test-wasm/Cargo.toml create mode 100644 core/runtime-interface/test-wasm/build.rs create mode 100644 core/runtime-interface/test-wasm/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 3afb40b744534..a438add18577a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5232,8 +5232,10 @@ version = "2.0.0" dependencies = [ "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", + "substrate-executor 2.0.0", "substrate-primitives 2.0.0", "substrate-runtime-interface-proc-macro 2.0.0", + "substrate-runtime-interface-test-wasm 2.0.0", "substrate-wasm-interface 2.0.0", ] @@ -5248,6 +5250,15 @@ dependencies = [ "syn 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "substrate-runtime-interface-test-wasm" +version = "2.0.0" +dependencies = [ + "sr-std 2.0.0", + "substrate-runtime-interface 2.0.0", + "substrate-wasm-builder-runner 1.0.3", +] + [[package]] name = "substrate-runtime-test" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index d93481769e1de..d0bca75846e9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ members = [ "core/rpc-servers", "core/runtime-interface", "core/runtime-interface/proc-macro", + "core/runtime-interface/test-wasm", "core/serializer", "core/service", "core/service/test", diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index a03129473ea11..1bf0d1ad67e3e 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -8,9 +8,13 @@ edition = "2018" wasm-interface = { package = "substrate-wasm-interface", path = "../wasm-interface", optional = true } rstd = { package = "sr-std", path = "../sr-std", default-features = false } primitives = { package = "substrate-primitives", path = "../primitives", default-features = false } -proc-macro = { package = "substrate-runtime-interface-proc-macro", path = "proc-macro" } +substrate-runtime-interface-proc-macro = { path = "proc-macro" } codec = { package = "parity-scale-codec", version = "1.0.5", default-features = false } +[dev-dependencies] +executor = { package = "substrate-executor", path = "../executor" } +test-wasm = { package = "substrate-runtime-interface-test-wasm", path = "test-wasm" } + [features] default = [ "std" ] std = [ "wasm-interface", "rstd/std", "codec/std", "primitives/std" ] diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index 890b54db3087d..9a0c032e9df5b 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -95,7 +95,7 @@ fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result< )* // Call the host function - let result = unsafe { #host_function_name( #( #arg_names2.into_ffi_arg() )* ) }; + let result = unsafe { #host_function_name( #( #arg_names2.into_ffi_arg(), )* ) }; #convert_return_value } @@ -114,7 +114,7 @@ fn function_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result Result { Ok( quote! { + #[cfg(feature = "std")] impl #trait_ for &mut dyn #crate_::Externalities<#crate_::Blake2Hasher> { #( #methods )* } diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index b71ee43f98131..d6dd815e48944 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -18,21 +18,29 @@ use rstd::{any::TypeId, borrow::Cow, mem}; +#[cfg(feature = "std")] use wasm_interface::{FunctionContext, IntoValue, TryFromValue, Pointer, Result}; use codec::{Encode, Decode}; #[doc(hidden)] -pub use primitives::{Blake2Hasher, traits::Externalities}; +pub use primitives::Blake2Hasher; #[doc(hidden)] +#[cfg(feature = "std")] +pub use primitives::traits::Externalities; + +#[doc(hidden)] +#[cfg(feature = "std")] pub use wasm_interface; -pub use proc_macro::runtime_interface; +pub use substrate_runtime_interface_proc_macro::runtime_interface; +#[cfg(feature = "std")] pub fn with_externalities) -> R, R>(f: F) -> R { unimplemented!() } + pub trait AsFFIArg { /// The owned rust type that converts into `Self::FFIType`. type RTOwned: IntoFFIArg; @@ -47,6 +55,7 @@ pub trait FromFFIArg: Sized { fn from_ffi_arg(arg: T) -> Self; } +#[cfg(feature = "std")] pub trait FromWasmFFIArg { /// The `Self` instance returned by `from_wasm_ffi_arg`. /// @@ -61,10 +70,12 @@ pub trait IntoFFIArg { fn into_ffi_arg(&self) -> T; } +#[cfg(feature = "std")] pub trait IntoWasmFFIArg { fn into_wasm_ffi_arg(self, context: &mut dyn FunctionContext) -> Result; } +#[cfg(feature = "std")] pub trait IntoPreAllocatedWasmFFIArg { type SelfInstance; @@ -174,6 +185,7 @@ impl FromFFIArg for Vec { } } +#[cfg(feature = "std")] impl IntoWasmFFIArg for Vec { fn into_wasm_ffi_arg(self, context: &mut dyn FunctionContext) -> Result { let vec: Cow<'_, [u8]> = if TypeId::of::() == TypeId::of::() { @@ -189,6 +201,7 @@ impl IntoWasmFFIArg for Vec { } } +#[cfg(feature = "std")] impl FromWasmFFIArg for Vec { type SelfInstance = Vec; @@ -197,6 +210,7 @@ impl FromWasmFFIArg for Vec { } } +#[cfg(feature = "std")] impl FromWasmFFIArg for [T] { type SelfInstance = Vec; @@ -215,21 +229,36 @@ impl FromWasmFFIArg for [T] { } } +#[cfg(feature = "std")] +impl IntoPreAllocatedWasmFFIArg for [u8] { + type SelfInstance = Vec; + + fn into_wasm_ffi_arg( + self_instance: Self::SelfInstance, + context: &mut dyn FunctionContext, + allocated: u64, + ) -> Result<()> { + let arg = u64::from_le(allocated); + let len = (arg & (!0u32 as u64)) as u32; + let ptr = (arg >> 32) as u32; + + if (len as usize) < self_instance.len() { + Err( + format!( + "Preallocated buffer is not big enough (given {} vs needed {})!", + len, + self_instance.len() + ) + ) + } else { + context.write_memory(Pointer::new(ptr), &self_instance) + } + } +} + #[cfg(test)] mod tests { use super::*; - #[runtime_interface] - trait Test { - fn test(lol: Vec) -> Vec { - Vec::new() - } - - fn test_with_self(&mut self, data: &[u8]) { - self.clear_storage(data); - } - fn test_with_self2(&self, data: &[u8]) { - } - } } diff --git a/core/runtime-interface/test-wasm/Cargo.toml b/core/runtime-interface/test-wasm/Cargo.toml new file mode 100644 index 0000000000000..98b2bef39d798 --- /dev/null +++ b/core/runtime-interface/test-wasm/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "substrate-runtime-interface-test-wasm" +version = "2.0.0" +authors = ["Parity Technologies "] +edition = "2018" +build = "build.rs" + +[dependencies] +runtime-interface = { package = "substrate-runtime-interface", path = "../", default-features = false } +rstd = { package = "sr-std", path = "../../sr-std", default-features = false } + +[build-dependencies] +wasm-builder-runner = { package = "substrate-wasm-builder-runner", version = "1.0.3", path = "../../utils/wasm-builder-runner" } + +[features] +default = [ "std" ] +std = [ "runtime-interface/std", "rstd/std" ] +no_std = [] diff --git a/core/runtime-interface/test-wasm/build.rs b/core/runtime-interface/test-wasm/build.rs new file mode 100644 index 0000000000000..fd4749b34c45e --- /dev/null +++ b/core/runtime-interface/test-wasm/build.rs @@ -0,0 +1,30 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource}; + +fn main() { + build_current_project_with_rustflags( + "wasm_binary.rs", + WasmBuilderSource::CratesOrPath { + path: "../../utils/wasm-builder", + version: "1.0.6", + }, + // This instructs LLD to export __heap_base as a global variable, which is used by the + // external memory allocator. + "-Clink-arg=--export=__heap_base", + ); +} diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs new file mode 100644 index 0000000000000..df50df921c8c7 --- /dev/null +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -0,0 +1,52 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#![cfg_attr(not(feature = "std"), no_std)] + +use runtime_interface::runtime_interface; + +use rstd::{vec, vec::Vec}; + +// Inlucde the WASM binary +#[cfg(feature = "std")] +include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); + +#[runtime_interface] +trait TestApi { + /// Returns the input data as result. + fn return_input(data: Vec) -> Vec { + data + } + + /// Set the storage at key with value. + fn set_storage(&mut self, key: &[u8], data: &[u8]) { + self.place_storage(key.to_vec(), Some(data.to_vec())); + } + + /// Copy `hello` into the given mutable reference + fn return_value_into_mutable_reference(&self, data: &mut [u8]) { + let res = "hello"; + data[..res.as_bytes().len()].copy_from_slice(res.as_bytes()); + } +} + +#[no_mangle] +pub fn test_return_data() { + let input = vec![1, 2, 3, 4, 5, 6]; + let res = test_api::return_input(input.clone()); + + assert_eq!(input, res); +} From 6bd6d99ecbf5e5b653c9546ee7de2a210c689f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Sep 2019 19:10:06 +0200 Subject: [PATCH 10/76] Make the test succeed --- Cargo.lock | 1 + core/executor/src/wasm_executor.rs | 19 ++++++++-------- core/runtime-interface/Cargo.toml | 1 + .../proc-macro/src/host_function_interface.rs | 9 +++++--- core/runtime-interface/src/lib.rs | 22 +++++++++++++++++-- 5 files changed, 38 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a438add18577a..97c9cdb3e6afd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5236,6 +5236,7 @@ dependencies = [ "substrate-primitives 2.0.0", "substrate-runtime-interface-proc-macro 2.0.0", "substrate-runtime-interface-test-wasm 2.0.0", + "substrate-state-machine 2.0.0", "substrate-wasm-interface 2.0.0", ] diff --git a/core/executor/src/wasm_executor.rs b/core/executor/src/wasm_executor.rs index f722549a28506..b8b9e78534285 100644 --- a/core/executor/src/wasm_executor.rs +++ b/core/executor/src/wasm_executor.rs @@ -25,7 +25,7 @@ use secp256k1; use wasmi::{ Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder, ModuleRef, - memory_units::Pages, RuntimeValue::{I32, I64, self}, + memory_units::Pages, }; use crate::error::{Error, Result}; use codec::{Encode, Decode}; @@ -40,7 +40,7 @@ use crate::allocator; use log::trace; use wasm_interface::{ FunctionContext, HostFunctions, Pointer, WordSize, Sandbox, MemoryId, - Result as WResult, ReadPrimitive, WritePrimitive, + Result as WResult, ReadPrimitive, WritePrimitive, Value, }; #[cfg(feature="wasm-extern-trace")] @@ -1401,8 +1401,8 @@ impl WasmExecutor { /// This should be used for tests only. pub fn call_with_custom_signature< E: Externalities, - F: FnOnce(&mut dyn FnMut(&[u8]) -> Result) -> Result>, - FR: FnOnce(Option, &MemoryRef) -> Result>, + F: FnOnce(&mut dyn FnMut(&[u8]) -> Result) -> Result>, + FR: FnOnce(Option, &MemoryRef) -> Result>, R, >( &self, @@ -1464,10 +1464,10 @@ impl WasmExecutor { method, |alloc| { let offset = alloc(data)?; - Ok(vec![I32(offset as i32), I32(data.len() as i32)]) + Ok(vec![Value::I32(offset as i32), Value::I32(data.len() as i32)]) }, |res, memory| { - if let Some(I64(r)) = res { + if let Some(Value::I64(r)) = res { let offset = r as u32; let length = (r as u64 >> 32) as usize; memory.get(offset, length).map_err(|_| Error::Runtime).map(Some) @@ -1481,8 +1481,8 @@ impl WasmExecutor { /// Call a given method in the given wasm-module runtime. fn call_in_wasm_module_with_custom_signature< E: Externalities, - F: FnOnce(&mut dyn FnMut(&[u8]) -> Result) -> Result>, - FR: FnOnce(Option, &MemoryRef) -> Result>, + F: FnOnce(&mut dyn FnMut(&[u8]) -> Result) -> Result>, + FR: FnOnce(Option, &MemoryRef) -> Result>, R, >( &self, @@ -1511,13 +1511,14 @@ impl WasmExecutor { fec.write_memory(offset, data).map(|_| offset.into()).map_err(Into::into) })?; + let parameters = parameters.into_iter().map(Into::into).collect::>(); let result = runtime_io::with_externalities( ext, || module_instance.invoke_export(method, ¶meters, &mut fec), ); match result { - Ok(val) => match filter_result(val, &memory)? { + Ok(val) => match filter_result(val.map(Into::into), &memory)? { Some(val) => Ok(val), None => Err(Error::InvalidReturn), }, diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index 1bf0d1ad67e3e..eeb87a2148caa 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -14,6 +14,7 @@ codec = { package = "parity-scale-codec", version = "1.0.5", default-features = [dev-dependencies] executor = { package = "substrate-executor", path = "../executor" } test-wasm = { package = "substrate-runtime-interface-test-wasm", path = "test-wasm" } +state_machine = { package = "substrate-state-machine", path = "../state-machine" } [features] default = [ "std" ] diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 386fd0b02383f..b5659bd0e2afb 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -96,7 +96,7 @@ fn generate_host_functions_struct(trait_def: &ItemTrait) -> Result TraitItem::Method(ref method) => Some(method), _ => None, }) - .map(generate_host_function_implementation) + .map(|m| generate_host_function_implementation(&trait_def.ident, m)) .collect::>>()?; let host_functions_count = trait_def .items @@ -132,8 +132,11 @@ fn generate_host_functions_struct(trait_def: &ItemTrait) -> Result /// /// When calling from wasm into the host, we will call the `execute` function that calls the native /// implementation of the function. -fn generate_host_function_implementation(method: &TraitItemMethod) -> Result { - let name = method.sig.ident.to_string(); +fn generate_host_function_implementation( + trait_name: &Ident, + method: &TraitItemMethod, +) -> Result { + let name = create_host_function_ident(&method.sig.ident, trait_name).to_string(); let struct_name = Ident::new(&name.to_pascal_case(), Span::call_site()); let crate_ = generate_crate_access(); let signature = generate_wasm_interface_signature_for_host_function(&method.sig)?; diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index d6dd815e48944..f2bb2c6fd465d 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -38,6 +38,7 @@ pub use substrate_runtime_interface_proc_macro::runtime_interface; #[cfg(feature = "std")] pub fn with_externalities) -> R, R>(f: F) -> R { + println!("HEY"); unimplemented!() } @@ -259,6 +260,23 @@ impl IntoPreAllocatedWasmFFIArg for [u8] { #[cfg(test)] mod tests { use super::*; - - + use test_wasm::{WASM_BINARY, test_api::HostFunctions}; + use executor::WasmExecutor; + + type TestExternalities = state_machine::TestExternalities; + + #[test] + fn test_return_data() { + let mut ext = TestExternalities::default(); + let executor = WasmExecutor::::new(); + + executor.call_with_custom_signature::<_, _, _, ()>( + &mut ext, + 8, + &WASM_BINARY[..], + "test_return_data", + |_| Ok(Vec::new()), + |res, _| if res.is_none() { Ok(Some(())) } else { Err("Invalid return value!".into()) }, + ).unwrap(); + } } From 1cf53101daceae5436095f771e2d159f830e0e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 25 Sep 2019 19:16:27 +0200 Subject: [PATCH 11/76] Add test to ensure that host functions are not found --- core/runtime-interface/src/lib.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index f2bb2c6fd465d..cfcae088a6373 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -279,4 +279,20 @@ mod tests { |res, _| if res.is_none() { Ok(Some(())) } else { Err("Invalid return value!".into()) }, ).unwrap(); } + + #[test] + #[should_panic(expected = "Wasmi(Instantiation(\"Export ext_test_api_return_input not found\"))")] + fn host_function_not_found() { + let mut ext = TestExternalities::default(); + let executor = ::new(); + + executor.call_with_custom_signature::<_, _, _, ()>( + &mut ext, + 8, + &WASM_BINARY[..], + "test_return_data", + |_| Ok(Vec::new()), + |res, _| Ok(None), + ).unwrap(); + } } From 8aaf8517903397648db0daa64ca79950c8fed170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 26 Sep 2019 10:39:22 +0200 Subject: [PATCH 12/76] It's alive! Make the `set_storage` test work --- Cargo.lock | 10 +-- core/executor/Cargo.toml | 3 +- core/executor/src/wasm_executor.rs | 4 +- core/runtime-interface/Cargo.toml | 3 +- .../proc-macro/src/bare_function_interface.rs | 21 +++++-- core/runtime-interface/src/externalities.rs | 40 ++++++++++++ core/runtime-interface/src/lib.rs | 47 +++++++------- core/runtime-interface/test-wasm/src/lib.rs | 8 +++ core/sr-io/Cargo.toml | 6 +- core/sr-io/with_std.rs | 61 ++++++++++--------- 10 files changed, 135 insertions(+), 68 deletions(-) create mode 100644 core/runtime-interface/src/externalities.rs diff --git a/Cargo.lock b/Cargo.lock index 97c9cdb3e6afd..f7e91e8b95cd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -798,7 +798,7 @@ dependencies = [ [[package]] name = "environmental" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -3825,7 +3825,6 @@ dependencies = [ name = "sr-io" version = "2.0.0" dependencies = [ - "environmental 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "libsecp256k1 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3833,6 +3832,7 @@ dependencies = [ "sr-std 2.0.0", "substrate-offchain 2.0.0", "substrate-primitives 2.0.0", + "substrate-runtime-interface 2.0.0", "substrate-state-machine 2.0.0", "substrate-trie 2.0.0", "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4895,7 +4895,7 @@ name = "substrate-executor" version = "2.0.0" dependencies = [ "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "derive_more 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "libsecp256k1 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4909,6 +4909,7 @@ dependencies = [ "substrate-offchain 2.0.0", "substrate-panic-handler 2.0.0", "substrate-primitives 2.0.0", + "substrate-runtime-interface 2.0.0", "substrate-runtime-test 2.0.0", "substrate-serializer 2.0.0", "substrate-state-machine 2.0.0", @@ -5230,6 +5231,7 @@ dependencies = [ name = "substrate-runtime-interface" version = "2.0.0" dependencies = [ + "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", "substrate-executor 2.0.0", @@ -6602,7 +6604,7 @@ dependencies = [ "checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" "checksum elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "073be79b6538296faf81c631872676600616073817dd9a440c477ad09b408983" "checksum env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" -"checksum environmental 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5c7464757b80de8930c91c9afe77ddce501826bf9d134a87db2c67d9dc177e2c" +"checksum environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "34f8467a0284de039e6bd0e25c14519538462ba5beb548bb1f03e645097837a8" "checksum erased-serde 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3beee4bc16478a1b26f2e80ad819a52d24745e292f521a63c16eea5f74b7eb60" "checksum error-chain 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ab49e9dcb602294bc42f9a7dfc9bc6e936fca4418ea300dbfb84fe16de0b7d9" "checksum exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d8013f441e38e31c670e7f34ec8f1d5d3a2bd9d303c1ff83976ca886005e8f48" diff --git a/core/executor/Cargo.toml b/core/executor/Cargo.toml index 3d8f047322fcb..a9ff172777854 100644 --- a/core/executor/Cargo.toml +++ b/core/executor/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Parity Technologies "] edition = "2018" [dependencies] -derive_more = "0.14.0" +derive_more = "0.15.0" codec = { package = "parity-scale-codec", version = "1.0.0" } runtime_io = { package = "sr-io", path = "../sr-io" } primitives = { package = "substrate-primitives", path = "../primitives" } @@ -14,6 +14,7 @@ serializer = { package = "substrate-serializer", path = "../serializer" } runtime_version = { package = "sr-version", path = "../sr-version" } panic-handler = { package = "substrate-panic-handler", path = "../panic-handler" } wasm-interface = { package = "substrate-wasm-interface", path = "../wasm-interface" } +runtime-interface = { package = "substrate-runtime-interface", path = "../runtime-interface" } wasmi = "0.5.0" parity-wasm = "0.31" lazy_static = "1.3" diff --git a/core/executor/src/wasm_executor.rs b/core/executor/src/wasm_executor.rs index b8b9e78534285..e7f075b3e0292 100644 --- a/core/executor/src/wasm_executor.rs +++ b/core/executor/src/wasm_executor.rs @@ -1512,9 +1512,9 @@ impl WasmExecutor { })?; let parameters = parameters.into_iter().map(Into::into).collect::>(); - let result = runtime_io::with_externalities( + let result = runtime_interface::set_and_run_with_externalities( ext, - || module_instance.invoke_export(method, ¶meters, &mut fec), + || module_instance.invoke_export(method, ¶meters, &mut fec) ); match result { diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index eeb87a2148caa..5dc5287d1f90f 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -10,6 +10,7 @@ rstd = { package = "sr-std", path = "../sr-std", default-features = false } primitives = { package = "substrate-primitives", path = "../primitives", default-features = false } substrate-runtime-interface-proc-macro = { path = "proc-macro" } codec = { package = "parity-scale-codec", version = "1.0.5", default-features = false } +environmental = { version = "1.0.2", optional = true } [dev-dependencies] executor = { package = "substrate-executor", path = "../executor" } @@ -18,4 +19,4 @@ state_machine = { package = "substrate-state-machine", path = "../state-machine" [features] default = [ "std" ] -std = [ "wasm-interface", "rstd/std", "codec/std", "primitives/std" ] +std = [ "wasm-interface", "rstd/std", "codec/std", "primitives/std", "environmental" ] diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index 9a0c032e9df5b..d569fa0145e45 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -21,11 +21,14 @@ use crate::utils::{ get_function_argument_names, get_function_argument_types_without_ref, }; -use syn::{Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Result, ReturnType, TraitItem}; +use syn::{ + Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Result, ReturnType, TraitItem, + spanned::Spanned, +}; use proc_macro2::{TokenStream, Span}; -use quote::quote; +use quote::{quote, quote_spanned}; /// Generate the bare-function interface. pub fn generate(trait_def: &ItemTrait) -> Result { @@ -111,13 +114,19 @@ fn function_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result + #crate_::with_externalities( + |mut ext| #trait_name::#method_name(&mut ext, #( #arg_names, )*) + ).expect(#expect_msg) } } else { - quote! { + quote_spanned! { method.span() => <&mut dyn #crate_::Externalities<#crate_::Blake2Hasher> as #trait_name>::#method_name( #( #arg_names, )* ) @@ -125,7 +134,7 @@ fn function_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result #[cfg(feature = "std")] fn #function_name( #( #args, )* ) #return_value { #call_to_trait diff --git a/core/runtime-interface/src/externalities.rs b/core/runtime-interface/src/externalities.rs new file mode 100644 index 0000000000000..a26185becb4c5 --- /dev/null +++ b/core/runtime-interface/src/externalities.rs @@ -0,0 +1,40 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Stores the externalities in an `environmental` value to make it scope limited available. + +use primitives::{Blake2Hasher, traits::Externalities}; + +environmental::environmental!(ext: trait Externalities); + +/// Set the given externalities while executing the given closure. To get access to the externalities +/// while executing the given closure [`with_externalities`] grants access to them. The externalities +/// are only set for the same thread this function was called from. +pub fn set_and_run_with_externalities( + ext: &mut dyn Externalities, + f: F, +) -> R where F: FnOnce() -> R { + ext::using(ext, f) +} + +/// Execute the given closure with the currently set externalities. +/// +/// Returns `None` if no externalities are set or `Some(_)` with the result of the closure. +pub fn with_externalities, +) -> R, R>(f: F) -> Option { + ext::with(f) +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index cfcae088a6373..b181caf1d6e14 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -19,7 +19,7 @@ use rstd::{any::TypeId, borrow::Cow, mem}; #[cfg(feature = "std")] -use wasm_interface::{FunctionContext, IntoValue, TryFromValue, Pointer, Result}; +use wasm_interface::{FunctionContext, Pointer, Result}; use codec::{Encode, Decode}; @@ -37,10 +37,10 @@ pub use wasm_interface; pub use substrate_runtime_interface_proc_macro::runtime_interface; #[cfg(feature = "std")] -pub fn with_externalities) -> R, R>(f: F) -> R { - println!("HEY"); - unimplemented!() -} +pub use externalities::{set_and_run_with_externalities, with_externalities}; + +#[cfg(feature = "std")] +mod externalities; pub trait AsFFIArg { /// The owned rust type that converts into `Self::FFIType`. @@ -262,37 +262,42 @@ mod tests { use super::*; use test_wasm::{WASM_BINARY, test_api::HostFunctions}; use executor::WasmExecutor; + use wasm_interface::HostFunctions as HostFunctionsT; type TestExternalities = state_machine::TestExternalities; - #[test] - fn test_return_data() { + fn call_wasm_method(method: &str) -> TestExternalities { let mut ext = TestExternalities::default(); - let executor = WasmExecutor::::new(); + let executor = WasmExecutor::::new(); executor.call_with_custom_signature::<_, _, _, ()>( &mut ext, 8, &WASM_BINARY[..], - "test_return_data", + method, |_| Ok(Vec::new()), |res, _| if res.is_none() { Ok(Some(())) } else { Err("Invalid return value!".into()) }, - ).unwrap(); + ).expect(&format!("Executes `{}`", method)); + + ext + } + + #[test] + fn test_return_data() { + call_wasm_method::("test_return_data"); + } + + #[test] + fn test_set_storage() { + let ext = call_wasm_method::("test_set_storage"); + + let expected = "world"; + assert_eq!(expected.as_bytes(), &ext.storage("hello".as_bytes()).unwrap()[..]); } #[test] #[should_panic(expected = "Wasmi(Instantiation(\"Export ext_test_api_return_input not found\"))")] fn host_function_not_found() { - let mut ext = TestExternalities::default(); - let executor = ::new(); - - executor.call_with_custom_signature::<_, _, _, ()>( - &mut ext, - 8, - &WASM_BINARY[..], - "test_return_data", - |_| Ok(Vec::new()), - |res, _| Ok(None), - ).unwrap(); + call_wasm_method::<()>("test_return_data"); } } diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index df50df921c8c7..19f79a3760077 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -50,3 +50,11 @@ pub fn test_return_data() { assert_eq!(input, res); } + +#[no_mangle] +pub fn test_set_storage() { + let key = "hello"; + let value = "world"; + + test_api::set_storage(key.as_bytes(), value.as_bytes()); +} diff --git a/core/sr-io/Cargo.toml b/core/sr-io/Cargo.toml index f3122f0e3029f..67e0f066a3948 100644 --- a/core/sr-io/Cargo.toml +++ b/core/sr-io/Cargo.toml @@ -15,8 +15,8 @@ codec = { package = "parity-scale-codec", version = "1.0.0", default-features = hash-db = { version = "0.15.2", default-features = false } libsecp256k1 = { version = "0.2.1", optional = true } tiny-keccak = { version = "1.4.2", optional = true } -environmental = { version = "1.0.1", optional = true } substrate-state-machine = { path = "../state-machine", optional = true } +runtime-interface = { package = "substrate-runtime-interface", path = "../runtime-interface", optional = true } trie = { package = "substrate-trie", path = "../trie", optional = true } [dev-dependencies] @@ -30,10 +30,10 @@ std = [ "rstd/std", "hash-db/std", "trie", - "environmental", "substrate-state-machine", "libsecp256k1", - "tiny-keccak" + "tiny-keccak", + "runtime-interface", ] nightly = [] strict = [] diff --git a/core/sr-io/with_std.rs b/core/sr-io/with_std.rs index f93194bb47409..1a44b17120eb4 100644 --- a/core/sr-io/with_std.rs +++ b/core/sr-io/with_std.rs @@ -22,12 +22,11 @@ use primitives::{ // pub use primitives::BlakeHasher; pub use substrate_state_machine::{BasicExternalities, TestExternalities}; -use environmental::environmental; use trie::{TrieConfiguration, trie_types::Layout}; use std::{collections::HashMap, convert::TryFrom}; -environmental!(ext: trait Externalities); +use runtime_interface::with_externalities as with_ext; /// Additional bounds for `Hasher` trait for with_std. pub trait HasherBounds {} @@ -47,12 +46,12 @@ fn child_storage_key_or_panic(storage_key: &[u8]) -> ChildStorageKey { impl StorageApi for () { fn storage(key: &[u8]) -> Option> { - ext::with(|ext| ext.storage(key).map(|s| s.to_vec())) + with_ext(|ext| ext.storage(key).map(|s| s.to_vec())) .expect("storage cannot be called outside of an Externalities-provided environment.") } fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { - ext::with(|ext| ext.storage(key).map(|value| { + with_ext(|ext| ext.storage(key).map(|value| { let data = &value[value_offset.min(value.len())..]; let written = std::cmp::min(data.len(), value_out.len()); value_out[..written].copy_from_slice(&data[..written]); @@ -61,7 +60,7 @@ impl StorageApi for () { } fn child_storage(storage_key: &[u8], key: &[u8]) -> Option> { - ext::with(|ext| { + with_ext(|ext| { let storage_key = child_storage_key_or_panic(storage_key); ext.child_storage(storage_key, key).map(|s| s.to_vec()) }) @@ -69,7 +68,7 @@ impl StorageApi for () { } fn set_storage(key: &[u8], value: &[u8]) { - ext::with(|ext| + with_ext(|ext| ext.set_storage(key.to_vec(), value.to_vec()) ); } @@ -80,7 +79,7 @@ impl StorageApi for () { value_out: &mut [u8], value_offset: usize, ) -> Option { - ext::with(|ext| { + with_ext(|ext| { let storage_key = child_storage_key_or_panic(storage_key); ext.child_storage(storage_key, key) .map(|value| { @@ -94,73 +93,73 @@ impl StorageApi for () { } fn set_child_storage(storage_key: &[u8], key: &[u8], value: &[u8]) { - ext::with(|ext| { + with_ext(|ext| { let storage_key = child_storage_key_or_panic(storage_key); ext.set_child_storage(storage_key, key.to_vec(), value.to_vec()) }); } fn clear_storage(key: &[u8]) { - ext::with(|ext| + with_ext(|ext| ext.clear_storage(key) ); } fn clear_child_storage(storage_key: &[u8], key: &[u8]) { - ext::with(|ext| { + with_ext(|ext| { let storage_key = child_storage_key_or_panic(storage_key); ext.clear_child_storage(storage_key, key) }); } fn kill_child_storage(storage_key: &[u8]) { - ext::with(|ext| { + with_ext(|ext| { let storage_key = child_storage_key_or_panic(storage_key); ext.kill_child_storage(storage_key) }); } fn exists_storage(key: &[u8]) -> bool { - ext::with(|ext| + with_ext(|ext| ext.exists_storage(key) ).unwrap_or(false) } fn exists_child_storage(storage_key: &[u8], key: &[u8]) -> bool { - ext::with(|ext| { + with_ext(|ext| { let storage_key = child_storage_key_or_panic(storage_key); ext.exists_child_storage(storage_key, key) }).unwrap_or(false) } fn clear_prefix(prefix: &[u8]) { - ext::with(|ext| + with_ext(|ext| ext.clear_prefix(prefix) ); } fn clear_child_prefix(storage_key: &[u8], prefix: &[u8]) { - ext::with(|ext| { + with_ext(|ext| { let storage_key = child_storage_key_or_panic(storage_key); ext.clear_child_prefix(storage_key, prefix) }); } fn storage_root() -> [u8; 32] { - ext::with(|ext| + with_ext(|ext| ext.storage_root() ).unwrap_or(H256::zero()).into() } fn child_storage_root(storage_key: &[u8]) -> Vec { - ext::with(|ext| { + with_ext(|ext| { let storage_key = child_storage_key_or_panic(storage_key); ext.child_storage_root(storage_key) }).expect("child_storage_root cannot be called outside of an Externalities-provided environment.") } fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]> { - ext::with(|ext| + with_ext(|ext| ext.storage_changes_root(parent_hash.into()).map(|h| h.map(|h| h.into())) ).unwrap_or(Ok(None)).expect("Invalid parent hash passed to storage_changes_root") } @@ -176,7 +175,7 @@ impl StorageApi for () { impl OtherApi for () { fn chain_id() -> u64 { - ext::with(|ext| + with_ext(|ext| ext.chain_id() ).unwrap_or(0) } @@ -198,7 +197,7 @@ impl OtherApi for () { impl CryptoApi for () { fn ed25519_public_keys(id: KeyTypeId) -> Vec { - ext::with(|ext| { + with_ext(|ext| { ext.keystore() .expect("No `keystore` associated for the current context!") .read() @@ -207,7 +206,7 @@ impl CryptoApi for () { } fn ed25519_generate(id: KeyTypeId, seed: Option<&str>) -> ed25519::Public { - ext::with(|ext| { + with_ext(|ext| { ext.keystore() .expect("No `keystore` associated for the current context!") .write() @@ -223,7 +222,7 @@ impl CryptoApi for () { ) -> Option { let pub_key = ed25519::Public::try_from(pubkey.as_ref()).ok()?; - ext::with(|ext| { + with_ext(|ext| { ext.keystore() .expect("No `keystore` associated for the current context!") .read() @@ -237,7 +236,7 @@ impl CryptoApi for () { } fn sr25519_public_keys(id: KeyTypeId) -> Vec { - ext::with(|ext| { + with_ext(|ext| { ext.keystore() .expect("No `keystore` associated for the current context!") .read() @@ -246,7 +245,7 @@ impl CryptoApi for () { } fn sr25519_generate(id: KeyTypeId, seed: Option<&str>) -> sr25519::Public { - ext::with(|ext| { + with_ext(|ext| { ext.keystore() .expect("No `keystore` associated for the current context!") .write() @@ -262,7 +261,7 @@ impl CryptoApi for () { ) -> Option { let pub_key = sr25519::Public::try_from(pubkey.as_ref()).ok()?; - ext::with(|ext| { + with_ext(|ext| { ext.keystore() .expect("No `keystore` associated for the current context!") .read() @@ -315,7 +314,7 @@ impl HashingApi for () { } fn with_offchain(f: impl FnOnce(&mut dyn offchain::Externalities) -> R, msg: &'static str) -> R { - ext::with(|ext| ext + with_ext(|ext| ext .offchain() .map(|ext| f(ext)) .expect(msg) @@ -446,7 +445,7 @@ impl Api for () {} /// externalities `ext`. Forwards the value that the closure returns. // NOTE: need a concrete hasher here due to limitations of the `environmental!` macro, otherwise a type param would have been fine I think. pub fn with_externalities R>(ext: &mut dyn Externalities, f: F) -> R { - ext::using(ext, f) + unimplemented!() } /// A set of key value pairs for storage. @@ -466,11 +465,13 @@ pub fn with_storage R>( rstd::mem::swap(&mut alt_storage, storage); let mut ext = BasicExternalities::new(alt_storage.0, alt_storage.1); - let r = ext::using(&mut ext, f); + // let r = ext::using(&mut ext, f); - *storage = ext.into_storages(); + // *storage = ext.into_storages(); - r + // r + + unimplemented!() } #[cfg(test)] From 8d88117a070217ca4a5703f8272c09a905b0d2f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 26 Sep 2019 11:10:28 +0200 Subject: [PATCH 13/76] Add test for mutable references --- core/runtime-interface/src/lib.rs | 5 +++++ core/runtime-interface/test-wasm/src/lib.rs | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index b181caf1d6e14..b457139efa559 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -295,6 +295,11 @@ mod tests { assert_eq!(expected.as_bytes(), &ext.storage("hello".as_bytes()).unwrap()[..]); } + #[test] + fn test_return_value_into_mutable_reference() { + call_wasm_method::("test_return_value_into_mutable_reference"); + } + #[test] #[should_panic(expected = "Wasmi(Instantiation(\"Export ext_test_api_return_input not found\"))")] fn host_function_not_found() { diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index 19f79a3760077..4bca7462f56ed 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -58,3 +58,13 @@ pub fn test_set_storage() { test_api::set_storage(key.as_bytes(), value.as_bytes()); } + +#[no_mangle] +pub fn test_return_value_into_mutable_reference() { + let mut data = vec![1, 2, 3, 4, 5, 6]; + + test_api::return_value_into_mutable_reference(&mut data); + + let expected = "hello"; + assert_eq!(expected.as_bytes(), &data[..expected.len()]); +} From d17cf4fb82cb8b745f0ddca7bb856e8d7789f936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 26 Sep 2019 15:57:24 +0200 Subject: [PATCH 14/76] Code cleanup and documentation etc --- Cargo.lock | 7 + core/runtime-interface/Cargo.toml | 1 + .../proc-macro/src/bare_function_interface.rs | 12 +- .../proc-macro/src/host_function_interface.rs | 27 +- core/runtime-interface/src/host.rs | 130 ++++++++++ core/runtime-interface/src/lib.rs | 239 +++--------------- core/runtime-interface/src/wasm.rs | 159 ++++++++++++ 7 files changed, 351 insertions(+), 224 deletions(-) create mode 100644 core/runtime-interface/src/host.rs create mode 100644 core/runtime-interface/src/wasm.rs diff --git a/Cargo.lock b/Cargo.lock index f7e91e8b95cd8..788281a59c21c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4429,6 +4429,11 @@ name = "static_assertions" version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "static_assertions" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "static_slice" version = "0.0.3" @@ -5234,6 +5239,7 @@ dependencies = [ "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", + "static_assertions 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-executor 2.0.0", "substrate-primitives 2.0.0", "substrate-runtime-interface-proc-macro 2.0.0", @@ -6886,6 +6892,7 @@ dependencies = [ "checksum spin 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44363f6f51401c34e7be73db0db371c04705d35efbe9f7d6082e03a921a32c55" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" "checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5" +"checksum static_assertions 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3" "checksum static_slice 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "92a7e0c5e3dfb52e8fbe0e63a1b947bbb17b4036408b151353c4491374931362" "checksum stream-cipher 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8861bc80f649f5b4c9bd38b696ae9af74499d479dbfb327f0607de6b326a36bc" "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index 5dc5287d1f90f..c932dd4c8c9a0 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -11,6 +11,7 @@ primitives = { package = "substrate-primitives", path = "../primitives", default substrate-runtime-interface-proc-macro = { path = "proc-macro" } codec = { package = "parity-scale-codec", version = "1.0.5", default-features = false } environmental = { version = "1.0.2", optional = true } +static_assertions = "0.3.4" [dev-dependencies] executor = { package = "substrate-executor", path = "../executor" } diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index d569fa0145e45..521b17b235019 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -82,7 +82,7 @@ fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result< let convert_return_value = match return_value { ReturnType::Default => quote!(), ReturnType::Type(_, ref ty) => quote! { - <#ty as #crate_::FromFFIArg<_>>::from_ffi_arg(result) + <#ty as #crate_::wasm::FromFFIValue>::from_ffi_value(result) } }; @@ -90,15 +90,17 @@ fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result< quote! { #[cfg(not(feature = "std"))] fn #function_name( #( #args, )* ) #return_value { - use #crate_::IntoFFIArg as _; + use #crate_::wasm::IntoFFIValue as _; - // Generate all ffi arg wrappers. + // Generate all wrapped ffi value. #( - let #arg_names = <#arg_types as #crate_::AsFFIArg>::as_ffi_arg(&#arg_names); + let #arg_names = <#arg_types as #crate_::wasm::AsWrappedFFIValue>::as_wrapped_ffi_value( + &#arg_names, + ); )* // Call the host function - let result = unsafe { #host_function_name( #( #arg_names2.into_ffi_arg(), )* ) }; + let result = unsafe { #host_function_name( #( #arg_names2.into_ffi_value(), )* ) }; #convert_return_value } diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index b5659bd0e2afb..0b48c41c652a0 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -32,6 +32,7 @@ use proc_macro2::{TokenStream, Span}; use quote::{quote, ToTokens}; use inflector::Inflector; + use std::iter::Iterator; /// Generate the extern host functions for wasm and the `HostFunctions` struct that provides the @@ -69,7 +70,7 @@ fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) - let output = match method.sig.output { ReturnType::Default => quote!(), ReturnType::Type(_, ref ty) => quote! { - -> <#ty as #crate_::AsFFIArg>::FFIType + -> <#ty as #crate_::RIType>::FFIType } }; @@ -78,7 +79,7 @@ fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) - #[cfg(not(feature = "std"))] extern "C" { pub fn #function ( - #( #arg_names: <#arg_types as #crate_::AsFFIArg>::FFIType ),* + #( #arg_names: <#arg_types as #crate_::RIType>::FFIType ),* ) #output; } } @@ -143,7 +144,7 @@ fn generate_host_function_implementation( let wasm_to_ffi_values = generate_wasm_to_ffi_values(&method.sig).collect::>>()?; let ffi_to_host_values = generate_ffi_to_host_value(&method.sig).collect::>>()?; let host_function_call = generate_host_function_call(&method.sig); - let into_preallocated_ffi_arg = generate_into_preallocated_ffi_arg(&method.sig)?; + let into_preallocated_ffi_value = generate_into_preallocated_ffi_value(&method.sig)?; let convert_return_value = generate_return_value_into_wasm_value(&method.sig); Ok( @@ -169,7 +170,7 @@ fn generate_host_function_implementation( #( #wasm_to_ffi_values )* #( #ffi_to_host_values )* #host_function_call - #into_preallocated_ffi_arg + #into_preallocated_ffi_value #convert_return_value } } @@ -186,13 +187,13 @@ fn generate_wasm_interface_signature_for_host_function(sig: &Signature) -> Resul let return_value = match &sig.output { ReturnType::Type(_, ty) => quote! { - Some( <<#ty as #crate_::AsFFIArg>::FFIType as #crate_::wasm_interface::IntoValue>::VALUE_TYPE ) + Some( <<#ty as #crate_::RIType>::FFIType as #crate_::wasm_interface::IntoValue>::VALUE_TYPE ) }, ReturnType::Default => quote!( None ), }; let arg_types = get_function_argument_types_without_ref(sig) .map(|ty| quote! { - <<#ty as #crate_::AsFFIArg>::FFIType as #crate_::wasm_interface::IntoValue>::VALUE_TYPE + <<#ty as #crate_::RIType>::FFIType as #crate_::wasm_interface::IntoValue>::VALUE_TYPE }); Ok( @@ -230,13 +231,13 @@ fn generate_wasm_to_ffi_values<'a>( Ok(quote! { let val = args.next().ok_or_else(|| #error_message)?; let #var_name = < - <#ty as #crate_::AsFFIArg>::FFIType as #crate_::wasm_interface::TryFromValue + <#ty as #crate_::RIType>::FFIType as #crate_::wasm_interface::TryFromValue >::try_from_value(val).ok_or_else(|| #try_from_error)?; }) }) } -/// Generate the code to convert the ffi values on the host to the host values using `FromWasmFFIArg`. +/// Generate the code to convert the ffi values on the host to the host values using `FromFFIValue`. fn generate_ffi_to_host_value<'a>( sig: &'a Signature, ) -> impl Iterator> + 'a { @@ -250,7 +251,7 @@ fn generate_ffi_to_host_value<'a>( Ok( quote! { - let #mut_access #name = <#ty as #crate_::FromWasmFFIArg<_>>::from_wasm_ffi_arg( + let #mut_access #name = <#ty as #crate_::host::FromFFIValue>::from_ffi_value( context, #ffi_value_var_name, )?; @@ -301,8 +302,8 @@ fn generate_ffi_value_var_name(pat: &Pat) -> Result { /// Generate code that copies data from the host back to preallocated wasm memory. /// /// Any argument that is given as `&mut` is interpreted as preallocated memory and it is expected -/// that the type implements `IntoPreAllocatedWasmFFIArg`. -fn generate_into_preallocated_ffi_arg(sig: &Signature) -> Result { +/// that the type implements `IntoPreAllocatedFFIValue`. +fn generate_into_preallocated_ffi_value(sig: &Signature) -> Result { let crate_ = generate_crate_access(); let ref_and_mut = get_function_argument_types_ref_and_mut(sig).map(|ram| ram.and_then(|(vr, vm)| vm.map(|v| (vr, v))) @@ -316,7 +317,7 @@ fn generate_into_preallocated_ffi_arg(sig: &Signature) -> Result { Ok( quote! { - <#ty as #crate_::IntoPreAllocatedWasmFFIArg<_>>::into_wasm_ffi_arg( + <#ty as #crate_::host::IntoPreallocatedFFIValue>::into_preallocated_ffi_value( #name, context, #ffi_var_name, @@ -337,7 +338,7 @@ fn generate_return_value_into_wasm_value(sig: &Signature) -> TokenStream { let result_var_name = generate_host_function_result_var_name(&sig.ident); quote! { - <#ty as #crate_::IntoWasmFFIArg<_>>::into_wasm_ffi_arg(#result_var_name, context) + <#ty as #crate_::host::IntoFFIValue>::into_ffi_value(#result_var_name, context) .map(#crate_::wasm_interface::IntoValue::into_value) .map(Some) } diff --git a/core/runtime-interface/src/host.rs b/core/runtime-interface/src/host.rs new file mode 100644 index 0000000000000..8e9197301de61 --- /dev/null +++ b/core/runtime-interface/src/host.rs @@ -0,0 +1,130 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Traits required by the runtime interface from the host side. + +use crate::{RIType, pointer_and_len_from_u64, pointer_and_len_to_u64}; + +use wasm_interface::{FunctionContext, Pointer, Result}; + +use rstd::{any::TypeId, borrow::Cow, mem}; + +use codec::{Encode, Decode}; + +/// Something that can be converted into a ffi value. +pub trait IntoFFIValue: RIType { + /// Convert `self` into a ffi value. + fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result; +} + +/// Something that can be converted into a preallocated ffi value. +/// +/// Every type parameter that should be given as `&mut` into a runtime interface function, needs +/// to implement this trait. After executing the host implementation of the runtime interface +/// function, the value is copied into the preallocated wasm memory. +/// +/// This should only be used for types which have a fixed size, like slices. Other types like a vec +/// do not work with this interface, as we can not call into wasm to reallocate memory. So, this +/// trait should be implemented carefully. +pub trait IntoPreallocatedFFIValue: RIType { + /// As `Self` can be an unsized type, it needs to be represented by a sized type at the host. + /// This `SelfInstance` is the sized type. + type SelfInstance; + + /// Convert `self_instance` into the given preallocated ffi value. + fn into_preallocated_ffi_value( + self_instance: Self::SelfInstance, + context: &mut dyn FunctionContext, + allocated: Self::FFIType, + ) -> Result<()>; +} + +/// Something that can be created from a ffi value. +pub trait FromFFIValue: RIType { + /// As `Self` can be an unsized type, it needs to be represented by a sized type at the host. + /// This `SelfInstance` is the sized type. + type SelfInstance; + + /// Create `SelfInstance` from the given + fn from_ffi_value( + context: &mut dyn FunctionContext, + arg: Self::FFIType, + ) -> Result; +} + +impl IntoFFIValue for Vec { + fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result { + let vec: Cow<'_, [u8]> = if TypeId::of::() == TypeId::of::() { + unsafe { Cow::Borrowed(std::mem::transmute(&self[..])) } + } else { + Cow::Owned(self.encode()) + }; + + let ptr = context.allocate_memory(vec.as_ref().len() as u32)?; + context.write_memory(ptr, &vec)?; + + Ok(pointer_and_len_to_u64(ptr.into(), vec.len() as u32)) + } +} + +impl FromFFIValue for Vec { + type SelfInstance = Vec; + + fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result> { + <[T] as FromFFIValue>::from_ffi_value(context, arg) + } +} + +impl FromFFIValue for [T] { + type SelfInstance = Vec; + + fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result> { + let (ptr, len) = pointer_and_len_from_u64(arg); + + let vec = context.read_memory(Pointer::new(ptr), len)?; + + if TypeId::of::() == TypeId::of::() { + Ok(unsafe { mem::transmute(vec) }) + } else { + Ok(Vec::::decode(&mut &vec[..]).expect("Wasm to host values are encoded correctly; qed")) + } + } +} + +impl IntoPreallocatedFFIValue for [u8] { + type SelfInstance = Vec; + + fn into_preallocated_ffi_value( + self_instance: Self::SelfInstance, + context: &mut dyn FunctionContext, + allocated: u64, + ) -> Result<()> { + let (ptr, len) = pointer_and_len_from_u64(allocated); + + if (len as usize) < self_instance.len() { + Err( + format!( + "Preallocated buffer is not big enough (given {} vs needed {})!", + len, + self_instance.len() + ) + ) + } else { + context.write_memory(Pointer::new(ptr), &self_instance) + } + } +} + diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index b457139efa559..20c1bdce6d9f4 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -15,13 +15,14 @@ // along with Substrate. If not, see . //! Traits and macros for creating interfaces between the runtime and the node. - -use rstd::{any::TypeId, borrow::Cow, mem}; - -#[cfg(feature = "std")] -use wasm_interface::{FunctionContext, Pointer, Result}; - -use codec::{Encode, Decode}; +//! +//! To make the communication between wasm and the host as fast as possible, we use a two-way +//! conversion strategy. First, we convert the value into a [`WrappedFFIValue`]. +//! This value stores a reference or an owned value that are convertible into the actual ffi value. +//! So, for values like `Vec` we just store a reference to the underlying slice and don't need +//! to allocate any memory. In the second step we get the actual ffi value from this +//! [`WrappedFFIValue`] to call into the host. The created [`WrappedFFIValue`] will remain on +//! the stack while we call into the host. #[doc(hidden)] pub use primitives::Blake2Hasher; @@ -41,220 +42,46 @@ pub use externalities::{set_and_run_with_externalities, with_externalities}; #[cfg(feature = "std")] mod externalities; - -pub trait AsFFIArg { - /// The owned rust type that converts into `Self::FFIType`. - type RTOwned: IntoFFIArg; - /// The borrowed rust type that converts into `Self::FFIType`. - type RTBorrowed: ?Sized + IntoFFIArg; - type FFIType; - - fn as_ffi_arg<'a>(&'a self) -> FFIArg<'a, Self::FFIType, Self::RTOwned, Self::RTBorrowed>; -} - -pub trait FromFFIArg: Sized { - fn from_ffi_arg(arg: T) -> Self; -} - -#[cfg(feature = "std")] -pub trait FromWasmFFIArg { - /// The `Self` instance returned by `from_wasm_ffi_arg`. - /// - /// For types that are do not implement `Sized` we can not return `Self`. So, we need to use a - /// wrapper type that stores `Self`. - type SelfInstance; - - fn from_wasm_ffi_arg(context: &mut dyn FunctionContext, arg: T) -> Result; -} - -pub trait IntoFFIArg { - fn into_ffi_arg(&self) -> T; -} - -#[cfg(feature = "std")] -pub trait IntoWasmFFIArg { - fn into_wasm_ffi_arg(self, context: &mut dyn FunctionContext) -> Result; -} - #[cfg(feature = "std")] -pub trait IntoPreAllocatedWasmFFIArg { - type SelfInstance; - - fn into_wasm_ffi_arg( - self_instance: Self::SelfInstance, - context: &mut dyn FunctionContext, - allocated: T, - ) -> Result<()>; -} - -pub enum FFIArg<'a, T, O, R: ?Sized = O> where O: IntoFFIArg, R: IntoFFIArg { - Ref(&'a R, std::marker::PhantomData), - Owned(O), -} - -impl<'a, T, O: IntoFFIArg, R: ?Sized + IntoFFIArg> FFIArg<'a, T, O, R> { - pub fn from_owned(o: O) -> Self { - Self::Owned(o) - } - - pub fn from_ref(r: &'a R) -> Self { - Self::Ref(r, Default::default()) - } - - pub fn into_ffi_arg(&self) -> T { - match self { - Self::Ref(data, _) => data.into_ffi_arg(), - Self::Owned(ref data) => data.into_ffi_arg(), - } - } +pub mod host; +#[cfg(not(feature = "std"))] +pub mod wasm; + +/// Something that can be used by the runtime interface as type to communicate between wasm and the host. +/// +/// Every type that should be used as in a runtime interface function needs to implement this trait. +pub trait RIType { + /// The ffi type that is used to represent `Self`. + #[cfg(feature = "std")] + type FFIType: wasm_interface::IntoValue + wasm_interface::TryFromValue; + #[cfg(not(feature = "std"))] + type FFIType; } -impl AsFFIArg for u32 { - type RTOwned = u32; - type RTBorrowed = u32; +impl RIType for u32 { type FFIType = u32; - - fn as_ffi_arg<'a>(&'a self) -> FFIArg<'a, u32, u32> { - FFIArg::from_owned(*self) - } -} - -impl IntoFFIArg for u32 { - fn into_ffi_arg(&self) -> u32 { - self.to_le() - } } -impl FromFFIArg for u32 { - fn from_ffi_arg(arg: u32) -> u32 { - u32::from_le(arg) - } -} - -impl> IntoFFIArg for T { - fn into_ffi_arg(&self) -> u64 { - let data = self.as_ref(); - - let ptr_address = data.as_ptr() as u64; - - ((data.len() as u64) | ptr_address << 32).to_le() - } -} - -impl AsFFIArg for [T] { - type RTOwned = Vec; - type RTBorrowed = [u8]; +impl RIType for Vec { type FFIType = u64; - - fn as_ffi_arg<'a>(&'a self) -> FFIArg<'a, u64, Vec, [u8]> { - if TypeId::of::() == TypeId::of::() { - let transmuted = unsafe { mem::transmute::<&[T], &[u8]>(self) }; - FFIArg::from_ref(transmuted) - } else { - FFIArg::from_owned(self.encode()) - } - } } -impl AsFFIArg for Vec { - type RTOwned = Vec; - type RTBorrowed = [u8]; +impl RIType for [T] { type FFIType = u64; - - fn as_ffi_arg<'a>(&'a self) -> FFIArg<'a, u64, Vec, [u8]> { - if TypeId::of::() == TypeId::of::() { - let transmuted = unsafe { mem::transmute::<&[T], &[u8]>(&self[..]) }; - FFIArg::from_ref(transmuted) - } else { - FFIArg::from_owned(self.encode()) - } - } } -impl FromFFIArg for Vec { - fn from_ffi_arg(arg: u64) -> Vec { - let arg = u64::from_le(arg); - let len: usize = (arg & (!0u32 as u64)) as usize; - let ptr: usize = (arg >> 32) as usize; - - if TypeId::of::() == TypeId::of::() { - unsafe { std::mem::transmute(Vec::from_raw_parts(ptr as *mut u8, len, len)) } - } else { - let slice = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; - Self::decode(&mut &slice[..]).expect("Host to Wasm values are encoded correctly; qed") - } - } +/// Converts a pointer and length into an `u64`. +fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { + ((len as u64) | u64::from(ptr) << 32).to_le() } -#[cfg(feature = "std")] -impl IntoWasmFFIArg for Vec { - fn into_wasm_ffi_arg(self, context: &mut dyn FunctionContext) -> Result { - let vec: Cow<'_, [u8]> = if TypeId::of::() == TypeId::of::() { - unsafe { Cow::Borrowed(std::mem::transmute(&self[..])) } - } else { - Cow::Owned(self.encode()) - }; - - let ptr = context.allocate_memory(vec.as_ref().len() as u32)?; - context.write_memory(ptr, &vec)?; +/// Splits an `u64` into the pointer and length. +fn pointer_and_len_from_u64(val: u64) -> (u32, u32) { + let val = u64::from_le(val); + let len = (val & (!0u32 as u64)) as u32; + let ptr = (val >> 32) as u32; - Ok((vec.len() as u64) | u64::from(ptr) << 32) - } -} - -#[cfg(feature = "std")] -impl FromWasmFFIArg for Vec { - type SelfInstance = Vec; - - fn from_wasm_ffi_arg(context: &mut dyn FunctionContext, arg: u64) -> Result> { - <[T] as FromWasmFFIArg>::from_wasm_ffi_arg(context, arg) - } -} - -#[cfg(feature = "std")] -impl FromWasmFFIArg for [T] { - type SelfInstance = Vec; - - fn from_wasm_ffi_arg(context: &mut dyn FunctionContext, arg: u64) -> Result> { - let arg = u64::from_le(arg); - let len = (arg & (!0u32 as u64)) as u32; - let ptr = (arg >> 32) as u32; - - let vec = context.read_memory(Pointer::new(ptr), len)?; - - if TypeId::of::() == TypeId::of::() { - Ok(unsafe { mem::transmute(vec) }) - } else { - Ok(Vec::::decode(&mut &vec[..]).expect("Wasm to Host values are encoded correctly; qed")) - } - } -} - -#[cfg(feature = "std")] -impl IntoPreAllocatedWasmFFIArg for [u8] { - type SelfInstance = Vec; - - fn into_wasm_ffi_arg( - self_instance: Self::SelfInstance, - context: &mut dyn FunctionContext, - allocated: u64, - ) -> Result<()> { - let arg = u64::from_le(allocated); - let len = (arg & (!0u32 as u64)) as u32; - let ptr = (arg >> 32) as u32; - - if (len as usize) < self_instance.len() { - Err( - format!( - "Preallocated buffer is not big enough (given {} vs needed {})!", - len, - self_instance.len() - ) - ) - } else { - context.write_memory(Pointer::new(ptr), &self_instance) - } - } + (ptr, len) } #[cfg(test)] diff --git a/core/runtime-interface/src/wasm.rs b/core/runtime-interface/src/wasm.rs new file mode 100644 index 0000000000000..ffd7902a136f9 --- /dev/null +++ b/core/runtime-interface/src/wasm.rs @@ -0,0 +1,159 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Traits required by the runtime interface from the wasm side. + +use crate::{RIType, pointer_and_len_to_u64, pointer_and_len_from_u64}; + +use rstd::{any::TypeId, mem, marker::PhantomData}; + +use codec::{Encode, Decode}; + +use static_assertions::assert_eq_size; + +/// Something that can be converted into a [`WrappedFFIValue`]. +pub trait AsWrappedFFIValue: RIType { + /// The owned rust type that converts into `Self::FFIType`. + type RTOwned; + /// The borrowed rust type that converts into `Self::FFIType`. + type RTBorrowed: ?Sized; + + /// Returns `self` as a [`WrappedFFIValue`] that can be converted into `Self::FFIType`. + fn as_wrapped_ffi_value<'a>( + &'a self, + ) -> WrappedFFIValue<'a, Self::FFIType, Self::RTOwned, Self::RTBorrowed>; +} + +/// Something that can be created from a ffi value. +pub trait FromFFIValue: Sized + RIType { + /// Create `Self` from the given ffi value. + fn from_ffi_value(arg: Self::FFIType) -> Self; +} + +/// Something that can be converted into a ffi value. +pub trait IntoFFIValue: RIType { + /// Convert `self` into a ffi value. + fn into_ffi_value(&self) -> Self::FFIType; +} + +/// Represents a wrapped ffi value that either holds a reference or an owned value of a rust type. +/// +/// The reference and the owned value need to be convertible into the same ffi value. +pub enum WrappedFFIValue<'a, T, O, R: ?Sized = O> { + Ref(&'a R, PhantomData), + Owned(O), +} + +impl<'a, T, O, R> WrappedFFIValue<'a, T, O, R> +where + O: IntoFFIValue, + R: ?Sized + IntoFFIValue, +{ + /// Create `Self` from an owned value. + pub fn from_owned(o: O) -> Self { + Self::Owned(o) + } + + /// Create `Self` from a reference value. + pub fn from_ref(r: &'a R) -> Self { + Self::Ref(r, Default::default()) + } + + /// Convert into the ffi value. + pub fn into_ffi_value(&self) -> T { + match self { + Self::Ref(data, _) => data.into_ffi_value(), + Self::Owned(ref data) => data.into_ffi_value(), + } + } +} + +impl AsWrappedFFIValue for u32 { + type RTOwned = u32; + type RTBorrowed = u32; + + fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u32, u32> { + WrappedFFIValue::from_owned(*self) + } +} + +impl IntoFFIValue for u32 { + fn into_ffi_value(&self) -> u32 { + self.to_le() + } +} + +impl FromFFIValue for u32 { + fn from_ffi_value(arg: u32) -> u32 { + u32::from_le(arg) + } +} + +// Make sure that our assumptions for storing a pointer + its size in `u64` is valid. +assert_eq_size!(usize_check; usize, u32); +assert_eq_size!(ptr_check; *const u8, u32); + +impl IntoFFIValue for [u8] { + fn into_ffi_value(&self) -> u64 { + let data = self.as_ref(); + let ptr_address = data.as_ptr() as u32; + + pointer_and_len_to_u64(ptr_address, data.len() as u32) + } +} + +impl IntoFFIValue for Vec { + fn into_ffi_value(&self) -> u64 { + self[..].into_ffi_value() + } +} + +impl AsWrappedFFIValue for [T] { + type RTOwned = Vec; + type RTBorrowed = [u8]; + + fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { + if TypeId::of::() == TypeId::of::() { + WrappedFFIValue::from_ref(unsafe { mem::transmute::<&[T], &[u8]>(self) }) + } else { + WrappedFFIValue::from_owned(self.encode()) + } + } +} + +impl AsWrappedFFIValue for Vec { + type RTOwned = Vec; + type RTBorrowed = [u8]; + + fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { + self[..].as_wrapped_ffi_value() + } +} + +impl FromFFIValue for Vec { + fn from_ffi_value(arg: u64) -> Vec { + let (ptr, len) = pointer_and_len_from_u64(arg); + let len = len as usize; + + if TypeId::of::() == TypeId::of::() { + unsafe { std::mem::transmute(Vec::from_raw_parts(ptr as *mut u8, len, len)) } + } else { + let slice = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; + Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") + } + } +} + From fcb6498b994a4f0639697a0db87546e6e368706e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 26 Sep 2019 20:22:14 +0200 Subject: [PATCH 15/76] Add marker trait for types that should be passed as SCALE encoded --- core/runtime-interface/src/host.rs | 21 +++++++++++++- core/runtime-interface/src/lib.rs | 31 ++++++++++++++++++++- core/runtime-interface/src/wasm.rs | 20 ++++++++++++- core/runtime-interface/test-wasm/src/lib.rs | 13 +++++++++ 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/core/runtime-interface/src/host.rs b/core/runtime-interface/src/host.rs index 8e9197301de61..d44973aee7dec 100644 --- a/core/runtime-interface/src/host.rs +++ b/core/runtime-interface/src/host.rs @@ -16,7 +16,7 @@ //! Traits required by the runtime interface from the host side. -use crate::{RIType, pointer_and_len_from_u64, pointer_and_len_to_u64}; +use crate::{RIType, pointer_and_len_from_u64, pointer_and_len_to_u64, PassedAsEncoded}; use wasm_interface::{FunctionContext, Pointer, Result}; @@ -128,3 +128,22 @@ impl IntoPreallocatedFFIValue for [u8] { } } +impl IntoFFIValue for T { + fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result { + let vec = self.encode(); + let ptr = context.allocate_memory(vec.len() as u32)?; + context.write_memory(ptr, &vec)?; + + Ok(pointer_and_len_to_u64(ptr.into(), vec.len() as u32)) + } +} + +impl FromFFIValue for T { + type SelfInstance = Self; + + fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result { + let (ptr, len) = pointer_and_len_from_u64(arg); + let vec = context.read_memory(Pointer::new(ptr), len)?; + Ok(Self::decode(&mut &vec[..]).expect("Wasm to host values are encoded correctly; qed")) + } +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 20c1bdce6d9f4..313fc363a4361 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -27,6 +27,8 @@ #[doc(hidden)] pub use primitives::Blake2Hasher; +use codec::Codec; + #[doc(hidden)] #[cfg(feature = "std")] pub use primitives::traits::Externalities; @@ -49,7 +51,8 @@ pub mod wasm; /// Something that can be used by the runtime interface as type to communicate between wasm and the host. /// -/// Every type that should be used as in a runtime interface function needs to implement this trait. +/// Every type that should be used in a runtime interface function signature needs to implement +/// this trait. pub trait RIType { /// The ffi type that is used to represent `Self`. #[cfg(feature = "std")] @@ -70,6 +73,27 @@ impl RIType for [T] { type FFIType = u64; } +/// Marker trait for types that should be passed to between wasm and the host by using SCALE codec. +/// +/// # Example +/// ``` +/// # use substrate_runtime_interface::PassedAsEncoded; +/// #[derive(codec::Encode, codec::Decode)] +/// struct Test; +/// +/// // It is sufficient to implement the trait for the desired type. +/// impl PassedAsEncoded for Test {} +/// ``` +pub trait PassedAsEncoded: Codec {} + +impl RIType for T { + type FFIType = u64; +} + +impl PassedAsEncoded for Option {} + +impl PassedAsEncoded for Result {} + /// Converts a pointer and length into an `u64`. fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { ((len as u64) | u64::from(ptr) << 32).to_le() @@ -114,6 +138,11 @@ mod tests { call_wasm_method::("test_return_data"); } + #[test] + fn test_return_option_data() { + call_wasm_method::("test_return_option_data"); + } + #[test] fn test_set_storage() { let ext = call_wasm_method::("test_set_storage"); diff --git a/core/runtime-interface/src/wasm.rs b/core/runtime-interface/src/wasm.rs index ffd7902a136f9..ab28eed96534c 100644 --- a/core/runtime-interface/src/wasm.rs +++ b/core/runtime-interface/src/wasm.rs @@ -16,7 +16,7 @@ //! Traits required by the runtime interface from the wasm side. -use crate::{RIType, pointer_and_len_to_u64, pointer_and_len_from_u64}; +use crate::{RIType, pointer_and_len_to_u64, pointer_and_len_from_u64, PassedAsEncoded}; use rstd::{any::TypeId, mem, marker::PhantomData}; @@ -157,3 +157,21 @@ impl FromFFIValue for Vec { } } +impl AsWrappedFFIValue for T { + type RTOwned = Vec; + type RTBorrowed = [u8]; + + fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { + WrappedFFIValue::from_owned(self.encode()) + } +} + +impl FromFFIValue for T { + fn from_ffi_value(arg: u64) -> Self { + let (ptr, len) = pointer_and_len_from_u64(arg); + let len = len as usize; + + let slice = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; + Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") + } +} diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index 4bca7462f56ed..bb59426902c0b 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -41,6 +41,11 @@ trait TestApi { let res = "hello"; data[..res.as_bytes().len()].copy_from_slice(res.as_bytes()); } + + /// Returns the input data wrapped in an `Option` as result. + fn return_option_input(data: Vec) -> Option> { + Some(data) + } } #[no_mangle] @@ -51,6 +56,14 @@ pub fn test_return_data() { assert_eq!(input, res); } +#[no_mangle] +pub fn test_return_option_data() { + let input = vec![1, 2, 3, 4, 5, 6]; + let res = test_api::return_option_input(input.clone()); + + assert_eq!(Some(input), res); +} + #[no_mangle] pub fn test_set_storage() { let key = "hello"; From 7c20e343f25e4a3b08702851c07bbfd69697077c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 26 Sep 2019 20:48:04 +0200 Subject: [PATCH 16/76] Inherit the visibility from the trait and more improvements --- .../proc-macro/src/host_function_interface.rs | 3 ++- core/runtime-interface/proc-macro/src/lib.rs | 3 ++- .../proc-macro/src/trait_decl_impl.rs | 16 ++++++++++++---- core/runtime-interface/test-wasm/src/lib.rs | 4 +++- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 0b48c41c652a0..787b1b05b38c4 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -14,7 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Generates the extern host function for wasm and the host functions interface for native. +//! Generates the extern host function declarations as well as the implementation for these host +//! functions. The implementation of these host functions will call the native bare functions. use crate::utils::{ generate_crate_access, create_host_function_ident, get_function_argument_names, diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index 5dac0ecd79333..b908cf37c1593 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -49,9 +49,10 @@ fn runtime_interface_impl(trait_def: ItemTrait) -> Result { let mod_name = Ident::new(&trait_def.ident.to_string().to_snake_case(), Span::call_site()); let trait_decl_impl = trait_decl_impl::process(&trait_def)?; let host_functions = host_function_interface::generate(&trait_def)?; + let vis = trait_def.vis; let res = quote! { - pub mod #mod_name { + #vis mod #mod_name { use super::*; #crate_include diff --git a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs index 33699393c9f60..8a50ff40c1ab6 100644 --- a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs +++ b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs @@ -16,19 +16,19 @@ //! Checks the trait declaration, folds and implements it. -use crate::utils::generate_crate_access; +use crate::utils::{generate_crate_access, get_function_argument_types_without_ref}; use syn::{ ItemTrait, TraitItemMethod, Result, TraitItem, Error, fold::{self, Fold}, spanned::Spanned, - Visibility, Receiver + Visibility, Receiver, Type, }; use proc_macro2::TokenStream; use quote::quote; -/// Process the given trait definition, by checking that the definition is valid, fold and -/// implement it. +/// Process the given trait definition, by checking that the definition is valid, fold it to the +/// essential definition and implement this essential definition for `dyn Externalities`. pub fn process(trait_def: &ItemTrait) -> Result { let impl_trait = impl_trait_for_externalities(trait_def)?; let essential_trait_def = ToEssentialTraitDef::convert(trait_def.clone())?; @@ -81,6 +81,14 @@ impl Fold for ToEssentialTraitDef { self.push_error(&method, "Methods need to have an implementation"); } + let arg_types = get_function_argument_types_without_ref(&method.sig); + arg_types.filter_map(|ty| + match &**ty { + Type::ImplTrait(impl_trait) => Some(impl_trait), + _ => None + } + ).for_each(|invalid| self.push_error(invalid, "`impl Trait` syntax not supported.")); + method } diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index bb59426902c0b..e930250b486b7 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +//! Tests for the runtime interface traits and proc macros. + #![cfg_attr(not(feature = "std"), no_std)] use runtime_interface::runtime_interface; @@ -25,7 +27,7 @@ use rstd::{vec, vec::Vec}; include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); #[runtime_interface] -trait TestApi { +pub trait TestApi { /// Returns the input data as result. fn return_input(data: Vec) -> Vec { data From 5eecdb02329a21344f9ea8c01742b1dd9e9de05e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 27 Sep 2019 12:14:55 +0200 Subject: [PATCH 17/76] More impls and move them into their own file --- core/runtime-interface/src/host.rs | 91 +------- core/runtime-interface/src/impls.rs | 308 ++++++++++++++++++++++++++++ core/runtime-interface/src/lib.rs | 4 +- core/runtime-interface/src/wasm.rs | 103 +--------- core/wasm-interface/src/lib.rs | 6 +- 5 files changed, 318 insertions(+), 194 deletions(-) create mode 100644 core/runtime-interface/src/impls.rs diff --git a/core/runtime-interface/src/host.rs b/core/runtime-interface/src/host.rs index d44973aee7dec..313aba3d855d6 100644 --- a/core/runtime-interface/src/host.rs +++ b/core/runtime-interface/src/host.rs @@ -16,13 +16,9 @@ //! Traits required by the runtime interface from the host side. -use crate::{RIType, pointer_and_len_from_u64, pointer_and_len_to_u64, PassedAsEncoded}; +use crate::RIType; -use wasm_interface::{FunctionContext, Pointer, Result}; - -use rstd::{any::TypeId, borrow::Cow, mem}; - -use codec::{Encode, Decode}; +use wasm_interface::{FunctionContext, Result}; /// Something that can be converted into a ffi value. pub trait IntoFFIValue: RIType { @@ -64,86 +60,3 @@ pub trait FromFFIValue: RIType { arg: Self::FFIType, ) -> Result; } - -impl IntoFFIValue for Vec { - fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result { - let vec: Cow<'_, [u8]> = if TypeId::of::() == TypeId::of::() { - unsafe { Cow::Borrowed(std::mem::transmute(&self[..])) } - } else { - Cow::Owned(self.encode()) - }; - - let ptr = context.allocate_memory(vec.as_ref().len() as u32)?; - context.write_memory(ptr, &vec)?; - - Ok(pointer_and_len_to_u64(ptr.into(), vec.len() as u32)) - } -} - -impl FromFFIValue for Vec { - type SelfInstance = Vec; - - fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result> { - <[T] as FromFFIValue>::from_ffi_value(context, arg) - } -} - -impl FromFFIValue for [T] { - type SelfInstance = Vec; - - fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result> { - let (ptr, len) = pointer_and_len_from_u64(arg); - - let vec = context.read_memory(Pointer::new(ptr), len)?; - - if TypeId::of::() == TypeId::of::() { - Ok(unsafe { mem::transmute(vec) }) - } else { - Ok(Vec::::decode(&mut &vec[..]).expect("Wasm to host values are encoded correctly; qed")) - } - } -} - -impl IntoPreallocatedFFIValue for [u8] { - type SelfInstance = Vec; - - fn into_preallocated_ffi_value( - self_instance: Self::SelfInstance, - context: &mut dyn FunctionContext, - allocated: u64, - ) -> Result<()> { - let (ptr, len) = pointer_and_len_from_u64(allocated); - - if (len as usize) < self_instance.len() { - Err( - format!( - "Preallocated buffer is not big enough (given {} vs needed {})!", - len, - self_instance.len() - ) - ) - } else { - context.write_memory(Pointer::new(ptr), &self_instance) - } - } -} - -impl IntoFFIValue for T { - fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result { - let vec = self.encode(); - let ptr = context.allocate_memory(vec.len() as u32)?; - context.write_memory(ptr, &vec)?; - - Ok(pointer_and_len_to_u64(ptr.into(), vec.len() as u32)) - } -} - -impl FromFFIValue for T { - type SelfInstance = Self; - - fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result { - let (ptr, len) = pointer_and_len_from_u64(arg); - let vec = context.read_memory(Pointer::new(ptr), len)?; - Ok(Self::decode(&mut &vec[..]).expect("Wasm to host values are encoded correctly; qed")) - } -} diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs new file mode 100644 index 0000000000000..a6f881f414e99 --- /dev/null +++ b/core/runtime-interface/src/impls.rs @@ -0,0 +1,308 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Provides implementations for the runtime interface traits. + +use crate::{RIType, pointer_and_len_to_u64, pointer_and_len_from_u64, PassedAsEncoded}; +#[cfg(feature = "std")] +use crate::host::*; +#[cfg(not(feature = "std"))] +use crate::wasm::*; + +#[cfg(not(feature = "std"))] +use static_assertions::assert_eq_size; + +#[cfg(feature = "std")] +use wasm_interface::{FunctionContext, Pointer, Result}; + +use codec::{Encode, Decode}; + +use rstd::{any::TypeId, mem, borrow::Cow}; + +/// Implement the traits for the given primitive traits. +macro_rules! impl_traits_for_primitives { + ( + $( + $rty:ty, $fty:ty, + )* + ) => { + $( + impl RIType for $rty { + type FFIType = $fty; + } + + #[cfg(not(feature = "std"))] + impl AsWrappedFFIValue for $rty { + type RTOwned = $fty; + type RTBorrowed = $fty; + + fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, $fty, $fty> { + WrappedFFIValue::from_owned(*self) + } + } + + #[cfg(not(feature = "std"))] + impl IntoFFIValue for $rty { + fn into_ffi_value(&self) -> $fty { + (*self as $fty).to_le() + } + } + + #[cfg(not(feature = "std"))] + impl FromFFIValue for $rty { + fn from_ffi_value(arg: $fty) -> $rty { + <$fty>::from_le(arg) as $rty + } + } + + #[cfg(feature = "std")] + impl FromFFIValue for $rty { + type SelfInstance = $rty; + + fn from_ffi_value(_: &mut dyn FunctionContext, arg: $fty) -> Result<$rty> { + Ok(<$fty>::from_le(arg) as $rty) + } + } + + #[cfg(feature = "std")] + impl IntoFFIValue for $rty { + fn into_ffi_value(self, _: &mut dyn FunctionContext) -> Result<$fty> { + Ok((self as $fty).to_le()) + } + } + )* + } +} + +impl_traits_for_primitives! { + u8, u8, + u16, u16, + u32, u32, + u64, u64, + i8, i8, + i16, i16, + i32, i32, + i64, i64, +} + +impl RIType for bool { + type FFIType = u8; +} + +#[cfg(not(feature = "std"))] +impl AsWrappedFFIValue for bool { + type RTOwned = u8; + type RTBorrowed = u8; + + fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u8, u8> { + WrappedFFIValue::from_owned(if *self { 1 } else { 0 }) + } +} + +#[cfg(not(feature = "std"))] +impl FromFFIValue for bool { + fn from_ffi_value(arg: u8) -> bool { + arg == 1 + } +} + +#[cfg(feature = "std")] +impl FromFFIValue for bool { + type SelfInstance = bool; + + fn from_ffi_value(_: &mut dyn FunctionContext, arg: u8) -> Result { + Ok(arg == 1) + } +} + +#[cfg(feature = "std")] +impl IntoFFIValue for bool { + fn into_ffi_value(self, _: &mut dyn FunctionContext) -> Result { + Ok(if self { 1 } else { 0 }) + } +} + +#[cfg(feature = "std")] +impl IntoFFIValue for Vec { + fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result { + let vec: Cow<'_, [u8]> = if TypeId::of::() == TypeId::of::() { + unsafe { Cow::Borrowed(std::mem::transmute(&self[..])) } + } else { + Cow::Owned(self.encode()) + }; + + let ptr = context.allocate_memory(vec.as_ref().len() as u32)?; + context.write_memory(ptr, &vec)?; + + Ok(pointer_and_len_to_u64(ptr.into(), vec.len() as u32)) + } +} + +#[cfg(feature = "std")] +impl FromFFIValue for Vec { + type SelfInstance = Vec; + + fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result> { + <[T] as FromFFIValue>::from_ffi_value(context, arg) + } +} + +#[cfg(feature = "std")] +impl FromFFIValue for [T] { + type SelfInstance = Vec; + + fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result> { + let (ptr, len) = pointer_and_len_from_u64(arg); + + let vec = context.read_memory(Pointer::new(ptr), len)?; + + if TypeId::of::() == TypeId::of::() { + Ok(unsafe { mem::transmute(vec) }) + } else { + Ok(Vec::::decode(&mut &vec[..]).expect("Wasm to host values are encoded correctly; qed")) + } + } +} + +#[cfg(feature = "std")] +impl IntoPreallocatedFFIValue for [u8] { + type SelfInstance = Vec; + + fn into_preallocated_ffi_value( + self_instance: Self::SelfInstance, + context: &mut dyn FunctionContext, + allocated: u64, + ) -> Result<()> { + let (ptr, len) = pointer_and_len_from_u64(allocated); + + if (len as usize) < self_instance.len() { + Err( + format!( + "Preallocated buffer is not big enough (given {} vs needed {})!", + len, + self_instance.len() + ) + ) + } else { + context.write_memory(Pointer::new(ptr), &self_instance) + } + } +} + +#[cfg(feature = "std")] +impl IntoFFIValue for T { + fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result { + let vec = self.encode(); + let ptr = context.allocate_memory(vec.len() as u32)?; + context.write_memory(ptr, &vec)?; + + Ok(pointer_and_len_to_u64(ptr.into(), vec.len() as u32)) + } +} + +#[cfg(feature = "std")] +impl FromFFIValue for T { + type SelfInstance = Self; + + fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result { + let (ptr, len) = pointer_and_len_from_u64(arg); + let vec = context.read_memory(Pointer::new(ptr), len)?; + Ok(Self::decode(&mut &vec[..]).expect("Wasm to host values are encoded correctly; qed")) + } +} + +// Make sure that our assumptions for storing a pointer + its size in `u64` is valid. +#[cfg(not(feature = "std"))] +assert_eq_size!(usize_check; usize, u32); +#[cfg(not(feature = "std"))] +assert_eq_size!(ptr_check; *const u8, u32); + +#[cfg(not(feature = "std"))] +impl IntoFFIValue for [u8] { + fn into_ffi_value(&self) -> u64 { + let data = self.as_ref(); + let ptr_address = data.as_ptr() as u32; + + pointer_and_len_to_u64(ptr_address, data.len() as u32) + } +} + +#[cfg(not(feature = "std"))] +impl IntoFFIValue for Vec { + fn into_ffi_value(&self) -> u64 { + self[..].into_ffi_value() + } +} + +#[cfg(not(feature = "std"))] +impl AsWrappedFFIValue for [T] { + type RTOwned = Vec; + type RTBorrowed = [u8]; + + fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { + if TypeId::of::() == TypeId::of::() { + WrappedFFIValue::from_ref(unsafe { mem::transmute::<&[T], &[u8]>(self) }) + } else { + WrappedFFIValue::from_owned(self.encode()) + } + } +} + +#[cfg(not(feature = "std"))] +impl AsWrappedFFIValue for Vec { + type RTOwned = Vec; + type RTBorrowed = [u8]; + + fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { + self[..].as_wrapped_ffi_value() + } +} + +#[cfg(not(feature = "std"))] +impl FromFFIValue for Vec { + fn from_ffi_value(arg: u64) -> Vec { + let (ptr, len) = pointer_and_len_from_u64(arg); + let len = len as usize; + + if TypeId::of::() == TypeId::of::() { + unsafe { std::mem::transmute(Vec::from_raw_parts(ptr as *mut u8, len, len)) } + } else { + let slice = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; + Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") + } + } +} + +#[cfg(not(feature = "std"))] +impl AsWrappedFFIValue for T { + type RTOwned = Vec; + type RTBorrowed = [u8]; + + fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { + WrappedFFIValue::from_owned(self.encode()) + } +} + +#[cfg(not(feature = "std"))] +impl FromFFIValue for T { + fn from_ffi_value(arg: u64) -> Self { + let (ptr, len) = pointer_and_len_from_u64(arg); + let len = len as usize; + + let slice = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; + Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") + } +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 313fc363a4361..f9679f66765a5 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -44,6 +44,7 @@ pub use externalities::{set_and_run_with_externalities, with_externalities}; #[cfg(feature = "std")] mod externalities; +mod impls; #[cfg(feature = "std")] pub mod host; #[cfg(not(feature = "std"))] @@ -61,9 +62,6 @@ pub trait RIType { type FFIType; } -impl RIType for u32 { - type FFIType = u32; -} impl RIType for Vec { type FFIType = u64; diff --git a/core/runtime-interface/src/wasm.rs b/core/runtime-interface/src/wasm.rs index ab28eed96534c..bd25c91b4bb84 100644 --- a/core/runtime-interface/src/wasm.rs +++ b/core/runtime-interface/src/wasm.rs @@ -16,13 +16,9 @@ //! Traits required by the runtime interface from the wasm side. -use crate::{RIType, pointer_and_len_to_u64, pointer_and_len_from_u64, PassedAsEncoded}; +use crate::RIType; -use rstd::{any::TypeId, mem, marker::PhantomData}; - -use codec::{Encode, Decode}; - -use static_assertions::assert_eq_size; +use rstd::marker::PhantomData; /// Something that can be converted into a [`WrappedFFIValue`]. pub trait AsWrappedFFIValue: RIType { @@ -80,98 +76,3 @@ where } } } - -impl AsWrappedFFIValue for u32 { - type RTOwned = u32; - type RTBorrowed = u32; - - fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u32, u32> { - WrappedFFIValue::from_owned(*self) - } -} - -impl IntoFFIValue for u32 { - fn into_ffi_value(&self) -> u32 { - self.to_le() - } -} - -impl FromFFIValue for u32 { - fn from_ffi_value(arg: u32) -> u32 { - u32::from_le(arg) - } -} - -// Make sure that our assumptions for storing a pointer + its size in `u64` is valid. -assert_eq_size!(usize_check; usize, u32); -assert_eq_size!(ptr_check; *const u8, u32); - -impl IntoFFIValue for [u8] { - fn into_ffi_value(&self) -> u64 { - let data = self.as_ref(); - let ptr_address = data.as_ptr() as u32; - - pointer_and_len_to_u64(ptr_address, data.len() as u32) - } -} - -impl IntoFFIValue for Vec { - fn into_ffi_value(&self) -> u64 { - self[..].into_ffi_value() - } -} - -impl AsWrappedFFIValue for [T] { - type RTOwned = Vec; - type RTBorrowed = [u8]; - - fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { - if TypeId::of::() == TypeId::of::() { - WrappedFFIValue::from_ref(unsafe { mem::transmute::<&[T], &[u8]>(self) }) - } else { - WrappedFFIValue::from_owned(self.encode()) - } - } -} - -impl AsWrappedFFIValue for Vec { - type RTOwned = Vec; - type RTBorrowed = [u8]; - - fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { - self[..].as_wrapped_ffi_value() - } -} - -impl FromFFIValue for Vec { - fn from_ffi_value(arg: u64) -> Vec { - let (ptr, len) = pointer_and_len_from_u64(arg); - let len = len as usize; - - if TypeId::of::() == TypeId::of::() { - unsafe { std::mem::transmute(Vec::from_raw_parts(ptr as *mut u8, len, len)) } - } else { - let slice = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; - Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") - } - } -} - -impl AsWrappedFFIValue for T { - type RTOwned = Vec; - type RTBorrowed = [u8]; - - fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { - WrappedFFIValue::from_owned(self.encode()) - } -} - -impl FromFFIValue for T { - fn from_ffi_value(arg: u64) -> Self { - let (ptr, len) = pointer_and_len_from_u64(arg); - let len = len as usize; - - let slice = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; - Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") - } -} diff --git a/core/wasm-interface/src/lib.rs b/core/wasm-interface/src/lib.rs index ae0599a1e8497..1ebf66a9f9dd7 100644 --- a/core/wasm-interface/src/lib.rs +++ b/core/wasm-interface/src/lib.rs @@ -340,9 +340,13 @@ macro_rules! impl_into_and_from_value { } impl_into_and_from_value! { + u8, I32, + u16, I32, u32, I32, - i32, I32, u64, I64, + i8, I32, + i16, I32, + i32, I32, i64, I64, } From 7a55140d88b24216a49c0e75c6803b8081669f3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 27 Sep 2019 13:47:15 +0200 Subject: [PATCH 18/76] Code simplification by dropping one trait --- .../proc-macro/src/bare_function_interface.rs | 6 +- core/runtime-interface/src/impls.rs | 81 +++++++------------ core/runtime-interface/src/wasm.rs | 68 +++++++--------- 3 files changed, 61 insertions(+), 94 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index 521b17b235019..5a5d3b3691ffc 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -90,17 +90,15 @@ fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result< quote! { #[cfg(not(feature = "std"))] fn #function_name( #( #args, )* ) #return_value { - use #crate_::wasm::IntoFFIValue as _; - // Generate all wrapped ffi value. #( - let #arg_names = <#arg_types as #crate_::wasm::AsWrappedFFIValue>::as_wrapped_ffi_value( + let #arg_names = <#arg_types as #crate_::wasm::IntoFFIValue>::into_ffi_value( &#arg_names, ); )* // Call the host function - let result = unsafe { #host_function_name( #( #arg_names2.into_ffi_value(), )* ) }; + let result = unsafe { #host_function_name( #( #arg_names2.get(), )* ) }; #convert_return_value } diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index a6f881f414e99..be2cb1c764b67 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -30,7 +30,10 @@ use wasm_interface::{FunctionContext, Pointer, Result}; use codec::{Encode, Decode}; -use rstd::{any::TypeId, mem, borrow::Cow}; +use rstd::{any::TypeId, mem}; + +#[cfg(feature = "std")] +use rstd::borrow::Cow; /// Implement the traits for the given primitive traits. macro_rules! impl_traits_for_primitives { @@ -44,20 +47,12 @@ macro_rules! impl_traits_for_primitives { type FFIType = $fty; } - #[cfg(not(feature = "std"))] - impl AsWrappedFFIValue for $rty { - type RTOwned = $fty; - type RTBorrowed = $fty; - - fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, $fty, $fty> { - WrappedFFIValue::from_owned(*self) - } - } - #[cfg(not(feature = "std"))] impl IntoFFIValue for $rty { - fn into_ffi_value(&self) -> $fty { - (*self as $fty).to_le() + type Owned = (); + + fn into_ffi_value(&self) -> WrappedFFIValue<$fty> { + (*self as $fty).to_le().into() } } @@ -103,12 +98,11 @@ impl RIType for bool { } #[cfg(not(feature = "std"))] -impl AsWrappedFFIValue for bool { - type RTOwned = u8; - type RTBorrowed = u8; +impl IntoFFIValue for bool { + type Owned = (); - fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u8, u8> { - WrappedFFIValue::from_owned(if *self { 1 } else { 0 }) + fn into_ffi_value(&self) -> WrappedFFIValue { + if *self { 1 } else { 0 }.into() } } @@ -231,43 +225,27 @@ assert_eq_size!(usize_check; usize, u32); assert_eq_size!(ptr_check; *const u8, u32); #[cfg(not(feature = "std"))] -impl IntoFFIValue for [u8] { - fn into_ffi_value(&self) -> u64 { - let data = self.as_ref(); - let ptr_address = data.as_ptr() as u32; - - pointer_and_len_to_u64(ptr_address, data.len() as u32) - } -} - -#[cfg(not(feature = "std"))] -impl IntoFFIValue for Vec { - fn into_ffi_value(&self) -> u64 { - self[..].into_ffi_value() - } -} +impl IntoFFIValue for [T] { + type Owned = Vec; -#[cfg(not(feature = "std"))] -impl AsWrappedFFIValue for [T] { - type RTOwned = Vec; - type RTBorrowed = [u8]; - - fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { + fn into_ffi_value(&self) -> WrappedFFIValue> { if TypeId::of::() == TypeId::of::() { - WrappedFFIValue::from_ref(unsafe { mem::transmute::<&[T], &[u8]>(self) }) + let slice = unsafe { mem::transmute::<&[T], &[u8]>(self) }; + pointer_and_len_to_u64(slice.as_ptr() as u32, slice.len() as u32).into() } else { - WrappedFFIValue::from_owned(self.encode()) + let data = self.encode(); + let ffi_value = pointer_and_len_to_u64(data.as_ptr() as u32, data.len() as u32); + (ffi_value, data).into() } } } #[cfg(not(feature = "std"))] -impl AsWrappedFFIValue for Vec { - type RTOwned = Vec; - type RTBorrowed = [u8]; +impl IntoFFIValue for Vec { + type Owned = Vec; - fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { - self[..].as_wrapped_ffi_value() + fn into_ffi_value(&self) -> WrappedFFIValue> { + self[..].into_ffi_value() } } @@ -287,12 +265,13 @@ impl FromFFIValue for Vec { } #[cfg(not(feature = "std"))] -impl AsWrappedFFIValue for T { - type RTOwned = Vec; - type RTBorrowed = [u8]; +impl IntoFFIValue for T { + type Owned = Vec; - fn as_wrapped_ffi_value<'a>(&'a self) -> WrappedFFIValue<'a, u64, Vec, [u8]> { - WrappedFFIValue::from_owned(self.encode()) + fn into_ffi_value(&self) -> WrappedFFIValue> { + let data = self.encode(); + let ffi_value = pointer_and_len_to_u64(data.as_ptr() as u32, data.len() as u32); + (ffi_value, data).into() } } diff --git a/core/runtime-interface/src/wasm.rs b/core/runtime-interface/src/wasm.rs index bd25c91b4bb84..80fd5a5e27e0e 100644 --- a/core/runtime-interface/src/wasm.rs +++ b/core/runtime-interface/src/wasm.rs @@ -18,21 +18,6 @@ use crate::RIType; -use rstd::marker::PhantomData; - -/// Something that can be converted into a [`WrappedFFIValue`]. -pub trait AsWrappedFFIValue: RIType { - /// The owned rust type that converts into `Self::FFIType`. - type RTOwned; - /// The borrowed rust type that converts into `Self::FFIType`. - type RTBorrowed: ?Sized; - - /// Returns `self` as a [`WrappedFFIValue`] that can be converted into `Self::FFIType`. - fn as_wrapped_ffi_value<'a>( - &'a self, - ) -> WrappedFFIValue<'a, Self::FFIType, Self::RTOwned, Self::RTBorrowed>; -} - /// Something that can be created from a ffi value. pub trait FromFFIValue: Sized + RIType { /// Create `Self` from the given ffi value. @@ -41,38 +26,43 @@ pub trait FromFFIValue: Sized + RIType { /// Something that can be converted into a ffi value. pub trait IntoFFIValue: RIType { - /// Convert `self` into a ffi value. - fn into_ffi_value(&self) -> Self::FFIType; + /// The owned rust type that is stored with the ffi value in [`WrappedFFIValue`]. + /// + /// If no owned value is required, `()` can be used as a type. + type Owned; + + /// Convert `self` into a [`WrappedFFIValue`]. + fn into_ffi_value(&self) -> WrappedFFIValue; } -/// Represents a wrapped ffi value that either holds a reference or an owned value of a rust type. +/// Represents a wrapped ffi value. /// -/// The reference and the owned value need to be convertible into the same ffi value. -pub enum WrappedFFIValue<'a, T, O, R: ?Sized = O> { - Ref(&'a R, PhantomData), - Owned(O), +/// It is either the ffi value itself or the ffi value plus some other owned value. By providing +/// support for storing another owned value besides the actual ffi value certain performance +/// optimizations can be applied. For example using the pointer to a `Vec`, while using the +/// pointer to a SCALE encoded `Vec` that is stored in this wrapper for any other `Vec`. +pub enum WrappedFFIValue { + Wrapped(T), + WrappedAndOwned(T, O), } -impl<'a, T, O, R> WrappedFFIValue<'a, T, O, R> -where - O: IntoFFIValue, - R: ?Sized + IntoFFIValue, -{ - /// Create `Self` from an owned value. - pub fn from_owned(o: O) -> Self { - Self::Owned(o) +impl WrappedFFIValue { + /// Returns the wrapped ffi value. + pub fn get(&self) -> T { + match self { + Self::Wrapped(data) | Self::WrappedAndOwned(data, _) => *data, + } } +} - /// Create `Self` from a reference value. - pub fn from_ref(r: &'a R) -> Self { - Self::Ref(r, Default::default()) +impl From for WrappedFFIValue { + fn from(val: T) -> Self { + WrappedFFIValue::Wrapped(val) } +} - /// Convert into the ffi value. - pub fn into_ffi_value(&self) -> T { - match self { - Self::Ref(data, _) => data.into_ffi_value(), - Self::Owned(ref data) => data.into_ffi_value(), - } +impl From<(T, O)> for WrappedFFIValue { + fn from(val: (T, O)) -> Self { + WrappedFFIValue::WrappedAndOwned(val.0, val.1) } } From 53f8de2b1179f3caad9d2f0e7fb9a9bcf0dec265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 27 Sep 2019 14:07:21 +0200 Subject: [PATCH 19/76] Give it a better name --- core/runtime-interface/src/impls.rs | 24 +++++++++++++++++----- core/runtime-interface/src/lib.rs | 31 +++++++++-------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index be2cb1c764b67..05868e908e124 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -16,7 +16,7 @@ //! Provides implementations for the runtime interface traits. -use crate::{RIType, pointer_and_len_to_u64, pointer_and_len_from_u64, PassedAsEncoded}; +use crate::{RIType, PassByCodec}; #[cfg(feature = "std")] use crate::host::*; #[cfg(not(feature = "std"))] @@ -35,6 +35,20 @@ use rstd::{any::TypeId, mem}; #[cfg(feature = "std")] use rstd::borrow::Cow; +/// Converts a pointer and length into an `u64`. +fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { + ((len as u64) | u64::from(ptr) << 32).to_le() +} + +/// Splits an `u64` into the pointer and length. +fn pointer_and_len_from_u64(val: u64) -> (u32, u32) { + let val = u64::from_le(val); + let len = (val & (!0u32 as u64)) as u32; + let ptr = (val >> 32) as u32; + + (ptr, len) +} + /// Implement the traits for the given primitive traits. macro_rules! impl_traits_for_primitives { ( @@ -197,7 +211,7 @@ impl IntoPreallocatedFFIValue for [u8] { } #[cfg(feature = "std")] -impl IntoFFIValue for T { +impl IntoFFIValue for T { fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result { let vec = self.encode(); let ptr = context.allocate_memory(vec.len() as u32)?; @@ -208,7 +222,7 @@ impl IntoFFIValue for T { } #[cfg(feature = "std")] -impl FromFFIValue for T { +impl FromFFIValue for T { type SelfInstance = Self; fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result { @@ -265,7 +279,7 @@ impl FromFFIValue for Vec { } #[cfg(not(feature = "std"))] -impl IntoFFIValue for T { +impl IntoFFIValue for T { type Owned = Vec; fn into_ffi_value(&self) -> WrappedFFIValue> { @@ -276,7 +290,7 @@ impl IntoFFIValue for T { } #[cfg(not(feature = "std"))] -impl FromFFIValue for T { +impl FromFFIValue for T { fn from_ffi_value(arg: u64) -> Self { let (ptr, len) = pointer_and_len_from_u64(arg); let len = len as usize; diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index f9679f66765a5..80db7bef1cff9 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -71,40 +71,27 @@ impl RIType for [T] { type FFIType = u64; } -/// Marker trait for types that should be passed to between wasm and the host by using SCALE codec. +/// Marker trait for types that should be passed between wasm and the host by using SCALE codec. /// /// # Example /// ``` -/// # use substrate_runtime_interface::PassedAsEncoded; +/// # use substrate_runtime_interface::PassByCodec; /// #[derive(codec::Encode, codec::Decode)] /// struct Test; /// -/// // It is sufficient to implement the trait for the desired type. -/// impl PassedAsEncoded for Test {} +/// // It is sufficient to implement the trait for the desired type and it will be usable with +/// // the runtime interface. +/// impl PassByCodec for Test {} /// ``` -pub trait PassedAsEncoded: Codec {} +pub trait PassByCodec: Codec {} -impl RIType for T { +impl RIType for T { type FFIType = u64; } -impl PassedAsEncoded for Option {} +impl PassByCodec for Option {} -impl PassedAsEncoded for Result {} - -/// Converts a pointer and length into an `u64`. -fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { - ((len as u64) | u64::from(ptr) << 32).to_le() -} - -/// Splits an `u64` into the pointer and length. -fn pointer_and_len_from_u64(val: u64) -> (u32, u32) { - let val = u64::from_le(val); - let len = (val & (!0u32 as u64)) as u32; - let ptr = (val >> 32) as u32; - - (ptr, len) -} +impl PassByCodec for Result {} #[cfg(test)] mod tests { From 0807fd9a2850d34d96f0256211c6bedcad2fb761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 30 Sep 2019 14:02:18 +0200 Subject: [PATCH 20/76] Implement traits for arrays --- core/runtime-interface/src/impls.rs | 81 +++++++++++++++++++++ core/runtime-interface/src/lib.rs | 10 +++ core/runtime-interface/src/wasm.rs | 6 ++ core/runtime-interface/test-wasm/src/lib.rs | 38 +++++++++- 4 files changed, 134 insertions(+), 1 deletion(-) diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index 05868e908e124..f2f2112cb0776 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -35,6 +35,9 @@ use rstd::{any::TypeId, mem}; #[cfg(feature = "std")] use rstd::borrow::Cow; +#[cfg(not(feature = "std"))] +use rstd::slice; + /// Converts a pointer and length into an `u64`. fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { ((len as u64) | u64::from(ptr) << 32).to_le() @@ -299,3 +302,81 @@ impl FromFFIValue for T { Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") } } + +/// Implement the traits for the `[u8; N]` arrays, where `N` is the input to this macro. +macro_rules! impl_traits_for_arrays { + ( + $( + $n:expr + ),* + $(,)? + ) => { + $( + impl RIType for [u8; $n] { + type FFIType = u32; + } + + #[cfg(not(feature = "std"))] + impl IntoFFIValue for [u8; $n] { + type Owned = (); + + fn into_ffi_value(&self) -> WrappedFFIValue { + (self.as_ptr() as u32).into() + } + } + + #[cfg(not(feature = "std"))] + impl FromFFIValue for [u8; $n] { + fn from_ffi_value(arg: u32) -> [u8; $n] { + let mut res = unsafe { mem::MaybeUninit::<[u8; $n]>::zeroed().assume_init() }; + res.copy_from_slice(unsafe { slice::from_raw_parts(arg as *const u8, $n) }); + + // Make sure we free the pointer. + let _ = unsafe { Box::from_raw(arg as *mut u8) }; + + res + } + } + + #[cfg(feature = "std")] + impl FromFFIValue for [u8; $n] { + type SelfInstance = [u8; $n]; + + fn from_ffi_value(context: &mut dyn FunctionContext, arg: u32) -> Result<[u8; $n]> { + let data = context.read_memory(Pointer::new(arg), $n)?; + let mut res = unsafe { mem::MaybeUninit::<[u8; $n]>::zeroed().assume_init() }; + res.copy_from_slice(&data); + Ok(res) + } + } + + #[cfg(feature = "std")] + impl IntoFFIValue for [u8; $n] { + fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result { + let addr = context.allocate_memory($n)?; + context.write_memory(addr, &self)?; + Ok(addr.into()) + } + } + + #[cfg(feature = "std")] + impl IntoPreallocatedFFIValue for [u8; $n] { + type SelfInstance = [u8; $n]; + + fn into_preallocated_ffi_value( + self_instance: Self::SelfInstance, + context: &mut dyn FunctionContext, + allocated: u32, + ) -> Result<()> { + context.write_memory(Pointer::new(allocated), &self_instance) + } + } + )* + } +} + +impl_traits_for_arrays! { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 80db7bef1cff9..5ba2799a43454 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -141,6 +141,16 @@ mod tests { call_wasm_method::("test_return_value_into_mutable_reference"); } + #[test] + fn test_get_and_return_array() { + call_wasm_method::("test_get_and_return_array"); + } + + #[test] + fn test_array_as_mutable_reference() { + call_wasm_method::("test_array_as_mutable_reference"); + } + #[test] #[should_panic(expected = "Wasmi(Instantiation(\"Export ext_test_api_return_input not found\"))")] fn host_function_not_found() { diff --git a/core/runtime-interface/src/wasm.rs b/core/runtime-interface/src/wasm.rs index 80fd5a5e27e0e..f5f13b7da0475 100644 --- a/core/runtime-interface/src/wasm.rs +++ b/core/runtime-interface/src/wasm.rs @@ -19,6 +19,12 @@ use crate::RIType; /// Something that can be created from a ffi value. +/// +/// # Safety +/// +/// It is unsafe behavior to call `Something::into_ffi_value().get()` and take this as input for +/// `from_ffi_value`. Implementations are safe to assume that the `arg` given to `from_ffi_value` +/// is only generated by the corresponding `host::IntoFFIValue` implementation. pub trait FromFFIValue: Sized + RIType { /// Create `Self` from the given ffi value. fn from_ffi_value(arg: Self::FFIType) -> Self; diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index e930250b486b7..48436bb887501 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -20,12 +20,15 @@ use runtime_interface::runtime_interface; -use rstd::{vec, vec::Vec}; +use rstd::{vec, vec::Vec, mem}; // Inlucde the WASM binary #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); +/// Used in the `test_array_as_mutable_reference` test. +const TEST_ARRAY: [u8; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + #[runtime_interface] pub trait TestApi { /// Returns the input data as result. @@ -48,6 +51,18 @@ pub trait TestApi { fn return_option_input(data: Vec) -> Option> { Some(data) } + + /// Get an array as input and returns a subset of this array. + fn get_and_return_array(data: [u8; 34]) -> [u8; 16] { + let mut res = [0u8; 16]; + res.copy_from_slice(&data[..16]); + res + } + + /// Take and fill mutable array. + fn array_as_mutable_reference(data: &mut [u8; 16]) { + data.copy_from_slice(&TEST_ARRAY); + } } #[no_mangle] @@ -83,3 +98,24 @@ pub fn test_return_value_into_mutable_reference() { let expected = "hello"; assert_eq!(expected.as_bytes(), &data[..expected.len()]); } + +#[no_mangle] +pub fn test_get_and_return_array() { + let mut input = unsafe { mem::MaybeUninit::<[u8; 34]>::zeroed().assume_init() }; + input.copy_from_slice(&[ + 24, 3, 23, 20, 2, 16, 32, 1, 12, 26, 27, 8, 29, 31, 6, 5, 4, 19, 10, 28, 34, 21, 18, 33, 9, + 13, 22, 25, 15, 11, 30, 7, 14, 17, + ]); + + let res = test_api::get_and_return_array(input); + + assert_eq!(&res, &input[..16]); +} + +#[no_mangle] +pub fn test_array_as_mutable_reference() { + let mut array = [0u8; 16]; + test_api::array_as_mutable_reference(&mut array); + + assert_eq!(array, TEST_ARRAY); +} From dbab5808c54267bd01753e7d3c3746d9445103d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 30 Sep 2019 19:11:57 +0200 Subject: [PATCH 21/76] Refactor code to support pass by codec/inner --- core/runtime-interface/src/impls.rs | 72 ++----- core/runtime-interface/src/lib.rs | 39 +--- core/runtime-interface/src/pass_by.rs | 284 ++++++++++++++++++++++++++ 3 files changed, 310 insertions(+), 85 deletions(-) create mode 100644 core/runtime-interface/src/pass_by.rs diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index f2f2112cb0776..408ac19a60e5d 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -16,7 +16,7 @@ //! Provides implementations for the runtime interface traits. -use crate::{RIType, PassByCodec}; +use crate::{RIType, pass_by::{PassBy, Inner, Codec}}; #[cfg(feature = "std")] use crate::host::*; #[cfg(not(feature = "std"))] @@ -39,12 +39,12 @@ use rstd::borrow::Cow; use rstd::slice; /// Converts a pointer and length into an `u64`. -fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { +pub fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { ((len as u64) | u64::from(ptr) << 32).to_le() } /// Splits an `u64` into the pointer and length. -fn pointer_and_len_from_u64(val: u64) -> (u32, u32) { +pub fn pointer_and_len_from_u64(val: u64) -> (u32, u32) { let val = u64::from_le(val); let len = (val & (!0u32 as u64)) as u32; let ptr = (val >> 32) as u32; @@ -146,11 +146,15 @@ impl IntoFFIValue for bool { } } +impl RIType for Vec { + type FFIType = u64; +} + #[cfg(feature = "std")] impl IntoFFIValue for Vec { fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result { let vec: Cow<'_, [u8]> = if TypeId::of::() == TypeId::of::() { - unsafe { Cow::Borrowed(std::mem::transmute(&self[..])) } + unsafe { Cow::Borrowed(mem::transmute(&self[..])) } } else { Cow::Owned(self.encode()) }; @@ -171,6 +175,10 @@ impl FromFFIValue for Vec { } } +impl RIType for [T] { + type FFIType = u64; +} + #[cfg(feature = "std")] impl FromFFIValue for [T] { type SelfInstance = Vec; @@ -213,28 +221,6 @@ impl IntoPreallocatedFFIValue for [u8] { } } -#[cfg(feature = "std")] -impl IntoFFIValue for T { - fn into_ffi_value(self, context: &mut dyn FunctionContext) -> Result { - let vec = self.encode(); - let ptr = context.allocate_memory(vec.len() as u32)?; - context.write_memory(ptr, &vec)?; - - Ok(pointer_and_len_to_u64(ptr.into(), vec.len() as u32)) - } -} - -#[cfg(feature = "std")] -impl FromFFIValue for T { - type SelfInstance = Self; - - fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result { - let (ptr, len) = pointer_and_len_from_u64(arg); - let vec = context.read_memory(Pointer::new(ptr), len)?; - Ok(Self::decode(&mut &vec[..]).expect("Wasm to host values are encoded correctly; qed")) - } -} - // Make sure that our assumptions for storing a pointer + its size in `u64` is valid. #[cfg(not(feature = "std"))] assert_eq_size!(usize_check; usize, u32); @@ -273,36 +259,14 @@ impl FromFFIValue for Vec { let len = len as usize; if TypeId::of::() == TypeId::of::() { - unsafe { std::mem::transmute(Vec::from_raw_parts(ptr as *mut u8, len, len)) } + unsafe { mem::transmute(Vec::from_raw_parts(ptr as *mut u8, len, len)) } } else { - let slice = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; + let slice = unsafe { slice::from_raw_parts(ptr as *const u8, len) }; Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") } } } -#[cfg(not(feature = "std"))] -impl IntoFFIValue for T { - type Owned = Vec; - - fn into_ffi_value(&self) -> WrappedFFIValue> { - let data = self.encode(); - let ffi_value = pointer_and_len_to_u64(data.as_ptr() as u32, data.len() as u32); - (ffi_value, data).into() - } -} - -#[cfg(not(feature = "std"))] -impl FromFFIValue for T { - fn from_ffi_value(arg: u64) -> Self { - let (ptr, len) = pointer_and_len_from_u64(arg); - let len = len as usize; - - let slice = unsafe { std::slice::from_raw_parts(ptr as *const u8, len) }; - Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") - } -} - /// Implement the traits for the `[u8; N]` arrays, where `N` is the input to this macro. macro_rules! impl_traits_for_arrays { ( @@ -380,3 +344,11 @@ impl_traits_for_arrays! { 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 } + +impl PassBy for Option { + type PassBy = Codec; +} + +impl PassBy for rstd::result::Result { + type PassBy = Codec; +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 5ba2799a43454..9665a2065dc8d 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -27,8 +27,6 @@ #[doc(hidden)] pub use primitives::Blake2Hasher; -use codec::Codec; - #[doc(hidden)] #[cfg(feature = "std")] pub use primitives::traits::Externalities; @@ -44,13 +42,15 @@ pub use externalities::{set_and_run_with_externalities, with_externalities}; #[cfg(feature = "std")] mod externalities; -mod impls; +pub(crate) mod impls; #[cfg(feature = "std")] pub mod host; #[cfg(not(feature = "std"))] pub mod wasm; +pub mod pass_by; -/// Something that can be used by the runtime interface as type to communicate between wasm and the host. +/// Something that can be used by the runtime interface as type to communicate between wasm and the +/// host. /// /// Every type that should be used in a runtime interface function signature needs to implement /// this trait. @@ -62,37 +62,6 @@ pub trait RIType { type FFIType; } - -impl RIType for Vec { - type FFIType = u64; -} - -impl RIType for [T] { - type FFIType = u64; -} - -/// Marker trait for types that should be passed between wasm and the host by using SCALE codec. -/// -/// # Example -/// ``` -/// # use substrate_runtime_interface::PassByCodec; -/// #[derive(codec::Encode, codec::Decode)] -/// struct Test; -/// -/// // It is sufficient to implement the trait for the desired type and it will be usable with -/// // the runtime interface. -/// impl PassByCodec for Test {} -/// ``` -pub trait PassByCodec: Codec {} - -impl RIType for T { - type FFIType = u64; -} - -impl PassByCodec for Option {} - -impl PassByCodec for Result {} - #[cfg(test)] mod tests { use super::*; diff --git a/core/runtime-interface/src/pass_by.rs b/core/runtime-interface/src/pass_by.rs new file mode 100644 index 0000000000000..6b9985cf4d2ab --- /dev/null +++ b/core/runtime-interface/src/pass_by.rs @@ -0,0 +1,284 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Provides implementations for the runtime interface traits. + +use crate::{RIType, impls::{pointer_and_len_from_u64, pointer_and_len_to_u64}}; + +#[cfg(feature = "std")] +use crate::host::*; +#[cfg(not(feature = "std"))] +use crate::wasm::*; + +#[cfg(feature = "std")] +use wasm_interface::{FunctionContext, Pointer, Result}; + +use rstd::marker::PhantomData; + +#[cfg(not(feature = "std"))] +use rstd::slice; + +/// Something that should be passed between wasm and the host using the given strategy. +/// +/// See [`Codec`] or [`Inner`] for more information about the provided strategies. +pub trait PassBy: Sized { + /// The strategy that should be used to pass the type. + type PassBy: PassByImpl; +} + +/// Something that provides a strategy for passing a type between wasm and the host. +/// +/// This trait exposes the same functionality as [`crate::host::IntoFFIValue`] and +/// [`crate::host::FromFFIValue`] to delegate the implementation for a type to a different type. +/// +/// This trait is used for the host implementation. +#[cfg(feature = "std")] +pub trait PassByImpl: RIType { + /// Convert the given instance to the ffi value. + /// + /// For more information see: [`crate::host::IntoFFIValue::into_ffi_value`] + fn into_ffi_value( + instance: T, + context: &mut dyn FunctionContext, + ) -> Result; + + /// Create `T` from the given ffi value. + /// + /// For more information see: [`crate::host::FromFFIValue::from_ffi_value`] + fn from_ffi_value( + context: &mut dyn FunctionContext, + arg: Self::FFIType, + ) -> Result; +} + +/// Something that provides a strategy for passing a type between wasm and the host. +/// +/// This trait exposes the same functionality as [`crate::wasm::IntoFFIValue`] and +/// [`crate::wasm::FromFFIValue`] to delegate the implementation for a type to a different type. +/// +/// This trait is used for the wasm implementation. +#[cfg(not(feature = "std"))] +pub trait PassByImpl: RIType { + /// The owned rust type that is stored with the ffi value in [`crate::wasm::WrappedFFIValue`]. + type Owned; + + /// Convert the given `instance` into [`crate::wasm::WrappedFFIValue`]. + /// + /// For more information see: [`crate::wasm::IntoFFIValue::into_ffi_value`] + fn into_ffi_value(instance: &T) -> WrappedFFIValue; + + /// Create `T` from the given ffi value. + /// + /// For more information see: [`crate::wasm::FromFFIValue::from_ffi_value`] + fn from_ffi_value(arg: Self::FFIType) -> T; +} + +/// The implementation of the pass by codec strategy. This strategy uses a SCALE encoded +/// representation of the type between wasm and the host. +/// +/// Use this type as associated type for [`PassBy`] to implement this strategy for a type. +/// +/// This type expects the type that wants to implement this strategy as generic parameter. +/// +/// # Example +/// ``` +/// # use substrate_runtime_interface::pass_by::{PassBy, Codec}; +/// #[derive(codec::Encode, codec::Decode)] +/// struct Test; +/// +/// impl PassBy for Test { +/// type PassBy = Codec; +/// } +/// ``` +pub struct Codec(PhantomData); + +#[cfg(feature = "std")] +impl PassByImpl for Codec { + fn into_ffi_value( + instance: T, + context: &mut dyn FunctionContext, + ) -> Result { + let vec = instance.encode(); + let ptr = context.allocate_memory(vec.len() as u32)?; + context.write_memory(ptr, &vec)?; + + Ok(pointer_and_len_to_u64(ptr.into(), vec.len() as u32)) + } + + fn from_ffi_value( + context: &mut dyn FunctionContext, + arg: Self::FFIType, + ) -> Result { + let (ptr, len) = pointer_and_len_from_u64(arg); + let vec = context.read_memory(Pointer::new(ptr), len)?; + Ok(T::decode(&mut &vec[..]).expect("Wasm to host values are encoded correctly; qed")) + } +} + +#[cfg(not(feature = "std"))] +impl PassByImpl for Codec { + type Owned = Vec; + + fn into_ffi_value(instance: &T) -> WrappedFFIValue { + let data = instance.encode(); + let ffi_value = pointer_and_len_to_u64(data.as_ptr() as u32, data.len() as u32); + (ffi_value, data).into() + } + + fn from_ffi_value(arg: Self::FFIType) -> T { + let (ptr, len) = pointer_and_len_from_u64(arg); + let len = len as usize; + + let slice = unsafe { slice::from_raw_parts(ptr as *const u8, len) }; + T::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") + } +} + +impl RIType for Codec { + type FFIType = u64; +} + +/// Trait that needs to be implemented by a type that should be passed between wasm and the host, +/// by using the inner type. See [`Inner`] for more information. +pub trait PassByInner: Sized { + /// The inner type that is wrapped by `Self`. + type Inner: RIType; + + /// Consumes `self` and returns the inner type. + fn into_inner(self) -> Self::Inner; + + /// Returns the reference to the inner type. + fn inner(&self) -> &Self::Inner; + + /// Construct `Self` from the given `inner`. + fn from_inner(inner: Self::Inner) -> Self; +} + +/// The implementation of the pass by inner type strategy. The type that uses this strategy will be +/// passed between wasm and the host by using the wrapped inner type. So, this strategy is only +/// usable by newtype structs. +/// +/// Use this type as associated type for [`PassBy`] to implement this strategy for a type. Besides +/// that the `PassByInner` trait need to be implemented as well. +/// +/// This type expects the type that wants to use this strategy as generic parameter `T` and the +/// inner type as generic parameter `I`. +/// +/// # Example +/// ``` +/// # use substrate_runtime_interface::pass_by::{PassBy, Inner, PassByInner}; +/// struct Test([u8; 32]); +/// +/// impl PassBy for Test { +/// type PassBy = Inner; +/// } +/// +/// impl PassByInner for Test { +/// type Inner = [u8; 32]; +/// +/// fn into_inner(self) -> [u8; 32] { +/// self.0 +/// } +/// fn inner(&self) -> &[u8; 32] { +/// &self.0 +/// } +/// fn from_inner(inner: [u8; 32]) -> Self { +/// Self(inner) +/// } +/// } +/// ``` +pub struct Inner, I: RIType>(PhantomData<(T, I)>); + +#[cfg(feature = "std")] +impl, I: RIType> PassByImpl for Inner + where I: IntoFFIValue + FromFFIValue +{ + fn into_ffi_value( + instance: T, + context: &mut dyn FunctionContext, + ) -> Result { + instance.into_inner().into_ffi_value(context) + } + + fn from_ffi_value( + context: &mut dyn FunctionContext, + arg: Self::FFIType, + ) -> Result { + I::from_ffi_value(context, arg).map(T::from_inner) + } +} + +#[cfg(not(feature = "std"))] +impl, I: RIType> PassByImpl for Inner + where I: IntoFFIValue + FromFFIValue +{ + type Owned = I::Owned; + + fn into_ffi_value(instance: &T) -> WrappedFFIValue { + instance.inner().into_ffi_value() + } + + fn from_ffi_value(arg: Self::FFIType) -> T { + T::from_inner(I::from_ffi_value(arg)) + } +} + +impl, I: RIType> RIType for Inner { + type FFIType = I::FFIType; +} + +impl RIType for T { + type FFIType = ::FFIType; +} + +#[cfg(feature = "std")] +impl IntoFFIValue for T { + fn into_ffi_value( + self, + context: &mut dyn FunctionContext, + ) -> Result<::FFIType> { + T::PassBy::into_ffi_value(self, context) + } +} + +#[cfg(feature = "std")] +impl FromFFIValue for T { + type SelfInstance = Self; + + fn from_ffi_value( + context: &mut dyn FunctionContext, + arg: ::FFIType, + ) -> Result { + T::PassBy::from_ffi_value(context, arg) + } +} + +#[cfg(not(feature = "std"))] +impl IntoFFIValue for T { + type Owned = >::Owned; + + fn into_ffi_value(&self) -> WrappedFFIValue<::FFIType, Self::Owned> { + T::PassBy::into_ffi_value(self) + } +} + +#[cfg(not(feature = "std"))] +impl FromFFIValue for T { + fn from_ffi_value(arg: ::FFIType) -> Self { + T::PassBy::from_ffi_value(arg) + } +} + From 69ffd260b39b633e9bceb7ba8735e6868f4a907b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 30 Sep 2019 19:15:43 +0200 Subject: [PATCH 22/76] Docs --- core/runtime-interface/src/pass_by.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/runtime-interface/src/pass_by.rs b/core/runtime-interface/src/pass_by.rs index 6b9985cf4d2ab..8accac3ebd84d 100644 --- a/core/runtime-interface/src/pass_by.rs +++ b/core/runtime-interface/src/pass_by.rs @@ -14,7 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Provides implementations for the runtime interface traits. +//! Provides the [`PassBy`] trait to simplify the implementation of the runtime interface traits +//! for custom types. [`Codec`] and [`Inner`] are the two provided strategy implementations. use crate::{RIType, impls::{pointer_and_len_from_u64, pointer_and_len_to_u64}}; From f7c1e5efa34863963603714703df2df1bcc450a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 30 Sep 2019 20:43:31 +0200 Subject: [PATCH 23/76] Implement pass by inner for some crypto types and add a test --- Cargo.lock | 1 + core/runtime-interface/src/impls.rs | 84 ++++++++++++++++++++- core/runtime-interface/src/lib.rs | 5 ++ core/runtime-interface/test-wasm/Cargo.toml | 3 +- core/runtime-interface/test-wasm/src/lib.rs | 24 +++++- 5 files changed, 114 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 788281a59c21c..a1ba0d4787d38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5264,6 +5264,7 @@ name = "substrate-runtime-interface-test-wasm" version = "2.0.0" dependencies = [ "sr-std 2.0.0", + "substrate-primitives 2.0.0", "substrate-runtime-interface 2.0.0", "substrate-wasm-builder-runner 1.0.3", ] diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index 408ac19a60e5d..3cd8cca04eceb 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -16,7 +16,7 @@ //! Provides implementations for the runtime interface traits. -use crate::{RIType, pass_by::{PassBy, Inner, Codec}}; +use crate::{RIType, pass_by::{PassBy, Inner, Codec, PassByInner}}; #[cfg(feature = "std")] use crate::host::*; #[cfg(not(feature = "std"))] @@ -38,6 +38,8 @@ use rstd::borrow::Cow; #[cfg(not(feature = "std"))] use rstd::slice; +use primitives::{sr25519, ed25519}; + /// Converts a pointer and length into an `u64`. pub fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { ((len as u64) | u64::from(ptr) << 32).to_le() @@ -352,3 +354,83 @@ impl PassBy for Option { impl PassBy for rstd::result::Result { type PassBy = Codec; } + +impl PassBy for sr25519::Public { + type PassBy = Inner; +} + +impl PassByInner for sr25519::Public { + type Inner = [u8; 32]; + + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn inner(&self) -> &Self::Inner { + &self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } +} + +impl PassBy for sr25519::Signature { + type PassBy = Inner; +} + +impl PassByInner for sr25519::Signature { + type Inner = [u8; 64]; + + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn inner(&self) -> &Self::Inner { + &self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } +} + +impl PassBy for ed25519::Public { + type PassBy = Inner; +} + +impl PassByInner for ed25519::Public { + type Inner = [u8; 32]; + + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn inner(&self) -> &Self::Inner { + &self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } +} + +impl PassBy for ed25519::Signature { + type PassBy = Inner; +} + +impl PassByInner for ed25519::Signature { + type Inner = [u8; 64]; + + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn inner(&self) -> &Self::Inner { + &self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 9665a2065dc8d..d37e0d836e70b 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -120,6 +120,11 @@ mod tests { call_wasm_method::("test_array_as_mutable_reference"); } + #[test] + fn test_return_input_public_key() { + call_wasm_method::("test_return_input_public_key"); + } + #[test] #[should_panic(expected = "Wasmi(Instantiation(\"Export ext_test_api_return_input not found\"))")] fn host_function_not_found() { diff --git a/core/runtime-interface/test-wasm/Cargo.toml b/core/runtime-interface/test-wasm/Cargo.toml index 98b2bef39d798..13abc15d8b1b8 100644 --- a/core/runtime-interface/test-wasm/Cargo.toml +++ b/core/runtime-interface/test-wasm/Cargo.toml @@ -8,11 +8,12 @@ build = "build.rs" [dependencies] runtime-interface = { package = "substrate-runtime-interface", path = "../", default-features = false } rstd = { package = "sr-std", path = "../../sr-std", default-features = false } +primitives = { package = "substrate-primitives", path = "../../primitives", default-features = false } [build-dependencies] wasm-builder-runner = { package = "substrate-wasm-builder-runner", version = "1.0.3", path = "../../utils/wasm-builder-runner" } [features] default = [ "std" ] -std = [ "runtime-interface/std", "rstd/std" ] +std = [ "runtime-interface/std", "rstd/std", "primitives/std" ] no_std = [] diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index 48436bb887501..d1812f02f3284 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -20,7 +20,9 @@ use runtime_interface::runtime_interface; -use rstd::{vec, vec::Vec, mem}; +use rstd::{vec, vec::Vec, mem, convert::TryFrom}; + +use primitives::sr25519::Public; // Inlucde the WASM binary #[cfg(feature = "std")] @@ -63,6 +65,11 @@ pub trait TestApi { fn array_as_mutable_reference(data: &mut [u8; 16]) { data.copy_from_slice(&TEST_ARRAY); } + + /// Returns the given public key as result. + fn return_input_public_key(key: Public) -> Public { + key + } } #[no_mangle] @@ -119,3 +126,18 @@ pub fn test_array_as_mutable_reference() { assert_eq!(array, TEST_ARRAY); } + +#[no_mangle] +pub fn test_return_input_public_key() { + let key = Public::try_from( + &[ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, + ][..], + ).unwrap(); + let ret_key = test_api::return_input_public_key(key.clone()); + + let key_data: &[u8] = key.as_ref(); + let ret_key_data: &[u8] = ret_key.as_ref(); + assert_eq!(key_data, ret_key_data); +} From 36ac4d3cc139a2125170c1822feafbadf63c7c41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 1 Oct 2019 09:58:28 +0200 Subject: [PATCH 24/76] Implement exchangeable function support --- .../proc-macro/src/bare_function_interface.rs | 24 +++--- .../proc-macro/src/host_function_interface.rs | 81 +++++++++++++++---- .../runtime-interface/proc-macro/src/utils.rs | 15 +++- core/runtime-interface/src/wasm.rs | 68 ++++++++++++++++ core/sr-io/Cargo.toml | 6 +- core/sr-io/src/lib.rs | 2 + core/sr-io/without_std.rs | 67 +-------------- 7 files changed, 164 insertions(+), 99 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index 5a5d3b3691ffc..f59043716f84a 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -18,12 +18,11 @@ use crate::utils::{ generate_crate_access, create_host_function_ident, get_function_arguments, - get_function_argument_names, get_function_argument_types_without_ref, + get_function_argument_names, get_function_argument_types_without_ref, get_trait_methods, }; use syn::{ - Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Result, ReturnType, TraitItem, - spanned::Spanned, + Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Result, ReturnType, spanned::Spanned, }; use proc_macro2::{TokenStream, Span}; @@ -33,17 +32,10 @@ use quote::{quote, quote_spanned}; /// Generate the bare-function interface. pub fn generate(trait_def: &ItemTrait) -> Result { let trait_name = &trait_def.ident; - trait_def - .items - .iter() - .filter_map(|i| match i { - TraitItem::Method(ref method) => Some(method), - _ => None, - }) - .try_fold(TokenStream::new(), |mut t, m| { - t.extend(function_for_method(trait_name, m)?); - Ok(t) - }) + get_trait_methods(trait_def).try_fold(TokenStream::new(), |mut t, m| { + t.extend(function_for_method(trait_name, m)?); + Ok(t) + }) } /// Generates the bare function implementation for the given method. @@ -55,9 +47,11 @@ fn function_for_method(trait_name: &Ident, method: &TraitItemMethod) -> Result Result< )* // Call the host function - let result = unsafe { #host_function_name( #( #arg_names2.get(), )* ) }; + let result = unsafe { #host_function_name.get()( #( #arg_names2.get(), )* ) }; #convert_return_value } diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 787b1b05b38c4..562b17ad04aa2 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -20,7 +20,7 @@ use crate::utils::{ generate_crate_access, create_host_function_ident, get_function_argument_names, get_function_argument_types_without_ref, get_function_argument_types_ref_and_mut, - get_function_argument_names_and_types_without_ref, + get_function_argument_names_and_types_without_ref, get_trait_methods, }; use syn::{ @@ -40,22 +40,32 @@ use std::iter::Iterator; /// implementations for the host functions on the host. pub fn generate(trait_def: &ItemTrait) -> Result { let trait_name = &trait_def.ident; - let extern_host_functions = trait_def - .items - .iter() - .filter_map(|i| match i { - TraitItem::Method(ref method) => Some(method), - _ => None, - }) + let extern_host_function_impls = get_trait_methods(trait_def) .try_fold(TokenStream::new(), |mut t, m| { t.extend(generate_extern_host_function(m, trait_name)?); Ok::<_, Error>(t) })?; + let extern_host_exchangeable_functions = get_trait_methods(trait_def) + .try_fold(TokenStream::new(), |mut t, m| { + t.extend(generate_extern_host_exchangeable_function(m, trait_name)?); + Ok::<_, Error>(t) + })?; let host_functions_struct = generate_host_functions_struct(trait_def)?; Ok( quote! { - #extern_host_functions + /// The implementations of the extern host functions. This special implementation module + /// is required to change the extern host functions signature to + /// `unsafe fn name(args) -> ret` to make the function implementations exchangeable. + #[cfg(not(feature = "std"))] + pub mod extern_host_function_impls { + use super::*; + + #extern_host_function_impls + } + + #extern_host_exchangeable_functions + #host_functions_struct } ) @@ -65,8 +75,12 @@ pub fn generate(trait_def: &ItemTrait) -> Result { fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) -> Result { let crate_ = generate_crate_access(); let arg_types = get_function_argument_types_without_ref(&method.sig); + let arg_types2 = get_function_argument_types_without_ref(&method.sig); let arg_names = get_function_argument_names(&method.sig); + let arg_names2 = get_function_argument_names(&method.sig); + let arg_names3 = get_function_argument_names(&method.sig); let function = create_host_function_ident(&method.sig.ident, trait_name); + let doc_string = format!(" Default extern host function implementation for [`../{}`].", function); let output = match method.sig.output { ReturnType::Default => quote!(), @@ -77,16 +91,55 @@ fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) - Ok( quote! { - #[cfg(not(feature = "std"))] - extern "C" { - pub fn #function ( - #( #arg_names: <#arg_types as #crate_::RIType>::FFIType ),* - ) #output; + #[doc(#doc_string)] + pub unsafe fn #function ( + #( #arg_names: <#arg_types as #crate_::RIType>::FFIType ),* + ) #output { + mod implementation { + use super::*; + + extern "C" { + pub fn #function ( + #( #arg_names2: <#arg_types2 as #crate_::RIType>::FFIType ),* + ) #output; + } + } + + implementation::#function( #( #arg_names3 ),* ) } } ) } +/// Generate the extern host exchangeable function for the given method. +fn generate_extern_host_exchangeable_function( + method: &TraitItemMethod, + trait_name: &Ident, +) -> Result { + let crate_ = generate_crate_access(); + let arg_types = get_function_argument_types_without_ref(&method.sig); + let function = create_host_function_ident(&method.sig.ident, trait_name); + let doc_string = format!(" Exchangeable extern host function used by [`{}`].", method.sig.ident); + + let output = match method.sig.output { + ReturnType::Default => quote!(), + ReturnType::Type(_, ref ty) => quote! { + -> <#ty as #crate_::RIType>::FFIType + } + }; + + Ok( + quote! { + #[cfg(not(feature = "std"))] + #[allow(non_upper_case_globals)] + #[doc(#doc_string)] + pub static #function : #crate_::wasm::ExchangeableFunction< + unsafe fn ( #( <#arg_types as #crate_::RIType>::FFIType ),* ) #output + > = #crate_::wasm::ExchangeableFunction::new(extern_host_function_impls::#function); + } + ) +} + /// Generate the `HostFunctions` struct that implements `wasm-interface::HostFunctions` to provide /// implementations for the extern host functions. fn generate_host_functions_struct(trait_def: &ItemTrait) -> Result { diff --git a/core/runtime-interface/proc-macro/src/utils.rs b/core/runtime-interface/proc-macro/src/utils.rs index 112ec79debd34..6f54572e1c615 100644 --- a/core/runtime-interface/proc-macro/src/utils.rs +++ b/core/runtime-interface/proc-macro/src/utils.rs @@ -18,7 +18,9 @@ use proc_macro2::{TokenStream, Span}; -use syn::{Ident, Error, Signature, Pat, PatType, FnArg, Type, token}; +use syn::{ + Ident, Error, Signature, Pat, PatType, FnArg, Type, token, TraitItemMethod, ItemTrait, TraitItem +}; use proc_macro_crate::crate_name; @@ -122,3 +124,14 @@ pub fn get_function_argument_types_ref_and_mut<'a>( _ => None, }) } + +/// Returns an iterator over all trait methods for the given trait definition. +pub fn get_trait_methods<'a>(trait_def: &'a ItemTrait) -> impl Iterator { + trait_def + .items + .iter() + .filter_map(|i| match i { + TraitItem::Method(ref method) => Some(method), + _ => None, + }) +} diff --git a/core/runtime-interface/src/wasm.rs b/core/runtime-interface/src/wasm.rs index f5f13b7da0475..7ac890a3ca3ba 100644 --- a/core/runtime-interface/src/wasm.rs +++ b/core/runtime-interface/src/wasm.rs @@ -18,6 +18,8 @@ use crate::RIType; +use rstd::cell::Cell; + /// Something that can be created from a ffi value. /// /// # Safety @@ -72,3 +74,69 @@ impl From<(T, O)> for WrappedFFIValue { WrappedFFIValue::WrappedAndOwned(val.0, val.1) } } + +/// The state of an exchangeable function. +#[derive(Clone, Copy)] +enum ExchangeableFunctionState { + /// Original function is present + Original, + /// The function has been replaced. + Replaced, +} + +/// A function which implementation can be exchanged. +/// +/// Internally this works by swapping function pointers. +pub struct ExchangeableFunction(Cell<(T, ExchangeableFunctionState)>); + +impl ExchangeableFunction { + /// Create a new instance of `ExchangeableFunction`. + pub const fn new(impl_: T) -> Self { + Self(Cell::new((impl_, ExchangeableFunctionState::Original))) + } +} + +impl ExchangeableFunction { + /// Replace the implementation with `new_impl`. + /// + /// # Panics + /// + /// Panics when trying to replace an already replaced implementation. + /// + /// # Returns + /// + /// Returns the original implementation wrapped in [`RestoreImplementation`]. + pub fn replace_implementation(&'static self, new_impl: T) -> RestoreImplementation { + if let ExchangeableFunctionState::Replaced = self.0.get().1 { + panic!("Trying to replace an already replaced implementation!") + } + + let old = self.0.replace((new_impl, ExchangeableFunctionState::Replaced)); + + RestoreImplementation(self, Some(old.0)) + } + + /// Restore the original implementation. + fn restore_orig_implementation(&self, orig: T) { + self.0.set((orig, ExchangeableFunctionState::Original)); + } + + /// Returns the internal function pointer. + pub fn get(&self) -> T { + self.0.get().0 + } +} + +// Wasm does not support threads, so this is safe; qed. +unsafe impl Sync for ExchangeableFunction {} + +/// Restores a function implementation on drop. +/// +/// Stores a static reference to the function object and the original implementation. +pub struct RestoreImplementation(&'static ExchangeableFunction, Option); + +impl Drop for RestoreImplementation { + fn drop(&mut self) { + self.0.restore_orig_implementation(self.1.take().expect("Value is only taken on drop; qed")); + } +} diff --git a/core/sr-io/Cargo.toml b/core/sr-io/Cargo.toml index 67e0f066a3948..70c56fe939a00 100644 --- a/core/sr-io/Cargo.toml +++ b/core/sr-io/Cargo.toml @@ -11,12 +11,12 @@ rustc_version = "0.2" [dependencies] rstd = { package = "sr-std", path = "../sr-std", default-features = false } primitives = { package = "substrate-primitives", path = "../primitives", default-features = false } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.0.5", default-features = false } hash-db = { version = "0.15.2", default-features = false } libsecp256k1 = { version = "0.2.1", optional = true } tiny-keccak = { version = "1.4.2", optional = true } substrate-state-machine = { path = "../state-machine", optional = true } -runtime-interface = { package = "substrate-runtime-interface", path = "../runtime-interface", optional = true } +runtime-interface = { package = "substrate-runtime-interface", path = "../runtime-interface", default-features = false } trie = { package = "substrate-trie", path = "../trie", optional = true } [dev-dependencies] @@ -33,7 +33,7 @@ std = [ "substrate-state-machine", "libsecp256k1", "tiny-keccak", - "runtime-interface", + "runtime-interface/std", ] nightly = [] strict = [] diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index f763bebb4cb39..e10c925c56f66 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -36,6 +36,8 @@ use primitives::{ }, }; +use runtime_interface::runtime_interface; + /// Error verifying ECDSA signature pub enum EcdsaVerifyError { /// Incorrect value of R or S diff --git a/core/sr-io/without_std.rs b/core/sr-io/without_std.rs index 6803b41ebd3fb..dfcf2a7e79023 100644 --- a/core/sr-io/without_std.rs +++ b/core/sr-io/without_std.rs @@ -22,6 +22,7 @@ use core::{intrinsics, panic::PanicInfo}; use rstd::{vec::Vec, cell::Cell, convert::TryInto}; use primitives::{offchain, Blake2Hasher}; use codec::Decode; +use runtime_interface::wasm::ExchangeableFunction; #[cfg(not(feature = "no_panic_handler"))] #[panic_handler] @@ -60,72 +61,6 @@ pub extern fn oom(_: core::alloc::Layout) -> ! { pub mod ext { use super::*; - /// The state of an exchangeable function. - #[derive(Clone, Copy)] - enum ExchangeableFunctionState { - /// Original function is present - Original, - /// The function has been replaced. - Replaced, - } - - /// A function which implementation can be exchanged. - /// - /// Internally this works by swapping function pointers. - pub struct ExchangeableFunction(Cell<(T, ExchangeableFunctionState)>); - - impl ExchangeableFunction { - /// Create a new instance of `ExchangeableFunction`. - pub const fn new(impl_: T) -> Self { - Self(Cell::new((impl_, ExchangeableFunctionState::Original))) - } - } - - impl ExchangeableFunction { - /// Replace the implementation with `new_impl`. - /// - /// # Panics - /// - /// Panics when trying to replace an already replaced implementation. - /// - /// # Returns - /// - /// Returns the original implementation wrapped in [`RestoreImplementation`]. - pub fn replace_implementation(&'static self, new_impl: T) -> RestoreImplementation { - if let ExchangeableFunctionState::Replaced = self.0.get().1 { - panic!("Trying to replace an already replaced implementation!") - } - - let old = self.0.replace((new_impl, ExchangeableFunctionState::Replaced)); - - RestoreImplementation(self, Some(old.0)) - } - - /// Restore the original implementation. - fn restore_orig_implementation(&self, orig: T) { - self.0.set((orig, ExchangeableFunctionState::Original)); - } - - /// Returns the internal function pointer. - pub fn get(&self) -> T { - self.0.get().0 - } - } - - // WASM does not support threads, so this is safe; qed. - unsafe impl Sync for ExchangeableFunction {} - - /// Restores a function implementation on drop. - /// - /// Stores a static reference to the function object and the original implementation. - pub struct RestoreImplementation(&'static ExchangeableFunction, Option); - - impl Drop for RestoreImplementation { - fn drop(&mut self) { - self.0.restore_orig_implementation(self.1.take().expect("Value is only taken on drop; qed")); - } - } - /// Declare extern functions macro_rules! extern_functions { ( From 5eb214372cb7634e09c9fc313f13204a3a190508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 2 Oct 2019 10:47:02 +0200 Subject: [PATCH 25/76] Rewrite sr-io with as runtime interface --- core/sr-io/src/lib.rs | 737 ++++++++++++++++++++++++-------------- core/sr-io/with_std.rs | 409 --------------------- core/sr-io/without_std.rs | 1 - 3 files changed, 467 insertions(+), 680 deletions(-) diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index e10c925c56f66..b67c5f313d6b2 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -27,6 +27,7 @@ #![cfg_attr(not(feature = "std"), doc = "Substrate's runtime standard library as compiled without Rust's standard library.")] use hash_db::Hasher; + use rstd::vec::Vec; use primitives::{ @@ -34,11 +35,17 @@ use primitives::{ offchain::{ Timestamp, HttpRequestId, HttpRequestStatus, HttpError, StorageKind, OpaqueNetworkState, }, + child_storage_key::ChildStorageKey, }; +use trie::{TrieConfiguration, trie_types::Layout}; + use runtime_interface::runtime_interface; +use codec::{Encode, Decode}; + /// Error verifying ECDSA signature +#[derive(Encode, Decode)] pub enum EcdsaVerifyError { /// Incorrect value of R or S BadRS, @@ -50,317 +57,507 @@ pub enum EcdsaVerifyError { pub mod offchain; -/// Converts a public trait definition into a private trait and set of public functions -/// that assume the trait is implemented for `()` for ease of calling. -macro_rules! export_api { - ( - $( #[$trait_attr:meta] )* - pub(crate) trait $trait_name:ident { - $( - $( #[$attr:meta] )* - fn $name:ident - ( $( $arg:ident : $arg_ty:ty ),* $(,)? ) - $( -> $ret:ty )? - $( where $( $w_name:path : $w_ty:path ),+ )?; - )* - } - ) => { - $( #[$trait_attr] )* - pub(crate) trait $trait_name { - $( - $( #[$attr] )* - fn $name ( $($arg : $arg_ty ),* ) $( -> $ret )? - $( where $( $w_name : $w_ty ),+ )?; - )* - } - - $( - $( #[$attr] )* - pub fn $name ( $($arg : $arg_ty ),* ) $( -> $ret )? - $( where $( $w_name : $w_ty ),+ )? - { - #[allow(deprecated)] - <()>:: $name ( $( $arg ),* ) - } - )* +/// Returns a `ChildStorageKey` if the given `storage_key` slice is a valid storage +/// key or panics otherwise. +/// +/// Panicking here is aligned with what the `without_std` environment would do +/// in the case of an invalid child storage key. +fn child_storage_key_or_panic(storage_key: &[u8]) -> ChildStorageKey { + match ChildStorageKey::from_slice(storage_key) { + Some(storage_key) => storage_key, + None => panic!("child storage key is invalid"), } } -export_api! { - pub(crate) trait StorageApi { - /// Get `key` from storage and return a `Vec`, empty if there's a problem. - fn storage(key: &[u8]) -> Option>; +/// Interface for accessing the storage from within the runtime. +#[runtime_interface] +pub trait Storage { + /// Returns the data for `key` in the storage or `None` if the key can not be found. + fn get(&self, key: &[u8]) -> Option> { + self.storage(key).map(|s| s.to_vec()) + } - /// Get `key` from child storage and return a `Vec`, empty if there's a problem. - fn child_storage(storage_key: &[u8], key: &[u8]) -> Option>; + /// Returns the data for `key` in the child storage or `None` if the key can not be found. + fn child_get(&self, child_storage_key: &[u8], key: &[u8]) -> Option> { + let storage_key = child_storage_key_or_panic(child_storage_key); + ext.child_storage(storage_key, key).map(|s| s.to_vec()) + } - /// Get `key` from storage, placing the value into `value_out` and return the number of - /// bytes that the entry in storage has beyond the offset or `None` if the storage entry - /// doesn't exist at all. - /// If `value_out` length is smaller than the returned length, only `value_out` length bytes - /// are copied into `value_out`. - fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option; + /// Get `key` from storage, placing the value into `value_out` and return the number of + /// bytes that the entry in storage has beyond the offset or `None` if the storage entry + /// doesn't exist at all. + /// If `value_out` length is smaller than the returned length, only `value_out` length bytes + /// are copied into `value_out`. + fn read(&self, key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option { + self.storage(key).map(|value| { + let value_offset = value_offset as usize; + let data = &value[value_offset.min(value.len())..]; + let written = std::cmp::min(data.len(), value_out.len()); + value_out[..written].copy_from_slice(&data[..written]); + value.len() as u32 + }) + } - /// Get `key` from child storage, placing the value into `value_out` and return the number - /// of bytes that the entry in storage has beyond the offset or `None` if the storage entry - /// doesn't exist at all. - /// If `value_out` length is smaller than the returned length, only `value_out` length bytes - /// are copied into `value_out`. - fn read_child_storage(storage_key: &[u8], key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option; + /// Get `key` from child storage, placing the value into `value_out` and return the number + /// of bytes that the entry in storage has beyond the offset or `None` if the storage entry + /// doesn't exist at all. + /// If `value_out` length is smaller than the returned length, only `value_out` length bytes + /// are copied into `value_out`. + fn child_read( + &self, + child_storage_key: &[u8], + key: &[u8], + value_out: &mut [u8], + value_offset: u32, + ) -> Option { + let storage_key = child_storage_key_or_panic(child_storage_key); + self.child_storage(storage_key, key) + .map(|value| { + let value_offset = value_offset as usize; + let data = &value[value_offset.min(value.len())..]; + let written = std::cmp::min(data.len(), value_out.len()); + value_out[..written].copy_from_slice(&data[..written]); + value.len() as u32 + }) + } - /// Set the storage of some particular key to Some value. - fn set_storage(key: &[u8], value: &[u8]); + /// Set `key` to `value` in the storage. + fn set(&mut self, key: &[u8], value: &[u8]) { + self.set_storage(key.to_vec(), value.to_vec()); + } - /// Set the child storage of some particular key to Some value. - fn set_child_storage(storage_key: &[u8], key: &[u8], value: &[u8]); + /// Set `key` to `value` in the child storage denoted by `child_storage_key`. + fn child_set(&mut self, child_storage_key: &[u8], key: &[u8], value: &[u8]) { + let storage_key = child_storage_key_or_panic(child_storage_key); + self.set_child_storage(storage_key, key.to_vec(), value.to_vec()); + } - /// Clear the storage of a key. - fn clear_storage(key: &[u8]); + /// Clear the storage of the given `key` and its value. + fn clear(&mut self, key: &[u8]) { + self.clear_storage(key) + } - /// Clear the storage of a key. - fn clear_child_storage(storage_key: &[u8], key: &[u8]); + /// Clear the storage of a key. + fn clear_child(&mut self, child_storage_key: &[u8], key: &[u8]) { + let storage_key = child_storage_key_or_panic(child_storage_key); + self.clear_child_storage(storage_key, key); + } - /// Clear an entire child storage. - fn kill_child_storage(storage_key: &[u8]); + /// Clear an entire child storage. + fn kill_child_storage(&mut self, child_storage_key: &[u8]) { + let storage_key = child_storage_key_or_panic(child_storage_key); + self.kill_child_storage(storage_key); + } - /// Check whether a given `key` exists in storage. - fn exists_storage(key: &[u8]) -> bool; + /// Check whether the given `key` exists in storage. + fn exists(&self, key: &[u8]) -> bool { + self.exists_storage(key) + } - /// Check whether a given `key` exists in storage. - fn exists_child_storage(storage_key: &[u8], key: &[u8]) -> bool; + /// Check whether the given `key` exists in storage. + fn child_exists(&self, child_storage_key: &[u8], key: &[u8]) -> bool { + let storage_key = child_storage_key_or_panic(child_storage_key); + self.exists_child_storage(storage_key, key) + } - /// Clear the storage entries with a key that starts with the given prefix. - fn clear_prefix(prefix: &[u8]); + /// Clear the storage of each key-value pair where the key starts with the given `prefix`. + fn clear_prefix(&mut self, prefix: &[u8]) { + self.clear_prefix(prefix) + } - /// Clear the child storage entries with a key that starts with the given prefix. - fn clear_child_prefix(storage_key: &[u8], prefix: &[u8]); + /// Clear the child storage of each key-value pair where the key starts with the given `prefix`. + fn child_clear_prefix(&mut self, child_storage_key: &[u8], prefix: &[u8]) { + let storage_key = child_storage_key_or_panic(child_storage_key); + self.clear_child_prefix(storage_key, prefix); + } - /// "Commit" all existing operations and compute the resultant storage root. - fn storage_root() -> [u8; 32]; + /// "Commit" all existing operations and compute the resulting storage root. + fn root(&mut self) -> [u8; 32] { + self.storage_root() + } - /// "Commit" all existing operations and compute the resultant child storage root. - fn child_storage_root(storage_key: &[u8]) -> Vec; + /// "Commit" all existing operations and compute the resulting child storage root. + fn child_root(&mut self, child_storage_key: &[u8]) -> Vec { + let storage_key = child_storage_key_or_panic(child_storage_key); + self.child_storage_root(storage_key) + } - /// "Commit" all existing operations and get the resultant storage change root. - fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]>; + /// "Commit" all existing operations and get the resulting storage change root. + fn changes_root(&mut self, parent_hash: [u8; 32]) -> Option<[u8; 32]> { + self.storage_changes_root(parent_hash.into()).map(|h| h.map(|h| h.into())).ok() + } - /// A trie root formed from the iterated items. - fn blake2_256_trie_root(input: Vec<(Vec, Vec)>) -> H256; + /// A trie root formed from the iterated items. + fn blake2_256_trie_root(input: Vec<(Vec, Vec)>) -> H256 { + Layout::::trie_root(input) + } - /// A trie root formed from the enumerated items. - fn blake2_256_ordered_trie_root(input: Vec>) -> H256; + /// A trie root formed from the enumerated items. + fn blake2_256_ordered_trie_root(input: Vec>) -> H256 { + Layout::::ordered_trie_root(input) } } -export_api! { - pub(crate) trait OtherApi { - /// The current relay chain identifier. - fn chain_id() -> u64; +/// Interface that provides miscellaneous functions for communicating between the runtime and the node. +#[runtime_interface] +pub trait Misc { + /// The current relay chain identifier. + fn chain_id(&self) -> u64 { + self.chain_id() + } + + /// Print a number. + fn print_num(val: u64) { + println!("{}", val); + } + + /// Print any valid `utf8` buffer. + fn print_utf8(utf8: &[u8]) { + if let Ok(data) = std::str::from_utf8(utf8) { + println!("{}", data) + } + } - /// Print a number. - fn print_num(val: u64); - /// Print any valid `utf8` buffer. - fn print_utf8(utf8: &[u8]); - /// Print any `u8` slice as hex. - fn print_hex(data: &[u8]); + /// Print any `u8` slice as hex. + fn print_hex(data: &[u8]) { + println!("{}", HexDisplay::from(&data)); } } -export_api! { - pub(crate) trait CryptoApi { - /// Returns all ed25519 public keys for the given key id from the keystore. - fn ed25519_public_keys(id: KeyTypeId) -> Vec; - /// Generate an ed22519 key for the given key type and store it in the keystore. - /// - /// Returns the raw public key. - fn ed25519_generate(id: KeyTypeId, seed: Option<&str>) -> ed25519::Public; - /// Sign the given `msg` with the ed25519 key that corresponds to the given public key and - /// key type in the keystore. - /// - /// Returns the raw signature. - fn ed25519_sign( - id: KeyTypeId, - pubkey: &ed25519::Public, - msg: &[u8], - ) -> Option; - /// Verify an ed25519 signature. - /// - /// Returns `true` when the verification in successful. - fn ed25519_verify(sig: &ed25519::Signature, msg: &[u8], pubkey: &ed25519::Public) -> bool; - - /// Returns all sr25519 public keys for the given key id from the keystore. - fn sr25519_public_keys(id: KeyTypeId) -> Vec; - /// Generate an sr22519 key for the given key type and store it in the keystore. - /// - /// Returns the raw public key. - fn sr25519_generate(id: KeyTypeId, seed: Option<&str>) -> sr25519::Public; - /// Sign the given `msg` with the sr25519 key that corresponds to the given public key and - /// key type in the keystore. - /// - /// Returns the raw signature. - fn sr25519_sign( - id: KeyTypeId, - pubkey: &sr25519::Public, - msg: &[u8], - ) -> Option; - /// Verify an sr25519 signature. - /// - /// Returns `true` when the verification in successful. - fn sr25519_verify(sig: &sr25519::Signature, msg: &[u8], pubkey: &sr25519::Public) -> bool; - - /// Verify and recover a SECP256k1 ECDSA signature. - /// - `sig` is passed in RSV format. V should be either 0/1 or 27/28. - /// - returns `Err` if the signature is bad, otherwise the 64-byte pubkey (doesn't include the 0x04 prefix). - fn secp256k1_ecdsa_recover(sig: &[u8; 65], msg: &[u8; 32]) -> Result<[u8; 64], EcdsaVerifyError>; +/// Interfaces for working with crypto related types from within the runtime. +pub trait Crypto { + /// Returns all `ed25519` public keys for the given key id from the keystore. + fn ed25519_public_keys(&self, id: KeyTypeId) -> Vec { + self.keystore() + .expect("No `keystore` associated for the current context!") + .read() + .ed25519_public_keys(id) + } + + /// Generate an `ed22519` key for the given key type and store it in the keystore. + /// + /// Returns the public key. + fn ed25519_generate(&self, id: KeyTypeId, seed: Option<&str>) -> ed25519::Public { + self.keystore() + .expect("No `keystore` associated for the current context!") + .write() + .ed25519_generate_new(id, seed) + .expect("`ed25519_generate` failed") + } + + /// Sign the given `msg` with the `ed25519` key that corresponds to the given public key and + /// key type in the keystore. + /// + /// Returns the signature. + fn ed25519_sign( + &self, + id: KeyTypeId, + pub_key: &ed25519::Public, + msg: &[u8], + ) -> Option { + self.keystore() + .expect("No `keystore` associated for the current context!") + .read() + .ed25519_key_pair(id, &pub_key) + .map(|k| k.sign(msg)) + } + + /// Verify an `ed25519` signature. + /// + /// Returns `true` when the verification in successful. + fn ed25519_verify( + &self, + sig: &ed25519::Signature, + msg: &[u8], + pub_key: &ed25519::Public, + ) -> bool { + ed25519::Pair::verify(sig, msg, pub_key) + } + + /// Returns all `sr25519` public keys for the given key id from the keystore. + fn sr25519_public_keys(&self, id: KeyTypeId) -> Vec { + self.keystore() + .expect("No `keystore` associated for the current context!") + .read() + .sr25519_public_keys(id) + } + + /// Generate an `sr22519` key for the given key type and store it in the keystore. + /// + /// Returns the public key. + fn sr25519_generate(&self, id: KeyTypeId, seed: Option<&str>) -> sr25519::Public { + self.keystore() + .expect("No `keystore` associated for the current context!") + .write() + .sr25519_generate_new(id, seed) + .expect("`sr25519_generate` failed") + } + + /// Sign the given `msg` with the `sr25519` key that corresponds to the given public key and + /// key type in the keystore. + /// + /// Returns the signature. + fn sr25519_sign( + self, + id: KeyTypeId, + pub_key: &sr25519::Public, + msg: &[u8], + ) -> Option { + self.keystore() + .expect("No `keystore` associated for the current context!") + .read() + .sr25519_key_pair(id, &pub_key) + .map(|k| k.sign(msg)) + } + + /// Verify an `sr25519` signature. + /// + /// Returns `true` when the verification in successful. + fn sr25519_verify(sig: &sr25519::Signature, msg: &[u8], pubkey: &sr25519::Public) -> bool { + sr25519::Pair::verify(sig, msg, pubkey) + } + + /// Verify and recover a SECP256k1 ECDSA signature. + /// - `sig` is passed in RSV format. V should be either 0/1 or 27/28. + /// Returns `Err` if the signature is bad, otherwise the 64-byte pubkey + /// (doesn't include the 0x04 prefix). + fn secp256k1_ecdsa_recover( + sig: &[u8; 65], + msg: &[u8; 32], + ) -> Result<[u8; 64], EcdsaVerifyError> { + let rs = secp256k1::Signature::parse_slice(&sig[0..64]) + .map_err(|_| EcdsaVerifyError::BadRS)?; + let v = secp256k1::RecoveryId::parse(if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8) + .map_err(|_| EcdsaVerifyError::BadV)?; + let pubkey = secp256k1::recover(&secp256k1::Message::parse(msg), &rs, &v) + .map_err(|_| EcdsaVerifyError::BadSignature)?; + let mut res = [0u8; 64]; + res.copy_from_slice(&pubkey.serialize()[1..65]); + Ok(res) } } -export_api! { - pub(crate) trait HashingApi { - /// Conduct a 256-bit Keccak hash. - fn keccak_256(data: &[u8]) -> [u8; 32]; +/// Interface that provides functions for hashing with different algorithms. +#[runtime_interface] +pub trait Hashing { + /// Conduct a 256-bit Keccak hash. + fn keccak_256(data: &[u8]) -> [u8; 32] { + tiny_keccak::keccak256(data) + } - /// Conduct a 128-bit Blake2 hash. - fn blake2_128(data: &[u8]) -> [u8; 16]; + /// Conduct a 128-bit Blake2 hash. + fn blake2_128(data: &[u8]) -> [u8; 16] { + blake2_128(data) + } - /// Conduct a 256-bit Blake2 hash. - fn blake2_256(data: &[u8]) -> [u8; 32]; + /// Conduct a 256-bit Blake2 hash. + fn blake2_256(data: &[u8]) -> [u8; 32] { + blake2_256(data) + } - /// Conduct four XX hashes to give a 256-bit result. - fn twox_256(data: &[u8]) -> [u8; 32]; + /// Conduct four XX hashes to give a 256-bit result. + fn twox_256(data: &[u8]) -> [u8; 32] { + twox_256(data) + } - /// Conduct two XX hashes to give a 128-bit result. - fn twox_128(data: &[u8]) -> [u8; 16]; + /// Conduct two XX hashes to give a 128-bit result. + fn twox_128(data: &[u8]) -> [u8; 16] { + twox_128(data) + } - /// Conduct two XX hashes to give a 64-bit result. - fn twox_64(data: &[u8]) -> [u8; 8]; + /// Conduct two XX hashes to give a 64-bit result. + fn twox_64(data: &[u8]) -> [u8; 8] { + twox_64(data) } } -export_api! { - pub(crate) trait OffchainApi { - /// Returns if the local node is a potential validator. - /// - /// Even if this function returns `true`, it does not mean that any keys are configured - /// and that the validator is registered in the chain. - fn is_validator() -> bool; - - /// Submit transaction to the pool. - /// - /// The transaction will end up in the pool. - fn submit_transaction(data: Vec) -> Result<(), ()>; - - /// Returns information about the local node's network state. - fn network_state() -> Result; - - /// Returns current UNIX timestamp (in millis) - fn timestamp() -> Timestamp; - - /// Pause the execution until `deadline` is reached. - fn sleep_until(deadline: Timestamp); - - /// Returns a random seed. - /// - /// This is a trully random non deterministic seed generated by host environment. - /// Obviously fine in the off-chain worker context. - fn random_seed() -> [u8; 32]; - - /// Sets a value in the local storage. - /// - /// Note this storage is not part of the consensus, it's only accessible by - /// offchain worker tasks running on the same machine. It IS persisted between runs. - fn local_storage_set(kind: StorageKind, key: &[u8], value: &[u8]); - - /// Sets a value in the local storage if it matches current value. - /// - /// Since multiple offchain workers may be running concurrently, to prevent - /// data races use CAS to coordinate between them. - /// - /// Returns `true` if the value has been set, `false` otherwise. - /// - /// Note this storage is not part of the consensus, it's only accessible by - /// offchain worker tasks running on the same machine. It IS persisted between runs. - fn local_storage_compare_and_set( - kind: StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool; - - /// Gets a value from the local storage. - /// - /// If the value does not exist in the storage `None` will be returned. - /// Note this storage is not part of the consensus, it's only accessible by - /// offchain worker tasks running on the same machine. It IS persisted between runs. - fn local_storage_get(kind: StorageKind, key: &[u8]) -> Option>; - - /// Initiates a http request given HTTP verb and the URL. - /// - /// Meta is a future-reserved field containing additional, parity-scale-codec encoded parameters. - /// Returns the id of newly started request. - fn http_request_start( - method: &str, - uri: &str, - meta: &[u8], - ) -> Result; - - /// Append header to the request. - fn http_request_add_header( - request_id: HttpRequestId, - name: &str, - value: &str, - ) -> Result<(), ()>; - - /// Write a chunk of request body. - /// - /// Writing an empty chunks finalises the request. - /// Passing `None` as deadline blocks forever. - /// - /// Returns an error in case deadline is reached or the chunk couldn't be written. - fn http_request_write_body( - request_id: HttpRequestId, - chunk: &[u8], - deadline: Option, - ) -> Result<(), HttpError>; - - /// Block and wait for the responses for given requests. - /// - /// Returns a vector of request statuses (the len is the same as ids). - /// Note that if deadline is not provided the method will block indefinitely, - /// otherwise unready responses will produce `DeadlineReached` status. - /// - /// Passing `None` as deadline blocks forever. - fn http_response_wait( - ids: &[HttpRequestId], - deadline: Option, - ) -> Vec; - - /// Read all response headers. - /// - /// Returns a vector of pairs `(HeaderKey, HeaderValue)`. - /// NOTE response headers have to be read before response body. - fn http_response_headers(request_id: HttpRequestId) -> Vec<(Vec, Vec)>; - - /// Read a chunk of body response to given buffer. - /// - /// Returns the number of bytes written or an error in case a deadline - /// is reached or server closed the connection. - /// If `0` is returned it means that the response has been fully consumed - /// and the `request_id` is now invalid. - /// NOTE this implies that response headers must be read before draining the body. - /// Passing `None` as a deadline blocks forever. - fn http_response_read_body( - request_id: HttpRequestId, - buffer: &mut [u8], - deadline: Option, - ) -> Result; +/// Interface that provides functions to access the offchain functionality. +#[runtime_interface] +pub trait Offchain { + /// Returns if the local node is a potential validator. + /// + /// Even if this function returns `true`, it does not mean that any keys are configured + /// and that the validator is registered in the chain. + fn is_validator(&self) -> bool { + self.offchain() + .expect("is_validator can be called only in the offchain worker context") + .is_validator() + } + + /// Submit an encoded transaction to the pool. + /// + /// The transaction will end up in the pool. + fn submit_transaction(&self, data: Vec) -> Result<(), ()> { + self.offchain() + .expect("submit_transaction can be called only in the offchain worker context") + .submit_transaction(data) + } + + /// Returns information about the local node's network state. + fn network_state(&self) -> Result { + self.offchain() + .expect("network_state can be called only in the offchain worker context") + .network_state() + } + + /// Returns current UNIX timestamp (in millis) + fn timestamp(&self) -> Timestamp { + self.offchain() + .expect("timestamp can be called only in the offchain worker context") + .timestamp() + } + + /// Pause the execution until `deadline` is reached. + fn sleep_until(&self, deadline: Timestamp) { + self.offchain() + .expect("sleep_until can be called only in the offchain worker context") + .sleep_until(deadline) + } + + /// Returns a random seed. + /// + /// This is a trully random non deterministic seed generated by host environment. + /// Obviously fine in the off-chain worker context. + fn random_seed(&self) -> [u8; 32] { + self.offchain() + .expect("random_seed can be called only in the offchain worker context") + .random_seed() + } + + /// Sets a value in the local storage. + /// + /// Note this storage is not part of the consensus, it's only accessible by + /// offchain worker tasks running on the same machine. It IS persisted between runs. + fn local_storage_set(&self, kind: StorageKind, key: &[u8], value: &[u8]) { + self.offchain() + .expect("random_seed can be called only in the offchain worker context") + .local_storage_set(kind, key, value) + } + + /// Sets a value in the local storage if it matches current value. + /// + /// Since multiple offchain workers may be running concurrently, to prevent + /// data races use CAS to coordinate between them. + /// + /// Returns `true` if the value has been set, `false` otherwise. + /// + /// Note this storage is not part of the consensus, it's only accessible by + /// offchain worker tasks running on the same machine. It IS persisted between runs. + fn local_storage_compare_and_set( + &self, + kind: StorageKind, + key: &[u8], + old_value: Option<&[u8]>, + new_value: &[u8], + ) -> bool { + self.offchain() + .expect("random_seed can be called only in the offchain worker context") + .local_storage_compare_and_set(kind, key, old_value, new_value) + } + + /// Gets a value from the local storage. + /// + /// If the value does not exist in the storage `None` will be returned. + /// Note this storage is not part of the consensus, it's only accessible by + /// offchain worker tasks running on the same machine. It IS persisted between runs. + fn local_storage_get(&self, kind: StorageKind, key: &[u8]) -> Option> { + self.offchain() + .expect("random_seed can be called only in the offchain worker context") + .local_storage_get(kind, key) + } + + /// Initiates a http request given HTTP verb and the URL. + /// + /// Meta is a future-reserved field containing additional, parity-scale-codec encoded parameters. + /// Returns the id of newly started request. + fn http_request_start( + &self, + method: &str, + uri: &str, + meta: &[u8], + ) -> Result { + self.offchain() + .expect("random_seed can be called only in the offchain worker context") + .http_request_start(method, uri, meta) + } + + /// Append header to the request. + fn http_request_add_header( + &self, + request_id: HttpRequestId, + name: &str, + value: &str, + ) -> Result<(), ()> { + self.offchain() + .expect("random_seed can be called only in the offchain worker context") + .http_request_add_header(request_id, name, value) + } + + /// Write a chunk of request body. + /// + /// Writing an empty chunks finalises the request. + /// Passing `None` as deadline blocks forever. + /// + /// Returns an error in case deadline is reached or the chunk couldn't be written. + fn http_request_write_body( + &self, + request_id: HttpRequestId, + chunk: &[u8], + deadline: Option, + ) -> Result<(), HttpError> { + self.offchain() + .expect("random_seed can be called only in the offchain worker context") + .http_request_write_body(request_id, chunk, deadline) + } + + /// Block and wait for the responses for given requests. + /// + /// Returns a vector of request statuses (the len is the same as ids). + /// Note that if deadline is not provided the method will block indefinitely, + /// otherwise unready responses will produce `DeadlineReached` status. + /// + /// Passing `None` as deadline blocks forever. + fn http_response_wait( + &self, + ids: &[HttpRequestId], + deadline: Option, + ) -> Vec { + self.offchain() + .expect("random_seed can be called only in the offchain worker context") + .http_response_wait(ids, deadline) + } + + /// Read all response headers. + /// + /// Returns a vector of pairs `(HeaderKey, HeaderValue)`. + /// NOTE response headers have to be read before response body. + fn http_response_headers(&self, request_id: HttpRequestId) -> Vec<(Vec, Vec)> { + self.offchain() + .expect("random_seed can be called only in the offchain worker context") + .http_response_headers(request_id) + } + + /// Read a chunk of body response to given buffer. + /// + /// Returns the number of bytes written or an error in case a deadline + /// is reached or server closed the connection. + /// If `0` is returned it means that the response has been fully consumed + /// and the `request_id` is now invalid. + /// NOTE this implies that response headers must be read before draining the body. + /// Passing `None` as a deadline blocks forever. + fn http_response_read_body( + &self, + request_id: HttpRequestId, + buffer: &mut [u8], + deadline: Option, + ) -> Result { + self.offchain() + .expect("random_seed can be called only in the offchain worker context") + .http_response_read_body(request_id, buffer, deadline) + .map(|r| r as u32) } } -/// API trait that should cover all other APIs. -/// -/// Implement this to make sure you implement all APIs. -trait Api: StorageApi + OtherApi + CryptoApi + HashingApi + OffchainApi {} mod imp { use super::*; diff --git a/core/sr-io/with_std.rs b/core/sr-io/with_std.rs index 1a44b17120eb4..44c9d4e1208fb 100644 --- a/core/sr-io/with_std.rs +++ b/core/sr-io/with_std.rs @@ -32,415 +32,6 @@ use runtime_interface::with_externalities as with_ext; pub trait HasherBounds {} impl HasherBounds for T {} -/// Returns a `ChildStorageKey` if the given `storage_key` slice is a valid storage -/// key or panics otherwise. -/// -/// Panicking here is aligned with what the `without_std` environment would do -/// in the case of an invalid child storage key. -fn child_storage_key_or_panic(storage_key: &[u8]) -> ChildStorageKey { - match ChildStorageKey::from_slice(storage_key) { - Some(storage_key) => storage_key, - None => panic!("child storage key is invalid"), - } -} - -impl StorageApi for () { - fn storage(key: &[u8]) -> Option> { - with_ext(|ext| ext.storage(key).map(|s| s.to_vec())) - .expect("storage cannot be called outside of an Externalities-provided environment.") - } - - fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { - with_ext(|ext| ext.storage(key).map(|value| { - let data = &value[value_offset.min(value.len())..]; - let written = std::cmp::min(data.len(), value_out.len()); - value_out[..written].copy_from_slice(&data[..written]); - value.len() - })).expect("read_storage cannot be called outside of an Externalities-provided environment.") - } - - fn child_storage(storage_key: &[u8], key: &[u8]) -> Option> { - with_ext(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.child_storage(storage_key, key).map(|s| s.to_vec()) - }) - .expect("storage cannot be called outside of an Externalities-provided environment.") - } - - fn set_storage(key: &[u8], value: &[u8]) { - with_ext(|ext| - ext.set_storage(key.to_vec(), value.to_vec()) - ); - } - - fn read_child_storage( - storage_key: &[u8], - key: &[u8], - value_out: &mut [u8], - value_offset: usize, - ) -> Option { - with_ext(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.child_storage(storage_key, key) - .map(|value| { - let data = &value[value_offset.min(value.len())..]; - let written = std::cmp::min(data.len(), value_out.len()); - value_out[..written].copy_from_slice(&data[..written]); - value.len() - }) - }) - .expect("read_child_storage cannot be called outside of an Externalities-provided environment.") - } - - fn set_child_storage(storage_key: &[u8], key: &[u8], value: &[u8]) { - with_ext(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.set_child_storage(storage_key, key.to_vec(), value.to_vec()) - }); - } - - fn clear_storage(key: &[u8]) { - with_ext(|ext| - ext.clear_storage(key) - ); - } - - fn clear_child_storage(storage_key: &[u8], key: &[u8]) { - with_ext(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.clear_child_storage(storage_key, key) - }); - } - - fn kill_child_storage(storage_key: &[u8]) { - with_ext(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.kill_child_storage(storage_key) - }); - } - - fn exists_storage(key: &[u8]) -> bool { - with_ext(|ext| - ext.exists_storage(key) - ).unwrap_or(false) - } - - fn exists_child_storage(storage_key: &[u8], key: &[u8]) -> bool { - with_ext(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.exists_child_storage(storage_key, key) - }).unwrap_or(false) - } - - fn clear_prefix(prefix: &[u8]) { - with_ext(|ext| - ext.clear_prefix(prefix) - ); - } - - fn clear_child_prefix(storage_key: &[u8], prefix: &[u8]) { - with_ext(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.clear_child_prefix(storage_key, prefix) - }); - } - - fn storage_root() -> [u8; 32] { - with_ext(|ext| - ext.storage_root() - ).unwrap_or(H256::zero()).into() - } - - fn child_storage_root(storage_key: &[u8]) -> Vec { - with_ext(|ext| { - let storage_key = child_storage_key_or_panic(storage_key); - ext.child_storage_root(storage_key) - }).expect("child_storage_root cannot be called outside of an Externalities-provided environment.") - } - - fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]> { - with_ext(|ext| - ext.storage_changes_root(parent_hash.into()).map(|h| h.map(|h| h.into())) - ).unwrap_or(Ok(None)).expect("Invalid parent hash passed to storage_changes_root") - } - - fn blake2_256_trie_root(input: Vec<(Vec, Vec)>) -> H256 { - Layout::::trie_root(input) - } - - fn blake2_256_ordered_trie_root(input: Vec>) -> H256 { - Layout::::ordered_trie_root(input) - } -} - -impl OtherApi for () { - fn chain_id() -> u64 { - with_ext(|ext| - ext.chain_id() - ).unwrap_or(0) - } - - fn print_num(val: u64) { - println!("{}", val); - } - - fn print_utf8(utf8: &[u8]) { - if let Ok(data) = std::str::from_utf8(utf8) { - println!("{}", data) - } - } - - fn print_hex(data: &[u8]) { - println!("{}", HexDisplay::from(&data)); - } -} - -impl CryptoApi for () { - fn ed25519_public_keys(id: KeyTypeId) -> Vec { - with_ext(|ext| { - ext.keystore() - .expect("No `keystore` associated for the current context!") - .read() - .ed25519_public_keys(id) - }).expect("`ed25519_public_keys` cannot be called outside of an Externalities-provided environment.") - } - - fn ed25519_generate(id: KeyTypeId, seed: Option<&str>) -> ed25519::Public { - with_ext(|ext| { - ext.keystore() - .expect("No `keystore` associated for the current context!") - .write() - .ed25519_generate_new(id, seed) - .expect("`ed25519_generate` failed") - }).expect("`ed25519_generate` cannot be called outside of an Externalities-provided environment.") - } - - fn ed25519_sign( - id: KeyTypeId, - pubkey: &ed25519::Public, - msg: &[u8], - ) -> Option { - let pub_key = ed25519::Public::try_from(pubkey.as_ref()).ok()?; - - with_ext(|ext| { - ext.keystore() - .expect("No `keystore` associated for the current context!") - .read() - .ed25519_key_pair(id, &pub_key) - .map(|k| k.sign(msg)) - }).expect("`ed25519_sign` cannot be called outside of an Externalities-provided environment.") - } - - fn ed25519_verify(sig: &ed25519::Signature, msg: &[u8], pubkey: &ed25519::Public) -> bool { - ed25519::Pair::verify(sig, msg, pubkey) - } - - fn sr25519_public_keys(id: KeyTypeId) -> Vec { - with_ext(|ext| { - ext.keystore() - .expect("No `keystore` associated for the current context!") - .read() - .sr25519_public_keys(id) - }).expect("`sr25519_public_keys` cannot be called outside of an Externalities-provided environment.") - } - - fn sr25519_generate(id: KeyTypeId, seed: Option<&str>) -> sr25519::Public { - with_ext(|ext| { - ext.keystore() - .expect("No `keystore` associated for the current context!") - .write() - .sr25519_generate_new(id, seed) - .expect("`sr25519_generate` failed") - }).expect("`sr25519_generate` cannot be called outside of an Externalities-provided environment.") - } - - fn sr25519_sign( - id: KeyTypeId, - pubkey: &sr25519::Public, - msg: &[u8], - ) -> Option { - let pub_key = sr25519::Public::try_from(pubkey.as_ref()).ok()?; - - with_ext(|ext| { - ext.keystore() - .expect("No `keystore` associated for the current context!") - .read() - .sr25519_key_pair(id, &pub_key) - .map(|k| k.sign(msg)) - }).expect("`sr25519_sign` cannot be called outside of an Externalities-provided environment.") - } - - fn sr25519_verify(sig: &sr25519::Signature, msg: &[u8], pubkey: &sr25519::Public) -> bool { - sr25519::Pair::verify(sig, msg, pubkey) - } - - fn secp256k1_ecdsa_recover(sig: &[u8; 65], msg: &[u8; 32]) -> Result<[u8; 64], EcdsaVerifyError> { - let rs = secp256k1::Signature::parse_slice(&sig[0..64]) - .map_err(|_| EcdsaVerifyError::BadRS)?; - let v = secp256k1::RecoveryId::parse(if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8) - .map_err(|_| EcdsaVerifyError::BadV)?; - let pubkey = secp256k1::recover(&secp256k1::Message::parse(msg), &rs, &v) - .map_err(|_| EcdsaVerifyError::BadSignature)?; - let mut res = [0u8; 64]; - res.copy_from_slice(&pubkey.serialize()[1..65]); - Ok(res) - } -} - -impl HashingApi for () { - fn keccak_256(data: &[u8]) -> [u8; 32] { - tiny_keccak::keccak256(data) - } - - fn blake2_128(data: &[u8]) -> [u8; 16] { - blake2_128(data) - } - - fn blake2_256(data: &[u8]) -> [u8; 32] { - blake2_256(data) - } - - fn twox_256(data: &[u8]) -> [u8; 32] { - twox_256(data) - } - - fn twox_128(data: &[u8]) -> [u8; 16] { - twox_128(data) - } - - fn twox_64(data: &[u8]) -> [u8; 8] { - twox_64(data) - } -} - -fn with_offchain(f: impl FnOnce(&mut dyn offchain::Externalities) -> R, msg: &'static str) -> R { - with_ext(|ext| ext - .offchain() - .map(|ext| f(ext)) - .expect(msg) - ).expect("offchain-worker functions cannot be called outside of an Externalities-provided environment.") -} - -impl OffchainApi for () { - fn is_validator() -> bool { - with_offchain(|ext| { - ext.is_validator() - }, "is_validator can be called only in the offchain worker context") - } - - fn submit_transaction(data: Vec) -> Result<(), ()> { - with_offchain(|ext| { - ext.submit_transaction(data) - }, "submit_transaction can be called only in the offchain worker context") - } - - fn network_state() -> Result { - with_offchain(|ext| { - ext.network_state() - }, "network_state can be called only in the offchain worker context") - } - - fn timestamp() -> offchain::Timestamp { - with_offchain(|ext| { - ext.timestamp() - }, "timestamp can be called only in the offchain worker context") - } - - fn sleep_until(deadline: offchain::Timestamp) { - with_offchain(|ext| { - ext.sleep_until(deadline) - }, "sleep_until can be called only in the offchain worker context") - } - - fn random_seed() -> [u8; 32] { - with_offchain(|ext| { - ext.random_seed() - }, "random_seed can be called only in the offchain worker context") - } - - fn local_storage_set(kind: offchain::StorageKind, key: &[u8], value: &[u8]) { - with_offchain(|ext| { - ext.local_storage_set(kind, key, value) - }, "local_storage_set can be called only in the offchain worker context") - } - - fn local_storage_compare_and_set( - kind: offchain::StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool { - with_offchain(|ext| { - ext.local_storage_compare_and_set(kind, key, old_value, new_value) - }, "local_storage_compare_and_set can be called only in the offchain worker context") - } - - fn local_storage_get(kind: offchain::StorageKind, key: &[u8]) -> Option> { - with_offchain(|ext| { - ext.local_storage_get(kind, key) - }, "local_storage_get can be called only in the offchain worker context") - } - - fn http_request_start( - method: &str, - uri: &str, - meta: &[u8], - ) -> Result { - with_offchain(|ext| { - ext.http_request_start(method, uri, meta) - }, "http_request_start can be called only in the offchain worker context") - } - - fn http_request_add_header( - request_id: offchain::HttpRequestId, - name: &str, - value: &str, - ) -> Result<(), ()> { - with_offchain(|ext| { - ext.http_request_add_header(request_id, name, value) - }, "http_request_add_header can be called only in the offchain worker context") - } - - fn http_request_write_body( - request_id: offchain::HttpRequestId, - chunk: &[u8], - deadline: Option, - ) -> Result<(), offchain::HttpError> { - with_offchain(|ext| { - ext.http_request_write_body(request_id, chunk, deadline) - }, "http_request_write_body can be called only in the offchain worker context") - } - - fn http_response_wait( - ids: &[offchain::HttpRequestId], - deadline: Option, - ) -> Vec { - with_offchain(|ext| { - ext.http_response_wait(ids, deadline) - }, "http_response_wait can be called only in the offchain worker context") - } - - fn http_response_headers( - request_id: offchain::HttpRequestId, - ) -> Vec<(Vec, Vec)> { - with_offchain(|ext| { - ext.http_response_headers(request_id) - }, "http_response_headers can be called only in the offchain worker context") - } - - fn http_response_read_body( - request_id: offchain::HttpRequestId, - buffer: &mut [u8], - deadline: Option, - ) -> Result { - with_offchain(|ext| { - ext.http_response_read_body(request_id, buffer, deadline) - }, "http_response_read_body can be called only in the offchain worker context") - } -} - -impl Api for () {} - /// Execute the given closure with global function available whose functionality routes into the /// externalities `ext`. Forwards the value that the closure returns. // NOTE: need a concrete hasher here due to limitations of the `environmental!` macro, otherwise a type param would have been fine I think. diff --git a/core/sr-io/without_std.rs b/core/sr-io/without_std.rs index dfcf2a7e79023..a909d1d1a20a8 100644 --- a/core/sr-io/without_std.rs +++ b/core/sr-io/without_std.rs @@ -22,7 +22,6 @@ use core::{intrinsics, panic::PanicInfo}; use rstd::{vec::Vec, cell::Cell, convert::TryInto}; use primitives::{offchain, Blake2Hasher}; use codec::Decode; -use runtime_interface::wasm::ExchangeableFunction; #[cfg(not(feature = "no_panic_handler"))] #[panic_handler] From 9e8869b7326c7e1db53305c6d3219e8dc7e95f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 9 Oct 2019 21:19:32 +0200 Subject: [PATCH 26/76] Start reworking after master merge --- Cargo.lock | 2 +- core/runtime-interface/Cargo.toml | 4 +- .../proc-macro/src/bare_function_interface.rs | 16 +++- core/runtime-interface/src/externalities.rs | 40 --------- core/runtime-interface/src/impls.rs | 84 +------------------ core/runtime-interface/src/lib.rs | 12 +-- 6 files changed, 20 insertions(+), 138 deletions(-) delete mode 100644 core/runtime-interface/src/externalities.rs diff --git a/Cargo.lock b/Cargo.lock index 8ff59d84cf6e3..8785559bd0357 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5355,7 +5355,7 @@ dependencies = [ "sr-std 2.0.0", "static_assertions 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-executor 2.0.0", - "substrate-primitives 2.0.0", + "substrate-externalities 2.0.0", "substrate-runtime-interface-proc-macro 2.0.0", "substrate-runtime-interface-test-wasm 2.0.0", "substrate-state-machine 2.0.0", diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index c932dd4c8c9a0..ca160cf6aa9f5 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -7,8 +7,8 @@ edition = "2018" [dependencies] wasm-interface = { package = "substrate-wasm-interface", path = "../wasm-interface", optional = true } rstd = { package = "sr-std", path = "../sr-std", default-features = false } -primitives = { package = "substrate-primitives", path = "../primitives", default-features = false } substrate-runtime-interface-proc-macro = { path = "proc-macro" } +externalities = { package = "substrate-externalities", path = "../externalities", optional = true } codec = { package = "parity-scale-codec", version = "1.0.5", default-features = false } environmental = { version = "1.0.2", optional = true } static_assertions = "0.3.4" @@ -20,4 +20,4 @@ state_machine = { package = "substrate-state-machine", path = "../state-machine" [features] default = [ "std" ] -std = [ "wasm-interface", "rstd/std", "codec/std", "primitives/std", "environmental" ] +std = [ "wasm-interface", "rstd/std", "codec/std", "externalities", "environmental" ] diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index f59043716f84a..17600b128cd20 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -15,6 +15,17 @@ // along with Substrate. If not, see . //! Generates the bare function interface for a given trait definition. +//! +//! The bare functions are the ones that will be called by the user. On the native/host side, these +//! functions directly execute the provided implementation. On the wasm side, these +//! functions will prepare the parameters for the FFI boundary, call the external host function +//! exported into wasm and convert back the result. +//! +//! [`generate`] is the entry point for generating for each trait method one bare function. +//! +//! [`function_for_method`] generates the bare function per trait method. Each bare function +//! contains both implementations. The implementations are feature-gated, so that one is compiled +//! for the native and the other for the wasm side. use crate::utils::{ generate_crate_access, create_host_function_ident, get_function_arguments, @@ -29,7 +40,8 @@ use proc_macro2::{TokenStream, Span}; use quote::{quote, quote_spanned}; -/// Generate the bare-function interface. +/// Generate one bare function per trait method. The name of the bare function is equal to the name +/// of the trait method. pub fn generate(trait_def: &ItemTrait) -> Result { let trait_name = &trait_def.ident; get_trait_methods(trait_def).try_fold(TokenStream::new(), |mut t, m| { @@ -121,7 +133,7 @@ fn function_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result - <&mut dyn #crate_::Externalities<#crate_::Blake2Hasher> as #trait_name>::#method_name( + <&mut dyn #crate_::Externalities as #trait_name>::#method_name( #( #arg_names, )* ) } diff --git a/core/runtime-interface/src/externalities.rs b/core/runtime-interface/src/externalities.rs deleted file mode 100644 index a26185becb4c5..0000000000000 --- a/core/runtime-interface/src/externalities.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Stores the externalities in an `environmental` value to make it scope limited available. - -use primitives::{Blake2Hasher, traits::Externalities}; - -environmental::environmental!(ext: trait Externalities); - -/// Set the given externalities while executing the given closure. To get access to the externalities -/// while executing the given closure [`with_externalities`] grants access to them. The externalities -/// are only set for the same thread this function was called from. -pub fn set_and_run_with_externalities( - ext: &mut dyn Externalities, - f: F, -) -> R where F: FnOnce() -> R { - ext::using(ext, f) -} - -/// Execute the given closure with the currently set externalities. -/// -/// Returns `None` if no externalities are set or `Some(_)` with the result of the closure. -pub fn with_externalities, -) -> R, R>(f: F) -> Option { - ext::with(f) -} diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index 3cd8cca04eceb..e1c3dd7ea708a 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -16,7 +16,7 @@ //! Provides implementations for the runtime interface traits. -use crate::{RIType, pass_by::{PassBy, Inner, Codec, PassByInner}}; +use crate::{RIType, pass_by::{PassBy, Codec}}; #[cfg(feature = "std")] use crate::host::*; #[cfg(not(feature = "std"))] @@ -38,8 +38,6 @@ use rstd::borrow::Cow; #[cfg(not(feature = "std"))] use rstd::slice; -use primitives::{sr25519, ed25519}; - /// Converts a pointer and length into an `u64`. pub fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { ((len as u64) | u64::from(ptr) << 32).to_le() @@ -354,83 +352,3 @@ impl PassBy for Option { impl PassBy for rstd::result::Result { type PassBy = Codec; } - -impl PassBy for sr25519::Public { - type PassBy = Inner; -} - -impl PassByInner for sr25519::Public { - type Inner = [u8; 32]; - - fn into_inner(self) -> Self::Inner { - self.0 - } - - fn inner(&self) -> &Self::Inner { - &self.0 - } - - fn from_inner(inner: Self::Inner) -> Self { - Self(inner) - } -} - -impl PassBy for sr25519::Signature { - type PassBy = Inner; -} - -impl PassByInner for sr25519::Signature { - type Inner = [u8; 64]; - - fn into_inner(self) -> Self::Inner { - self.0 - } - - fn inner(&self) -> &Self::Inner { - &self.0 - } - - fn from_inner(inner: Self::Inner) -> Self { - Self(inner) - } -} - -impl PassBy for ed25519::Public { - type PassBy = Inner; -} - -impl PassByInner for ed25519::Public { - type Inner = [u8; 32]; - - fn into_inner(self) -> Self::Inner { - self.0 - } - - fn inner(&self) -> &Self::Inner { - &self.0 - } - - fn from_inner(inner: Self::Inner) -> Self { - Self(inner) - } -} - -impl PassBy for ed25519::Signature { - type PassBy = Inner; -} - -impl PassByInner for ed25519::Signature { - type Inner = [u8; 64]; - - fn into_inner(self) -> Self::Inner { - self.0 - } - - fn inner(&self) -> &Self::Inner { - &self.0 - } - - fn from_inner(inner: Self::Inner) -> Self { - Self(inner) - } -} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index d37e0d836e70b..7183267ddc264 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -24,24 +24,16 @@ //! [`WrappedFFIValue`] to call into the host. The created [`WrappedFFIValue`] will remain on //! the stack while we call into the host. -#[doc(hidden)] -pub use primitives::Blake2Hasher; - -#[doc(hidden)] -#[cfg(feature = "std")] -pub use primitives::traits::Externalities; - #[doc(hidden)] #[cfg(feature = "std")] pub use wasm_interface; pub use substrate_runtime_interface_proc_macro::runtime_interface; +#[doc(hidden)] #[cfg(feature = "std")] -pub use externalities::{set_and_run_with_externalities, with_externalities}; +pub use externalities::{set_and_run_with_externalities, with_externalities, Externalities}; -#[cfg(feature = "std")] -mod externalities; pub(crate) mod impls; #[cfg(feature = "std")] pub mod host; From 0402799c5d013a5c16de4a485ebecd4ab46f1a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 10 Oct 2019 19:31:16 +0200 Subject: [PATCH 27/76] Adds `PassByCodec` derive --- core/runtime-interface/proc-macro/src/lib.rs | 9 ++- .../proc-macro/src/pass_by_codec.rs | 58 +++++++++++++++++++ core/runtime-interface/src/lib.rs | 3 + core/runtime-interface/src/pass_by.rs | 2 + 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 core/runtime-interface/proc-macro/src/pass_by_codec.rs diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index b908cf37c1593..71cd6cd2f7139 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -20,7 +20,7 @@ extern crate proc_macro; use proc_macro2::{Span, TokenStream}; -use syn::{parse_macro_input, Ident, ItemTrait, Result}; +use syn::{parse_macro_input, Ident, ItemTrait, Result, DeriveInput}; use quote::quote; @@ -30,6 +30,7 @@ use utils::generate_runtime_interface_include; mod bare_function_interface; mod host_function_interface; +mod pass_by_codec; mod trait_decl_impl; mod utils; @@ -68,3 +69,9 @@ fn runtime_interface_impl(trait_def: ItemTrait) -> Result { Ok(res) } + +#[proc_macro_derive(PassByCodec)] +pub fn pass_by_codec(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = parse_macro_input!(input as DeriveInput); + pass_by_codec::derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() +} diff --git a/core/runtime-interface/proc-macro/src/pass_by_codec.rs b/core/runtime-interface/proc-macro/src/pass_by_codec.rs new file mode 100644 index 0000000000000..7f41ec543524e --- /dev/null +++ b/core/runtime-interface/proc-macro/src/pass_by_codec.rs @@ -0,0 +1,58 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Derive macro implementation of `PassBy` with the associated type set to `Codec`. +//! +//! It is required that the type implements `Encode` and `Decode` from the `parity-scale-codec` +//! crate. + +use crate::utils::{generate_crate_access, generate_runtime_interface_include}; + +use syn::{DeriveInput, Result, Generics, parse_quote}; + +use quote::quote; + +use proc_macro2::TokenStream; + +/// The derive implementation for `PassBy` with `Codec`. +pub fn derive_impl(mut input: DeriveInput) -> Result { + add_trait_bounds(&mut input.generics); + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + let crate_include = generate_runtime_interface_include(); + let crate_ = generate_crate_access(); + let ident = input.ident; + + let res = quote! { + const _ = { + #crate_include + + impl #impl_generics #crate_::pass_by::PassBy for #ident #ty_generics #where_clause { + type PassBy = #crate_::pass_by::Codec<#ident>; + } + }; + }; + + Ok(res) +} + +/// Add the `codec::Codec` trait bound to every type parameter. +fn add_trait_bounds(generics: &mut Generics) { + let crate_ = generate_crate_access(); + + generics.type_params_mut() + .for_each(|type_param| type_param.bounds.push(parse_quote!(#crate_::codec::Codec))); +} + diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 7183267ddc264..9b9779629c4ab 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -34,6 +34,9 @@ pub use substrate_runtime_interface_proc_macro::runtime_interface; #[cfg(feature = "std")] pub use externalities::{set_and_run_with_externalities, with_externalities, Externalities}; +#[doc(hidden)] +pub use codec; + pub(crate) mod impls; #[cfg(feature = "std")] pub mod host; diff --git a/core/runtime-interface/src/pass_by.rs b/core/runtime-interface/src/pass_by.rs index 8accac3ebd84d..a5e6e4492d957 100644 --- a/core/runtime-interface/src/pass_by.rs +++ b/core/runtime-interface/src/pass_by.rs @@ -32,6 +32,8 @@ use rstd::marker::PhantomData; #[cfg(not(feature = "std"))] use rstd::slice; +pub use substrate_runtime_interface_proc_macro::PassByCodec; + /// Something that should be passed between wasm and the host using the given strategy. /// /// See [`Codec`] or [`Inner`] for more information about the provided strategies. From 78a7092c110c6d6154ed93e156a7da686cd5ab44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 10 Oct 2019 20:49:28 +0200 Subject: [PATCH 28/76] Adds `PassByInner` derive --- core/runtime-interface/proc-macro/src/lib.rs | 7 ++ .../proc-macro/src/pass_by_inner.rs | 105 ++++++++++++++++++ core/runtime-interface/src/pass_by.rs | 2 +- 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 core/runtime-interface/proc-macro/src/pass_by_inner.rs diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index 71cd6cd2f7139..cb1fb59582682 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -31,6 +31,7 @@ use utils::generate_runtime_interface_include; mod bare_function_interface; mod host_function_interface; mod pass_by_codec; +mod pass_by_inner; mod trait_decl_impl; mod utils; @@ -75,3 +76,9 @@ pub fn pass_by_codec(input: proc_macro::TokenStream) -> proc_macro::TokenStream let input = parse_macro_input!(input as DeriveInput); pass_by_codec::derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() } + +#[proc_macro_derive(PassByInner)] +pub fn pass_by_inner(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = parse_macro_input!(input as DeriveInput); + pass_by_inner::derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() +} diff --git a/core/runtime-interface/proc-macro/src/pass_by_inner.rs b/core/runtime-interface/proc-macro/src/pass_by_inner.rs new file mode 100644 index 0000000000000..661a809eb26bd --- /dev/null +++ b/core/runtime-interface/proc-macro/src/pass_by_inner.rs @@ -0,0 +1,105 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Derive macro implementation of `PassBy` with the associated type set to `Inner` and of the +//! helper trait `PassByInner`. +//! +//! It is required that the type is a newtype struct, otherwise an error is generated. + +use crate::utils::{generate_crate_access, generate_runtime_interface_include}; + +use syn::{DeriveInput, Result, Generics, parse_quote, Type, Data, Error, Fields, Ident}; + +use quote::quote; + +use proc_macro2::{TokenStream, Span}; + +/// The derive implementation for `PassBy` with `Inner` and `PassByInner`. +pub fn derive_impl(mut input: DeriveInput) -> Result { + add_trait_bounds(&mut input.generics); + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + let crate_include = generate_runtime_interface_include(); + let crate_ = generate_crate_access(); + let ident = input.ident; + let (inner_ty, inner_name) = extract_inner_ty_and_name(&input.data)?; + + let access_inner = match inner_name { + Some(ref name) => quote!(self.#name), + None => quote!(self.0), + }; + + let from_inner = match inner_name { + Some(name) => quote!(Self { #name: inner }), + None => quote!(Self(inner)), + }; + + let res = quote! { + const _ = { + #crate_include + + impl #impl_generics #crate_::pass_by::PassBy for #ident #ty_generics #where_clause { + type PassBy = #crate_::pass_by::Inner<#ident>; + } + + impl #impl_generics #crate_::pass_by::PassByInner for #ident #ty_generics #where_clause { + type Inner = #inner_ty; + + fn into_inner(self) -> Self::Inner { + #access_inner + } + + fn inner(&self) -> &Self::Inner { + &#access_inner + } + + fn from_inner(inner: Self::Inner) -> Self { + #from_inner + } + } + }; + }; + + Ok(res) +} + +/// Add the `RIType` trait bound to every type parameter. +fn add_trait_bounds(generics: &mut Generics) { + let crate_ = generate_crate_access(); + + generics.type_params_mut() + .for_each(|type_param| type_param.bounds.push(parse_quote!(#crate_::RIType))); +} + +/// Extract the inner type and optional name from given input data. +/// +/// It also checks that the input data is a newtype struct. +fn extract_inner_ty_and_name(data: &Data) -> Result<(Type, Option)> { + if let Data::Struct(ref struct_data) = data { + match struct_data.fields { + Fields::Named(ref named) if named.named.len() == 1 => { + let field = &named.named[0]; + return Ok((field.ty.clone(), field.ident.clone())) + }, + Fields::Unnamed(ref unnamed) if unnamed.unnamed.len() == 1 => { + let field = &unnamed.unnamed[0]; + return Ok((field.ty.clone(), field.ident.clone())) + } + _ => {}, + } + } + + Err(Error::new(Span::call_site(), "Only newtype structs are supported by `PassByInner`!")) +} diff --git a/core/runtime-interface/src/pass_by.rs b/core/runtime-interface/src/pass_by.rs index a5e6e4492d957..6145c0f5474a0 100644 --- a/core/runtime-interface/src/pass_by.rs +++ b/core/runtime-interface/src/pass_by.rs @@ -32,7 +32,7 @@ use rstd::marker::PhantomData; #[cfg(not(feature = "std"))] use rstd::slice; -pub use substrate_runtime_interface_proc_macro::PassByCodec; +pub use substrate_runtime_interface_proc_macro::{PassByCodec, PassByInner}; /// Something that should be passed between wasm and the host using the given strategy. /// From 8a1987a802f0448e3345f01d42abafd2e3e9d06a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 14 Oct 2019 13:17:33 +0200 Subject: [PATCH 29/76] Fix compilation errors --- Cargo.lock | 7 +- core/externalities/src/lib.rs | 6 +- core/primitives/Cargo.toml | 2 + core/primitives/src/ed25519.rs | 5 +- core/primitives/src/offchain.rs | 83 +++--------------- core/primitives/src/sr25519.rs | 5 +- core/runtime-interface/Cargo.toml | 4 +- core/runtime-interface/proc-macro/src/lib.rs | 2 +- .../proc-macro/src/pass_by_codec.rs | 2 +- .../proc-macro/src/pass_by_inner.rs | 4 +- .../proc-macro/src/trait_decl_impl.rs | 9 +- core/runtime-interface/src/impls.rs | 64 +++++++------- core/runtime-interface/src/lib.rs | 6 +- core/sr-io/src/lib.rs | 87 +++++++++++-------- core/sr-io/with_std.rs | 4 +- 15 files changed, 128 insertions(+), 162 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f95fa5cbaabeb..2cb70f82f6fe8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4490,7 +4490,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "static_assertions" -version = "0.3.4" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -5265,6 +5265,7 @@ dependencies = [ "substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-externalities 2.0.0", "substrate-primitives-storage 2.0.0", + "substrate-runtime-interface 2.0.0", "substrate-serializer 2.0.0", "tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5363,7 +5364,7 @@ dependencies = [ "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", - "static_assertions 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "static_assertions 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-executor 2.0.0", "substrate-externalities 2.0.0", "substrate-runtime-interface-proc-macro 2.0.0", @@ -7024,7 +7025,7 @@ dependencies = [ "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" "checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5" -"checksum static_assertions 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3" +"checksum static_assertions 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0fa13613355688665b68639b1c378a62dbedea78aff0fc59a4fa656cbbdec657" "checksum static_slice 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "92a7e0c5e3dfb52e8fbe0e63a1b947bbb17b4036408b151353c4491374931362" "checksum stream-cipher 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8131256a5896cabcf5eb04f4d6dacbe1aefda854b0d9896e09cb58829ec5638c" "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" diff --git a/core/externalities/src/lib.rs b/core/externalities/src/lib.rs index ecd5f7a7a751d..4efbc54a4edc9 100644 --- a/core/externalities/src/lib.rs +++ b/core/externalities/src/lib.rs @@ -132,8 +132,8 @@ pub trait ExternalitiesExt { fn extension(&mut self) -> Option<&mut T>; } -impl ExternalitiesExt for T { - fn extension(&mut self) -> Option<&mut A> { - self.extension_by_type_id(TypeId::of::()).and_then(Any::downcast_mut) +impl ExternalitiesExt for &mut dyn Externalities { + fn extension(&mut self) -> Option<&mut T> { + self.extension_by_type_id(TypeId::of::()).and_then(Any::downcast_mut) } } diff --git a/core/primitives/Cargo.toml b/core/primitives/Cargo.toml index a527f16c1ce5a..62b43e9b67163 100644 --- a/core/primitives/Cargo.toml +++ b/core/primitives/Cargo.toml @@ -33,6 +33,7 @@ lazy_static = { version = "1.4.0", optional = true } parking_lot = { version = "0.9.0", optional = true } externalities = { package = "substrate-externalities", path = "../externalities", optional = true } primitives-storage = { package = "substrate-primitives-storage", path = "storage", default-features = false } +runtime-interface = { package = "substrate-runtime-interface", path = "../runtime-interface", default-features = false } [dev-dependencies] substrate-serializer = { path = "../serializer" } @@ -83,4 +84,5 @@ std = [ "num-traits/std", "externalities", "primitives-storage/std", + "runtime-interface/std", ] diff --git a/core/primitives/src/ed25519.rs b/core/primitives/src/ed25519.rs index c40499a3a182e..7a2c5200eaf38 100644 --- a/core/primitives/src/ed25519.rs +++ b/core/primitives/src/ed25519.rs @@ -33,6 +33,7 @@ use crate::crypto::{Pair as TraitPair, DeriveJunction, SecretStringError, Ss58Co #[cfg(feature = "std")] use serde::{de, Serializer, Serialize, Deserializer, Deserialize}; use crate::{crypto::{Public as TraitPublic, UncheckedFrom, CryptoType, Derive}}; +use runtime_interface::pass_by::PassByInner; /// A secret seed. It's not called a "secret key" because ring doesn't expose the secret keys /// of the key pair (yeah, dumb); as such we're forced to remember the seed manually if we @@ -41,7 +42,7 @@ use crate::{crypto::{Public as TraitPublic, UncheckedFrom, CryptoType, Derive}}; type Seed = [u8; 32]; /// A public key. -#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, Default)] +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, Default, PassByInner)] pub struct Public(pub [u8; 32]); /// A key pair. @@ -160,7 +161,7 @@ impl std::hash::Hash for Public { } /// A signature (a 512-bit value). -#[derive(Encode, Decode)] +#[derive(Encode, Decode, PassByInner)] pub struct Signature(pub [u8; 64]); impl rstd::convert::TryFrom<&[u8]> for Signature { diff --git a/core/primitives/src/offchain.rs b/core/primitives/src/offchain.rs index 84fee530f602f..84c278ad708ec 100644 --- a/core/primitives/src/offchain.rs +++ b/core/primitives/src/offchain.rs @@ -17,12 +17,13 @@ //! Offchain workers types use codec::{Encode, Decode}; -use rstd::{prelude::{Vec, Box}, convert::TryFrom}; +use rstd::prelude::{Vec, Box}; +use runtime_interface::pass_by::{PassByCodec, PassByInner}; pub use crate::crypto::KeyTypeId; /// A type of supported crypto. -#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode)] +#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, PassByCodec)] #[cfg_attr(feature = "std", derive(Debug))] #[repr(C)] pub enum StorageKind { @@ -40,26 +41,8 @@ pub enum StorageKind { LOCAL = 2, } -impl TryFrom for StorageKind { - type Error = (); - - fn try_from(kind: u32) -> Result { - match kind { - e if e == u32::from(StorageKind::PERSISTENT as u8) => Ok(StorageKind::PERSISTENT), - e if e == u32::from(StorageKind::LOCAL as u8) => Ok(StorageKind::LOCAL), - _ => Err(()), - } - } -} - -impl From for u32 { - fn from(c: StorageKind) -> Self { - c as u8 as u32 - } -} - /// Opaque type for offchain http requests. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, PassByInner, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug, Hash))] pub struct HttpRequestId(pub u16); @@ -70,7 +53,7 @@ impl From for u32 { } /// An error enum returned by some http methods. -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, PassByCodec)] #[cfg_attr(feature = "std", derive(Debug))] #[repr(C)] pub enum HttpError { @@ -82,27 +65,8 @@ pub enum HttpError { Invalid = 3, } -impl TryFrom for HttpError { - type Error = (); - - fn try_from(error: u32) -> Result { - match error { - e if e == HttpError::DeadlineReached as u8 as u32 => Ok(HttpError::DeadlineReached), - e if e == HttpError::IoError as u8 as u32 => Ok(HttpError::IoError), - e if e == HttpError::Invalid as u8 as u32 => Ok(HttpError::Invalid), - _ => Err(()) - } - } -} - -impl From for u32 { - fn from(c: HttpError) -> Self { - c as u8 as u32 - } -} - /// Status of the HTTP request -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, PassByCodec)] #[cfg_attr(feature = "std", derive(Debug))] pub enum HttpRequestStatus { /// Deadline was reached while we waited for this request to finish. @@ -122,34 +86,9 @@ pub enum HttpRequestStatus { Finished(u16), } -impl From for u32 { - fn from(status: HttpRequestStatus) -> Self { - match status { - HttpRequestStatus::Invalid => 0, - HttpRequestStatus::DeadlineReached => 10, - HttpRequestStatus::IoError => 20, - HttpRequestStatus::Finished(code) => u32::from(code), - } - } -} - -impl TryFrom for HttpRequestStatus { - type Error = (); - - fn try_from(status: u32) -> Result { - match status { - 0 => Ok(HttpRequestStatus::Invalid), - 10 => Ok(HttpRequestStatus::DeadlineReached), - 20 => Ok(HttpRequestStatus::IoError), - 100..=999 => u16::try_from(status).map(HttpRequestStatus::Finished).map_err(|_| ()), - _ => Err(()), - } - } -} - /// A blob to hold information about the local node's network state /// without committing to its format. -#[derive(Clone, Eq, PartialEq, Encode, Decode)] +#[derive(Clone, Eq, PartialEq, Encode, Decode, PassByCodec)] #[cfg_attr(feature = "std", derive(Debug))] pub struct OpaqueNetworkState { /// PeerId of the local node. @@ -159,7 +98,7 @@ pub struct OpaqueNetworkState { } /// Simple blob to hold a `PeerId` without committing to its format. -#[derive(Default, Clone, Eq, PartialEq, Encode, Decode)] +#[derive(Default, Clone, Eq, PartialEq, Encode, Decode, PassByInner)] #[cfg_attr(feature = "std", derive(Debug))] pub struct OpaquePeerId(pub Vec); @@ -171,7 +110,7 @@ impl OpaquePeerId { } /// Simple blob to hold a `Multiaddr` without committing to its format. -#[derive(Clone, Eq, PartialEq, Encode, Decode)] +#[derive(Clone, Eq, PartialEq, Encode, Decode, PassByInner)] #[cfg_attr(feature = "std", derive(Debug))] pub struct OpaqueMultiaddr(pub Vec); @@ -183,12 +122,12 @@ impl OpaqueMultiaddr { } /// Opaque timestamp type -#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default)] +#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, PassByInner)] #[cfg_attr(feature = "std", derive(Debug))] pub struct Timestamp(u64); /// Duration type -#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default)] +#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, PassByInner)] #[cfg_attr(feature = "std", derive(Debug))] pub struct Duration(u64); diff --git a/core/primitives/src/sr25519.rs b/core/primitives/src/sr25519.rs index 0e573f49ce34c..1e53f7a4e8fd1 100644 --- a/core/primitives/src/sr25519.rs +++ b/core/primitives/src/sr25519.rs @@ -41,13 +41,14 @@ use codec::{Encode, Decode}; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; #[cfg(feature = "std")] use schnorrkel::keys::{MINI_SECRET_KEY_LENGTH, SECRET_KEY_LENGTH}; +use runtime_interface::pass_by::PassByInner; // signing context #[cfg(feature = "std")] const SIGNING_CTX: &[u8] = b"substrate"; /// An Schnorrkel/Ristretto x25519 ("sr25519") public key. -#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, Default)] +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, Default, PassByInner)] pub struct Public(pub [u8; 32]); /// An Schnorrkel/Ristretto x25519 ("sr25519") key pair. @@ -161,7 +162,7 @@ impl std::hash::Hash for Public { /// An Schnorrkel/Ristretto x25519 ("sr25519") signature. /// /// Instead of importing it for the local module, alias it to be available as a public type -#[derive(Encode, Decode)] +#[derive(Encode, Decode, PassByInner)] pub struct Signature(pub [u8; 64]); impl rstd::convert::TryFrom<&[u8]> for Signature { diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index ca160cf6aa9f5..fb5b3856a6cf1 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -9,9 +9,9 @@ wasm-interface = { package = "substrate-wasm-interface", path = "../wasm-interfa rstd = { package = "sr-std", path = "../sr-std", default-features = false } substrate-runtime-interface-proc-macro = { path = "proc-macro" } externalities = { package = "substrate-externalities", path = "../externalities", optional = true } -codec = { package = "parity-scale-codec", version = "1.0.5", default-features = false } +codec = { package = "parity-scale-codec", version = "1.0.6", default-features = false } environmental = { version = "1.0.2", optional = true } -static_assertions = "0.3.4" +static_assertions = "1.0.0" [dev-dependencies] executor = { package = "substrate-executor", path = "../executor" } diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index cb1fb59582682..6d85c2a486e99 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -66,7 +66,7 @@ fn runtime_interface_impl(trait_def: ItemTrait) -> Result { } }; - println!("{}", res); + // println!("{}", res); Ok(res) } diff --git a/core/runtime-interface/proc-macro/src/pass_by_codec.rs b/core/runtime-interface/proc-macro/src/pass_by_codec.rs index 7f41ec543524e..c5c6980d2c652 100644 --- a/core/runtime-interface/proc-macro/src/pass_by_codec.rs +++ b/core/runtime-interface/proc-macro/src/pass_by_codec.rs @@ -36,7 +36,7 @@ pub fn derive_impl(mut input: DeriveInput) -> Result { let ident = input.ident; let res = quote! { - const _ = { + const _: () = { #crate_include impl #impl_generics #crate_::pass_by::PassBy for #ident #ty_generics #where_clause { diff --git a/core/runtime-interface/proc-macro/src/pass_by_inner.rs b/core/runtime-interface/proc-macro/src/pass_by_inner.rs index 661a809eb26bd..5fa9577ca3104 100644 --- a/core/runtime-interface/proc-macro/src/pass_by_inner.rs +++ b/core/runtime-interface/proc-macro/src/pass_by_inner.rs @@ -47,11 +47,11 @@ pub fn derive_impl(mut input: DeriveInput) -> Result { }; let res = quote! { - const _ = { + const _: () = { #crate_include impl #impl_generics #crate_::pass_by::PassBy for #ident #ty_generics #where_clause { - type PassBy = #crate_::pass_by::Inner<#ident>; + type PassBy = #crate_::pass_by::Inner<#ident, #inner_ty>; } impl #impl_generics #crate_::pass_by::PassByInner for #ident #ty_generics #where_clause { diff --git a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs index 8a50ff40c1ab6..cc69daa517fb1 100644 --- a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs +++ b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs @@ -14,7 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Checks the trait declaration, folds and implements it. +//! Checks the trait declaration, makes the trait declaration module local, removes all method +//! default implementations and implements the trait for `&mut dyn Externalities`. use crate::utils::{generate_crate_access, get_function_argument_types_without_ref}; @@ -28,7 +29,7 @@ use proc_macro2::TokenStream; use quote::quote; /// Process the given trait definition, by checking that the definition is valid, fold it to the -/// essential definition and implement this essential definition for `dyn Externalities`. +/// essential definition and implement this essential definition for `dyn Externalities`. pub fn process(trait_def: &ItemTrait) -> Result { let impl_trait = impl_trait_for_externalities(trait_def)?; let essential_trait_def = ToEssentialTraitDef::convert(trait_def.clone())?; @@ -110,7 +111,7 @@ impl Fold for ToEssentialTraitDef { } } -/// Implements the given trait definition for `dyn Externalities`. +/// Implements the given trait definition for `dyn Externalities`. fn impl_trait_for_externalities(trait_def: &ItemTrait) -> Result { let trait_ = &trait_def.ident; let crate_ = generate_crate_access(); @@ -125,7 +126,7 @@ fn impl_trait_for_externalities(trait_def: &ItemTrait) -> Result { Ok( quote! { #[cfg(feature = "std")] - impl #trait_ for &mut dyn #crate_::Externalities<#crate_::Blake2Hasher> { + impl #trait_ for &mut dyn #crate_::Externalities { #( #methods )* } } diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index e1c3dd7ea708a..b8ef5fbe64f7a 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -38,6 +38,12 @@ use rstd::borrow::Cow; #[cfg(not(feature = "std"))] use rstd::slice; +// Make sure that our assumptions for storing a pointer + its size in `u64` is valid. +#[cfg(not(feature = "std"))] +assert_eq_size!(usize, u32); +#[cfg(not(feature = "std"))] +assert_eq_size!(*const u8, u32); + /// Converts a pointer and length into an `u64`. pub fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { ((len as u64) | u64::from(ptr) << 32).to_le() @@ -175,6 +181,30 @@ impl FromFFIValue for Vec { } } +#[cfg(not(feature = "std"))] +impl IntoFFIValue for Vec { + type Owned = Vec; + + fn into_ffi_value(&self) -> WrappedFFIValue> { + self[..].into_ffi_value() + } +} + +#[cfg(not(feature = "std"))] +impl FromFFIValue for Vec { + fn from_ffi_value(arg: u64) -> Vec { + let (ptr, len) = pointer_and_len_from_u64(arg); + let len = len as usize; + + if TypeId::of::() == TypeId::of::() { + unsafe { mem::transmute(Vec::from_raw_parts(ptr as *mut u8, len, len)) } + } else { + let slice = unsafe { slice::from_raw_parts(ptr as *const u8, len) }; + Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") + } + } +} + impl RIType for [T] { type FFIType = u64; } @@ -221,12 +251,6 @@ impl IntoPreallocatedFFIValue for [u8] { } } -// Make sure that our assumptions for storing a pointer + its size in `u64` is valid. -#[cfg(not(feature = "std"))] -assert_eq_size!(usize_check; usize, u32); -#[cfg(not(feature = "std"))] -assert_eq_size!(ptr_check; *const u8, u32); - #[cfg(not(feature = "std"))] impl IntoFFIValue for [T] { type Owned = Vec; @@ -243,30 +267,6 @@ impl IntoFFIValue for [T] { } } -#[cfg(not(feature = "std"))] -impl IntoFFIValue for Vec { - type Owned = Vec; - - fn into_ffi_value(&self) -> WrappedFFIValue> { - self[..].into_ffi_value() - } -} - -#[cfg(not(feature = "std"))] -impl FromFFIValue for Vec { - fn from_ffi_value(arg: u64) -> Vec { - let (ptr, len) = pointer_and_len_from_u64(arg); - let len = len as usize; - - if TypeId::of::() == TypeId::of::() { - unsafe { mem::transmute(Vec::from_raw_parts(ptr as *mut u8, len, len)) } - } else { - let slice = unsafe { slice::from_raw_parts(ptr as *const u8, len) }; - Self::decode(&mut &slice[..]).expect("Host to wasm values are encoded correctly; qed") - } - } -} - /// Implement the traits for the `[u8; N]` arrays, where `N` is the input to this macro. macro_rules! impl_traits_for_arrays { ( @@ -345,10 +345,10 @@ impl_traits_for_arrays! { 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 } -impl PassBy for Option { +impl PassBy for rstd::result::Result { type PassBy = Codec; } -impl PassBy for rstd::result::Result { +impl PassBy for Option { type PassBy = Codec; } diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 9b9779629c4ab..bc2ef4633888e 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -32,7 +32,9 @@ pub use substrate_runtime_interface_proc_macro::runtime_interface; #[doc(hidden)] #[cfg(feature = "std")] -pub use externalities::{set_and_run_with_externalities, with_externalities, Externalities}; +pub use externalities::{ + set_and_run_with_externalities, with_externalities, Externalities, ExternalitiesExt, ExtensionStore, +}; #[doc(hidden)] pub use codec; @@ -66,7 +68,7 @@ mod tests { type TestExternalities = state_machine::TestExternalities; - fn call_wasm_method(method: &str) -> TestExternalities { + fn call_wasm_method(method: &str) -> TestExternalities { let mut ext = TestExternalities::default(); let executor = WasmExecutor::::new(); diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 5da6d83fc3b81..29a7ca0772e50 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -29,11 +29,12 @@ use rstd::vec::Vec; use primitives::{ - crypto::KeyTypeId, ed25519, sr25519, H256, + crypto::{KeyTypeId, Pair}, ed25519, sr25519, H256, hexdisplay::HexDisplay, offchain::{ Timestamp, HttpRequestId, HttpRequestStatus, HttpError, StorageKind, OpaqueNetworkState, + OffchainExt, }, - child_storage_key::ChildStorageKey, + storage::ChildStorageKey, traits::KeystoreExt, }; use trie::{TrieConfiguration, trie_types::Layout}; @@ -42,6 +43,8 @@ use runtime_interface::runtime_interface; use codec::{Encode, Decode}; +use externalities::ExternalitiesExt; + /// Error verifying ECDSA signature #[derive(Encode, Decode)] pub enum EcdsaVerifyError { @@ -53,6 +56,18 @@ pub enum EcdsaVerifyError { BadSignature, } +/// Returns a `ChildStorageKey` if the given `storage_key` slice is a valid storage +/// key or panics otherwise. +/// +/// Panicking here is aligned with what the `without_std` environment would do +/// in the case of an invalid child storage key. +fn child_storage_key_or_panic(storage_key: &[u8]) -> ChildStorageKey { + match ChildStorageKey::from_slice(storage_key) { + Some(storage_key) => storage_key, + None => panic!("child storage key is invalid"), + } +} + /// Interface for accessing the storage from within the runtime. #[runtime_interface] pub trait Storage { @@ -64,7 +79,7 @@ pub trait Storage { /// Returns the data for `key` in the child storage or `None` if the key can not be found. fn child_get(&self, child_storage_key: &[u8], key: &[u8]) -> Option> { let storage_key = child_storage_key_or_panic(child_storage_key); - ext.child_storage(storage_key, key).map(|s| s.to_vec()) + self.child_storage(storage_key, key).map(|s| s.to_vec()) } /// Get `key` from storage, placing the value into `value_out` and return the number of @@ -121,14 +136,14 @@ pub trait Storage { self.clear_storage(key) } - /// Clear the storage of a key. - fn clear_child(&mut self, child_storage_key: &[u8], key: &[u8]) { + /// Clear the given child storage of the given `key` and its value. + fn child_clear(&mut self, child_storage_key: &[u8], key: &[u8]) { let storage_key = child_storage_key_or_panic(child_storage_key); self.clear_child_storage(storage_key, key); } /// Clear an entire child storage. - fn kill_child_storage(&mut self, child_storage_key: &[u8]) { + fn child_storage_kill(&mut self, child_storage_key: &[u8]) { let storage_key = child_storage_key_or_panic(child_storage_key); self.kill_child_storage(storage_key); } @@ -156,7 +171,7 @@ pub trait Storage { } /// "Commit" all existing operations and compute the resulting storage root. - fn root(&mut self) -> [u8; 32] { + fn root(&mut self) -> H256 { self.storage_root() } @@ -167,18 +182,18 @@ pub trait Storage { } /// "Commit" all existing operations and get the resulting storage change root. - fn changes_root(&mut self, parent_hash: [u8; 32]) -> Option<[u8; 32]> { - self.storage_changes_root(parent_hash.into()).map(|h| h.map(|h| h.into())).ok() + fn changes_root(&mut self, parent_hash: [u8; 32]) -> Option { + self.storage_changes_root(parent_hash.into()).ok().and_then(|h| h) } /// A trie root formed from the iterated items. fn blake2_256_trie_root(input: Vec<(Vec, Vec)>) -> H256 { - Layout::::trie_root(input) + Layout::::trie_root(input) } /// A trie root formed from the enumerated items. fn blake2_256_ordered_trie_root(input: Vec>) -> H256 { - Layout::::ordered_trie_root(input) + Layout::::ordered_trie_root(input) } } @@ -187,6 +202,7 @@ pub trait Storage { pub trait Misc { /// The current relay chain identifier. fn chain_id(&self) -> u64 { + self.extension::(); self.chain_id() } @@ -209,10 +225,11 @@ pub trait Misc { } /// Interfaces for working with crypto related types from within the runtime. +#[runtime_interface] pub trait Crypto { /// Returns all `ed25519` public keys for the given key id from the keystore. - fn ed25519_public_keys(&self, id: KeyTypeId) -> Vec { - self.keystore() + fn ed25519_public_keys(&mut self, id: KeyTypeId) -> Vec { + self.extension::() .expect("No `keystore` associated for the current context!") .read() .ed25519_public_keys(id) @@ -221,8 +238,8 @@ pub trait Crypto { /// Generate an `ed22519` key for the given key type and store it in the keystore. /// /// Returns the public key. - fn ed25519_generate(&self, id: KeyTypeId, seed: Option<&str>) -> ed25519::Public { - self.keystore() + fn ed25519_generate(&mut self, id: KeyTypeId, seed: Option<&str>) -> ed25519::Public { + self.extension::() .expect("No `keystore` associated for the current context!") .write() .ed25519_generate_new(id, seed) @@ -239,7 +256,7 @@ pub trait Crypto { pub_key: &ed25519::Public, msg: &[u8], ) -> Option { - self.keystore() + self.extension::() .expect("No `keystore` associated for the current context!") .read() .ed25519_key_pair(id, &pub_key) @@ -260,7 +277,7 @@ pub trait Crypto { /// Returns all `sr25519` public keys for the given key id from the keystore. fn sr25519_public_keys(&self, id: KeyTypeId) -> Vec { - self.keystore() + self.extension::() .expect("No `keystore` associated for the current context!") .read() .sr25519_public_keys(id) @@ -270,7 +287,7 @@ pub trait Crypto { /// /// Returns the public key. fn sr25519_generate(&self, id: KeyTypeId, seed: Option<&str>) -> sr25519::Public { - self.keystore() + self.extension::() .expect("No `keystore` associated for the current context!") .write() .sr25519_generate_new(id, seed) @@ -282,12 +299,12 @@ pub trait Crypto { /// /// Returns the signature. fn sr25519_sign( - self, + &mut self, id: KeyTypeId, pub_key: &sr25519::Public, msg: &[u8], ) -> Option { - self.keystore() + self.extension::() .expect("No `keystore` associated for the current context!") .read() .sr25519_key_pair(id, &pub_key) @@ -363,7 +380,7 @@ pub trait Offchain { /// Even if this function returns `true`, it does not mean that any keys are configured /// and that the validator is registered in the chain. fn is_validator(&self) -> bool { - self.offchain() + self.extension::() .expect("is_validator can be called only in the offchain worker context") .is_validator() } @@ -372,28 +389,28 @@ pub trait Offchain { /// /// The transaction will end up in the pool. fn submit_transaction(&self, data: Vec) -> Result<(), ()> { - self.offchain() + self.extension::() .expect("submit_transaction can be called only in the offchain worker context") .submit_transaction(data) } /// Returns information about the local node's network state. fn network_state(&self) -> Result { - self.offchain() + self.extension::() .expect("network_state can be called only in the offchain worker context") .network_state() } /// Returns current UNIX timestamp (in millis) fn timestamp(&self) -> Timestamp { - self.offchain() + self.extension::() .expect("timestamp can be called only in the offchain worker context") .timestamp() } /// Pause the execution until `deadline` is reached. fn sleep_until(&self, deadline: Timestamp) { - self.offchain() + self.extension::() .expect("sleep_until can be called only in the offchain worker context") .sleep_until(deadline) } @@ -403,7 +420,7 @@ pub trait Offchain { /// This is a trully random non deterministic seed generated by host environment. /// Obviously fine in the off-chain worker context. fn random_seed(&self) -> [u8; 32] { - self.offchain() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .random_seed() } @@ -413,7 +430,7 @@ pub trait Offchain { /// Note this storage is not part of the consensus, it's only accessible by /// offchain worker tasks running on the same machine. It IS persisted between runs. fn local_storage_set(&self, kind: StorageKind, key: &[u8], value: &[u8]) { - self.offchain() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .local_storage_set(kind, key, value) } @@ -434,7 +451,7 @@ pub trait Offchain { old_value: Option<&[u8]>, new_value: &[u8], ) -> bool { - self.offchain() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .local_storage_compare_and_set(kind, key, old_value, new_value) } @@ -445,7 +462,7 @@ pub trait Offchain { /// Note this storage is not part of the consensus, it's only accessible by /// offchain worker tasks running on the same machine. It IS persisted between runs. fn local_storage_get(&self, kind: StorageKind, key: &[u8]) -> Option> { - self.offchain() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .local_storage_get(kind, key) } @@ -460,7 +477,7 @@ pub trait Offchain { uri: &str, meta: &[u8], ) -> Result { - self.offchain() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .http_request_start(method, uri, meta) } @@ -472,7 +489,7 @@ pub trait Offchain { name: &str, value: &str, ) -> Result<(), ()> { - self.offchain() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .http_request_add_header(request_id, name, value) } @@ -489,7 +506,7 @@ pub trait Offchain { chunk: &[u8], deadline: Option, ) -> Result<(), HttpError> { - self.offchain() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .http_request_write_body(request_id, chunk, deadline) } @@ -506,7 +523,7 @@ pub trait Offchain { ids: &[HttpRequestId], deadline: Option, ) -> Vec { - self.offchain() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .http_response_wait(ids, deadline) } @@ -516,7 +533,7 @@ pub trait Offchain { /// Returns a vector of pairs `(HeaderKey, HeaderValue)`. /// NOTE response headers have to be read before response body. fn http_response_headers(&self, request_id: HttpRequestId) -> Vec<(Vec, Vec)> { - self.offchain() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .http_response_headers(request_id) } @@ -535,7 +552,7 @@ pub trait Offchain { buffer: &mut [u8], deadline: Option, ) -> Result { - self.offchain() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .http_response_read_body(request_id, buffer, deadline) .map(|r| r as u32) diff --git a/core/sr-io/with_std.rs b/core/sr-io/with_std.rs index cfe2d7edb1223..5573c15424ab8 100644 --- a/core/sr-io/with_std.rs +++ b/core/sr-io/with_std.rs @@ -27,6 +27,8 @@ use trie::{TrieConfiguration, trie_types::Layout}; use std::{collections::HashMap, convert::TryFrom}; +use externalities::Externalities; + /// Additional bounds for `Hasher` trait for with_std. pub trait HasherBounds {} impl HasherBounds for T {} @@ -34,7 +36,7 @@ impl HasherBounds for T {} /// Execute the given closure with global function available whose functionality routes into the /// externalities `ext`. Forwards the value that the closure returns. // NOTE: need a concrete hasher here due to limitations of the `environmental!` macro, otherwise a type param would have been fine I think. -pub fn with_externalities R>(ext: &mut dyn Externalities, f: F) -> R { +pub fn with_externalities R>(ext: &mut dyn Externalities, f: F) -> R { unimplemented!() } From 450a9d739474aaf1c29fbf4451e86b4653738638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 14 Oct 2019 20:48:51 +0200 Subject: [PATCH 30/76] More implementations --- Cargo.lock | 1 + core/primitives/src/crypto.rs | 3 ++- core/primitives/src/offchain.rs | 2 +- core/runtime-interface/Cargo.toml | 3 ++- core/runtime-interface/src/impls.rs | 38 +++++++++++++++++++++++++++-- 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2cb70f82f6fe8..e4fa2055c4e86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5363,6 +5363,7 @@ version = "2.0.0" dependencies = [ "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", "static_assertions 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-executor 2.0.0", diff --git a/core/primitives/src/crypto.rs b/core/primitives/src/crypto.rs index 8a336783b7965..a0e206e4f72b4 100644 --- a/core/primitives/src/crypto.rs +++ b/core/primitives/src/crypto.rs @@ -35,6 +35,7 @@ use std::hash::Hash; use zeroize::Zeroize; #[doc(hidden)] pub use rstd::ops::Deref; +use runtime_interface::pass_by::PassByInner; /// The root phrase for our publicly known keys. pub const DEV_PHRASE: &str = "bottom drive obey lake curtain smoke basket hold race lonely fit walk"; @@ -742,7 +743,7 @@ pub trait CryptoType { /// /// Values whose first character is `_` are reserved for private use and won't conflict with any /// public modules. -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode, PassByInner)] #[cfg_attr(feature = "std", derive(Debug))] pub struct KeyTypeId(pub [u8; 4]); diff --git a/core/primitives/src/offchain.rs b/core/primitives/src/offchain.rs index 84c278ad708ec..277bca6c0f078 100644 --- a/core/primitives/src/offchain.rs +++ b/core/primitives/src/offchain.rs @@ -122,7 +122,7 @@ impl OpaqueMultiaddr { } /// Opaque timestamp type -#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, PassByInner)] +#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, Encode, Decode, PassByInner)] #[cfg_attr(feature = "std", derive(Debug))] pub struct Timestamp(u64); diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index fb5b3856a6cf1..86edd6c5d0df0 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -12,6 +12,7 @@ externalities = { package = "substrate-externalities", path = "../externalities" codec = { package = "parity-scale-codec", version = "1.0.6", default-features = false } environmental = { version = "1.0.2", optional = true } static_assertions = "1.0.0" +primitive-types = { version = "0.5.1", default-features = false } [dev-dependencies] executor = { package = "substrate-executor", path = "../executor" } @@ -20,4 +21,4 @@ state_machine = { package = "substrate-state-machine", path = "../state-machine" [features] default = [ "std" ] -std = [ "wasm-interface", "rstd/std", "codec/std", "externalities", "environmental" ] +std = [ "wasm-interface", "rstd/std", "codec/std", "externalities", "environmental", "primitive-types/std" ] diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index b8ef5fbe64f7a..7c88d52826f06 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -16,7 +16,7 @@ //! Provides implementations for the runtime interface traits. -use crate::{RIType, pass_by::{PassBy, Codec}}; +use crate::{RIType, pass_by::{PassBy, Codec, Inner, PassByInner}}; #[cfg(feature = "std")] use crate::host::*; #[cfg(not(feature = "std"))] @@ -342,7 +342,8 @@ macro_rules! impl_traits_for_arrays { impl_traits_for_arrays! { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 + 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, } impl PassBy for rstd::result::Result { @@ -352,3 +353,36 @@ impl PassBy for rstd::result::Result { impl PassBy for Option { type PassBy = Codec; } + +/// Implement `PassBy` with `Inner` for the given fixed sized hash types. +macro_rules! for_primitive_types { + { $( $hash:ident $n:expr ),* $(,)? } => { + $( + impl PassBy for primitive_types::$hash { + type PassBy = Inner; + } + + impl PassByInner for primitive_types::$hash { + type Inner = [u8; $n]; + + fn inner(&self) -> &Self::Inner { + &self.0 + } + + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } + } + )* + } +} + +for_primitive_types! { + H160 20, + H256 32, + H512 64, +} From 0e9167f4d2dceedb0949bf7ead06b6cb83e2a0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 14 Oct 2019 23:58:00 +0200 Subject: [PATCH 31/76] Implement runtime interface traits for `str` --- core/runtime-interface/src/impls.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index 7c88d52826f06..d106e15232713 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -386,3 +386,31 @@ for_primitive_types! { H256 32, H512 64, } + +impl RIType for str { + type FFIType = u64; +} + +#[cfg(feature = "std")] +impl FromFFIValue for str { + type SelfInstance = String; + + fn from_ffi_value(context: &mut dyn FunctionContext, arg: u64) -> Result { + let (ptr, len) = pointer_and_len_from_u64(arg); + + let vec = context.read_memory(Pointer::new(ptr), len)?; + + // The data is valid utf8, as it is stored as `&str` in wasm. + Ok(unsafe { String::from_utf8_unchecked(vec) }) + } +} + +#[cfg(not(feature = "std"))] +impl IntoFFIValue for str { + type Owned = (); + + fn into_ffi_value(&self) -> WrappedFFIValue { + let bytes = self.as_bytes(); + pointer_and_len_to_u64(bytes.as_ptr() as u32, bytes.len() as u32).into() + } +} From c0d81327943fbf396dd3e5f732d7cab284f133f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 15 Oct 2019 09:51:36 +0200 Subject: [PATCH 32/76] Make `sr-io` compile again --- core/application-crypto/src/ed25519.rs | 10 ++-- core/application-crypto/src/lib.rs | 2 +- core/application-crypto/src/sr25519.rs | 10 ++-- core/application-crypto/src/traits.rs | 13 +++-- core/sr-io/src/lib.rs | 57 +++++++++++-------- .../src/generic/unchecked_extrinsic.rs | 2 +- core/sr-primitives/src/lib.rs | 4 +- core/sr-primitives/src/offchain/http.rs | 26 ++++++--- core/sr-primitives/src/testing.rs | 2 +- core/sr-primitives/src/traits.rs | 26 ++++----- 10 files changed, 87 insertions(+), 65 deletions(-) diff --git a/core/application-crypto/src/ed25519.rs b/core/application-crypto/src/ed25519.rs index b0113718b5ed0..446746a19d757 100644 --- a/core/application-crypto/src/ed25519.rs +++ b/core/application-crypto/src/ed25519.rs @@ -34,19 +34,19 @@ impl RuntimePublic for Public { type Signature = Signature; fn all(key_type: KeyTypeId) -> crate::Vec { - runtime_io::ed25519_public_keys(key_type) + runtime_io::crypto::ed25519_public_keys(key_type) } - fn generate_pair(key_type: KeyTypeId, seed: Option<&str>) -> Self { - runtime_io::ed25519_generate(key_type, seed) + fn generate_pair(key_type: KeyTypeId, seed: Option>) -> Self { + runtime_io::crypto::ed25519_generate(key_type, seed) } fn sign>(&self, key_type: KeyTypeId, msg: &M) -> Option { - runtime_io::ed25519_sign(key_type, self, msg.as_ref()) + runtime_io::crypto::ed25519_sign(key_type, self, msg.as_ref()) } fn verify>(&self, msg: &M, signature: &Self::Signature) -> bool { - runtime_io::ed25519_verify(&signature, msg.as_ref(), self) + runtime_io::crypto::ed25519_verify(&signature, msg.as_ref(), self) } } diff --git a/core/application-crypto/src/lib.rs b/core/application-crypto/src/lib.rs index e3366a461a064..b14f2b17c9872 100644 --- a/core/application-crypto/src/lib.rs +++ b/core/application-crypto/src/lib.rs @@ -220,7 +220,7 @@ macro_rules! app_crypto { <$public as $crate::RuntimePublic>::all($key_type).into_iter().map(Self).collect() } - fn generate_pair(seed: Option<&str>) -> Self { + fn generate_pair(seed: Option>) -> Self { Self(<$public as $crate::RuntimePublic>::generate_pair($key_type, seed)) } diff --git a/core/application-crypto/src/sr25519.rs b/core/application-crypto/src/sr25519.rs index 40f6c6b22ec02..f1c7f39998257 100644 --- a/core/application-crypto/src/sr25519.rs +++ b/core/application-crypto/src/sr25519.rs @@ -34,19 +34,19 @@ impl RuntimePublic for Public { type Signature = Signature; fn all(key_type: KeyTypeId) -> crate::Vec { - runtime_io::sr25519_public_keys(key_type) + runtime_io::crypto::sr25519_public_keys(key_type) } - fn generate_pair(key_type: KeyTypeId, seed: Option<&str>) -> Self { - runtime_io::sr25519_generate(key_type, seed) + fn generate_pair(key_type: KeyTypeId, seed: Option>) -> Self { + runtime_io::crypto::sr25519_generate(key_type, seed) } fn sign>(&self, key_type: KeyTypeId, msg: &M) -> Option { - runtime_io::sr25519_sign(key_type, self, msg.as_ref()) + runtime_io::crypto::sr25519_sign(key_type, self, msg.as_ref()) } fn verify>(&self, msg: &M, signature: &Self::Signature) -> bool { - runtime_io::sr25519_verify(&signature, msg.as_ref(), self) + runtime_io::crypto::sr25519_verify(&signature, msg.as_ref(), self) } } diff --git a/core/application-crypto/src/traits.rs b/core/application-crypto/src/traits.rs index aad076bd90012..d9140288c2b3c 100644 --- a/core/application-crypto/src/traits.rs +++ b/core/application-crypto/src/traits.rs @@ -78,10 +78,13 @@ pub trait RuntimePublic: Sized { /// Returns all public keys for the given key type in the keystore. fn all(key_type: KeyTypeId) -> crate::Vec; - /// Generate a public/private pair for the given key type and store it in the keystore. + /// Generate a public/private pair for the given key type with an optional `seed` and + /// store it in the keystore. + /// + /// The `seed` needs to be valid utf8. /// /// Returns the generated public key. - fn generate_pair(key_type: KeyTypeId, seed: Option<&str>) -> Self; + fn generate_pair(key_type: KeyTypeId, seed: Option>) -> Self; /// Sign the given message with the corresponding private key of this public key. /// @@ -106,10 +109,12 @@ pub trait RuntimeAppPublic: Sized { /// Returns all public keys for this application in the keystore. fn all() -> crate::Vec; - /// Generate a public/private pair and store it in the keystore. + /// Generate a public/private pair with an optional `seed` and store it in the keystore. + /// + /// The `seed` needs to be valid utf8. /// /// Returns the generated public key. - fn generate_pair(seed: Option<&str>) -> Self; + fn generate_pair(seed: Option>) -> Self; /// Sign the given message with the corresponding private key of this public key. /// diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 29a7ca0772e50..383276cb26fc1 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -202,8 +202,7 @@ pub trait Storage { pub trait Misc { /// The current relay chain identifier. fn chain_id(&self) -> u64 { - self.extension::(); - self.chain_id() + externalities::Externalities::chain_id(*self) } /// Print a number. @@ -235,10 +234,14 @@ pub trait Crypto { .ed25519_public_keys(id) } - /// Generate an `ed22519` key for the given key type and store it in the keystore. + /// Generate an `ed22519` key for the given key type using an optional `seed` and + /// store it in the keystore. + /// + /// The `seed` needs to be a valid utf8. /// /// Returns the public key. - fn ed25519_generate(&mut self, id: KeyTypeId, seed: Option<&str>) -> ed25519::Public { + fn ed25519_generate(&mut self, id: KeyTypeId, seed: Option>) -> ed25519::Public { + let seed = seed.as_ref().map(|s| std::str::from_utf8(&s).expect("Seed is valid utf8!")); self.extension::() .expect("No `keystore` associated for the current context!") .write() @@ -251,7 +254,7 @@ pub trait Crypto { /// /// Returns the signature. fn ed25519_sign( - &self, + &mut self, id: KeyTypeId, pub_key: &ed25519::Public, msg: &[u8], @@ -276,17 +279,21 @@ pub trait Crypto { } /// Returns all `sr25519` public keys for the given key id from the keystore. - fn sr25519_public_keys(&self, id: KeyTypeId) -> Vec { + fn sr25519_public_keys(&mut self, id: KeyTypeId) -> Vec { self.extension::() .expect("No `keystore` associated for the current context!") .read() .sr25519_public_keys(id) } - /// Generate an `sr22519` key for the given key type and store it in the keystore. + /// Generate an `sr22519` key for the given key type using an optional seed and + /// store it in the keystore. + /// + /// The `seed` needs to be a valid utf8. /// /// Returns the public key. - fn sr25519_generate(&self, id: KeyTypeId, seed: Option<&str>) -> sr25519::Public { + fn sr25519_generate(&mut self, id: KeyTypeId, seed: Option>) -> sr25519::Public { + let seed = seed.as_ref().map(|s| std::str::from_utf8(&s).expect("Seed is valid utf8!")); self.extension::() .expect("No `keystore` associated for the current context!") .write() @@ -379,7 +386,7 @@ pub trait Offchain { /// /// Even if this function returns `true`, it does not mean that any keys are configured /// and that the validator is registered in the chain. - fn is_validator(&self) -> bool { + fn is_validator(&mut self) -> bool { self.extension::() .expect("is_validator can be called only in the offchain worker context") .is_validator() @@ -388,28 +395,28 @@ pub trait Offchain { /// Submit an encoded transaction to the pool. /// /// The transaction will end up in the pool. - fn submit_transaction(&self, data: Vec) -> Result<(), ()> { + fn submit_transaction(&mut self, data: Vec) -> Result<(), ()> { self.extension::() .expect("submit_transaction can be called only in the offchain worker context") .submit_transaction(data) } /// Returns information about the local node's network state. - fn network_state(&self) -> Result { + fn network_state(&mut self) -> Result { self.extension::() .expect("network_state can be called only in the offchain worker context") .network_state() } /// Returns current UNIX timestamp (in millis) - fn timestamp(&self) -> Timestamp { + fn timestamp(&mut self) -> Timestamp { self.extension::() .expect("timestamp can be called only in the offchain worker context") .timestamp() } /// Pause the execution until `deadline` is reached. - fn sleep_until(&self, deadline: Timestamp) { + fn sleep_until(&mut self, deadline: Timestamp) { self.extension::() .expect("sleep_until can be called only in the offchain worker context") .sleep_until(deadline) @@ -419,7 +426,7 @@ pub trait Offchain { /// /// This is a trully random non deterministic seed generated by host environment. /// Obviously fine in the off-chain worker context. - fn random_seed(&self) -> [u8; 32] { + fn random_seed(&mut self) -> [u8; 32] { self.extension::() .expect("random_seed can be called only in the offchain worker context") .random_seed() @@ -429,7 +436,7 @@ pub trait Offchain { /// /// Note this storage is not part of the consensus, it's only accessible by /// offchain worker tasks running on the same machine. It IS persisted between runs. - fn local_storage_set(&self, kind: StorageKind, key: &[u8], value: &[u8]) { + fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { self.extension::() .expect("random_seed can be called only in the offchain worker context") .local_storage_set(kind, key, value) @@ -445,15 +452,15 @@ pub trait Offchain { /// Note this storage is not part of the consensus, it's only accessible by /// offchain worker tasks running on the same machine. It IS persisted between runs. fn local_storage_compare_and_set( - &self, + &mut self, kind: StorageKind, key: &[u8], - old_value: Option<&[u8]>, + old_value: Option>, new_value: &[u8], ) -> bool { self.extension::() .expect("random_seed can be called only in the offchain worker context") - .local_storage_compare_and_set(kind, key, old_value, new_value) + .local_storage_compare_and_set(kind, key, old_value.as_deref(), new_value) } /// Gets a value from the local storage. @@ -461,7 +468,7 @@ pub trait Offchain { /// If the value does not exist in the storage `None` will be returned. /// Note this storage is not part of the consensus, it's only accessible by /// offchain worker tasks running on the same machine. It IS persisted between runs. - fn local_storage_get(&self, kind: StorageKind, key: &[u8]) -> Option> { + fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { self.extension::() .expect("random_seed can be called only in the offchain worker context") .local_storage_get(kind, key) @@ -472,7 +479,7 @@ pub trait Offchain { /// Meta is a future-reserved field containing additional, parity-scale-codec encoded parameters. /// Returns the id of newly started request. fn http_request_start( - &self, + &mut self, method: &str, uri: &str, meta: &[u8], @@ -484,7 +491,7 @@ pub trait Offchain { /// Append header to the request. fn http_request_add_header( - &self, + &mut self, request_id: HttpRequestId, name: &str, value: &str, @@ -501,7 +508,7 @@ pub trait Offchain { /// /// Returns an error in case deadline is reached or the chunk couldn't be written. fn http_request_write_body( - &self, + &mut self, request_id: HttpRequestId, chunk: &[u8], deadline: Option, @@ -519,7 +526,7 @@ pub trait Offchain { /// /// Passing `None` as deadline blocks forever. fn http_response_wait( - &self, + &mut self, ids: &[HttpRequestId], deadline: Option, ) -> Vec { @@ -532,7 +539,7 @@ pub trait Offchain { /// /// Returns a vector of pairs `(HeaderKey, HeaderValue)`. /// NOTE response headers have to be read before response body. - fn http_response_headers(&self, request_id: HttpRequestId) -> Vec<(Vec, Vec)> { + fn http_response_headers(&mut self, request_id: HttpRequestId) -> Vec<(Vec, Vec)> { self.extension::() .expect("random_seed can be called only in the offchain worker context") .http_response_headers(request_id) @@ -547,7 +554,7 @@ pub trait Offchain { /// NOTE this implies that response headers must be read before draining the body. /// Passing `None` as a deadline blocks forever. fn http_response_read_body( - &self, + &mut self, request_id: HttpRequestId, buffer: &mut [u8], deadline: Option, diff --git a/core/sr-primitives/src/generic/unchecked_extrinsic.rs b/core/sr-primitives/src/generic/unchecked_extrinsic.rs index c5ee76e21c011..a12d4286a0006 100644 --- a/core/sr-primitives/src/generic/unchecked_extrinsic.rs +++ b/core/sr-primitives/src/generic/unchecked_extrinsic.rs @@ -20,7 +20,7 @@ use std::fmt; use rstd::prelude::*; -use runtime_io::blake2_256; +use runtime_io::hashing::blake2_256; use codec::{Decode, Encode, EncodeLike, Input, Error}; use crate::{ traits::{self, Member, MaybeDisplay, SignedExtension, Checkable, Extrinsic}, diff --git a/core/sr-primitives/src/lib.rs b/core/sr-primitives/src/lib.rs index 7032560d0f6b1..0a919a2cd456f 100644 --- a/core/sr-primitives/src/lib.rs +++ b/core/sr-primitives/src/lib.rs @@ -269,11 +269,11 @@ impl Verify for AnySignature { type Signer = sr25519::Public; fn verify>(&self, mut msg: L, signer: &sr25519::Public) -> bool { sr25519::Signature::try_from(self.0.as_fixed_bytes().as_ref()) - .map(|s| runtime_io::sr25519_verify(&s, msg.get(), &signer)) + .map(|s| runtime_io::crypto::sr25519_verify(&s, msg.get(), &signer)) .unwrap_or(false) || ed25519::Signature::try_from(self.0.as_fixed_bytes().as_ref()) .and_then(|s| ed25519::Public::try_from(signer.0.as_ref()).map(|p| (s, p))) - .map(|(s, p)| runtime_io::ed25519_verify(&s, msg.get(), &p)) + .map(|(s, p)| runtime_io::crypto::ed25519_verify(&s, msg.get(), &p)) .unwrap_or(false) } } diff --git a/core/sr-primitives/src/offchain/http.rs b/core/sr-primitives/src/offchain/http.rs index b68cf28365b59..49846e51652de 100644 --- a/core/sr-primitives/src/offchain/http.rs +++ b/core/sr-primitives/src/offchain/http.rs @@ -223,11 +223,15 @@ impl<'a, I: AsRef<[u8]>, T: IntoIterator> Request<'a, T> { let meta = &[]; // start an http request. - let id = runtime_io::http_request_start(self.method.as_ref(), self.url, meta).map_err(|_| HttpError::IoError)?; + let id = runtime_io::offchain::http_request_start( + self.method.as_ref(), + self.url, + meta, + ).map_err(|_| HttpError::IoError)?; // add custom headers for header in &self.headers { - runtime_io::http_request_add_header( + runtime_io::offchain::http_request_add_header( id, header.name(), header.value(), @@ -236,11 +240,11 @@ impl<'a, I: AsRef<[u8]>, T: IntoIterator> Request<'a, T> { // write body for chunk in self.body { - runtime_io::http_request_write_body(id, chunk.as_ref(), self.deadline)?; + runtime_io::offchain::http_request_write_body(id, chunk.as_ref(), self.deadline)?; } // finalise the request - runtime_io::http_request_write_body(id, &[], self.deadline)?; + runtime_io::offchain::http_request_write_body(id, &[], self.deadline)?; Ok(PendingRequest { id, @@ -307,7 +311,7 @@ impl PendingRequest { deadline: impl Into> ) -> Vec> { let ids = requests.iter().map(|r| r.id).collect::>(); - let statuses = runtime_io::http_response_wait(&ids, deadline.into()); + let statuses = runtime_io::offchain::http_response_wait(&ids, deadline.into()); statuses .into_iter() @@ -345,7 +349,9 @@ impl Response { /// Retrieve the headers for this response. pub fn headers(&mut self) -> &Headers { if self.headers.is_none() { - self.headers = Some(Headers { raw: runtime_io::http_response_headers(self.id) }); + self.headers = Some( + Headers { raw: runtime_io::offchain::http_response_headers(self.id) }, + ); } self.headers.as_ref().expect("Headers were just set; qed") } @@ -424,7 +430,11 @@ impl Iterator for ResponseBody { } if self.filled_up_to.is_none() { - let result = runtime_io::http_response_read_body(self.id, &mut self.buffer, self.deadline); + let result = runtime_io::offchain::http_response_read_body( + self.id, + &mut self.buffer, + self.deadline, + ); match result { Err(e) => { self.error = Some(e); @@ -435,7 +445,7 @@ impl Iterator for ResponseBody { } Ok(size) => { self.position = 0; - self.filled_up_to = Some(size); + self.filled_up_to = Some(size as usize); } } } diff --git a/core/sr-primitives/src/testing.rs b/core/sr-primitives/src/testing.rs index 5391735576aa2..93fcc3bd993dd 100644 --- a/core/sr-primitives/src/testing.rs +++ b/core/sr-primitives/src/testing.rs @@ -88,7 +88,7 @@ impl app_crypto::RuntimeAppPublic for UintAuthorityId { ALL_KEYS.with(|l| l.borrow().clone()) } - fn generate_pair(_: Option<&str>) -> Self { + fn generate_pair(_: Option>) -> Self { use rand::RngCore; UintAuthorityId(rand::thread_rng().next_u64()) } diff --git a/core/sr-primitives/src/traits.rs b/core/sr-primitives/src/traits.rs index 10c2e806584ba..6b6381571c102 100644 --- a/core/sr-primitives/src/traits.rs +++ b/core/sr-primitives/src/traits.rs @@ -65,14 +65,14 @@ pub trait Verify { impl Verify for primitives::ed25519::Signature { type Signer = primitives::ed25519::Public; fn verify>(&self, mut msg: L, signer: &Self::Signer) -> bool { - runtime_io::ed25519_verify(self, msg.get(), signer) + runtime_io::crypto::ed25519_verify(self, msg.get(), signer) } } impl Verify for primitives::sr25519::Signature { type Signer = primitives::sr25519::Public; fn verify>(&self, mut msg: L, signer: &Self::Signer) -> bool { - runtime_io::sr25519_verify(self, msg.get(), signer) + runtime_io::crypto::sr25519_verify(self, msg.get(), signer) } } @@ -479,23 +479,23 @@ impl Hash for BlakeTwo256 { type Output = primitives::H256; type Hasher = Blake2Hasher; fn hash(s: &[u8]) -> Self::Output { - runtime_io::blake2_256(s).into() + runtime_io::hashing::blake2_256(s).into() } fn trie_root(input: Vec<(Vec, Vec)>) -> Self::Output { - runtime_io::blake2_256_trie_root(input) + runtime_io::storage::blake2_256_trie_root(input) } fn ordered_trie_root(input: Vec>) -> Self::Output { - runtime_io::blake2_256_ordered_trie_root(input) + runtime_io::storage::blake2_256_ordered_trie_root(input) } fn storage_root() -> Self::Output { - runtime_io::storage_root().into() + runtime_io::storage::root().into() } fn storage_changes_root(parent_hash: Self::Output) -> Option { - runtime_io::storage_changes_root(parent_hash.into()).map(Into::into) + runtime_io::storage::changes_root(parent_hash.into()).map(Into::into) } } @@ -617,9 +617,9 @@ pub trait IsMember { /// You can also create a `new` one from those fields. pub trait Header: Clone + Send + Sync + Codec + Eq + MaybeSerializeDebugButNotDeserialize + 'static { /// Header number. - type Number: Member + MaybeSerializeDebug + ::rstd::hash::Hash + Copy + MaybeDisplay + SimpleArithmetic + Codec; + type Number: Member + MaybeSerializeDebug + rstd::hash::Hash + Copy + MaybeDisplay + SimpleArithmetic + Codec; /// Header hash type - type Hash: Member + MaybeSerializeDebug + ::rstd::hash::Hash + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]> + AsMut<[u8]>; + type Hash: Member + MaybeSerializeDebug + rstd::hash::Hash + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]> + AsMut<[u8]>; /// Hashing algorithm type Hashing: Hash; @@ -673,7 +673,7 @@ pub trait Block: Clone + Send + Sync + Codec + Eq + MaybeSerializeDebugButNotDes /// Header type. type Header: Header; /// Block hash type. - type Hash: Member + MaybeSerializeDebug + ::rstd::hash::Hash + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]> + AsMut<[u8]>; + type Hash: Member + MaybeSerializeDebug + rstd::hash::Hash + Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]> + AsMut<[u8]>; /// Returns a reference to the header. fn header(&self) -> &Self::Header; @@ -1253,19 +1253,19 @@ impl Printable for u8 { impl Printable for &[u8] { fn print(&self) { - runtime_io::print_hex(self); + runtime_io::misc::print_hex(self); } } impl Printable for &str { fn print(&self) { - runtime_io::print_utf8(self.as_bytes()); + runtime_io::misc::print_utf8(self.as_bytes()); } } impl Printable for u64 { fn print(&self) { - runtime_io::print_num(*self); + runtime_io::misc::print_num(*self); } } From 323bf15900473f77a13885931cd5b8f7b4db856e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 18 Oct 2019 17:16:32 +0200 Subject: [PATCH 33/76] Fix more compilation errors --- core/executor/src/host_interface.rs | 104 +++++++++++------------- core/sr-primitives/src/offchain/http.rs | 3 +- 2 files changed, 49 insertions(+), 58 deletions(-) diff --git a/core/executor/src/host_interface.rs b/core/executor/src/host_interface.rs index ff4e42f78049d..e4ec319e18f96 100644 --- a/core/executor/src/host_interface.rs +++ b/core/executor/src/host_interface.rs @@ -27,7 +27,10 @@ use primitives::{ crypto::KeyTypeId, offchain, }; use trie::{TrieConfiguration, trie_types::Layout}; -use wasm_interface::{FunctionContext, Pointer, PointerType, Result as WResult, WordSize}; +use wasm_interface::{ + FunctionContext, Pointer, PointerType, Result as WResult, WordSize, + WritePrimitive, ReadPrimitive, +}; #[cfg(feature="wasm-extern-trace")] macro_rules! debug_trace { @@ -134,20 +137,20 @@ impl_wasm_host_interface! { ext_print_utf8(utf8_data: Pointer, utf8_len: WordSize) { if let Ok(utf8) = context.read_memory(utf8_data, utf8_len) { - runtime_io::print_utf8(&utf8); + runtime_io::misc::print_utf8(&utf8); } Ok(()) } ext_print_hex(data: Pointer, len: WordSize) { if let Ok(hex) = context.read_memory(data, len) { - runtime_io::print_hex(&hex); + runtime_io::misc::print_hex(&hex); } Ok(()) } ext_print_num(number: u64) { - runtime_io::print_num(number); + runtime_io::misc::print_num(number); Ok(()) } @@ -162,7 +165,7 @@ impl_wasm_host_interface! { let value = context.read_memory(value_data, value_len) .map_err(|_| "Invalid attempt to determine value in ext_set_storage")?; with_external_storage(move || - Ok(runtime_io::set_storage(&key, &value)) + Ok(runtime_io::storage::set(&key, &value)) )?; Ok(()) } @@ -183,7 +186,7 @@ impl_wasm_host_interface! { .map_err(|_| "Invalid attempt to determine value in ext_set_child_storage")?; with_external_storage(move || - Ok(runtime_io::set_child_storage(&storage_key, &key, &value)) + Ok(runtime_io::storage::child_set(&storage_key, &key, &value)) )?; Ok(()) } @@ -200,7 +203,7 @@ impl_wasm_host_interface! { .map_err(|_| "Invalid attempt to determine key in ext_clear_child_storage")?; with_external_storage(move || - Ok(runtime_io::clear_child_storage(&storage_key, &key)) + Ok(runtime_io::storage::child_clear(&storage_key, &key)) )?; Ok(()) } @@ -209,7 +212,7 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_clear_storage")?; with_external_storage(move || - Ok(runtime_io::clear_storage(&key)) + Ok(runtime_io::storage::clear(&key)) )?; Ok(()) } @@ -218,7 +221,7 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_exists_storage")?; with_external_storage(move || - Ok(if runtime_io::exists_storage(&key) { 1 } else { 0 }) + Ok(if runtime_io::storage::exists(&key) { 1 } else { 0 }) ) } @@ -234,7 +237,7 @@ impl_wasm_host_interface! { .map_err(|_| "Invalid attempt to determine key in ext_exists_child_storage")?; with_external_storage(move || - Ok(if runtime_io::exists_child_storage(&storage_key, &key) { 1 } else { 0 }) + Ok(if runtime_io::storage::child_exists(&storage_key, &key) { 1 } else { 0 }) ) } @@ -242,7 +245,7 @@ impl_wasm_host_interface! { let prefix = context.read_memory(prefix_data, prefix_len) .map_err(|_| "Invalid attempt to determine prefix in ext_clear_prefix")?; with_external_storage(move || - Ok(runtime_io::clear_prefix(&prefix)) + Ok(runtime_io::storage::clear_prefix(&prefix)) )?; Ok(()) } @@ -258,7 +261,7 @@ impl_wasm_host_interface! { let prefix = context.read_memory(prefix_data, prefix_len) .map_err(|_| "Invalid attempt to determine prefix in ext_clear_child_prefix")?; with_external_storage(move || - Ok(runtime_io::clear_child_prefix(&storage_key, &prefix)) + Ok(runtime_io::storage::child_clear_prefix(&storage_key, &prefix)) )?; Ok(()) @@ -268,7 +271,7 @@ impl_wasm_host_interface! { let storage_key = context.read_memory(storage_key_data, storage_key_len) .map_err(|_| "Invalid attempt to determine storage_key in ext_kill_child_storage")?; with_external_storage(move || - Ok(runtime_io::kill_child_storage(&storage_key)) + Ok(runtime_io::storage::child_storage_kill(&storage_key)) )?; Ok(()) @@ -282,7 +285,7 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_get_allocated_storage")?; let maybe_value = with_external_storage(move || - Ok(runtime_io::storage(&key)) + Ok(runtime_io::storage::get(&key)) )?; if let Some(value) = maybe_value { @@ -312,7 +315,7 @@ impl_wasm_host_interface! { .map_err(|_| "Invalid attempt to determine key in ext_get_allocated_child_storage")?; let maybe_value = with_external_storage(move || - Ok(runtime_io::child_storage(&storage_key, &key)) + Ok(runtime_io::storage::child_get(&storage_key, &key)) )?; if let Some(value) = maybe_value { @@ -339,7 +342,7 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to get key in ext_get_storage_into")?; let maybe_value = with_external_storage(move || - Ok(runtime_io::storage(&key)) + Ok(runtime_io::storage::get(&key)) )?; if let Some(value) = maybe_value { @@ -368,7 +371,7 @@ impl_wasm_host_interface! { .map_err(|_| "Invalid attempt to get key in ext_get_child_storage_into")?; let maybe_value = with_external_storage(move || - Ok(runtime_io::child_storage(&storage_key, &key)) + Ok(runtime_io::storage::child_get(&storage_key, &key)) )?; if let Some(value) = maybe_value { @@ -384,7 +387,7 @@ impl_wasm_host_interface! { ext_storage_root(result: Pointer) { let r = with_external_storage(move || - Ok(runtime_io::storage_root()) + Ok(runtime_io::storage::root()) )?; context.write_memory(result, r.as_ref()) .map_err(|_| "Invalid attempt to set memory in ext_storage_root")?; @@ -399,7 +402,7 @@ impl_wasm_host_interface! { let storage_key = context.read_memory(storage_key_data, storage_key_len) .map_err(|_| "Invalid attempt to determine storage_key in ext_child_storage_root")?; let value = with_external_storage(move || - Ok(runtime_io::child_storage_root(&storage_key)) + Ok(runtime_io::storage::child_root(&storage_key)) )?; let offset = context.allocate_memory(value.len() as u32)?; @@ -419,7 +422,7 @@ impl_wasm_host_interface! { context.read_memory_into(parent_hash_data, &mut parent_hash[..]) .map_err(|_| "Invalid attempt to get parent_hash in ext_storage_changes_root")?; let r = with_external_storage(move || - Ok(runtime_io::storage_changes_root(parent_hash)) + Ok(runtime_io::storage::changes_root(parent_hash)) )?; if let Some(r) = r { @@ -456,7 +459,7 @@ impl_wasm_host_interface! { } ext_chain_id() -> u64 { - Ok(runtime_io::chain_id()) + Ok(runtime_io::misc::chain_id()) } ext_twox_64(data: Pointer, len: WordSize, out: Pointer) { @@ -552,7 +555,7 @@ impl_wasm_host_interface! { .map_err(|_| "Invalid attempt to get id in ext_ed25519_public_keys")?; let key_type = KeyTypeId(id); - let keys = runtime_io::ed25519_public_keys(key_type).encode(); + let keys = runtime_io::crypto::ed25519_public_keys(key_type).encode(); let len = keys.len() as u32; let offset = context.allocate_memory(len)?; @@ -607,13 +610,7 @@ impl_wasm_host_interface! { ) }; - let seed = seed.as_ref() - .map(|seed| - std::str::from_utf8(&seed) - .map_err(|_| "Seed not a valid utf8 string in ext_sr25119_generate") - ).transpose()?; - - let pubkey = runtime_io::ed25519_generate(key_type, seed); + let pubkey = runtime_io::crypto::ed25519_generate(key_type, seed); context.write_memory(out, pubkey.as_ref()) .map_err(|_| "Invalid attempt to set out in ext_ed25519_generate".into()) @@ -641,7 +638,7 @@ impl_wasm_host_interface! { let pub_key = ed25519::Public::try_from(pubkey.as_ref()) .map_err(|_| "Invalid `ed25519` public key")?; - let signature = runtime_io::ed25519_sign(key_type, &pub_key, &msg); + let signature = runtime_io::crypto::ed25519_sign(key_type, &pub_key, &msg); match signature { Some(signature) => { @@ -659,7 +656,7 @@ impl_wasm_host_interface! { .map_err(|_| "Invalid attempt to get id in ext_sr25519_public_keys")?; let key_type = KeyTypeId(id); - let keys = runtime_io::sr25519_public_keys(key_type).encode(); + let keys = runtime_io::crypto::sr25519_public_keys(key_type).encode(); let len = keys.len() as u32; let offset = context.allocate_memory(len)?; @@ -713,14 +710,7 @@ impl_wasm_host_interface! { ) }; - let seed = seed.as_ref() - .map(|seed| - std::str::from_utf8(&seed) - .map_err(|_| "Seed not a valid utf8 string in ext_sr25119_generate") - ) - .transpose()?; - - let pubkey = runtime_io::sr25519_generate(key_type, seed); + let pubkey = runtime_io::crypto::sr25519_generate(key_type, seed); context.write_memory(out, pubkey.as_ref()) .map_err(|_| "Invalid attempt to set out in ext_sr25519_generate".into()) @@ -748,7 +738,7 @@ impl_wasm_host_interface! { let pub_key = sr25519::Public::try_from(pubkey.as_ref()) .map_err(|_| "Invalid `sr25519` public key")?; - let signature = runtime_io::sr25519_sign(key_type, &pub_key, &msg); + let signature = runtime_io::crypto::sr25519_sign(key_type, &pub_key, &msg); match signature { Some(signature) => { @@ -795,20 +785,20 @@ impl_wasm_host_interface! { } ext_is_validator() -> u32 { - if runtime_io::is_validator() { Ok(1) } else { Ok(0) } + if runtime_io::offchain::is_validator() { Ok(1) } else { Ok(0) } } ext_submit_transaction(msg_data: Pointer, len: WordSize) -> u32 { let extrinsic = context.read_memory(msg_data, len) .map_err(|_| "OOB while ext_submit_transaction: wasm")?; - let res = runtime_io::submit_transaction(extrinsic); + let res = runtime_io::offchain::submit_transaction(extrinsic); Ok(if res.is_ok() { 0 } else { 1 }) } ext_network_state(written_out: Pointer) -> Pointer { - let res = runtime_io::network_state(); + let res = runtime_io::offchain::network_state(); let encoded = res.encode(); let len = encoded.len() as u32; @@ -823,17 +813,17 @@ impl_wasm_host_interface! { } ext_timestamp() -> u64 { - Ok(runtime_io::timestamp().unix_millis()) + Ok(runtime_io::offchain::timestamp().unix_millis()) } ext_sleep_until(deadline: u64) { - runtime_io::sleep_until(offchain::Timestamp::from_unix_millis(deadline)); + runtime_io::offchain::sleep_until(offchain::Timestamp::from_unix_millis(deadline)); Ok(()) } ext_random_seed(seed_data: Pointer) { // NOTE the runtime as assumptions about seed size. - let seed = runtime_io::random_seed(); + let seed = runtime_io::offchain::random_seed(); context.write_memory(seed_data, &seed) .map_err(|_| "Invalid attempt to set value in ext_random_seed")?; @@ -854,7 +844,7 @@ impl_wasm_host_interface! { let value = context.read_memory(value, value_len) .map_err(|_| "OOB while ext_local_storage_set: wasm")?; - runtime_io::local_storage_set(kind, &key, &value); + runtime_io::offchain::local_storage_set(kind, &key, &value); Ok(()) } @@ -870,7 +860,7 @@ impl_wasm_host_interface! { let key = context.read_memory(key, key_len) .map_err(|_| "OOB while ext_local_storage_get: wasm")?; - let maybe_value = runtime_io::local_storage_get(kind, &key); + let maybe_value = runtime_io::offchain::local_storage_get(kind, &key); let (offset, len) = if let Some(value) = maybe_value { let offset = context.allocate_memory(value.len() as u32)?; @@ -912,10 +902,10 @@ impl_wasm_host_interface! { ) }; - let res = runtime_io::local_storage_compare_and_set( + let res = runtime_io::offchain::local_storage_compare_and_set( kind, &key, - old_value.as_ref().map(|v| v.as_ref()), + old_value, &new_value, ); @@ -942,7 +932,7 @@ impl_wasm_host_interface! { let url_str = str::from_utf8(&url) .map_err(|_| "invalid str while ext_http_request_start: wasm")?; - let id = runtime_io::http_request_start(method_str, url_str, &meta); + let id = runtime_io::offchain::http_request_start(method_str, url_str, &meta); if let Ok(id) = id { Ok(id.into()) @@ -968,7 +958,7 @@ impl_wasm_host_interface! { let value_str = str::from_utf8(&value) .map_err(|_| "Invalid str while ext_http_request_add_header: wasm")?; - let res = runtime_io::http_request_add_header( + let res = runtime_io::offchain::http_request_add_header( offchain::HttpRequestId(request_id as u16), name_str, value_str, @@ -986,7 +976,7 @@ impl_wasm_host_interface! { let chunk = context.read_memory(chunk, chunk_len) .map_err(|_| "OOB while ext_http_request_write_body: wasm")?; - let res = runtime_io::http_request_write_body( + let res = runtime_io::offchain::http_request_write_body( offchain::HttpRequestId(request_id as u16), &chunk, deadline_to_timestamp(deadline), @@ -1012,7 +1002,7 @@ impl_wasm_host_interface! { ) .collect::, _>>()?; - let res = runtime_io::http_response_wait(&ids, deadline_to_timestamp(deadline)) + let res = runtime_io::offchain::http_response_wait(&ids, deadline_to_timestamp(deadline)) .into_iter() .map(|status| u32::from(status)) .enumerate() @@ -1033,7 +1023,9 @@ impl_wasm_host_interface! { ) -> Pointer { use codec::Encode; - let headers = runtime_io::http_response_headers(offchain::HttpRequestId(request_id as u16)); + let headers = runtime_io::offchain::http_response_headers( + offchain::HttpRequestId(request_id as u16), + ); let encoded = headers.encode(); let len = encoded.len() as u32; @@ -1056,7 +1048,7 @@ impl_wasm_host_interface! { let mut internal_buffer = Vec::with_capacity(buffer_len as usize); internal_buffer.resize(buffer_len as usize, 0); - let res = runtime_io::http_response_read_body( + let res = runtime_io::offchain::http_response_read_body( offchain::HttpRequestId(request_id as u16), &mut internal_buffer, deadline_to_timestamp(deadline), diff --git a/core/sr-primitives/src/offchain/http.rs b/core/sr-primitives/src/offchain/http.rs index 49846e51652de..b54e52489ac91 100644 --- a/core/sr-primitives/src/offchain/http.rs +++ b/core/sr-primitives/src/offchain/http.rs @@ -433,8 +433,7 @@ impl Iterator for ResponseBody { let result = runtime_io::offchain::http_response_read_body( self.id, &mut self.buffer, - self.deadline, - ); + self.deadline); match result { Err(e) => { self.error = Some(e); From a27038eceb4f6164bc81484155f63332f2508817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 22 Oct 2019 13:00:59 +0200 Subject: [PATCH 34/76] More progress on getting stuff back to compile --- Cargo.lock | 1 + core/application-crypto/src/ed25519.rs | 5 +- core/application-crypto/src/lib.rs | 2 +- core/application-crypto/src/sr25519.rs | 5 +- core/application-crypto/src/traits.rs | 1 + core/primitives/src/offchain.rs | 64 +- .../proc-macro/src/host_function_interface.rs | 5 +- core/runtime-interface/proc-macro/src/lib.rs | 2 + core/runtime-interface/src/impls.rs | 2 +- core/runtime-interface/src/lib.rs | 2 + core/runtime-interface/src/pass_by.rs | 2 +- core/runtime-interface/test-wasm/Cargo.toml | 4 +- core/runtime-interface/test-wasm/src/lib.rs | 7 + core/sr-io/Cargo.toml | 2 - core/sr-io/src/lib.rs | 18 +- core/sr-io/without_std.rs | 1092 +---------------- 16 files changed, 104 insertions(+), 1110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4fa2055c4e86..2cacd4237d89f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5389,6 +5389,7 @@ dependencies = [ name = "substrate-runtime-interface-test-wasm" version = "2.0.0" dependencies = [ + "sr-io 2.0.0", "sr-std 2.0.0", "substrate-primitives 2.0.0", "substrate-runtime-interface 2.0.0", diff --git a/core/application-crypto/src/ed25519.rs b/core/application-crypto/src/ed25519.rs index 446746a19d757..2b7f2f425b32c 100644 --- a/core/application-crypto/src/ed25519.rs +++ b/core/application-crypto/src/ed25519.rs @@ -18,6 +18,8 @@ use crate::{RuntimePublic, KeyTypeId}; +use rstd::vec::Vec; + pub use primitives::ed25519::*; mod app { @@ -25,8 +27,7 @@ mod app { crate::app_crypto!(super, ED25519); } -pub use app::Public as AppPublic; -pub use app::Signature as AppSignature; +pub use app::{Public as AppPublic, Signature as AppSignature}; #[cfg(feature="std")] pub use app::Pair as AppPair; diff --git a/core/application-crypto/src/lib.rs b/core/application-crypto/src/lib.rs index b14f2b17c9872..152d902f58099 100644 --- a/core/application-crypto/src/lib.rs +++ b/core/application-crypto/src/lib.rs @@ -220,7 +220,7 @@ macro_rules! app_crypto { <$public as $crate::RuntimePublic>::all($key_type).into_iter().map(Self).collect() } - fn generate_pair(seed: Option>) -> Self { + fn generate_pair(seed: Option<$crate::Vec>) -> Self { Self(<$public as $crate::RuntimePublic>::generate_pair($key_type, seed)) } diff --git a/core/application-crypto/src/sr25519.rs b/core/application-crypto/src/sr25519.rs index f1c7f39998257..ee8a754e65c58 100644 --- a/core/application-crypto/src/sr25519.rs +++ b/core/application-crypto/src/sr25519.rs @@ -18,6 +18,8 @@ use crate::{RuntimePublic, KeyTypeId}; +use rstd::vec::Vec; + pub use primitives::sr25519::*; mod app { @@ -25,8 +27,7 @@ mod app { crate::app_crypto!(super, SR25519); } -pub use app::Public as AppPublic; -pub use app::Signature as AppSignature; +pub use app::{Public as AppPublic, Signature as AppSignature}; #[cfg(feature="std")] pub use app::Pair as AppPair; diff --git a/core/application-crypto/src/traits.rs b/core/application-crypto/src/traits.rs index d9140288c2b3c..816821c1fdb19 100644 --- a/core/application-crypto/src/traits.rs +++ b/core/application-crypto/src/traits.rs @@ -18,6 +18,7 @@ use primitives::crypto::{KeyTypeId, CryptoType, IsWrappedBy, Public}; #[cfg(feature = "std")] use primitives::crypto::Pair; use codec::Codec; +use rstd::vec::Vec; /// An application-specific key. pub trait AppKey: 'static + Send + Sync + Sized + CryptoType + Clone { diff --git a/core/primitives/src/offchain.rs b/core/primitives/src/offchain.rs index 277bca6c0f078..26f4d97979229 100644 --- a/core/primitives/src/offchain.rs +++ b/core/primitives/src/offchain.rs @@ -17,7 +17,7 @@ //! Offchain workers types use codec::{Encode, Decode}; -use rstd::prelude::{Vec, Box}; +use rstd::{convert::TryFrom, prelude::{Vec, Box}}; use runtime_interface::pass_by::{PassByCodec, PassByInner}; pub use crate::crypto::KeyTypeId; @@ -41,6 +41,24 @@ pub enum StorageKind { LOCAL = 2, } +impl TryFrom for StorageKind { + type Error = (); + + fn try_from(kind: u32) -> Result { + match kind { + e if e == u32::from(StorageKind::PERSISTENT as u8) => Ok(StorageKind::PERSISTENT), + e if e == u32::from(StorageKind::LOCAL as u8) => Ok(StorageKind::LOCAL), + _ => Err(()), + } + } +} + +impl From for u32 { + fn from(c: StorageKind) -> Self { + c as u8 as u32 + } +} + /// Opaque type for offchain http requests. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, PassByInner, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug, Hash))] @@ -65,6 +83,25 @@ pub enum HttpError { Invalid = 3, } +impl TryFrom for HttpError { + type Error = (); + + fn try_from(error: u32) -> Result { + match error { + e if e == HttpError::DeadlineReached as u8 as u32 => Ok(HttpError::DeadlineReached), + e if e == HttpError::IoError as u8 as u32 => Ok(HttpError::IoError), + e if e == HttpError::Invalid as u8 as u32 => Ok(HttpError::Invalid), + _ => Err(()) + } + } +} + +impl From for u32 { + fn from(c: HttpError) -> Self { + c as u8 as u32 + } +} + /// Status of the HTTP request #[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, PassByCodec)] #[cfg_attr(feature = "std", derive(Debug))] @@ -86,6 +123,31 @@ pub enum HttpRequestStatus { Finished(u16), } +impl From for u32 { + fn from(status: HttpRequestStatus) -> Self { + match status { + HttpRequestStatus::Invalid => 0, + HttpRequestStatus::DeadlineReached => 10, + HttpRequestStatus::IoError => 20, + HttpRequestStatus::Finished(code) => u32::from(code), + } + } +} + +impl TryFrom for HttpRequestStatus { + type Error = (); + + fn try_from(status: u32) -> Result { + match status { + 0 => Ok(HttpRequestStatus::Invalid), + 10 => Ok(HttpRequestStatus::DeadlineReached), + 20 => Ok(HttpRequestStatus::IoError), + 100..=999 => u16::try_from(status).map(HttpRequestStatus::Finished).map_err(|_| ()), + _ => Err(()), + } + } +} + /// A blob to hold information about the local node's network state /// without committing to its format. #[derive(Clone, Eq, PartialEq, Encode, Decode, PassByCodec)] diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 562b17ad04aa2..fc0e8967d35be 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -58,7 +58,7 @@ pub fn generate(trait_def: &ItemTrait) -> Result { /// is required to change the extern host functions signature to /// `unsafe fn name(args) -> ret` to make the function implementations exchangeable. #[cfg(not(feature = "std"))] - pub mod extern_host_function_impls { + mod extern_host_function_impls { use super::*; #extern_host_function_impls @@ -86,7 +86,7 @@ fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) - ReturnType::Default => quote!(), ReturnType::Type(_, ref ty) => quote! { -> <#ty as #crate_::RIType>::FFIType - } + }, }; Ok( @@ -95,6 +95,7 @@ fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) - pub unsafe fn #function ( #( #arg_names: <#arg_types as #crate_::RIType>::FFIType ),* ) #output { + /// Contains the actual extern function. mod implementation { use super::*; diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index 6d85c2a486e99..e51c3674795f1 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -52,8 +52,10 @@ fn runtime_interface_impl(trait_def: ItemTrait) -> Result { let trait_decl_impl = trait_decl_impl::process(&trait_def)?; let host_functions = host_function_interface::generate(&trait_def)?; let vis = trait_def.vis; + let attrs = &trait_def.attrs; let res = quote! { + #( #attrs )* #vis mod #mod_name { use super::*; #crate_include diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index d106e15232713..241bfee18c0e1 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -30,7 +30,7 @@ use wasm_interface::{FunctionContext, Pointer, Result}; use codec::{Encode, Decode}; -use rstd::{any::TypeId, mem}; +use rstd::{any::TypeId, mem, vec::Vec, boxed::Box}; #[cfg(feature = "std")] use rstd::borrow::Cow; diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index bc2ef4633888e..d159aee95a7d4 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -24,6 +24,8 @@ //! [`WrappedFFIValue`] to call into the host. The created [`WrappedFFIValue`] will remain on //! the stack while we call into the host. +#![cfg_attr(not(feature = "std"), no_std)] + #[doc(hidden)] #[cfg(feature = "std")] pub use wasm_interface; diff --git a/core/runtime-interface/src/pass_by.rs b/core/runtime-interface/src/pass_by.rs index 6145c0f5474a0..10a58418860a9 100644 --- a/core/runtime-interface/src/pass_by.rs +++ b/core/runtime-interface/src/pass_by.rs @@ -27,7 +27,7 @@ use crate::wasm::*; #[cfg(feature = "std")] use wasm_interface::{FunctionContext, Pointer, Result}; -use rstd::marker::PhantomData; +use rstd::{marker::PhantomData, vec::Vec}; #[cfg(not(feature = "std"))] use rstd::slice; diff --git a/core/runtime-interface/test-wasm/Cargo.toml b/core/runtime-interface/test-wasm/Cargo.toml index 13abc15d8b1b8..dd870164e98c2 100644 --- a/core/runtime-interface/test-wasm/Cargo.toml +++ b/core/runtime-interface/test-wasm/Cargo.toml @@ -8,6 +8,7 @@ build = "build.rs" [dependencies] runtime-interface = { package = "substrate-runtime-interface", path = "../", default-features = false } rstd = { package = "sr-std", path = "../../sr-std", default-features = false } +runtime-io = { package = "sr-io", path = "../../sr-io", default-features = false } primitives = { package = "substrate-primitives", path = "../../primitives", default-features = false } [build-dependencies] @@ -15,5 +16,4 @@ wasm-builder-runner = { package = "substrate-wasm-builder-runner", version = "1. [features] default = [ "std" ] -std = [ "runtime-interface/std", "rstd/std", "primitives/std" ] -no_std = [] +std = [ "runtime-interface/std", "rstd/std", "primitives/std", "runtime-io/std" ] diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index d1812f02f3284..faa836b7ee0b1 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -72,6 +72,13 @@ pub trait TestApi { } } +/// This function is not used, but we require it for the compiler to include `runtime-io`. +/// `runtime-io` is required for its panic and oom handler. +#[no_mangle] +pub fn import_runtime_io() { + runtime_io::misc::print_utf8(&[]); +} + #[no_mangle] pub fn test_return_data() { let input = vec![1, 2, 3, 4, 5, 6]; diff --git a/core/sr-io/Cargo.toml b/core/sr-io/Cargo.toml index eeff31a110614..878ed7d8b6eed 100644 --- a/core/sr-io/Cargo.toml +++ b/core/sr-io/Cargo.toml @@ -34,8 +34,6 @@ std = [ "runtime-interface/std", "externalities", ] -nightly = [] -strict = [] # These two features are used for `no_std` builds for the environments which already provides # `#[panic_handler]` and `#[alloc_error_handler]`. diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 383276cb26fc1..8ded1cad23cd4 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -28,21 +28,29 @@ use rstd::vec::Vec; +#[cfg(feature = "std")] +use rstd::ops::Deref; + +#[cfg(feature = "std")] +use primitives::{ + crypto::Pair, traits::KeystoreExt, offchain::OffchainExt, hexdisplay::HexDisplay, +}; + use primitives::{ - crypto::{KeyTypeId, Pair}, ed25519, sr25519, H256, hexdisplay::HexDisplay, + crypto::KeyTypeId, ed25519, sr25519, H256, storage::ChildStorageKey, offchain::{ Timestamp, HttpRequestId, HttpRequestStatus, HttpError, StorageKind, OpaqueNetworkState, - OffchainExt, }, - storage::ChildStorageKey, traits::KeystoreExt, }; +#[cfg(feature = "std")] use trie::{TrieConfiguration, trie_types::Layout}; use runtime_interface::runtime_interface; use codec::{Encode, Decode}; +#[cfg(feature = "std")] use externalities::ExternalitiesExt; /// Error verifying ECDSA signature @@ -460,7 +468,7 @@ pub trait Offchain { ) -> bool { self.extension::() .expect("random_seed can be called only in the offchain worker context") - .local_storage_compare_and_set(kind, key, old_value.as_deref(), new_value) + .local_storage_compare_and_set(kind, key, old_value.as_ref().map(|v| v.deref()), new_value) } /// Gets a value from the local storage. @@ -579,8 +587,6 @@ mod imp { #[cfg(feature = "std")] pub use self::imp::{StorageOverlay, ChildrenStorageOverlay, with_storage}; -#[cfg(not(feature = "std"))] -pub use self::imp::ext::*; /// Type alias for Externalities implementation used in tests. #[cfg(feature = "std")] diff --git a/core/sr-io/without_std.rs b/core/sr-io/without_std.rs index 7b6dcdc7af98b..7b4246fc1739e 100644 --- a/core/sr-io/without_std.rs +++ b/core/sr-io/without_std.rs @@ -14,14 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -#[doc(hidden)] -pub use rstd; -pub use rstd::{mem, slice}; - use core::{intrinsics, panic::PanicInfo}; -use rstd::{vec::Vec, cell::Cell, convert::TryInto}; -use primitives::offchain; -use codec::Decode; #[cfg(not(feature = "no_panic_handler"))] #[panic_handler] @@ -29,7 +22,7 @@ use codec::Decode; pub fn panic(info: &PanicInfo) -> ! { unsafe { let message = rstd::alloc::format!("{}", info); - extern_functions_host_impl::ext_print_utf8(message.as_ptr() as *const u8, message.len() as u32); + misc::print_utf8(message.as_bytes()); intrinsics::abort() } } @@ -40,1088 +33,7 @@ pub extern fn oom(_: core::alloc::Layout) -> ! { static OOM_MSG: &str = "Runtime memory exhausted. Aborting"; unsafe { - extern_functions_host_impl::ext_print_utf8(OOM_MSG.as_ptr(), OOM_MSG.len() as u32); + misc::print_utf8(OOM_MSG.as_bytes()); intrinsics::abort(); } } - -/// External (Host) APIs -pub mod ext { - use super::*; - - /// Declare extern functions - macro_rules! extern_functions { - ( - $( - $( #[$attr:meta] )* - fn $name:ident ( $( $arg:ident : $arg_ty:ty ),* $(,)? ) $( -> $ret:ty )?; - )* - ) => { - $( - $( #[$attr] )* - #[allow(non_upper_case_globals)] - pub static $name: ExchangeableFunction $ret )?> = - ExchangeableFunction::new(extern_functions_host_impl::$name); - )* - - /// The exchangeable extern functions host implementations. - pub(crate) mod extern_functions_host_impl { - $( - pub unsafe fn $name ( $( $arg : $arg_ty ),* ) $( -> $ret )? { - implementation::$name ( $( $arg ),* ) - } - )* - - mod implementation { - extern "C" { - $( - pub fn $name ( $( $arg : $arg_ty ),* ) $( -> $ret )?; - )* - } - } - } - }; - } - - /// Host functions, provided by the executor. - /// A WebAssembly runtime module would "import" these to access the execution environment - /// (most importantly, storage) or perform heavy hash calculations. - /// See also "ext_" functions in sr-sandbox and sr-std - extern_functions! { - /// Host functions for printing, useful for debugging. - fn ext_print_utf8(utf8_data: *const u8, utf8_len: u32); - /// Print data as hex. - fn ext_print_hex(data: *const u8, len: u32); - /// Print a number - fn ext_print_num(value: u64); - - /// Set value for key in storage. - fn ext_set_storage(key_data: *const u8, key_len: u32, value_data: *const u8, value_len: u32); - /// Remove key and value from storage. - fn ext_clear_storage(key_data: *const u8, key_len: u32); - /// Checks if the given key exists in the storage. - /// - /// # Returns - /// - /// - `1` if the value exists. - /// - `0` if the value does not exists. - fn ext_exists_storage(key_data: *const u8, key_len: u32) -> u32; - /// Remove storage entries which key starts with given prefix. - fn ext_clear_prefix(prefix_data: *const u8, prefix_len: u32); - /// Remove child storage entries which key starts with given prefix. - fn ext_clear_child_prefix( - storage_key_data: *const u8, - storage_key_len: u32, - prefix_data: *const u8, - prefix_len: u32, - ); - /// Gets the value of the given key from storage. - /// - /// The host allocates the memory for storing the value. - /// - /// # Returns - /// - /// - `0` if no value exists to the given key. `written_out` is set to `u32::max_value()`. - /// - Otherwise, pointer to the value in memory. `written_out` contains the length of the value. - fn ext_get_allocated_storage(key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8; - /// Gets the value of the given key from storage. - /// - /// The value is written into `value` starting at `value_offset`. - /// - /// If the value length is greater than `value_len - value_offset`, the value is written partially. - /// - /// # Returns - /// - /// - `u32::max_value()` if the value does not exists. - /// - /// - Otherwise, the number of bytes written for value. - fn ext_get_storage_into( - key_data: *const u8, - key_len: u32, - value_data: *mut u8, - value_len: u32, - value_offset: u32, - ) -> u32; - /// Gets the trie root of the storage. - fn ext_storage_root(result: *mut u8); - /// Get the change trie root of the current storage overlay at a block with given parent. - /// - /// # Returns - /// - /// - `1` if the change trie root was found. - /// - `0` if the change trie root was not found. - fn ext_storage_changes_root( - parent_hash_data: *const u8, - parent_hash_len: u32, - result: *mut u8, - ) -> u32; - - /// A child storage function. - /// - /// See [`ext_set_storage`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_set_child_storage( - storage_key_data: *const u8, - storage_key_len: u32, - key_data: *const u8, - key_len: u32, - value_data: *const u8, - value_len: u32, - ); - /// A child storage function. - /// - /// See [`ext_clear_storage`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_clear_child_storage( - storage_key_data: *const u8, - storage_key_len: u32, - key_data: *const u8, - key_len: u32, - ); - /// A child storage function. - /// - /// See [`ext_exists_storage`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_exists_child_storage( - storage_key_data: *const u8, - storage_key_len: u32, - key_data: *const u8, - key_len: u32, - ) -> u32; - /// A child storage function. - /// - /// See [`ext_kill_storage`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_kill_child_storage(storage_key_data: *const u8, storage_key_len: u32); - /// A child storage function. - /// - /// See [`ext_get_allocated_storage`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_get_allocated_child_storage( - storage_key_data: *const u8, - storage_key_len: u32, - key_data: *const u8, - key_len: u32, - written_out: *mut u32, - ) -> *mut u8; - /// A child storage function. - /// - /// See [`ext_get_storage_into`] for details. - /// - /// A child storage is used e.g. by a contract. - fn ext_get_child_storage_into( - storage_key_data: *const u8, - storage_key_len: u32, - key_data: *const u8, - key_len: u32, - value_data: *mut u8, - value_len: u32, - value_offset: u32, - ) -> u32; - /// Commits all changes and calculates the child-storage root. - /// - /// A child storage is used e.g. by a contract. - /// - /// # Returns - /// - /// - The pointer to the result vector and `written_out` contains its length. - fn ext_child_storage_root( - storage_key_data: *const u8, - storage_key_len: u32, - written_out: *mut u32 - ) -> *mut u8; - - /// The current relay chain identifier. - fn ext_chain_id() -> u64; - - /// Calculate a blake2_256 merkle trie root. - fn ext_blake2_256_enumerated_trie_root( - values_data: *const u8, - lens_data: *const u32, - lens_len: u32, - result: *mut u8 - ); - /// BLAKE2_128 hash - fn ext_blake2_128(data: *const u8, len: u32, out: *mut u8); - /// BLAKE2_256 hash - fn ext_blake2_256(data: *const u8, len: u32, out: *mut u8); - /// XX64 hash - fn ext_twox_64(data: *const u8, len: u32, out: *mut u8); - /// XX128 hash - fn ext_twox_128(data: *const u8, len: u32, out: *mut u8); - /// XX256 hash - fn ext_twox_256(data: *const u8, len: u32, out: *mut u8); - /// Keccak256 hash - fn ext_keccak_256(data: *const u8, len: u32, out: *mut u8); - - /// Returns all `ed25519` public keys for the given key type from the keystore. - fn ext_ed25519_public_keys(id: *const u8, result_len: *mut u32) -> *mut u8; - - /// Note: `ext_ed25519_verify` returns `0` if the signature is correct, nonzero otherwise. - fn ext_ed25519_verify( - msg_data: *const u8, - msg_len: u32, - sig_data: *const u8, - pubkey_data: *const u8, - ) -> u32; - - /// Generate an `ed25519` key pair for the given key type id and store the public key - /// in `out`. - fn ext_ed25519_generate(id: *const u8, seed: *const u8, seed_len: u32, out: *mut u8); - - /// Sign the given `msg` with the `ed25519` key pair that corresponds to then given key - /// type id and public key. The raw signature is stored in `out`. - /// - /// # Returns - /// - /// - `0` on success - /// - nonezero if something failed, e.g. retrieving of the key. - fn ext_ed25519_sign( - id: *const u8, - pubkey: *const u8, - msg: *const u8, - msg_len: u32, - out: *mut u8, - ) -> u32; - - /// Returns all `sr25519` public keys for the given key type from the keystore. - fn ext_sr25519_public_keys(id: *const u8, result_len: *mut u32) -> *mut u8; - - /// Note: `ext_sr25519_verify` returns 0 if the signature is correct, nonzero otherwise. - fn ext_sr25519_verify( - msg_data: *const u8, - msg_len: u32, - sig_data: *const u8, - pubkey_data: *const u8, - ) -> u32; - - /// Generate an `sr25519` key pair for the given key type id and store the public - /// key in `out`. - fn ext_sr25519_generate(id: *const u8, seed: *const u8, seed_len: u32, out: *mut u8); - - /// Sign the given `msg` with the `sr25519` key pair that corresponds to then given key - /// type id and public key. The raw signature is stored in `out`. - /// - /// # Returns - /// - /// - `0` on success - /// - nonezero if something failed, e.g. retrieving of the key. - fn ext_sr25519_sign( - id: *const u8, - pubkey: *const u8, - msg: *const u8, - msg_len: u32, - out: *mut u8, - ) -> u32; - - /// Note: ext_secp256k1_ecdsa_recover returns 0 if the signature is correct, nonzero otherwise. - fn ext_secp256k1_ecdsa_recover( - msg_data: *const u8, - sig_data: *const u8, - pubkey_data: *mut u8, - ) -> u32; - - //================================ - // Offchain-worker Context - //================================ - - /// Returns if the local node is a potential validator. - /// - /// - `1` == `true` - /// - `0` == `false` - fn ext_is_validator() -> u32; - - /// Submit transaction. - /// - /// # Returns - /// - /// - 0 if it was successfuly added to the pool - /// - nonzero otherwise. - fn ext_submit_transaction(data: *const u8, len: u32) -> u32; - - /// Returns information about the local node's network state. - /// - /// # Returns - /// - /// The encoded `Result`. - /// `written_out` contains the length of the message. - /// - /// The ownership of the returned buffer is transferred to the runtime - /// code and the runtime is responsible for freeing it. This is always - /// a properly allocated pointer (which cannot be NULL), hence the - /// runtime code can always rely on it. - fn ext_network_state(written_out: *mut u32) -> *mut u8; - - /// Returns current UNIX timestamp (milliseconds) - fn ext_timestamp() -> u64; - - /// Pause execution until given timestamp (milliseconds; `deadline`) is reached. - /// - /// The deadline is obtained by querying the current timestamp via `ext_timestamp` - /// and then adding some time to it. - fn ext_sleep_until(deadline: u64); - - /// Generate a random seed - /// - /// `data` has to be a pointer to a slice of 32 bytes. - fn ext_random_seed(data: *mut u8); - - /// Write a value to local storage. - fn ext_local_storage_set(kind: u32, key: *const u8, key_len: u32, value: *const u8, value_len: u32); - - /// Write a value to local storage in atomic fashion. - /// - /// # Returns - /// - `0` in case the value has been set - /// - `1` if the `old_value` didn't match - fn ext_local_storage_compare_and_set( - kind: u32, - key: *const u8, - key_len: u32, - old_value: *const u8, - old_value_len: u32, - new_value: *const u8, - new_value_len: u32, - ) -> u32; - - /// Read a value from local storage. - /// - /// - /// # Returns - /// - /// - 0 if the value has not been found, the `value_len` is set to `u32::max_value`. - /// - Otherwise, pointer to the value in memory. `value_len` contains the length of the value. - fn ext_local_storage_get(kind: u32, key: *const u8, key_len: u32, value_len: *mut u32) -> *mut u8; - - /// Initiates a http request. - /// - /// `meta` is parity-scale-codec encoded additional parameters to the request (like redirection policy, - /// timeouts, certificates policy, etc). The format is not yet specified and the field is currently - /// only reserved for future use. - /// - /// # Returns - /// - /// `RequestId(u16)` of initiated request, any value beyond `u16::max_value` - /// signifies an error. - fn ext_http_request_start( - method: *const u8, - method_len: u32, - url: *const u8, - url_len: u32, - meta: *const u8, - meta_len: u32, - ) -> u32; - - /// Add a header to the request. - /// - /// # Returns - /// - /// - `0` if successful (and the request id exists) - /// - nonzero otherwise - fn ext_http_request_add_header( - request_id: u32, - name: *const u8, - name_len: u32, - value: *const u8, - value_len: u32, - ) -> u32; - - /// Write a chunk of request body. - /// - /// Writing an empty chunks finalises the request. - /// Passing `0` as deadline blocks forever. - /// - /// # Returns - /// - /// - `0` if successful, - /// - nonzero otherwise (see HttpError for the codes) - fn ext_http_request_write_body( - request_id: u32, - chunk: *const u8, - chunk_len: u32, - deadline: u64, - ) -> u32; - - /// Block and wait for the responses for given requests. - /// - /// Note that if deadline is 0 the method will block indefinitely, - /// otherwise unready responses will produce `DeadlineReached` status. - /// (see #primitives::offchain::HttpRequestStatus) - /// - /// Make sure that `statuses` have the same length as ids. - fn ext_http_response_wait( - ids: *const u32, - ids_len: u32, - statuses: *mut u32, - deadline: u64, - ); - - /// Read all response headers. - /// - /// Note the headers are only available before response body is fully consumed. - /// - /// # Returns - /// - /// - A pointer to parity-scale-codec encoded vector of pairs `(HeaderKey, HeaderValue)`. - /// - In case invalid `id` is passed it returns a pointer to parity-encoded empty vector. - fn ext_http_response_headers( - id: u32, - written_out: *mut u32, - ) -> *mut u8; - - /// Read a chunk of body response to given buffer. - /// - /// Passing `0` as deadline blocks forever. - /// - /// # Returns - /// - /// The number of bytes written if successful, - /// - if it's `0` it means response has been fully consumed, - /// - if it's greater than `u32::max_value() - 255` it means reading body failed. - /// - /// In case of failure, the error code should be mapped to `HttpError` - /// in a following manner: - /// - `u32::max_value()` HttpError code 1 (DeadlineReached) - /// - `u32::max_value() - 1` HttpError code 2 (IoError) - /// The rest is reserved for potential future errors. - fn ext_http_response_read_body( - id: u32, - buffer: *mut u8, - buffer_len: u32, - deadline: u64, - ) -> u32; - } -} - -pub use self::ext::*; - -impl StorageApi for () { - fn storage(key: &[u8]) -> Option> { - let mut length: u32 = 0; - unsafe { - let ptr = ext_get_allocated_storage.get()(key.as_ptr(), key.len() as u32, &mut length); - from_raw_parts(ptr, length) - } - } - - fn child_storage(storage_key: &[u8], key: &[u8]) -> Option> { - let mut length: u32 = 0; - unsafe { - let ptr = ext_get_allocated_child_storage.get()( - storage_key.as_ptr(), - storage_key.len() as u32, - key.as_ptr(), - key.len() as u32, - &mut length - ); - from_raw_parts(ptr, length) - } - } - - fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { - unsafe { - match ext_get_storage_into.get()( - key.as_ptr(), - key.len() as u32, - value_out.as_mut_ptr(), - value_out.len() as u32, - value_offset as u32, - ) { - none if none == u32::max_value() => None, - length => Some(length as usize), - } - } - } - - fn read_child_storage(storage_key: &[u8], key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { - unsafe { - match ext_get_child_storage_into.get()( - storage_key.as_ptr(), storage_key.len() as u32, - key.as_ptr(), key.len() as u32, - value_out.as_mut_ptr(), value_out.len() as u32, - value_offset as u32 - ) { - none if none == u32::max_value() => None, - length => Some(length as usize), - } - } - } - - fn set_storage(key: &[u8], value: &[u8]) { - unsafe { - ext_set_storage.get()( - key.as_ptr(), key.len() as u32, - value.as_ptr(), value.len() as u32 - ); - } - } - - fn set_child_storage(storage_key: &[u8], key: &[u8], value: &[u8]) { - unsafe { - ext_set_child_storage.get()( - storage_key.as_ptr(), storage_key.len() as u32, - key.as_ptr(), key.len() as u32, - value.as_ptr(), value.len() as u32 - ); - } - } - - fn clear_storage(key: &[u8]) { - unsafe { - ext_clear_storage.get()( - key.as_ptr(), key.len() as u32 - ); - } - } - - fn clear_child_storage(storage_key: &[u8], key: &[u8]) { - unsafe { - ext_clear_child_storage.get()( - storage_key.as_ptr(), storage_key.len() as u32, - key.as_ptr(), key.len() as u32 - ); - } - } - - fn exists_storage(key: &[u8]) -> bool { - unsafe { - ext_exists_storage.get()( - key.as_ptr(), key.len() as u32 - ) != 0 - } - } - - fn exists_child_storage(storage_key: &[u8], key: &[u8]) -> bool { - unsafe { - ext_exists_child_storage.get()( - storage_key.as_ptr(), storage_key.len() as u32, - key.as_ptr(), key.len() as u32 - ) != 0 - } - } - - fn clear_prefix(prefix: &[u8]) { - unsafe { - ext_clear_prefix.get()( - prefix.as_ptr(), - prefix.len() as u32 - ); - } - } - - fn clear_child_prefix(storage_key: &[u8], prefix: &[u8]) { - unsafe { - ext_clear_child_prefix.get()( - storage_key.as_ptr(), storage_key.len() as u32, - prefix.as_ptr(), prefix.len() as u32 - ); - } - } - - fn kill_child_storage(storage_key: &[u8]) { - unsafe { - ext_kill_child_storage.get()( - storage_key.as_ptr(), - storage_key.len() as u32 - ); - } - } - - fn storage_root() -> [u8; 32] { - let mut result: [u8; 32] = Default::default(); - unsafe { - ext_storage_root.get()(result.as_mut_ptr()); - } - result - } - - fn child_storage_root(storage_key: &[u8]) -> Vec { - let mut length: u32 = 0; - unsafe { - let ptr = ext_child_storage_root.get()( - storage_key.as_ptr(), - storage_key.len() as u32, - &mut length - ); - from_raw_parts(ptr, length).expect("ext_child_storage_root never returns u32::max_value; qed") - } - } - - fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]> { - let mut result: [u8; 32] = Default::default(); - let is_set = unsafe { - ext_storage_changes_root.get()(parent_hash.as_ptr(), parent_hash.len() as u32, result.as_mut_ptr()) - }; - - if is_set != 0 { - Some(result) - } else { - None - } - } - - - fn blake2_256_trie_root(_input: Vec<(Vec, Vec)>) -> H256 { - unimplemented!() - } - - fn blake2_256_ordered_trie_root(input: Vec>) -> H256 { - let mut values = Vec::with_capacity(input.len()); - let mut lengths = Vec::with_capacity(input.len()); - for v in input { - values.extend_from_slice(&v); - lengths.push((v.len() as u32).to_le()); - } - let mut result: [u8; 32] = Default::default(); - unsafe { - ext_blake2_256_enumerated_trie_root.get()( - values.as_ptr(), - lengths.as_ptr(), - lengths.len() as u32, - result.as_mut_ptr(), - ); - } - result.into() - } -} - -impl OtherApi for () { - fn chain_id() -> u64 { - unsafe { - ext_chain_id.get()() - } - } - - fn print_num(val: u64) { - unsafe { - ext_print_num.get()(val); - } - } - - fn print_utf8(utf8: &[u8]) { - unsafe { - ext_print_utf8.get()(utf8.as_ptr(), utf8.len() as u32); - } - } - - fn print_hex(data: &[u8]) { - unsafe { - ext_print_hex.get()(data.as_ptr(), data.len() as u32); - } - } -} - -impl HashingApi for () { - fn keccak_256(data: &[u8]) -> [u8; 32] { - let mut result: [u8; 32] = Default::default(); - unsafe { - ext_keccak_256.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } - - fn blake2_128(data: &[u8]) -> [u8; 16] { - let mut result: [u8; 16] = Default::default(); - unsafe { - ext_blake2_128.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } - - fn blake2_256(data: &[u8]) -> [u8; 32] { - let mut result: [u8; 32] = Default::default(); - unsafe { - ext_blake2_256.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } - - fn twox_256(data: &[u8]) -> [u8; 32] { - let mut result: [u8; 32] = Default::default(); - unsafe { - ext_twox_256.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } - - fn twox_128(data: &[u8]) -> [u8; 16] { - let mut result: [u8; 16] = Default::default(); - unsafe { - ext_twox_128.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } - - fn twox_64(data: &[u8]) -> [u8; 8] { - let mut result: [u8; 8] = Default::default(); - unsafe { - ext_twox_64.get()(data.as_ptr(), data.len() as u32, result.as_mut_ptr()); - } - result - } -} - -impl CryptoApi for () { - fn ed25519_public_keys(id: KeyTypeId) -> Vec { - let mut res_len = 0u32; - unsafe { - let res_ptr = ext_ed25519_public_keys.get()(id.0.as_ptr(), &mut res_len); - Vec::decode(&mut rstd::slice::from_raw_parts(res_ptr, res_len as usize)).unwrap_or_default() - } - } - - fn ed25519_generate(id: KeyTypeId, seed: Option<&str>) -> ed25519::Public { - let mut res = [0u8; 32]; - let seed = seed.as_ref().map(|s| s.as_bytes()).unwrap_or(&[]); - unsafe { - ext_ed25519_generate.get()(id.0.as_ptr(), seed.as_ptr(), seed.len() as u32, res.as_mut_ptr()) - }; - ed25519::Public(res) - } - - fn ed25519_sign( - id: KeyTypeId, - pubkey: &ed25519::Public, - msg: &[u8], - ) -> Option { - let mut res = [0u8; 64]; - let success = unsafe { - ext_ed25519_sign.get()( - id.0.as_ptr(), - pubkey.0.as_ptr(), - msg.as_ptr(), - msg.len() as u32, - res.as_mut_ptr(), - ) == 0 - }; - - if success { - Some(ed25519::Signature(res)) - } else { - None - } - } - - fn ed25519_verify(sig: &ed25519::Signature, msg: &[u8], pubkey: &ed25519::Public) -> bool { - unsafe { - ext_ed25519_verify.get()( - msg.as_ptr(), - msg.len() as u32, - sig.0.as_ptr(), - pubkey.0.as_ptr(), - ) == 0 - } - } - - fn sr25519_public_keys(id: KeyTypeId) -> Vec { - let mut res_len = 0u32; - unsafe { - let res_ptr = ext_sr25519_public_keys.get()(id.0.as_ptr(), &mut res_len); - Vec::decode(&mut rstd::slice::from_raw_parts(res_ptr, res_len as usize)).unwrap_or_default() - } - } - - fn sr25519_generate(id: KeyTypeId, seed: Option<&str>) -> sr25519::Public { - let mut res = [0u8;32]; - let seed = seed.as_ref().map(|s| s.as_bytes()).unwrap_or(&[]); - unsafe { - ext_sr25519_generate.get()(id.0.as_ptr(), seed.as_ptr(), seed.len() as u32, res.as_mut_ptr()) - }; - sr25519::Public(res) - } - - fn sr25519_sign( - id: KeyTypeId, - pubkey: &sr25519::Public, - msg: &[u8], - ) -> Option { - let mut res = [0u8; 64]; - let success = unsafe { - ext_sr25519_sign.get()( - id.0.as_ptr(), - pubkey.0.as_ptr(), - msg.as_ptr(), - msg.len() as u32, - res.as_mut_ptr(), - ) == 0 - }; - - if success { - Some(sr25519::Signature(res)) - } else { - None - } - } - - fn sr25519_verify(sig: &sr25519::Signature, msg: &[u8], pubkey: &sr25519::Public) -> bool { - unsafe { - ext_sr25519_verify.get()( - msg.as_ptr(), - msg.len() as u32, - sig.0.as_ptr(), - pubkey.0.as_ptr(), - ) == 0 - } - } - - fn secp256k1_ecdsa_recover(sig: &[u8; 65], msg: &[u8; 32]) -> Result<[u8; 64], EcdsaVerifyError> { - let mut pubkey = [0u8; 64]; - match unsafe { - ext_secp256k1_ecdsa_recover.get()(msg.as_ptr(), sig.as_ptr(), pubkey.as_mut_ptr()) - } { - 0 => Ok(pubkey), - 1 => Err(EcdsaVerifyError::BadRS), - 2 => Err(EcdsaVerifyError::BadV), - 3 => Err(EcdsaVerifyError::BadSignature), - _ => unreachable!("`ext_secp256k1_ecdsa_recover` only returns 0, 1, 2 or 3; qed"), - } - } -} - -impl OffchainApi for () { - fn is_validator() -> bool { - unsafe { ext_is_validator.get()() == 1 } - } - - fn submit_transaction(data: Vec) -> Result<(), ()> { - let ret = unsafe { - ext_submit_transaction.get()(data.as_ptr(), data.len() as u32) - }; - - if ret == 0 { - Ok(()) - } else { - Err(()) - } - } - - fn network_state() -> Result { - let mut len = 0_u32; - let raw_result = unsafe { - let ptr = ext_network_state.get()(&mut len); - - from_raw_parts(ptr, len) - }; - - match raw_result { - Some(raw_result) => codec::Decode::decode(&mut &*raw_result).unwrap_or(Err(())), - None => Err(()) - } - } - - fn timestamp() -> offchain::Timestamp { - offchain::Timestamp::from_unix_millis(unsafe { - ext_timestamp.get()() - }) - } - - fn sleep_until(deadline: offchain::Timestamp) { - unsafe { - ext_sleep_until.get()(deadline.unix_millis()) - } - } - - fn random_seed() -> [u8; 32] { - let mut result = [0_u8; 32]; - unsafe { - ext_random_seed.get()(result.as_mut_ptr()) - } - result - } - - fn local_storage_set(kind: offchain::StorageKind, key: &[u8], value: &[u8]) { - unsafe { - ext_local_storage_set.get()( - kind.into(), - key.as_ptr(), - key.len() as u32, - value.as_ptr(), - value.len() as u32, - ) - } - } - - fn local_storage_compare_and_set( - kind: offchain::StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool { - let (ptr, len) = match old_value { - Some(old_value) => ( - old_value.as_ptr(), - old_value.len() as u32, - ), - None => (0 as *const u8, u32::max_value()), - }; - - unsafe { - ext_local_storage_compare_and_set.get()( - kind.into(), - key.as_ptr(), - key.len() as u32, - ptr, - len, - new_value.as_ptr(), - new_value.len() as u32, - ) == 0 - } - } - - fn local_storage_get(kind: offchain::StorageKind, key: &[u8]) -> Option> { - let mut len = 0u32; - unsafe { - let ptr = ext_local_storage_get.get()( - kind.into(), - key.as_ptr(), - key.len() as u32, - &mut len, - ); - - from_raw_parts(ptr, len) - } - } - - fn http_request_start(method: &str, url: &str, meta: &[u8]) -> Result { - let method = method.as_bytes(); - let url = url.as_bytes(); - - let result = unsafe { - ext_http_request_start.get()( - method.as_ptr(), - method.len() as u32, - url.as_ptr(), - url.len() as u32, - meta.as_ptr(), - meta.len() as u32, - ) - }; - - if result > u16::max_value() as u32 { - Err(()) - } else { - Ok(offchain::HttpRequestId(result as u16)) - } - } - - fn http_request_add_header(request_id: offchain::HttpRequestId, name: &str, value: &str) -> Result<(), ()> { - let name = name.as_bytes(); - let value = value.as_bytes(); - - let result = unsafe { - ext_http_request_add_header.get()( - request_id.into(), - name.as_ptr(), - name.len() as u32, - value.as_ptr(), - value.len() as u32, - ) - }; - - if result == 0 { - Ok(()) - } else { - Err(()) - } - } - - fn http_request_write_body( - request_id: offchain::HttpRequestId, - chunk: &[u8], - deadline: Option - ) -> Result<(), offchain::HttpError> { - let res = unsafe { - ext_http_request_write_body.get()( - request_id.into(), - chunk.as_ptr(), - chunk.len() as u32, - deadline.map_or(0, |x| x.unix_millis()), - ) - }; - - if res == 0 { - Ok(()) - } else { - Err(res.try_into().unwrap_or(offchain::HttpError::IoError)) - } - } - - fn http_response_wait( - ids: &[offchain::HttpRequestId], - deadline: Option - ) -> Vec { - let ids = ids.iter().map(|x| x.0 as u32).collect::>(); - let mut statuses = Vec::new(); - statuses.resize(ids.len(), 0u32); - - unsafe { - ext_http_response_wait.get()( - ids.as_ptr(), - ids.len() as u32, - statuses.as_mut_ptr(), - deadline.map_or(0, |x| x.unix_millis()), - ) - } - - statuses - .into_iter() - .map(|status| status.try_into().unwrap_or(offchain::HttpRequestStatus::Invalid)) - .collect() - } - - fn http_response_headers( - request_id: offchain::HttpRequestId, - ) -> Vec<(Vec, Vec)> { - let mut len = 0u32; - let raw_result = unsafe { - let ptr = ext_http_response_headers.get()( - request_id.into(), - &mut len, - ); - - from_raw_parts(ptr, len).expect("ext_http_response_headers never return u32::max_value; qed") - }; - - codec::Decode::decode(&mut &*raw_result).unwrap_or_default() - } - - fn http_response_read_body( - request_id: offchain::HttpRequestId, - buffer: &mut [u8], - deadline: Option, - ) -> Result { - let res = unsafe { - ext_http_response_read_body.get()( - request_id.into(), - buffer.as_mut_ptr(), - buffer.len() as u32, - deadline.map_or(0, |x| x.unix_millis()), - ) - }; - - if res >= u32::max_value() - 255 { - let code = (u32::max_value() - res) + 1; - code.try_into().map_err(|_| offchain::HttpError::IoError) - } else { - Ok(res as usize) - } - } -} - -unsafe fn from_raw_parts(ptr: *mut u8, len: u32) -> Option> { - if len == u32::max_value() { - None - } else { - // Invariants required by Vec::from_raw_parts are not formally fulfilled. - // We don't allocate via String/Vec, but use a custom allocator instead. - // See #300 for more details. - Some(>::from_raw_parts(ptr, len as usize, len as usize)) - } -} - -impl Api for () {} From 87eb710f1179674e4a2986fe91c2ee2f4d8bbfe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 25 Oct 2019 07:28:28 +0200 Subject: [PATCH 35/76] More compilation fixes --- Cargo.lock | 1 + core/executor/src/host_interface.rs | 2 +- core/executor/src/lib.rs | 4 ++-- core/executor/src/native_executor.rs | 8 +++++-- core/executor/src/wasm_runtime.rs | 24 +++++++++++++-------- core/executor/src/wasmi_execution.rs | 31 ++++++++++++++-------------- core/runtime-interface/Cargo.toml | 1 + core/runtime-interface/src/lib.rs | 21 +++++++++---------- core/sr-io/src/lib.rs | 12 +++++++++++ 9 files changed, 63 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68506bf140123..c303245bd6edd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5523,6 +5523,7 @@ dependencies = [ "static_assertions 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-executor 2.0.0", "substrate-externalities 2.0.0", + "substrate-primitives 2.0.0", "substrate-runtime-interface-proc-macro 2.0.0", "substrate-runtime-interface-test-wasm 2.0.0", "substrate-state-machine 2.0.0", diff --git a/core/executor/src/host_interface.rs b/core/executor/src/host_interface.rs index e4ec319e18f96..7e0e98f606a90 100644 --- a/core/executor/src/host_interface.rs +++ b/core/executor/src/host_interface.rs @@ -1056,7 +1056,7 @@ impl_wasm_host_interface! { Ok(match res { Ok(read) => { - context.write_memory(buffer, &internal_buffer[..read]) + context.write_memory(buffer, &internal_buffer[..read as usize]) .map_err(|_| "Invalid attempt to set memory in ext_http_response_read_body")?; read as u32 diff --git a/core/executor/src/lib.rs b/core/executor/src/lib.rs index ac98388cd7bbe..b94e0ff086f93 100644 --- a/core/executor/src/lib.rs +++ b/core/executor/src/lib.rs @@ -60,7 +60,7 @@ pub use wasm_runtime::WasmExecutionMethod; /// - `heap_pages`: The number of heap pages to allocate. /// /// Returns the `Vec` that contains the return value of the function. -pub fn call_in_wasm( +pub fn call_in_wasm( function: &str, call_data: &[u8], execution_method: WasmExecutionMethod, @@ -68,7 +68,7 @@ pub fn call_in_wasm( code: &[u8], heap_pages: u64, ) -> error::Result> { - let mut instance = wasm_runtime::create_wasm_runtime_with_code( + let mut instance = wasm_runtime::create_wasm_runtime_with_code::<_, HF>( ext, execution_method, heap_pages, diff --git a/core/executor/src/native_executor.rs b/core/executor/src/native_executor.rs index 7323703ed9656..f525a61d45e6a 100644 --- a/core/executor/src/native_executor.rs +++ b/core/executor/src/native_executor.rs @@ -65,7 +65,7 @@ pub trait NativeExecutionDispatch: Send + Sync { #[derive(Debug)] pub struct NativeExecutor { /// Dummy field to avoid the compiler complaining about us not using `D`. - _dummy: ::std::marker::PhantomData, + _dummy: std::marker::PhantomData, /// Method used to execute fallback Wasm code. fallback_method: WasmExecutionMethod, /// Native runtime version info. @@ -99,7 +99,11 @@ impl NativeExecutor { ) -> Result where E: Externalities { RUNTIMES_CACHE.with(|cache| { let mut cache = cache.borrow_mut(); - let runtime = cache.fetch_runtime(ext, self.fallback_method, self.default_heap_pages)?; + let runtime = cache.fetch_runtime::<_, runtime_io::SubstrateHostFunctions>( + ext, + self.fallback_method, + self.default_heap_pages, + )?; f(runtime, ext) }) } diff --git a/core/executor/src/wasm_runtime.rs b/core/executor/src/wasm_runtime.rs index 8d2291fe04893..4e95557f77b8f 100644 --- a/core/executor/src/wasm_runtime.rs +++ b/core/executor/src/wasm_runtime.rs @@ -19,14 +19,20 @@ //! The primary means of accessing the runtimes is through a cache which saves the reusable //! components of the runtime that are expensive to initialize. -use crate::error::{Error, WasmError}; -use crate::wasmi_execution; +use crate::{wasmi_execution, error::{Error, WasmError}}; + use log::{trace, warn}; + use codec::Decode; + use primitives::{storage::well_known_keys, traits::Externalities}; + use runtime_version::RuntimeVersion; + use std::{collections::hash_map::{Entry, HashMap}}; +use wasm_interface::HostFunctions; + /// The Substrate Wasm runtime. pub trait WasmRuntime { /// Attempt to update the number of heap pages available during execution. @@ -109,7 +115,7 @@ impl RuntimesCache { /// /// `Error::InvalidMemoryReference` is returned if no memory export with the /// identifier `memory` can be found in the runtime. - pub fn fetch_runtime( + pub fn fetch_runtime( &mut self, ext: &mut E, wasm_method: WasmExecutionMethod, @@ -133,7 +139,7 @@ impl RuntimesCache { target: "runtimes_cache", "heap_pages were changed. Reinstantiating the instance" ); - *result = create_wasm_runtime(ext, wasm_method, heap_pages); + *result = create_wasm_runtime::<_, HF>(ext, wasm_method, heap_pages); if let Err(ref err) = result { warn!(target: "runtimes_cache", "cannot create a runtime: {:?}", err); } @@ -143,7 +149,7 @@ impl RuntimesCache { }, Entry::Vacant(v) => { trace!(target: "runtimes_cache", "no instance found in cache, creating now."); - let result = create_wasm_runtime(ext, wasm_method, heap_pages); + let result = create_wasm_runtime::<_, HF>(ext, wasm_method, heap_pages); if let Err(ref err) = result { warn!(target: "runtimes_cache", "cannot create a runtime: {:?}", err); } @@ -158,7 +164,7 @@ impl RuntimesCache { } /// Create a wasm runtime with the given `code`. -pub fn create_wasm_runtime_with_code( +pub fn create_wasm_runtime_with_code( ext: &mut E, wasm_method: WasmExecutionMethod, heap_pages: u64, @@ -166,12 +172,12 @@ pub fn create_wasm_runtime_with_code( ) -> Result, WasmError> { match wasm_method { WasmExecutionMethod::Interpreted => - wasmi_execution::create_instance(ext, code, heap_pages) + wasmi_execution::create_instance::<_, HF>(ext, code, heap_pages) .map(|runtime| -> Box { Box::new(runtime) }), } } -fn create_wasm_runtime( +fn create_wasm_runtime( ext: &mut E, wasm_method: WasmExecutionMethod, heap_pages: u64, @@ -179,5 +185,5 @@ fn create_wasm_runtime( let code = ext .original_storage(well_known_keys::CODE) .ok_or(WasmError::CodeNotFound)?; - create_wasm_runtime_with_code(ext, wasm_method, heap_pages, &code) + create_wasm_runtime_with_code::<_, HF>(ext, wasm_method, heap_pages, &code) } diff --git a/core/executor/src/wasmi_execution.rs b/core/executor/src/wasmi_execution.rs index ca90726e726eb..9a9c63fdcd669 100644 --- a/core/executor/src/wasmi_execution.rs +++ b/core/executor/src/wasmi_execution.rs @@ -313,7 +313,7 @@ impl wasmi::Externals for FunctionExecutor { -> Result, wasmi::Trap> { let mut args = args.as_ref().iter().copied().map(Into::into); - let function = SubstrateExternals::functions().get(index).ok_or_else(|| + let function = SubstrateExternals::get_function(index).ok_or_else(|| Error::from( format!("Could not find host function with index: {}", index), ) @@ -352,13 +352,13 @@ fn get_heap_base(module: &ModuleRef) -> Result { } /// Call a given method in the given wasm-module runtime. -fn call_in_wasm_module( +fn call_in_wasm_module( ext: &mut dyn Externalities, module_instance: &ModuleRef, method: &str, data: &[u8], ) -> Result, Error> { - call_in_wasm_module_with_custom_signature( + call_in_wasm_module_with_custom_signature::( ext, module_instance, method, @@ -380,6 +380,7 @@ fn call_in_wasm_module( /// Call a given method in the given wasm-module runtime. fn call_in_wasm_module_with_custom_signature< + HF: HostFunctions, F: FnOnce(&mut dyn FnMut(&[u8]) -> Result) -> Result, Error>, FR: FnOnce(Option, &MemoryRef) -> Result, Error>, R, @@ -398,11 +399,7 @@ fn call_in_wasm_module_with_custom_signature< .and_then(|e| e.as_table().cloned()); let heap_base = get_heap_base(module_instance)?; - let mut fec = FunctionExecutor::new( - memory.clone(), - heap_base, - table, - )?; + let mut fec = FunctionExecutor::::new(memory.clone(), heap_base, table)?; let parameters = create_parameters(&mut |data: &[u8]| { let offset = fec.allocate_memory(data.len() as u32)?; @@ -431,7 +428,7 @@ fn call_in_wasm_module_with_custom_signature< } /// Prepare module instance -fn instantiate_module( +fn instantiate_module( heap_pages: usize, module: &Module, ) -> Result { @@ -439,7 +436,7 @@ fn instantiate_module( let intermediate_instance = ModuleInstance::new( module, &ImportsBuilder::new() - .with_resolver("env", FunctionExecutor::resolver()) + .with_resolver("env", FunctionExecutor::::resolver()) )?; // Verify that the module has the heap base global variable. @@ -603,7 +600,7 @@ impl WasmRuntime for WasmiRuntime { -> Result, Error> { self.with(|module| { - call_in_wasm_module(ext, module, method, data) + call_in_wasm_module::(ext, module, method, data) }) } @@ -612,9 +609,11 @@ impl WasmRuntime for WasmiRuntime { } } -pub fn create_instance(ext: &mut E, code: &[u8], heap_pages: u64) - -> Result -{ +pub fn create_instance( + ext: &mut E, + code: &[u8], + heap_pages: u64, +) -> Result { let module = Module::from_buffer(&code).map_err(|_| WasmError::InvalidModule)?; // Extract the data segments from the wasm code. @@ -624,7 +623,7 @@ pub fn create_instance(ext: &mut E, code: &[u8], heap_pages: u let data_segments = extract_data_segments(&code)?; // Instantiate this module. - let instance = instantiate_module(heap_pages as usize, &module) + let instance = instantiate_module::(heap_pages as usize, &module) .map_err(WasmError::Instantiation)?; // Take state snapshot before executing anything. @@ -636,7 +635,7 @@ pub fn create_instance(ext: &mut E, code: &[u8], heap_pages: u ", ); - let version = call_in_wasm_module(ext, &instance, "Core_version", &[]) + let version = call_in_wasm_module::(ext, &instance, "Core_version", &[]) .ok() .and_then(|v| RuntimeVersion::decode(&mut v.as_slice()).ok()); Ok(WasmiRuntime { diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index 86edd6c5d0df0..3e9bb6e6b53d9 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -18,6 +18,7 @@ primitive-types = { version = "0.5.1", default-features = false } executor = { package = "substrate-executor", path = "../executor" } test-wasm = { package = "substrate-runtime-interface-test-wasm", path = "test-wasm" } state_machine = { package = "substrate-state-machine", path = "../state-machine" } +primitives = { package = "substrate-primitives", path = "../primitives" } [features] default = [ "std" ] diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index d159aee95a7d4..1280a0361fe6c 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -65,22 +65,21 @@ pub trait RIType { mod tests { use super::*; use test_wasm::{WASM_BINARY, test_api::HostFunctions}; - use executor::WasmExecutor; use wasm_interface::HostFunctions as HostFunctionsT; - type TestExternalities = state_machine::TestExternalities; + type TestExternalities = state_machine::TestExternalities; fn call_wasm_method(method: &str) -> TestExternalities { let mut ext = TestExternalities::default(); - let executor = WasmExecutor::::new(); + let mut ext_ext = ext.ext(); - executor.call_with_custom_signature::<_, _, _, ()>( - &mut ext, - 8, - &WASM_BINARY[..], + executor::call_in_wasm::<_, HF>( method, - |_| Ok(Vec::new()), - |res, _| if res.is_none() { Ok(Some(())) } else { Err("Invalid return value!".into()) }, + &[], + executor::WasmExecutionMethod::Interpreted, + &mut ext_ext, + &WASM_BINARY[..], + 8, ).expect(&format!("Executes `{}`", method)); ext @@ -98,10 +97,10 @@ mod tests { #[test] fn test_set_storage() { - let ext = call_wasm_method::("test_set_storage"); + let mut ext = call_wasm_method::("test_set_storage"); let expected = "world"; - assert_eq!(expected.as_bytes(), &ext.storage("hello".as_bytes()).unwrap()[..]); + assert_eq!(expected.as_bytes(), &ext.ext().storage("hello".as_bytes()).unwrap()[..]); } #[test] diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 8ded1cad23cd4..e9ba9c170c1c8 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -591,3 +591,15 @@ pub use self::imp::{StorageOverlay, ChildrenStorageOverlay, with_storage}; /// Type alias for Externalities implementation used in tests. #[cfg(feature = "std")] pub type TestExternalities = self::imp::TestExternalities; + +/// The host functions Substrate provides for the Wasm runtime environment. +/// +/// All these host functions will be callable from inside the Wasm environment. +#[cfg(feature = "std")] +pub type SubstrateHostFunctions = ( + storage::HostFunctions, + misc::HostFunctions, + offchain::HostFunctions, + crypto::HostFunctions, + hashing::HostFunctions, +); From cd7b9a9da646680c16fa717661ed0b3a019f94e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 25 Oct 2019 07:31:22 +0200 Subject: [PATCH 36/76] Fix warnings --- core/executor/src/host_interface.rs | 5 +---- core/runtime-interface/src/impls.rs | 4 ++-- core/runtime-interface/src/pass_by.rs | 4 ++-- core/sr-io/with_std.rs | 9 --------- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/core/executor/src/host_interface.rs b/core/executor/src/host_interface.rs index 7e0e98f606a90..af5ca094458ff 100644 --- a/core/executor/src/host_interface.rs +++ b/core/executor/src/host_interface.rs @@ -27,10 +27,7 @@ use primitives::{ crypto::KeyTypeId, offchain, }; use trie::{TrieConfiguration, trie_types::Layout}; -use wasm_interface::{ - FunctionContext, Pointer, PointerType, Result as WResult, WordSize, - WritePrimitive, ReadPrimitive, -}; +use wasm_interface::{Pointer, WordSize, WritePrimitive, ReadPrimitive}; #[cfg(feature="wasm-extern-trace")] macro_rules! debug_trace { diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index 241bfee18c0e1..c950e57de043e 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -30,13 +30,13 @@ use wasm_interface::{FunctionContext, Pointer, Result}; use codec::{Encode, Decode}; -use rstd::{any::TypeId, mem, vec::Vec, boxed::Box}; +use rstd::{any::TypeId, mem, vec::Vec}; #[cfg(feature = "std")] use rstd::borrow::Cow; #[cfg(not(feature = "std"))] -use rstd::slice; +use rstd::{slice, boxed::Box}; // Make sure that our assumptions for storing a pointer + its size in `u64` is valid. #[cfg(not(feature = "std"))] diff --git a/core/runtime-interface/src/pass_by.rs b/core/runtime-interface/src/pass_by.rs index 10a58418860a9..53ea6e6bba3bd 100644 --- a/core/runtime-interface/src/pass_by.rs +++ b/core/runtime-interface/src/pass_by.rs @@ -27,10 +27,10 @@ use crate::wasm::*; #[cfg(feature = "std")] use wasm_interface::{FunctionContext, Pointer, Result}; -use rstd::{marker::PhantomData, vec::Vec}; +use rstd::{marker::PhantomData}; #[cfg(not(feature = "std"))] -use rstd::slice; +use rstd::{slice, vec::Vec}; pub use substrate_runtime_interface_proc_macro::{PassByCodec, PassByInner}; diff --git a/core/sr-io/with_std.rs b/core/sr-io/with_std.rs index 5573c15424ab8..2edf438a0d088 100644 --- a/core/sr-io/with_std.rs +++ b/core/sr-io/with_std.rs @@ -14,11 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use primitives::{ - blake2_128, blake2_256, twox_128, twox_256, twox_64, ed25519, Blake2Hasher, sr25519, Pair, H256, - traits::KeystoreExt, storage::ChildStorageKey, hexdisplay::HexDisplay, Hasher, - offchain::{self, OffchainExt}, -}; // Switch to this after PoC-3 // pub use primitives::BlakeHasher; pub use substrate_state_machine::{BasicExternalities, TestExternalities}; @@ -29,10 +24,6 @@ use std::{collections::HashMap, convert::TryFrom}; use externalities::Externalities; -/// Additional bounds for `Hasher` trait for with_std. -pub trait HasherBounds {} -impl HasherBounds for T {} - /// Execute the given closure with global function available whose functionality routes into the /// externalities `ext`. Forwards the value that the closure returns. // NOTE: need a concrete hasher here due to limitations of the `environmental!` macro, otherwise a type param would have been fine I think. From a2f9d31c0706962727836589eff1cb6662212d73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 25 Oct 2019 07:34:51 +0200 Subject: [PATCH 37/76] Remove le conversions --- core/runtime-interface/src/impls.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index c950e57de043e..eabae82246e20 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -46,14 +46,13 @@ assert_eq_size!(*const u8, u32); /// Converts a pointer and length into an `u64`. pub fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { - ((len as u64) | u64::from(ptr) << 32).to_le() + (u64::from(len) << 32) | u64::from(ptr) } /// Splits an `u64` into the pointer and length. pub fn pointer_and_len_from_u64(val: u64) -> (u32, u32) { - let val = u64::from_le(val); - let len = (val & (!0u32 as u64)) as u32; - let ptr = (val >> 32) as u32; + let ptr = (val & (!0u32 as u64)) as u32; + let len = (val >> 32) as u32; (ptr, len) } @@ -75,14 +74,14 @@ macro_rules! impl_traits_for_primitives { type Owned = (); fn into_ffi_value(&self) -> WrappedFFIValue<$fty> { - (*self as $fty).to_le().into() + (*self as $fty).into() } } #[cfg(not(feature = "std"))] impl FromFFIValue for $rty { fn from_ffi_value(arg: $fty) -> $rty { - <$fty>::from_le(arg) as $rty + arg as $rty } } @@ -91,14 +90,14 @@ macro_rules! impl_traits_for_primitives { type SelfInstance = $rty; fn from_ffi_value(_: &mut dyn FunctionContext, arg: $fty) -> Result<$rty> { - Ok(<$fty>::from_le(arg) as $rty) + Ok(arg as $rty) } } #[cfg(feature = "std")] impl IntoFFIValue for $rty { fn into_ffi_value(self, _: &mut dyn FunctionContext) -> Result<$fty> { - Ok((self as $fty).to_le()) + Ok(self as $fty) } } )* From 2200ca11f22a3a18a79c338f4d4a334a92fd5dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 25 Oct 2019 14:34:14 +0200 Subject: [PATCH 38/76] Add support for `wasm_only` interfaces --- .../proc-macro/src/bare_function_interface.rs | 134 ++++++++++++------ .../proc-macro/src/host_function_interface.rs | 47 +++--- core/runtime-interface/proc-macro/src/lib.rs | 20 ++- .../proc-macro/src/trait_decl_impl.rs | 14 +- 4 files changed, 140 insertions(+), 75 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index 17600b128cd20..7eeaa8d56d3b2 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -34,50 +34,46 @@ use crate::utils::{ use syn::{ Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Result, ReturnType, spanned::Spanned, + parse_quote, }; use proc_macro2::{TokenStream, Span}; use quote::{quote, quote_spanned}; +use std::iter; + /// Generate one bare function per trait method. The name of the bare function is equal to the name /// of the trait method. -pub fn generate(trait_def: &ItemTrait) -> Result { +pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result { let trait_name = &trait_def.ident; get_trait_methods(trait_def).try_fold(TokenStream::new(), |mut t, m| { - t.extend(function_for_method(trait_name, m)?); + t.extend(function_for_method(trait_name, m, is_wasm_only)?); Ok(t) }) } -/// Generates the bare function implementation for the given method. -fn function_for_method(trait_name: &Ident, method: &TraitItemMethod) -> Result { - let std_impl = function_std_impl(trait_name, method)?; +/// Generates the bare function implementation for the given method for the host and wasm side. +fn function_for_method( + trait_name: &Ident, + method: &TraitItemMethod, + is_wasm_only: bool, +) -> Result { + let std_impl = function_std_impl(trait_name, method, is_wasm_only)?; let no_std_impl = function_no_std_impl(trait_name, method)?; - let function_impl_name = create_function_impl_ident(&method.sig.ident); - let function_name = &method.sig.ident; - let args = get_function_arguments(&method.sig); - let arg_names = get_function_argument_names(&method.sig); - let return_value = &method.sig.output; - let attrs = &method.attrs; Ok( quote! { - #( #attrs )* - pub fn #function_name( #( #args, )* ) #return_value { - #std_impl - - #no_std_impl + #std_impl - #function_impl_name( #( #arg_names, )* ) - } + #no_std_impl } ) } /// Generates the bare function implementation for `cfg(not(feature = "std"))`. fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result { - let function_name = create_function_impl_ident(&method.sig.ident); + let function_name = &method.sig.ident; let host_function_name = create_host_function_ident(&method.sig.ident, trait_name); let args = get_function_arguments(&method.sig); let arg_names = get_function_argument_names(&method.sig); @@ -95,8 +91,8 @@ fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result< Ok( quote! { #[cfg(not(feature = "std"))] - fn #function_name( #( #args, )* ) #return_value { - // Generate all wrapped ffi value. + pub fn #function_name( #( #args, )* ) #return_value { + // Generate all wrapped ffi values. #( let #arg_names = <#arg_types as #crate_::wasm::IntoFFIValue>::into_ffi_value( &#arg_names, @@ -113,45 +109,89 @@ fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result< } /// Generates the bare function implementation for `cfg(feature = "std")`. -fn function_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result { - let method_name = &method.sig.ident; - let function_name = create_function_impl_ident(&method.sig.ident); - let args = get_function_arguments(&method.sig); - let arg_names = get_function_argument_names(&method.sig); +fn function_std_impl( + trait_name: &Ident, + method: &TraitItemMethod, + is_wasm_only: bool, +) -> Result { + let function_name = &method.sig.ident; let crate_ = generate_crate_access(); + let args = get_function_arguments(&method.sig).cloned().map(FnArg::Typed).chain( + // Add the function context as last parameter when this is a wasm only interface. + iter::from_fn(|| if is_wasm_only { + Some( + parse_quote!( + __function_context__: &mut dyn #crate_::wasm_interface::FunctionContext + ) + ) + } else { + None + } + ).take(1), + ); let return_value = &method.sig.output; + let attrs = &method.attrs; + // Don't make the function public accessible when this is a wasm only interface. + let vis = if is_wasm_only { quote!() } else { quote!(pub) }; + let call_to_trait = generate_call_to_trait(trait_name, method, is_wasm_only); + + Ok( + quote_spanned! { method.span() => + #[cfg(feature = "std")] + #( #attrs )* + #vis fn #function_name( #( #args, )* ) #return_value { + #call_to_trait + } + } + ) +} + +/// Generate the call to the interface trait. +fn generate_call_to_trait( + trait_name: &Ident, + method: &TraitItemMethod, + is_wasm_only: bool, +) -> TokenStream { + let crate_ = generate_crate_access(); + let method_name = &method.sig.ident; let expect_msg = format!( "`{}` called outside of an Externalities-provided environment.", method_name, ); + let arg_names = get_function_argument_names(&method.sig); - let call_to_trait = if takes_self_argument(&method.sig) { - quote_spanned! { method.span() => - #crate_::with_externalities( - |mut ext| #trait_name::#method_name(&mut ext, #( #arg_names, )*) - ).expect(#expect_msg) + if takes_self_argument(&method.sig) { + let instance = if is_wasm_only { + Ident::new("__function_context__", Span::call_site()) + } else { + Ident::new("__externalities__", Span::call_site()) + }; + + let impl_ = quote!( #trait_name::#method_name(&mut #instance, #( #arg_names, )*) ); + + if is_wasm_only { + quote_spanned! { method.span() => #impl_ } + } else { + quote_spanned! { method.span() => + #crate_::with_externalities( + |mut #instance| #impl_ + ).expect(#expect_msg) + } } } else { + // The name of the trait the interface trait is implemented for + let impl_trait_name = if is_wasm_only { + quote!( #crate_::wasm_interface::FunctionContext ) + } else { + quote!( #crate_::Externalities ) + }; + quote_spanned! { method.span() => - <&mut dyn #crate_::Externalities as #trait_name>::#method_name( + <&mut dyn #impl_trait_name as #trait_name>::#method_name( #( #arg_names, )* ) } - }; - - Ok( - quote_spanned! { method.span() => - #[cfg(feature = "std")] - fn #function_name( #( #args, )* ) #return_value { - #call_to_trait - } - } - ) -} - -/// Create the function identifier for the internal implementation function. -fn create_function_impl_ident(method_name: &Ident) -> Ident { - Ident::new(&format!("{}_impl", method_name), Span::call_site()) + } } /// Returns if the given `Signature` takes a `self` argument. diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index fc0e8967d35be..2e078e9dc562d 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -14,8 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Generates the extern host function declarations as well as the implementation for these host -//! functions. The implementation of these host functions will call the native bare functions. +//! Generates the extern host functions and the implementation for these host functions. +//! +//! The extern host functions will be called by the bare function interface from the Wasm side. +//! The implementation of these host functions will be called on the host side from the Wasm +//! executor. These implementations call the bare function interface. use crate::utils::{ generate_crate_access, create_host_function_ident, get_function_argument_names, @@ -34,11 +37,11 @@ use quote::{quote, ToTokens}; use inflector::Inflector; -use std::iter::Iterator; +use std::iter::{Iterator, self}; /// Generate the extern host functions for wasm and the `HostFunctions` struct that provides the /// implementations for the host functions on the host. -pub fn generate(trait_def: &ItemTrait) -> Result { +pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result { let trait_name = &trait_def.ident; let extern_host_function_impls = get_trait_methods(trait_def) .try_fold(TokenStream::new(), |mut t, m| { @@ -50,7 +53,7 @@ pub fn generate(trait_def: &ItemTrait) -> Result { t.extend(generate_extern_host_exchangeable_function(m, trait_name)?); Ok::<_, Error>(t) })?; - let host_functions_struct = generate_host_functions_struct(trait_def)?; + let host_functions_struct = generate_host_functions_struct(trait_def, is_wasm_only)?; Ok( quote! { @@ -143,7 +146,7 @@ fn generate_extern_host_exchangeable_function( /// Generate the `HostFunctions` struct that implements `wasm-interface::HostFunctions` to provide /// implementations for the extern host functions. -fn generate_host_functions_struct(trait_def: &ItemTrait) -> Result { +fn generate_host_functions_struct(trait_def: &ItemTrait, is_wasm_only: bool) -> Result { let crate_ = generate_crate_access(); let host_functions = trait_def .items @@ -152,7 +155,7 @@ fn generate_host_functions_struct(trait_def: &ItemTrait) -> Result TraitItem::Method(ref method) => Some(method), _ => None, }) - .map(|m| generate_host_function_implementation(&trait_def.ident, m)) + .map(|m| generate_host_function_implementation(&trait_def.ident, m, is_wasm_only)) .collect::>>()?; let host_functions_count = trait_def .items @@ -191,6 +194,7 @@ fn generate_host_functions_struct(trait_def: &ItemTrait) -> Result fn generate_host_function_implementation( trait_name: &Ident, method: &TraitItemMethod, + is_wasm_only: bool, ) -> Result { let name = create_host_function_ident(&method.sig.ident, trait_name).to_string(); let struct_name = Ident::new(&name.to_pascal_case(), Span::call_site()); @@ -198,7 +202,7 @@ fn generate_host_function_implementation( let signature = generate_wasm_interface_signature_for_host_function(&method.sig)?; let wasm_to_ffi_values = generate_wasm_to_ffi_values(&method.sig).collect::>>()?; let ffi_to_host_values = generate_ffi_to_host_value(&method.sig).collect::>>()?; - let host_function_call = generate_host_function_call(&method.sig); + let host_function_call = generate_host_function_call(&method.sig, is_wasm_only); let into_preallocated_ffi_value = generate_into_preallocated_ffi_value(&method.sig)?; let convert_return_value = generate_return_value_into_wasm_value(&method.sig); @@ -219,7 +223,7 @@ fn generate_host_function_implementation( fn execute( &self, - context: &mut dyn #crate_::wasm_interface::FunctionContext, + __function_context__: &mut dyn #crate_::wasm_interface::FunctionContext, args: &mut dyn Iterator, ) -> std::result::Result, String> { #( #wasm_to_ffi_values )* @@ -307,7 +311,7 @@ fn generate_ffi_to_host_value<'a>( Ok( quote! { let #mut_access #name = <#ty as #crate_::host::FromFFIValue>::from_ffi_value( - context, + __function_context__, #ffi_value_var_name, )?; } @@ -316,7 +320,7 @@ fn generate_ffi_to_host_value<'a>( } /// Generate the code to call the host function and the ident that stores the result. -fn generate_host_function_call(sig: &Signature) -> TokenStream { +fn generate_host_function_call(sig: &Signature, is_wasm_only: bool) -> TokenStream { let host_function_name = &sig.ident; let result_var_name = generate_host_function_result_var_name(&sig.ident); let ref_and_mut = get_function_argument_types_ref_and_mut(sig).map(|ram| @@ -324,9 +328,15 @@ fn generate_host_function_call(sig: &Signature) -> TokenStream { ); let names = get_function_argument_names(sig); - let var_access = names.zip(ref_and_mut).map(|(n, ref_and_mut)| { - quote!( #ref_and_mut #n ) - }); + let var_access = names.zip(ref_and_mut) + .map(|(n, ref_and_mut)| { + quote!( #ref_and_mut #n ) + }) + // If this is a wasm only interface, we add the function context as last parameter. + .chain( + iter::from_fn(|| if is_wasm_only { Some(quote!(__function_context__)) } else { None }) + .take(1) + ); quote! { let #result_var_name = #host_function_name ( #( #var_access ),* ); @@ -374,7 +384,7 @@ fn generate_into_preallocated_ffi_value(sig: &Signature) -> Result quote! { <#ty as #crate_::host::IntoPreallocatedFFIValue>::into_preallocated_ffi_value( #name, - context, + __function_context__, #ffi_var_name, )?; } @@ -393,9 +403,10 @@ fn generate_return_value_into_wasm_value(sig: &Signature) -> TokenStream { let result_var_name = generate_host_function_result_var_name(&sig.ident); quote! { - <#ty as #crate_::host::IntoFFIValue>::into_ffi_value(#result_var_name, context) - .map(#crate_::wasm_interface::IntoValue::into_value) - .map(Some) + <#ty as #crate_::host::IntoFFIValue>::into_ffi_value( + #result_var_name, + __function_context__, + ).map(#crate_::wasm_interface::IntoValue::into_value).map(Some) } } } diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index e51c3674795f1..6f37f5f71f534 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -35,22 +35,30 @@ mod pass_by_inner; mod trait_decl_impl; mod utils; +mod kw { + // Custom keyword `wasm_only` that can be given as attribute to [`runtime_interface`]. + syn::custom_keyword!(wasm_only); +} + #[proc_macro_attribute] pub fn runtime_interface( - _: proc_macro::TokenStream, + attrs: proc_macro::TokenStream, input: proc_macro::TokenStream, ) -> proc_macro::TokenStream { let trait_def = parse_macro_input!(input as ItemTrait); + let wasm_only = parse_macro_input!(attrs as Option); - runtime_interface_impl(trait_def).unwrap_or_else(|e| e.to_compile_error()).into() + runtime_interface_impl(trait_def, wasm_only.is_some()) + .unwrap_or_else(|e| e.to_compile_error()) + .into() } -fn runtime_interface_impl(trait_def: ItemTrait) -> Result { - let bare_functions = bare_function_interface::generate(&trait_def)?; +fn runtime_interface_impl(trait_def: ItemTrait, is_wasm_only: bool) -> Result { + let bare_functions = bare_function_interface::generate(&trait_def, is_wasm_only)?; let crate_include = generate_runtime_interface_include(); let mod_name = Ident::new(&trait_def.ident.to_string().to_snake_case(), Span::call_site()); - let trait_decl_impl = trait_decl_impl::process(&trait_def)?; - let host_functions = host_function_interface::generate(&trait_def)?; + let trait_decl_impl = trait_decl_impl::process(&trait_def, is_wasm_only)?; + let host_functions = host_function_interface::generate(&trait_def, is_wasm_only)?; let vis = trait_def.vis; let attrs = &trait_def.attrs; diff --git a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs index cc69daa517fb1..5098904ec55ed 100644 --- a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs +++ b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs @@ -30,8 +30,8 @@ use quote::quote; /// Process the given trait definition, by checking that the definition is valid, fold it to the /// essential definition and implement this essential definition for `dyn Externalities`. -pub fn process(trait_def: &ItemTrait) -> Result { - let impl_trait = impl_trait_for_externalities(trait_def)?; +pub fn process(trait_def: &ItemTrait, is_wasm_only: bool) -> Result { + let impl_trait = impl_trait_for_externalities(trait_def, is_wasm_only)?; let essential_trait_def = ToEssentialTraitDef::convert(trait_def.clone())?; Ok( @@ -112,7 +112,7 @@ impl Fold for ToEssentialTraitDef { } /// Implements the given trait definition for `dyn Externalities`. -fn impl_trait_for_externalities(trait_def: &ItemTrait) -> Result { +fn impl_trait_for_externalities(trait_def: &ItemTrait, is_wasm_only: bool) -> Result { let trait_ = &trait_def.ident; let crate_ = generate_crate_access(); let methods = trait_def @@ -123,10 +123,16 @@ fn impl_trait_for_externalities(trait_def: &ItemTrait) -> Result { _ => None, }); + let impl_type = if is_wasm_only { + quote!( &mut dyn #crate_::wasm_interface::FunctionContext ) + } else { + quote!( &mut dyn #crate_::Externalities ) + }; + Ok( quote! { #[cfg(feature = "std")] - impl #trait_ for &mut dyn #crate_::Externalities { + impl #trait_ for #impl_type { #( #methods )* } } From a91273020642fd54754410dd50a8fe7b8648cdd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 25 Oct 2019 21:03:13 +0200 Subject: [PATCH 39/76] Implement `Allocator` interface --- .../proc-macro/src/bare_function_interface.rs | 9 ++-- .../proc-macro/src/trait_decl_impl.rs | 6 +-- core/runtime-interface/src/impls.rs | 46 ++++++++++++++++++- core/runtime-interface/src/lib.rs | 6 +++ core/sr-io/src/lib.rs | 12 ++++- core/sr-std/with_std.rs | 1 + core/sr-std/without_std.rs | 1 - 7 files changed, 69 insertions(+), 12 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index 7eeaa8d56d3b2..53068ad53822f 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -118,10 +118,11 @@ fn function_std_impl( let crate_ = generate_crate_access(); let args = get_function_arguments(&method.sig).cloned().map(FnArg::Typed).chain( // Add the function context as last parameter when this is a wasm only interface. - iter::from_fn(|| if is_wasm_only { + iter::from_fn(|| + if is_wasm_only { Some( parse_quote!( - __function_context__: &mut dyn #crate_::wasm_interface::FunctionContext + mut __function_context__: &mut dyn #crate_::wasm_interface::FunctionContext ) ) } else { @@ -173,9 +174,7 @@ fn generate_call_to_trait( quote_spanned! { method.span() => #impl_ } } else { quote_spanned! { method.span() => - #crate_::with_externalities( - |mut #instance| #impl_ - ).expect(#expect_msg) + #crate_::with_externalities(|mut #instance| #impl_).expect(#expect_msg) } } } else { diff --git a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs index 5098904ec55ed..29f94c4556a88 100644 --- a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs +++ b/core/runtime-interface/proc-macro/src/trait_decl_impl.rs @@ -90,7 +90,7 @@ impl Fold for ToEssentialTraitDef { } ).for_each(|invalid| self.push_error(invalid, "`impl Trait` syntax not supported.")); - method + fold::fold_trait_item_method(self, method) } fn fold_item_trait(&mut self, mut trait_def: ItemTrait) -> ItemTrait { @@ -99,7 +99,7 @@ impl Fold for ToEssentialTraitDef { } trait_def.vis = Visibility::Inherited; - trait_def + fold::fold_item_trait(self, trait_def) } fn fold_receiver(&mut self, receiver: Receiver) -> Receiver { @@ -107,7 +107,7 @@ impl Fold for ToEssentialTraitDef { self.push_error(&receiver, "Taking `Self` by value is not allowed"); } - receiver + fold::fold_receiver(self, receiver) } } diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index eabae82246e20..7a835b1c66fd5 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -16,7 +16,7 @@ //! Provides implementations for the runtime interface traits. -use crate::{RIType, pass_by::{PassBy, Codec, Inner, PassByInner}}; +use crate::{RIType, Pointer, pass_by::{PassBy, Codec, Inner, PassByInner}}; #[cfg(feature = "std")] use crate::host::*; #[cfg(not(feature = "std"))] @@ -26,7 +26,7 @@ use crate::wasm::*; use static_assertions::assert_eq_size; #[cfg(feature = "std")] -use wasm_interface::{FunctionContext, Pointer, Result}; +use wasm_interface::{FunctionContext, Result}; use codec::{Encode, Decode}; @@ -413,3 +413,45 @@ impl IntoFFIValue for str { pointer_and_len_to_u64(bytes.as_ptr() as u32, bytes.len() as u32).into() } } + +#[cfg(feature = "std")] +impl RIType for Pointer { + type FFIType = u32; +} + +#[cfg(not(feature = "std"))] +impl RIType for Pointer { + type FFIType = u32; +} + +#[cfg(not(feature = "std"))] +impl IntoFFIValue for Pointer { + type Owned = (); + + fn into_ffi_value(&self) -> WrappedFFIValue { + (*self as u32).into() + } +} + +#[cfg(not(feature = "std"))] +impl FromFFIValue for Pointer { + fn from_ffi_value(arg: u32) -> Self { + arg as _ + } +} + +#[cfg(feature = "std")] +impl FromFFIValue for Pointer { + type SelfInstance = Self; + + fn from_ffi_value(_: &mut dyn FunctionContext, arg: u32) -> Result { + Ok(Pointer::new(arg)) + } +} + +#[cfg(feature = "std")] +impl IntoFFIValue for Pointer { + fn into_ffi_value(self, _: &mut dyn FunctionContext) -> Result { + Ok(self.into()) + } +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 1280a0361fe6c..a8c631cacef94 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -61,6 +61,12 @@ pub trait RIType { type FFIType; } +#[cfg(not(feature = "std"))] +pub type Pointer = *mut T; + +#[cfg(feature = "std")] +pub type Pointer = wasm_interface::Pointer; + #[cfg(test)] mod tests { use super::*; diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index e9ba9c170c1c8..ba2a7e93ce32e 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -46,7 +46,7 @@ use primitives::{ #[cfg(feature = "std")] use trie::{TrieConfiguration, trie_types::Layout}; -use runtime_interface::runtime_interface; +use runtime_interface::{runtime_interface, Pointer}; use codec::{Encode, Decode}; @@ -574,6 +574,16 @@ pub trait Offchain { } } +#[runtime_interface(wasm_only)] +trait Allocator { + fn malloc(&mut self, size: u32) -> Pointer { + self.allocate_memory(size).unwrap() + } + + fn free(&mut self, ptr: Pointer) { + self.deallocate_memory(ptr).unwrap() + } +} mod imp { use super::*; diff --git a/core/sr-std/with_std.rs b/core/sr-std/with_std.rs index bbb932022bdaa..e41a9d7ad6c5c 100644 --- a/core/sr-std/with_std.rs +++ b/core/sr-std/with_std.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +pub use std::alloc; pub use std::any; pub use std::borrow; pub use std::boxed; diff --git a/core/sr-std/without_std.rs b/core/sr-std/without_std.rs index 97f22bc2ef5b3..b52b3bd333326 100755 --- a/core/sr-std/without_std.rs +++ b/core/sr-std/without_std.rs @@ -14,7 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -#[doc(hidden)] pub extern crate alloc; extern "C" { From 0313975207f779e8f3dfbf868e22d2de937d44b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 25 Oct 2019 21:10:58 +0200 Subject: [PATCH 40/76] Improve error message --- .../proc-macro/src/host_function_interface.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 2e078e9dc562d..5bd13d1fff4da 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -200,7 +200,10 @@ fn generate_host_function_implementation( let struct_name = Ident::new(&name.to_pascal_case(), Span::call_site()); let crate_ = generate_crate_access(); let signature = generate_wasm_interface_signature_for_host_function(&method.sig)?; - let wasm_to_ffi_values = generate_wasm_to_ffi_values(&method.sig).collect::>>()?; + let wasm_to_ffi_values = generate_wasm_to_ffi_values( + &method.sig, + trait_name, + ).collect::>>()?; let ffi_to_host_values = generate_ffi_to_host_value(&method.sig).collect::>>()?; let host_function_call = generate_host_function_call(&method.sig, is_wasm_only); let into_preallocated_ffi_value = generate_into_preallocated_ffi_value(&method.sig)?; @@ -269,6 +272,7 @@ fn generate_wasm_interface_signature_for_host_function(sig: &Signature) -> Resul /// values. fn generate_wasm_to_ffi_values<'a>( sig: &'a Signature, + trait_name: &'a Ident, ) -> impl Iterator> + 'a { let crate_ = generate_crate_access(); let function_name = &sig.ident; @@ -280,9 +284,10 @@ fn generate_wasm_to_ffi_values<'a>( get_function_argument_names_and_types_without_ref(sig) .map(move |(name, ty)| { let try_from_error = format!( - "Could not instantiate `{}` from wasm value while executing `{}`!", + "Could not instantiate `{}` from wasm value while executing `{}` from interface `{}`!", name.to_token_stream(), function_name, + trait_name, ); let var_name = generate_ffi_value_var_name(name)?; From f2c35bff486416aefcfc3f00d0c9f4391bf61711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 25 Oct 2019 21:35:36 +0200 Subject: [PATCH 41/76] Move `WasmAllocator` to `sr-io` and more clean ups --- core/sr-io/Cargo.toml | 7 +++--- core/sr-io/src/lib.rs | 50 +++++++++++++++++++++++++++++++++++--- core/sr-io/without_std.rs | 39 ----------------------------- core/sr-std/Cargo.toml | 7 ------ core/sr-std/build.rs | 13 ---------- core/sr-std/without_std.rs | 28 --------------------- 6 files changed, 51 insertions(+), 93 deletions(-) delete mode 100644 core/sr-io/without_std.rs delete mode 100644 core/sr-std/build.rs diff --git a/core/sr-io/Cargo.toml b/core/sr-io/Cargo.toml index 878ed7d8b6eed..9880f680c4963 100644 --- a/core/sr-io/Cargo.toml +++ b/core/sr-io/Cargo.toml @@ -36,8 +36,9 @@ std = [ ] # These two features are used for `no_std` builds for the environments which already provides -# `#[panic_handler]` and `#[alloc_error_handler]`. +# `#[panic_handler]`, `#[alloc_error_handler]` and `#[global_allocator]`. # # For the regular wasm runtime builds those are not used. -no_panic_handler = [] -no_oom = [] +disable_panic_handler = [] +disable_oom = [] +disable_allocator = [] diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index ba2a7e93ce32e..b243a19b1a12b 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -585,14 +585,57 @@ trait Allocator { } } +/// Allocator used by Substrate when executing the Wasm runtime. +#[cfg(not(feature = "std"))] +struct WasmAllocator; + +#[cfg(all(not(feature = "disable_global_allocator"), not(feature = "std")))] +#[global_allocator] +static ALLOCATOR: WasmAllocator = WasmAllocator; + +#[cfg(not(feature = "std"))] +mod allocator_impl { + use super::*; + use core::alloc::{GlobalAlloc, Layout}; + + unsafe impl GlobalAlloc for WasmAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + allocator::malloc(layout.size() as u32) + } + + unsafe fn dealloc(&self, ptr: *mut u8, _: Layout) { + allocator::free(ptr) + } + } +} + +#[cfg(all(not(feature = "disable_panic_handler"), not(feature = "std")))] +#[panic_handler] +#[no_mangle] +pub fn panic(info: &core::panic::PanicInfo) -> ! { + unsafe { + let message = rstd::alloc::format!("{}", info); + misc::print_utf8(message.as_bytes()); + core::intrinsics::abort() + } +} + +#[cfg(all(not(feature = "disable_oom"), not(feature = "std")))] +#[alloc_error_handler] +pub extern fn oom(_: core::alloc::Layout) -> ! { + static OOM_MSG: &str = "Runtime memory exhausted. Aborting"; + + unsafe { + misc::print_utf8(OOM_MSG.as_bytes()); + core::intrinsics::abort(); + } +} + mod imp { use super::*; #[cfg(feature = "std")] include!("../with_std.rs"); - - #[cfg(not(feature = "std"))] - include!("../without_std.rs"); } #[cfg(feature = "std")] @@ -612,4 +655,5 @@ pub type SubstrateHostFunctions = ( offchain::HostFunctions, crypto::HostFunctions, hashing::HostFunctions, + allocator::HostFunctions, ); diff --git a/core/sr-io/without_std.rs b/core/sr-io/without_std.rs deleted file mode 100644 index 7b4246fc1739e..0000000000000 --- a/core/sr-io/without_std.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017-2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use core::{intrinsics, panic::PanicInfo}; - -#[cfg(not(feature = "no_panic_handler"))] -#[panic_handler] -#[no_mangle] -pub fn panic(info: &PanicInfo) -> ! { - unsafe { - let message = rstd::alloc::format!("{}", info); - misc::print_utf8(message.as_bytes()); - intrinsics::abort() - } -} - -#[cfg(not(feature = "no_oom"))] -#[alloc_error_handler] -pub extern fn oom(_: core::alloc::Layout) -> ! { - static OOM_MSG: &str = "Runtime memory exhausted. Aborting"; - - unsafe { - misc::print_utf8(OOM_MSG.as_bytes()); - intrinsics::abort(); - } -} diff --git a/core/sr-std/Cargo.toml b/core/sr-std/Cargo.toml index 2a8b7d37ca272..77021af935ae1 100644 --- a/core/sr-std/Cargo.toml +++ b/core/sr-std/Cargo.toml @@ -2,15 +2,8 @@ name = "sr-std" version = "2.0.0" authors = ["Parity Technologies "] -build = "build.rs" edition = "2018" -[build-dependencies] -rustc_version = "0.2.3" - [features] default = ["std"] std = [] -nightly = [] -strict = [] -no_global_allocator = [] diff --git a/core/sr-std/build.rs b/core/sr-std/build.rs deleted file mode 100644 index af9c91db877dd..0000000000000 --- a/core/sr-std/build.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Set a nightly feature - -use rustc_version::{version, version_meta, Channel}; - -fn main() { - // Assert we haven't traveled back in time - assert!(version().unwrap().major >= 1); - - // Set cfg flags depending on release channel - if let Channel::Nightly = version_meta().unwrap().channel { - println!("cargo:rustc-cfg=feature=\"nightly\""); - } -} diff --git a/core/sr-std/without_std.rs b/core/sr-std/without_std.rs index b52b3bd333326..841ade1bb0514 100755 --- a/core/sr-std/without_std.rs +++ b/core/sr-std/without_std.rs @@ -16,34 +16,6 @@ pub extern crate alloc; -extern "C" { - fn ext_malloc(size: u32) -> *mut u8; - fn ext_free(ptr: *mut u8); -} - -/// Wasm allocator -pub struct WasmAllocator; - -#[cfg(not(feature = "no_global_allocator"))] -#[global_allocator] -static ALLOCATOR: WasmAllocator = WasmAllocator; - -mod __impl { - use core::alloc::{GlobalAlloc, Layout}; - - use super::WasmAllocator; - - unsafe impl GlobalAlloc for WasmAllocator { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - super::ext_malloc(layout.size() as u32) as *mut u8 - } - - unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { - super::ext_free(ptr as *mut u8) - } - } -} - pub use alloc::boxed; pub use alloc::rc; pub use alloc::vec; From e71f3e512d97d89e651f3cfcaa63f036465d7731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 25 Oct 2019 22:38:29 +0200 Subject: [PATCH 42/76] Use correct function signature for wasm functions --- core/executor/src/wasmi_execution.rs | 11 +- core/runtime-interface/Cargo.toml | 1 + core/runtime-interface/src/lib.rs | 2 +- core/runtime-interface/test-wasm/src/lib.rs | 106 ++++++++++---------- 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/core/executor/src/wasmi_execution.rs b/core/executor/src/wasmi_execution.rs index 9a9c63fdcd669..ce66af26e8b17 100644 --- a/core/executor/src/wasmi_execution.rs +++ b/core/executor/src/wasmi_execution.rs @@ -313,7 +313,7 @@ impl wasmi::Externals for FunctionExecutor { -> Result, wasmi::Trap> { let mut args = args.as_ref().iter().copied().map(Into::into); - let function = SubstrateExternals::get_function(index).ok_or_else(|| + let function = HF::get_function(index).ok_or_else(|| Error::from( format!("Could not find host function with index: {}", index), ) @@ -596,9 +596,12 @@ impl WasmRuntime for WasmiRuntime { self.state_snapshot.heap_pages == heap_pages } - fn call(&mut self, ext: &mut dyn Externalities, method: &str, data: &[u8]) - -> Result, Error> - { + fn call( + &mut self, + ext: &mut dyn Externalities, + method: &str, + data: &[u8], + ) -> Result, Error> { self.with(|module| { call_in_wasm_module::(ext, module, method, data) }) diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index 3e9bb6e6b53d9..da211de0df554 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -19,6 +19,7 @@ executor = { package = "substrate-executor", path = "../executor" } test-wasm = { package = "substrate-runtime-interface-test-wasm", path = "test-wasm" } state_machine = { package = "substrate-state-machine", path = "../state-machine" } primitives = { package = "substrate-primitives", path = "../primitives" } +runtime-io = { package = "sr-io", path = "../sr-io" } [features] default = [ "std" ] diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index a8c631cacef94..b7cd2f805d3d3 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -79,7 +79,7 @@ mod tests { let mut ext = TestExternalities::default(); let mut ext_ext = ext.ext(); - executor::call_in_wasm::<_, HF>( + executor::call_in_wasm::<_, (HF, runtime_io::SubstrateHostFunctions)>( method, &[], executor::WasmExecutionMethod::Interpreted, diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index faa836b7ee0b1..75a789c05ba17 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -20,9 +20,10 @@ use runtime_interface::runtime_interface; +#[cfg(not(feature = "std"))] use rstd::{vec, vec::Vec, mem, convert::TryFrom}; -use primitives::sr25519::Public; +use primitives::{sr25519::Public, wasm_export_functions}; // Inlucde the WASM binary #[cfg(feature = "std")] @@ -79,72 +80,67 @@ pub fn import_runtime_io() { runtime_io::misc::print_utf8(&[]); } -#[no_mangle] -pub fn test_return_data() { - let input = vec![1, 2, 3, 4, 5, 6]; - let res = test_api::return_input(input.clone()); +wasm_export_functions! { + fn test_return_data() { + let input = vec![1, 2, 3, 4, 5, 6]; + let res = test_api::return_input(input.clone()); - assert_eq!(input, res); -} + assert_eq!(input, res); + } -#[no_mangle] -pub fn test_return_option_data() { - let input = vec![1, 2, 3, 4, 5, 6]; - let res = test_api::return_option_input(input.clone()); + fn test_return_option_data() { + let input = vec![1, 2, 3, 4, 5, 6]; + let res = test_api::return_option_input(input.clone()); - assert_eq!(Some(input), res); -} + assert_eq!(Some(input), res); + } -#[no_mangle] -pub fn test_set_storage() { - let key = "hello"; - let value = "world"; + fn test_set_storage() { + let key = "hello"; + let value = "world"; - test_api::set_storage(key.as_bytes(), value.as_bytes()); -} + test_api::set_storage(key.as_bytes(), value.as_bytes()); + } -#[no_mangle] -pub fn test_return_value_into_mutable_reference() { - let mut data = vec![1, 2, 3, 4, 5, 6]; + fn test_return_value_into_mutable_reference() { + let mut data = vec![1, 2, 3, 4, 5, 6]; - test_api::return_value_into_mutable_reference(&mut data); + test_api::return_value_into_mutable_reference(&mut data); - let expected = "hello"; - assert_eq!(expected.as_bytes(), &data[..expected.len()]); -} + let expected = "hello"; + assert_eq!(expected.as_bytes(), &data[..expected.len()]); + } -#[no_mangle] -pub fn test_get_and_return_array() { - let mut input = unsafe { mem::MaybeUninit::<[u8; 34]>::zeroed().assume_init() }; - input.copy_from_slice(&[ - 24, 3, 23, 20, 2, 16, 32, 1, 12, 26, 27, 8, 29, 31, 6, 5, 4, 19, 10, 28, 34, 21, 18, 33, 9, - 13, 22, 25, 15, 11, 30, 7, 14, 17, - ]); + fn test_get_and_return_array() { + let mut input = unsafe { mem::MaybeUninit::<[u8; 34]>::zeroed().assume_init() }; + input.copy_from_slice(&[ + 24, 3, 23, 20, 2, 16, 32, 1, 12, 26, 27, 8, 29, 31, 6, 5, 4, 19, 10, 28, 34, 21, 18, 33, 9, + 13, 22, 25, 15, 11, 30, 7, 14, 17, + ]); - let res = test_api::get_and_return_array(input); + let res = test_api::get_and_return_array(input); - assert_eq!(&res, &input[..16]); -} + assert_eq!(&res, &input[..16]); + } -#[no_mangle] -pub fn test_array_as_mutable_reference() { - let mut array = [0u8; 16]; - test_api::array_as_mutable_reference(&mut array); + fn test_array_as_mutable_reference() { + let mut array = [0u8; 16]; + test_api::array_as_mutable_reference(&mut array); - assert_eq!(array, TEST_ARRAY); -} + assert_eq!(array, TEST_ARRAY); + } -#[no_mangle] -pub fn test_return_input_public_key() { - let key = Public::try_from( - &[ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, - ][..], - ).unwrap(); - let ret_key = test_api::return_input_public_key(key.clone()); - - let key_data: &[u8] = key.as_ref(); - let ret_key_data: &[u8] = ret_key.as_ref(); - assert_eq!(key_data, ret_key_data); + fn test_return_input_public_key() { + let key = Public::try_from( + &[ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, + ][..], + ).unwrap(); + let ret_key = test_api::return_input_public_key(key.clone()); + + let key_data: &[u8] = key.as_ref(); + let ret_key_data: &[u8] = ret_key.as_ref(); + assert_eq!(key_data, ret_key_data); + } } From 4e667eb460bf15afdf3672c33a9ecbf607252959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 26 Oct 2019 13:39:25 +0200 Subject: [PATCH 43/76] Store the host functions with the Wasm runtime --- Cargo.lock | 4 +- core/executor/src/lib.rs | 3 +- core/executor/src/native_executor.rs | 20 +- core/executor/src/wasm_runtime.rs | 37 +++- core/executor/src/wasm_utils.rs | 15 +- core/executor/src/wasmi_execution.rs | 173 ++++++++---------- .../proc-macro/src/host_function_interface.rs | 16 +- core/runtime-interface/src/lib.rs | 2 +- core/wasm-interface/src/lib.rs | 44 ++--- 9 files changed, 136 insertions(+), 178 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c303245bd6edd..f0e9c900528e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3959,9 +3959,6 @@ dependencies = [ [[package]] name = "sr-std" version = "2.0.0" -dependencies = [ - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "sr-version" @@ -5519,6 +5516,7 @@ dependencies = [ "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "primitive-types 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sr-io 2.0.0", "sr-std 2.0.0", "static_assertions 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-executor 2.0.0", diff --git a/core/executor/src/lib.rs b/core/executor/src/lib.rs index b94e0ff086f93..9b7ca542c9676 100644 --- a/core/executor/src/lib.rs +++ b/core/executor/src/lib.rs @@ -68,11 +68,12 @@ pub fn call_in_wasm( code: &[u8], heap_pages: u64, ) -> error::Result> { - let mut instance = wasm_runtime::create_wasm_runtime_with_code::<_, HF>( + let mut instance = wasm_runtime::create_wasm_runtime_with_code( ext, execution_method, heap_pages, code, + HF::host_functions(), )?; instance.call(ext, function, call_data) } diff --git a/core/executor/src/native_executor.rs b/core/executor/src/native_executor.rs index f525a61d45e6a..8ed7a0313f1a4 100644 --- a/core/executor/src/native_executor.rs +++ b/core/executor/src/native_executor.rs @@ -14,15 +14,23 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::{result, cell::RefCell, panic::UnwindSafe}; -use crate::error::{Error, Result}; -use crate::wasm_runtime::{RuntimesCache, WasmExecutionMethod, WasmRuntime}; -use crate::RuntimeInfo; +use crate::{ + RuntimeInfo, error::{Error, Result}, host_interface::SubstrateExternals, + wasm_runtime::{RuntimesCache, WasmExecutionMethod, WasmRuntime}, +}; + use runtime_version::{NativeVersion, RuntimeVersion}; + use codec::{Decode, Encode}; + use primitives::{NativeOrEncoded, traits::{CodeExecutor, Externalities}}; + use log::{trace, warn}; +use std::{result, cell::RefCell, panic::UnwindSafe}; + +use wasm_interface::HostFunctions; + thread_local! { static RUNTIMES_CACHE: RefCell = RefCell::new(RuntimesCache::new()); } @@ -99,10 +107,12 @@ impl NativeExecutor { ) -> Result where E: Externalities { RUNTIMES_CACHE.with(|cache| { let mut cache = cache.borrow_mut(); - let runtime = cache.fetch_runtime::<_, runtime_io::SubstrateHostFunctions>( + let runtime = cache.fetch_runtime( ext, self.fallback_method, self.default_heap_pages, + // Use the `SubstrateExternals` as well, to be backwards compatible. + <(runtime_io::SubstrateHostFunctions, SubstrateExternals)>::host_functions(), )?; f(runtime, ext) }) diff --git a/core/executor/src/wasm_runtime.rs b/core/executor/src/wasm_runtime.rs index 4e95557f77b8f..9cd3b1c7dd8dd 100644 --- a/core/executor/src/wasm_runtime.rs +++ b/core/executor/src/wasm_runtime.rs @@ -31,7 +31,7 @@ use runtime_version::RuntimeVersion; use std::{collections::hash_map::{Entry, HashMap}}; -use wasm_interface::HostFunctions; +use wasm_interface::Function; /// The Substrate Wasm runtime. pub trait WasmRuntime { @@ -41,6 +41,9 @@ pub trait WasmRuntime { /// the heap pages would not change from its current value. fn update_heap_pages(&mut self, heap_pages: u64) -> bool; + /// Return the host functions that are registered for this Wasm runtime. + fn host_functions(&self) -> &[&'static dyn Function]; + /// Call a method in the Substrate runtime by name. Returns the encoded result on success. fn call(&mut self, ext: &mut dyn Externalities, method: &str, data: &[u8]) -> Result, Error>; @@ -103,6 +106,8 @@ impl RuntimesCache { /// /// `default_heap_pages` - Number of 64KB pages to allocate for Wasm execution. /// + /// `host_functions` - The host functions that should be registered for the Wasm runtime. + /// /// # Return value /// /// If no error occurred a tuple `(wasmi::ModuleRef, Option)` is @@ -115,11 +120,12 @@ impl RuntimesCache { /// /// `Error::InvalidMemoryReference` is returned if no memory export with the /// identifier `memory` can be found in the runtime. - pub fn fetch_runtime( + pub fn fetch_runtime( &mut self, ext: &mut E, wasm_method: WasmExecutionMethod, default_heap_pages: u64, + host_functions: Vec<&'static dyn Function>, ) -> Result<&mut (dyn WasmRuntime + 'static), Error> { let code_hash = ext .original_storage_hash(well_known_keys::CODE) @@ -134,12 +140,21 @@ impl RuntimesCache { Entry::Occupied(o) => { let result = o.into_mut(); if let Ok(ref mut cached_runtime) = result { - if !cached_runtime.update_heap_pages(heap_pages) { + let heap_pages_changed = !cached_runtime.update_heap_pages(heap_pages); + let host_functions_changed = cached_runtime.host_functions() != &host_functions[..]; + if heap_pages_changed || host_functions_changed { + let changed = if heap_pages_changed { + "heap_pages" + } else { + "host functions" + }; + trace!( target: "runtimes_cache", - "heap_pages were changed. Reinstantiating the instance" + "{} were changed. Reinstantiating the instance", + changed, ); - *result = create_wasm_runtime::<_, HF>(ext, wasm_method, heap_pages); + *result = create_wasm_runtime(ext, wasm_method, heap_pages, host_functions); if let Err(ref err) = result { warn!(target: "runtimes_cache", "cannot create a runtime: {:?}", err); } @@ -149,7 +164,7 @@ impl RuntimesCache { }, Entry::Vacant(v) => { trace!(target: "runtimes_cache", "no instance found in cache, creating now."); - let result = create_wasm_runtime::<_, HF>(ext, wasm_method, heap_pages); + let result = create_wasm_runtime(ext, wasm_method, heap_pages, host_functions); if let Err(ref err) = result { warn!(target: "runtimes_cache", "cannot create a runtime: {:?}", err); } @@ -164,26 +179,28 @@ impl RuntimesCache { } /// Create a wasm runtime with the given `code`. -pub fn create_wasm_runtime_with_code( +pub fn create_wasm_runtime_with_code( ext: &mut E, wasm_method: WasmExecutionMethod, heap_pages: u64, code: &[u8], + host_functions: Vec<&'static dyn Function>, ) -> Result, WasmError> { match wasm_method { WasmExecutionMethod::Interpreted => - wasmi_execution::create_instance::<_, HF>(ext, code, heap_pages) + wasmi_execution::create_instance(ext, code, heap_pages, host_functions) .map(|runtime| -> Box { Box::new(runtime) }), } } -fn create_wasm_runtime( +fn create_wasm_runtime( ext: &mut E, wasm_method: WasmExecutionMethod, heap_pages: u64, + host_functions: Vec<&'static dyn Function>, ) -> Result, WasmError> { let code = ext .original_storage(well_known_keys::CODE) .ok_or(WasmError::CodeNotFound)?; - create_wasm_runtime_with_code::<_, HF>(ext, wasm_method, heap_pages, &code) + create_wasm_runtime_with_code(ext, wasm_method, heap_pages, &code, host_functions) } diff --git a/core/executor/src/wasm_utils.rs b/core/executor/src/wasm_utils.rs index 5172d4924ec9e..fabb0b9729cc1 100644 --- a/core/executor/src/wasm_utils.rs +++ b/core/executor/src/wasm_utils.rs @@ -45,7 +45,7 @@ macro_rules! gen_functions { { $( $generated:tt )* } $context:ident, ) => ( - &[ $( $generated )* ] + vec![ $( $generated )* ] ); (@INTERNAL { $( $generated:tt )* } @@ -162,22 +162,11 @@ macro_rules! impl_wasm_host_interface { ) => ( impl $crate::wasm_interface::HostFunctions for $interface_name { #[allow(non_camel_case_types)] - fn get_function(index: usize) -> Option<&'static dyn $crate::wasm_interface::Function> { + fn host_functions() -> Vec<&'static dyn $crate::wasm_interface::Function> { gen_functions!( $context, $( $name( $( $names: $params ),* ) $( -> $returns )? { $( $body )* } )* ) - .get(index) - .map(|f| *f) - } - - #[allow(non_camel_case_types)] - fn num_functions() -> usize { - gen_functions!( - $context, - $( $name( $( $names: $params ),* ) $( -> $returns )? { $( $body )* } )* - ) - .len() } } ); diff --git a/core/executor/src/wasmi_execution.rs b/core/executor/src/wasmi_execution.rs index ce66af26e8b17..16fa6deb2cec0 100644 --- a/core/executor/src/wasmi_execution.rs +++ b/core/executor/src/wasmi_execution.rs @@ -16,7 +16,7 @@ //! Implementation of a Wasm runtime using the Wasmi interpreter. -use std::{str, mem, marker::PhantomData}; +use std::{str, mem}; use wasmi::{ Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder, ModuleRef, memory_units::Pages, RuntimeValue::{I32, I64, self}, @@ -24,7 +24,6 @@ use wasmi::{ use crate::error::{Error, WasmError}; use codec::{Encode, Decode}; use primitives::{sandbox as sandbox_primitives, traits::Externalities}; -use crate::host_interface::SubstrateExternals; use crate::sandbox; use crate::allocator; use crate::wasm_runtime::WasmRuntime; @@ -32,30 +31,35 @@ use log::trace; use parity_wasm::elements::{deserialize_buffer, DataSegment, Instruction, Module as RawModule}; use runtime_version::RuntimeVersion; use wasm_interface::{ - FunctionContext, HostFunctions, Pointer, WordSize, Sandbox, MemoryId, Result as WResult, + FunctionContext, Pointer, WordSize, Sandbox, MemoryId, Result as WResult, Function, }; -struct FunctionExecutor { +struct FunctionExecutor<'a> { sandbox_store: sandbox::Store, heap: allocator::FreeingBumpHeapAllocator, memory: MemoryRef, table: Option, - _marker: PhantomData, + host_functions: &'a [&'static dyn Function], } -impl FunctionExecutor { - fn new(m: MemoryRef, heap_base: u32, t: Option) -> Result { +impl<'a> FunctionExecutor<'a> { + fn new( + m: MemoryRef, + heap_base: u32, + t: Option, + host_functions: &'a [&'static dyn Function], + ) -> Result { Ok(FunctionExecutor { sandbox_store: sandbox::Store::new(), heap: allocator::FreeingBumpHeapAllocator::new(heap_base), memory: m, table: t, - _marker: PhantomData, + host_functions, }) } } -impl sandbox::SandboxCapabilities for FunctionExecutor { +impl<'a> sandbox::SandboxCapabilities for FunctionExecutor<'a> { type SupervisorFuncRef = wasmi::FuncRef; fn store(&self) -> &sandbox::Store { @@ -110,7 +114,7 @@ impl sandbox::SandboxCapabilities for FunctionExecutor { } } -impl FunctionContext for FunctionExecutor { +impl<'a> FunctionContext for FunctionExecutor<'a> { fn read_memory_into(&self, address: Pointer, dest: &mut [u8]) -> WResult<()> { self.memory.get_into(address.into(), dest).map_err(|e| format!("{:?}", e)) } @@ -138,7 +142,7 @@ impl FunctionContext for FunctionExecutor { } } -impl Sandbox for FunctionExecutor { +impl<'a> Sandbox for FunctionExecutor<'a> { fn memory_get( &self, memory_id: MemoryId, @@ -263,57 +267,44 @@ impl Sandbox for FunctionExecutor { } } -impl FunctionExecutor { - fn resolver() -> &'static dyn wasmi::ModuleImportResolver { - struct Resolver(PhantomData); - impl wasmi::ModuleImportResolver for Resolver { - fn resolve_func(&self, name: &str, signature: &wasmi::Signature) - -> std::result::Result - { - let signature = wasm_interface::Signature::from(signature); - let num_functions = HF::num_functions(); - let mut function_index = 0; - - while function_index < num_functions { - let function = HF::get_function(function_index) - .expect("Any index below the number of functions is valid; qed"); - - if name == function.name() { - if signature == function.signature() { - return Ok( - wasmi::FuncInstance::alloc_host(signature.into(), function_index), - ) - } else { - return Err(wasmi::Error::Instantiation( - format!( - "Invalid signature for function `{}` expected `{:?}`, got `{:?}`", - function.name(), - signature, - function.signature(), - ), - )) - } - } +struct Resolver<'a>(&'a[&'static dyn Function]); - function_index += 1; +impl<'a> wasmi::ModuleImportResolver for Resolver<'a> { + fn resolve_func(&self, name: &str, signature: &wasmi::Signature) + -> std::result::Result + { + let signature = wasm_interface::Signature::from(signature); + for (function_index, function) in self.0.iter().enumerate() { + if name == function.name() { + if signature == function.signature() { + return Ok( + wasmi::FuncInstance::alloc_host(signature.into(), function_index), + ) + } else { + return Err(wasmi::Error::Instantiation( + format!( + "Invalid signature for function `{}` expected `{:?}`, got `{:?}`", + function.name(), + signature, + function.signature(), + ), + )) } - - Err(wasmi::Error::Instantiation( - format!("Export {} not found", name), - )) } } - &Resolver::(PhantomData::) + Err(wasmi::Error::Instantiation( + format!("Export {} not found", name), + )) } } -impl wasmi::Externals for FunctionExecutor { +impl<'a> wasmi::Externals for FunctionExecutor<'a> { fn invoke_index(&mut self, index: usize, args: wasmi::RuntimeArgs) -> Result, wasmi::Trap> { let mut args = args.as_ref().iter().copied().map(Into::into); - let function = HF::get_function(index).ok_or_else(|| + let function = self.host_functions.get(index).ok_or_else(|| Error::from( format!("Could not find host function with index: {}", index), ) @@ -352,45 +343,13 @@ fn get_heap_base(module: &ModuleRef) -> Result { } /// Call a given method in the given wasm-module runtime. -fn call_in_wasm_module( +fn call_in_wasm_module( ext: &mut dyn Externalities, module_instance: &ModuleRef, method: &str, data: &[u8], + host_functions: &[&'static dyn Function], ) -> Result, Error> { - call_in_wasm_module_with_custom_signature::( - ext, - module_instance, - method, - |alloc| { - let offset = alloc(data)?; - Ok(vec![I32(offset as i32), I32(data.len() as i32)]) - }, - |res, memory| { - if let Some(I64(r)) = res { - let offset = r as u32; - let length = (r as u64 >> 32) as usize; - memory.get(offset, length).map_err(|_| Error::Runtime).map(Some) - } else { - Ok(None) - } - } - ) -} - -/// Call a given method in the given wasm-module runtime. -fn call_in_wasm_module_with_custom_signature< - HF: HostFunctions, - F: FnOnce(&mut dyn FnMut(&[u8]) -> Result) -> Result, Error>, - FR: FnOnce(Option, &MemoryRef) -> Result, Error>, - R, ->( - ext: &mut dyn Externalities, - module_instance: &ModuleRef, - method: &str, - create_parameters: F, - filter_result: FR, -) -> Result { // extract a reference to a linear memory, optional reference to a table // and then initialize FunctionExecutor. let memory = get_mem_instance(module_instance)?; @@ -399,22 +358,26 @@ fn call_in_wasm_module_with_custom_signature< .and_then(|e| e.as_table().cloned()); let heap_base = get_heap_base(module_instance)?; - let mut fec = FunctionExecutor::::new(memory.clone(), heap_base, table)?; + let mut fec = FunctionExecutor::new(memory.clone(), heap_base, table, host_functions)?; - let parameters = create_parameters(&mut |data: &[u8]| { - let offset = fec.allocate_memory(data.len() as u32)?; - fec.write_memory(offset, data).map(|_| offset.into()).map_err(Into::into) - })?; + // Write the call data + let offset = fec.allocate_memory(data.len() as u32)?; + fec.write_memory(offset, data)?; let result = externalities::set_and_run_with_externalities( ext, - || module_instance.invoke_export(method, ¶meters, &mut fec), + || module_instance.invoke_export( + method, + &[I32(u32::from(offset) as i32), I32(data.len() as i32)], + &mut fec, + ), ); match result { - Ok(val) => match filter_result(val, &memory)? { - Some(val) => Ok(val), - None => Err(Error::InvalidReturn), + Ok(Some(I64(r))) => { + let offset = r as u32; + let length = (r as u64 >> 32) as usize; + memory.get(offset, length).map_err(|_| Error::Runtime) }, Err(e) => { trace!( @@ -424,19 +387,21 @@ fn call_in_wasm_module_with_custom_signature< ); Err(e.into()) }, + _ => Err(Error::InvalidReturn), } } /// Prepare module instance -fn instantiate_module( +fn instantiate_module( heap_pages: usize, module: &Module, + host_functions: &[&'static dyn Function], ) -> Result { + let resolver = Resolver(host_functions); // start module instantiation. Don't run 'start' function yet. let intermediate_instance = ModuleInstance::new( module, - &ImportsBuilder::new() - .with_resolver("env", FunctionExecutor::::resolver()) + &ImportsBuilder::new().with_resolver("env", &resolver), )?; // Verify that the module has the heap base global variable. @@ -572,6 +537,8 @@ pub struct WasmiRuntime { version: Option, /// The snapshot of the instance's state taken just after the instantiation. state_snapshot: StateSnapshot, + /// The host functions registered for this instance. + host_functions: Vec<&'static dyn Function>, } impl WasmiRuntime { @@ -596,6 +563,10 @@ impl WasmRuntime for WasmiRuntime { self.state_snapshot.heap_pages == heap_pages } + fn host_functions(&self) -> &[&'static dyn Function] { + &self.host_functions + } + fn call( &mut self, ext: &mut dyn Externalities, @@ -603,7 +574,7 @@ impl WasmRuntime for WasmiRuntime { data: &[u8], ) -> Result, Error> { self.with(|module| { - call_in_wasm_module::(ext, module, method, data) + call_in_wasm_module(ext, module, method, data, &self.host_functions) }) } @@ -612,10 +583,11 @@ impl WasmRuntime for WasmiRuntime { } } -pub fn create_instance( +pub fn create_instance( ext: &mut E, code: &[u8], heap_pages: u64, + host_functions: Vec<&'static dyn Function>, ) -> Result { let module = Module::from_buffer(&code).map_err(|_| WasmError::InvalidModule)?; @@ -626,7 +598,7 @@ pub fn create_instance( let data_segments = extract_data_segments(&code)?; // Instantiate this module. - let instance = instantiate_module::(heap_pages as usize, &module) + let instance = instantiate_module(heap_pages as usize, &module, &host_functions) .map_err(WasmError::Instantiation)?; // Take state snapshot before executing anything. @@ -638,13 +610,14 @@ pub fn create_instance( ", ); - let version = call_in_wasm_module::(ext, &instance, "Core_version", &[]) + let version = call_in_wasm_module(ext, &instance, "Core_version", &[], &host_functions) .ok() .and_then(|v| RuntimeVersion::decode(&mut v.as_slice()).ok()); Ok(WasmiRuntime { instance, version, state_snapshot, + host_functions, }) } diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 5bd13d1fff4da..95f1afd9d7a42 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -157,14 +157,6 @@ fn generate_host_functions_struct(trait_def: &ItemTrait, is_wasm_only: bool) -> }) .map(|m| generate_host_function_implementation(&trait_def.ident, m, is_wasm_only)) .collect::>>()?; - let host_functions_count = trait_def - .items - .iter() - .filter(|i| match i { - TraitItem::Method(_) => true, - _ => false, - }) - .count(); Ok( quote! { @@ -174,12 +166,8 @@ fn generate_host_functions_struct(trait_def: &ItemTrait, is_wasm_only: bool) -> #[cfg(feature = "std")] impl #crate_::wasm_interface::HostFunctions for HostFunctions { - fn get_function(index: usize) -> Option<&'static dyn #crate_::wasm_interface::Function> { - [ #( #host_functions ),* ].get(index).map(|f| *f) - } - - fn num_functions() -> usize { - #host_functions_count + fn host_functions() -> Vec<&'static dyn #crate_::wasm_interface::Function> { + vec![ #( #host_functions ),* ] } } } diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index b7cd2f805d3d3..23876c24fc7bc 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -130,7 +130,7 @@ mod tests { } #[test] - #[should_panic(expected = "Wasmi(Instantiation(\"Export ext_test_api_return_input not found\"))")] + #[should_panic(expected = "Other(\"Instantiation: Export ext_test_api_return_input not found\")")] fn host_function_not_found() { call_wasm_method::<()>("test_return_data"); } diff --git a/core/wasm-interface/src/lib.rs b/core/wasm-interface/src/lib.rs index 1ebf66a9f9dd7..e45139ff8a528 100644 --- a/core/wasm-interface/src/lib.rs +++ b/core/wasm-interface/src/lib.rs @@ -190,6 +190,12 @@ pub trait Function { ) -> Result>; } +impl PartialEq for dyn Function { + fn eq(&self, other: &Self) -> bool { + other.name() == self.name() && other.signature() == self.signature() + } +} + /// Context used by `Function` to interact with the allocator and the memory of the wasm instance. pub trait FunctionContext { /// Read memory from `address` into a vector. @@ -261,42 +267,18 @@ pub trait Sandbox { /// Something that provides implementations for host functions. pub trait HostFunctions: 'static { - /// Returns the function at the given index or `None` if the index is invalid. - fn get_function(index: usize) -> Option<&'static dyn Function>; - /// Returns the number of host functions. - fn num_functions() -> usize; + /// Returns the host functions `Self` provides. + fn host_functions() -> Vec<&'static dyn Function>; } -#[impl_trait_for_tuples::impl_for_tuples(1, 30)] +#[impl_trait_for_tuples::impl_for_tuples(30)] impl HostFunctions for Tuple { - fn get_function(mut index: usize) -> Option<&'static dyn Function> { - for_tuples!( - #( - let num_functions = Tuple::num_functions(); - - if index < num_functions { - return Tuple::get_function(index) - } - - index -= num_functions; - )* - ); + fn host_functions() -> Vec<&'static dyn Function> { + let mut host_functions = Vec::new(); - None - } - - fn num_functions() -> usize { - for_tuples!( #( Tuple::num_functions() )+* ) - } -} - -impl HostFunctions for () { - fn get_function(_: usize) -> Option<&'static dyn Function> { - None - } + for_tuples!( #( host_functions.extend(Tuple::host_functions()); )* ); - fn num_functions() -> usize { - 0 + host_functions } } From 08d478c539017758272bb60b571bc736c0be8ffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 26 Oct 2019 14:04:09 +0200 Subject: [PATCH 44/76] Docs update --- core/runtime-interface/src/lib.rs | 2 ++ core/sr-io/with_std.rs | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 23876c24fc7bc..67fdebbe94a6c 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -61,9 +61,11 @@ pub trait RIType { type FFIType; } +/// A pointer that can be used in a runtime interface function signature. #[cfg(not(feature = "std"))] pub type Pointer = *mut T; +/// A pointer that can be used in a runtime interface function signature. #[cfg(feature = "std")] pub type Pointer = wasm_interface::Pointer; diff --git a/core/sr-io/with_std.rs b/core/sr-io/with_std.rs index 2edf438a0d088..72d2f30915902 100644 --- a/core/sr-io/with_std.rs +++ b/core/sr-io/with_std.rs @@ -18,8 +18,6 @@ // pub use primitives::BlakeHasher; pub use substrate_state_machine::{BasicExternalities, TestExternalities}; -use trie::{TrieConfiguration, trie_types::Layout}; - use std::{collections::HashMap, convert::TryFrom}; use externalities::Externalities; From bcc04e6512afcde96007cd8e04fe808f71adc442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 26 Oct 2019 14:57:30 +0200 Subject: [PATCH 45/76] Fix compilation after master merge --- core/executor/src/host_interface.rs | 11 +++++++++-- core/primitives/src/offchain.rs | 4 ++-- core/sr-io/src/lib.rs | 4 ++-- core/sr-primitives/src/lib.rs | 10 ++++++---- core/sr-primitives/src/traits.rs | 7 +++++-- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/core/executor/src/host_interface.rs b/core/executor/src/host_interface.rs index 2aa524a259866..9f209abba39bf 100644 --- a/core/executor/src/host_interface.rs +++ b/core/executor/src/host_interface.rs @@ -27,7 +27,9 @@ use primitives::{ crypto::KeyTypeId, offchain, }; use trie::{TrieConfiguration, trie_types::Layout}; -use wasm_interface::{Pointer, WordSize, WritePrimitive, ReadPrimitive}; +use wasm_interface::{ + Pointer, WordSize, WritePrimitive, ReadPrimitive, FunctionContext, Result as WResult, +}; #[cfg(feature="wasm-extern-trace")] macro_rules! debug_trace { @@ -197,7 +199,12 @@ impl_wasm_host_interface! { let message = context.read_memory(message_data, message_len) .map_err(|_| "Invalid attempt to determine message in ext_log")?; - runtime_io::log(level.into(), &target, &message); + let target_str = std::str::from_utf8(&target) + .map_err(|_| "Target invalid utf8 in ext_log")?; + let message_str = std::str::from_utf8(&message) + .map_err(|_| "Message invalid utf8 in ext_log")?; + + runtime_io::log::log(level.into(), &target_str, &message_str); Ok(()) } diff --git a/core/primitives/src/offchain.rs b/core/primitives/src/offchain.rs index 20eddba4fb43a..3de2adfb9617f 100644 --- a/core/primitives/src/offchain.rs +++ b/core/primitives/src/offchain.rs @@ -179,11 +179,11 @@ impl OpaqueMultiaddr { } /// Opaque timestamp type -#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner)] +#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner, Encode, Decode)] pub struct Timestamp(u64); /// Duration type -#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner)] +#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner, Encode, Decode)] pub struct Duration(u64); impl Duration { diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index da61f529c7f52..a486f5310223b 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -611,9 +611,9 @@ pub trait Log { /// /// Instead of using directly, prefer setting up `RuntimeLogger` and using `log` macros. fn log(level: LogLevel, target: &str, message: &str) { - log::log!( + ::log::log!( target: target, - log::Level::from(level), + ::log::Level::from(level), "{}", message, ) diff --git a/core/sr-primitives/src/lib.rs b/core/sr-primitives/src/lib.rs index 4341e3543add5..ad523f6b92a71 100644 --- a/core/sr-primitives/src/lib.rs +++ b/core/sr-primitives/src/lib.rs @@ -244,7 +244,7 @@ impl traits::IdentifyAccount for MultiSigner { match self { MultiSigner::Ed25519(who) => <[u8; 32]>::from(who).into(), MultiSigner::Sr25519(who) => <[u8; 32]>::from(who).into(), - MultiSigner::Ecdsa(who) => runtime_io::blake2_256(who.as_ref()).into(), + MultiSigner::Ecdsa(who) => runtime_io::hashing::blake2_256(who.as_ref()).into(), } } } @@ -307,9 +307,11 @@ impl Verify for MultiSignature { (MultiSignature::Ed25519(ref sig), who) => sig.verify(msg, &ed25519::Public::from_slice(who.as_ref())), (MultiSignature::Sr25519(ref sig), who) => sig.verify(msg, &sr25519::Public::from_slice(who.as_ref())), (MultiSignature::Ecdsa(ref sig), who) => { - let m = runtime_io::blake2_256(msg.get()); - match runtime_io::secp256k1_ecdsa_recover_compressed(sig.as_ref(), &m) { - Ok(pubkey) => &runtime_io::blake2_256(pubkey.as_ref()) == >::as_ref(who), + let m = runtime_io::hashing::blake2_256(msg.get()); + match runtime_io::crypto::secp256k1_ecdsa_recover_compressed(sig.as_ref(), &m) { + Ok(pubkey) => + &runtime_io::hashing::blake2_256(pubkey.as_ref()) + == >::as_ref(who), _ => false, } } diff --git a/core/sr-primitives/src/traits.rs b/core/sr-primitives/src/traits.rs index ea7cc103797ae..afffee7f09a74 100644 --- a/core/sr-primitives/src/traits.rs +++ b/core/sr-primitives/src/traits.rs @@ -99,7 +99,10 @@ impl Verify for primitives::sr25519::Signature { impl Verify for primitives::ecdsa::Signature { type Signer = primitives::ecdsa::Public; fn verify>(&self, mut msg: L, signer: &primitives::ecdsa::Public) -> bool { - match runtime_io::secp256k1_ecdsa_recover_compressed(self.as_ref(), &runtime_io::blake2_256(msg.get())) { + match runtime_io::crypto::secp256k1_ecdsa_recover_compressed( + self.as_ref(), + &runtime_io::hashing::blake2_256(msg.get()), + ) { Ok(pubkey) => >::as_ref(signer) == &pubkey[..], _ => false, } @@ -1189,7 +1192,7 @@ impl Printable for usize { impl Printable for u64 { fn print(&self) { - runtime_io::print_num(*self); + runtime_io::misc::print_num(*self); } } From df8f9fc834ad16a870fb174872dbb62ac22ae433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 27 Oct 2019 12:55:14 +0100 Subject: [PATCH 46/76] Remove `sr-io/without_std` --- Cargo.lock | 1 + core/primitives/storage/src/lib.rs | 8 ++ core/sr-io/src/lib.rs | 89 ++++++++++--- core/sr-io/with_std.rs | 118 ------------------ core/sr-primitives/src/lib.rs | 7 +- core/state-machine/src/basic.rs | 40 ++++-- core/state-machine/src/testing.rs | 1 - srml/support/Cargo.toml | 2 + .../src/storage/genesis_config/mod.rs | 2 +- srml/support/src/lib.rs | 2 +- srml/support/src/storage/storage_items.rs | 13 +- srml/support/test/tests/final_keys.rs | 8 +- 12 files changed, 134 insertions(+), 157 deletions(-) delete mode 100644 core/sr-io/with_std.rs diff --git a/Cargo.lock b/Cargo.lock index 772f140dde36f..abefb21410ac2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4491,6 +4491,7 @@ dependencies = [ "srml-system 2.0.0", "substrate-inherents 2.0.0", "substrate-primitives 2.0.0", + "substrate-state-machine 2.0.0", ] [[package]] diff --git a/core/primitives/storage/src/lib.rs b/core/primitives/storage/src/lib.rs index dcdc223994e26..ba36e2c80f81e 100644 --- a/core/primitives/storage/src/lib.rs +++ b/core/primitives/storage/src/lib.rs @@ -40,6 +40,14 @@ pub struct StorageData( pub Vec, ); +/// A set of key value pairs for storage. +#[cfg(feature = "std")] +pub type StorageOverlay = std::collections::HashMap, Vec>; + +/// A set of key value pairs for children storage; +#[cfg(feature = "std")] +pub type ChildrenStorageOverlay = std::collections::HashMap, StorageOverlay>; + /// Storage change set #[derive(RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize, PartialEq, Eq))] diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index a486f5310223b..1ad22f2e561b4 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -52,7 +52,7 @@ use runtime_interface::{runtime_interface, Pointer}; use codec::{Encode, Decode}; #[cfg(feature = "std")] -use externalities::ExternalitiesExt; +use externalities::{ExternalitiesExt, Externalities}; /// Error verifying ECDSA signature #[derive(Encode, Decode)] @@ -170,7 +170,7 @@ pub trait Storage { /// Clear the storage of each key-value pair where the key starts with the given `prefix`. fn clear_prefix(&mut self, prefix: &[u8]) { - self.clear_prefix(prefix) + Externalities::clear_prefix(*self, prefix) } /// Clear the child storage of each key-value pair where the key starts with the given `prefix`. @@ -591,17 +591,21 @@ pub trait Offchain { } } +/// Wasm only interface that provides functions for calling into the allocator. #[runtime_interface(wasm_only)] trait Allocator { + /// Malloc the given number of bytes and return the pointer to the allocated memory location. fn malloc(&mut self, size: u32) -> Pointer { - self.allocate_memory(size).unwrap() + self.allocate_memory(size).expect("Failed to allocate memory") } + /// Free the given pointer. fn free(&mut self, ptr: Pointer) { - self.deallocate_memory(ptr).unwrap() + self.deallocate_memory(ptr).expect("Failed to free allocated memory") } } +/// Interface that provides functions for logging from within the runtime. #[runtime_interface] pub trait Log { /// Request to print a log message on the host. @@ -666,19 +670,9 @@ pub extern fn oom(_: core::alloc::Layout) -> ! { } } -mod imp { - use super::*; - - #[cfg(feature = "std")] - include!("../with_std.rs"); -} - -#[cfg(feature = "std")] -pub use self::imp::{StorageOverlay, ChildrenStorageOverlay, with_storage}; - /// Type alias for Externalities implementation used in tests. #[cfg(feature = "std")] -pub type TestExternalities = self::imp::TestExternalities; +pub type TestExternalities = substrate_state_machine::TestExternalities; /// The host functions Substrate provides for the Wasm runtime environment. /// @@ -693,3 +687,68 @@ pub type SubstrateHostFunctions = ( allocator::HostFunctions, log::HostFunctions, ); + +#[cfg(test)] +mod tests { + use super::*; + use primitives::map; + use substrate_state_machine::BasicExternalities; + + #[test] + fn storage_works() { + let mut t = BasicExternalities::default(); + t.execute_with(|| { + assert_eq!(storage::get(b"hello"), None); + storage::set(b"hello", b"world"); + assert_eq!(storage::get(b"hello"), Some(b"world".to_vec())); + assert_eq!(storage::get(b"foo"), None); + storage::set(b"foo", &[1, 2, 3][..]); + }); + + t = BasicExternalities::new(map![b"foo".to_vec() => b"bar".to_vec()], map![]); + + t.execute_with(|| { + assert_eq!(storage::get(b"hello"), None); + assert_eq!(storage::get(b"foo"), Some(b"bar".to_vec())); + }); + } + + #[test] + fn read_storage_works() { + let mut t = BasicExternalities::new( + map![b":test".to_vec() => b"\x0b\0\0\0Hello world".to_vec()], + map![], + ); + + t.execute_with(|| { + let mut v = [0u8; 4]; + assert!(storage::read(b":test", &mut v[..], 0).unwrap() >= 4); + assert_eq!(v, [11u8, 0, 0, 0]); + let mut w = [0u8; 11]; + assert!(storage::read(b":test", &mut w[..], 4).unwrap() >= 11); + assert_eq!(&w, b"Hello world"); + }); + } + + #[test] + fn clear_prefix_works() { + let mut t = BasicExternalities::new( + map![ + b":a".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), + b":abcd".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), + b":abc".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), + b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() + ], + map![], + ); + + t.execute_with(|| { + storage::clear_prefix(b":abc"); + + assert!(storage::get(b":a").is_some()); + assert!(storage::get(b":abdd").is_some()); + assert!(storage::get(b":abcd").is_none()); + assert!(storage::get(b":abc").is_none()); + }); + } +} diff --git a/core/sr-io/with_std.rs b/core/sr-io/with_std.rs deleted file mode 100644 index ef50433048e3d..0000000000000 --- a/core/sr-io/with_std.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017-2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -// Switch to this after PoC-3 -// pub use primitives::BlakeHasher; -pub use substrate_state_machine::{BasicExternalities, TestExternalities}; - -use std::collections::HashMap; - -use externalities::Externalities; - -/// Execute the given closure with global function available whose functionality routes into the -/// externalities `ext`. Forwards the value that the closure returns. -// NOTE: need a concrete hasher here due to limitations of the `environmental!` macro, otherwise a type param would have been fine I think. -pub fn with_externalities R>(ext: &mut dyn Externalities, f: F) -> R { - unimplemented!() -} - -/// A set of key value pairs for storage. -pub type StorageOverlay = HashMap, Vec>; - -/// A set of key value pairs for children storage; -pub type ChildrenStorageOverlay = HashMap, StorageOverlay>; - -/// Execute the given closure with global functions available whose functionality routes into -/// externalities that draw from and populate `storage` and `children_storage`. -/// Forwards the value that the closure returns. -pub fn with_storage R>( - storage: &mut (StorageOverlay, ChildrenStorageOverlay), - f: F -) -> R { - let mut alt_storage = Default::default(); - rstd::mem::swap(&mut alt_storage, storage); - - let mut ext = BasicExternalities::new(alt_storage.0, alt_storage.1); - // let r = ext::using(&mut ext, f); - - // *storage = ext.into_storages(); - - // r - - unimplemented!() -} - -#[cfg(test)] -mod std_tests { - use super::*; - use primitives::map; - - #[test] - fn storage_works() { - let mut t = BasicExternalities::default(); - assert!(set_and_run_with_externalities(&mut t, || { - assert_eq!(storage(b"hello"), None); - set_storage(b"hello", b"world"); - assert_eq!(storage(b"hello"), Some(b"world".to_vec())); - assert_eq!(storage(b"foo"), None); - set_storage(b"foo", &[1, 2, 3][..]); - true - })); - - t = BasicExternalities::new(map![b"foo".to_vec() => b"bar".to_vec()], map![]); - - assert!(!set_and_run_with_externalities(&mut t, || { - assert_eq!(storage(b"hello"), None); - assert_eq!(storage(b"foo"), Some(b"bar".to_vec())); - false - })); - } - - #[test] - fn read_storage_works() { - let mut t = BasicExternalities::new(map![ - b":test".to_vec() => b"\x0b\0\0\0Hello world".to_vec() - ], map![]); - - set_and_run_with_externalities(&mut t, || { - let mut v = [0u8; 4]; - assert!(read_storage(b":test", &mut v[..], 0).unwrap() >= 4); - assert_eq!(v, [11u8, 0, 0, 0]); - let mut w = [0u8; 11]; - assert!(read_storage(b":test", &mut w[..], 4).unwrap() >= 11); - assert_eq!(&w, b"Hello world"); - }); - } - - #[test] - fn clear_prefix_works() { - let mut t = BasicExternalities::new(map![ - b":a".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), - b":abcd".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), - b":abc".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), - b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() - ], map![]); - - set_and_run_with_externalities(&mut t, || { - clear_prefix(b":abc"); - - assert!(storage(b":a").is_some()); - assert!(storage(b":abdd").is_some()); - assert!(storage(b":abcd").is_none()); - assert!(storage(b":abc").is_none()); - }); - } -} diff --git a/core/sr-primitives/src/lib.rs b/core/sr-primitives/src/lib.rs index ad523f6b92a71..25ee023f88e7f 100644 --- a/core/sr-primitives/src/lib.rs +++ b/core/sr-primitives/src/lib.rs @@ -38,7 +38,7 @@ pub use paste; pub use app_crypto; #[cfg(feature = "std")] -pub use runtime_io::{StorageOverlay, ChildrenStorageOverlay}; +pub use primitives::storage::{StorageOverlay, ChildrenStorageOverlay}; use rstd::prelude::*; use rstd::convert::TryFrom; @@ -66,10 +66,7 @@ pub use app_crypto::RuntimeAppPublic; pub use primitives::RuntimeDebug; /// Re-export top-level arithmetic stuff. -pub use arithmetic::{ - Perquintill, Perbill, Permill, Percent, - Rational128, Fixed64 -}; +pub use arithmetic::{Perquintill, Perbill, Permill, Percent, Rational128, Fixed64}; /// Re-export 128 bit helpers. pub use arithmetic::helpers_128bit; /// Re-export big_uint stuff. diff --git a/core/state-machine/src/basic.rs b/core/state-machine/src/basic.rs index c2d1a0e3950d0..e758b3dd3b814 100644 --- a/core/state-machine/src/basic.rs +++ b/core/state-machine/src/basic.rs @@ -22,7 +22,10 @@ use hash_db::Hasher; use trie::{TrieConfiguration, default_child_trie_root}; use trie::trie_types::Layout; use primitives::{ - storage::{well_known_keys::is_child_storage_key, ChildStorageKey}, + storage::{ + well_known_keys::is_child_storage_key, ChildStorageKey, StorageOverlay, + ChildrenStorageOverlay + }, traits::Externalities, Blake2Hasher, hash::H256, }; use log::warn; @@ -30,16 +33,13 @@ use log::warn; /// Simple HashMap-based Externalities impl. #[derive(Debug)] pub struct BasicExternalities { - top: HashMap, Vec>, - children: HashMap, HashMap, Vec>>, + top: StorageOverlay, + children: ChildrenStorageOverlay, } impl BasicExternalities { /// Create a new instance of `BasicExternalities` - pub fn new( - top: HashMap, Vec>, - children: HashMap, HashMap, Vec>>, - ) -> Self { + pub fn new(top: StorageOverlay, children: ChildrenStorageOverlay) -> Self { BasicExternalities { top, children, @@ -58,6 +58,32 @@ impl BasicExternalities { ) { (self.top, self.children) } + + /// Execute the given closure `f` with the externalities set and initialized with `storage`. + /// + /// Returns the result of the closure and updates `storage` with all changes. + pub fn execute_with_storage( + storage: &mut (StorageOverlay, ChildrenStorageOverlay), + f: impl FnOnce() -> R, + ) -> R { + let mut ext = Self { + top: storage.0.drain().collect(), + children: storage.1.drain().collect(), + }; + + let r = ext.execute_with(f); + + *storage = ext.into_storages(); + + r + } + + /// Execute the given closure while `self` is set as externalities. + /// + /// Returns the result of the given closure. + pub fn execute_with(&mut self, f: impl FnOnce() -> R) -> R { + externalities::set_and_run_with_externalities(self, f) + } } impl PartialEq for BasicExternalities { diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 16ff62020b594..7ffac614c24c7 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -46,7 +46,6 @@ pub struct TestExternalities=Blake2Hasher, N: ChangesTrieBlo } impl, N: ChangesTrieBlockNumber> TestExternalities { - /// Get externalities implementation. pub fn ext(&mut self) -> Ext, ChangesTrieInMemoryStorage> { Ext::new( diff --git a/srml/support/Cargo.toml b/srml/support/Cargo.toml index 2b8c4cb9f98b8..518f1f23fbc7d 100644 --- a/srml/support/Cargo.toml +++ b/srml/support/Cargo.toml @@ -17,6 +17,7 @@ inherents = { package = "substrate-inherents", path = "../../core/inherents", de srml-support-procedural = { package = "srml-support-procedural", path = "./procedural" } paste = "0.1.6" once_cell = { version = "0.2.4", default-features = false, optional = true } +state-machine = { package = "substrate-state-machine", path = "../../core/state-machine", optional = true } bitmask = { version = "0.5.0", default-features = false } impl-trait-for-tuples = "0.1.2" @@ -36,6 +37,7 @@ std = [ "sr-primitives/std", "srml-metadata/std", "inherents/std", + "state-machine", ] nightly = [] strict = [] diff --git a/srml/support/procedural/src/storage/genesis_config/mod.rs b/srml/support/procedural/src/storage/genesis_config/mod.rs index 2d4d4af3861ed..43dc047255645 100644 --- a/srml/support/procedural/src/storage/genesis_config/mod.rs +++ b/srml/support/procedural/src/storage/genesis_config/mod.rs @@ -158,7 +158,7 @@ fn impl_build_storage( #scrate::sr_primitives::ChildrenStorageOverlay, ), ) -> std::result::Result<(), String> #fn_where_clause { - #scrate::with_storage(tuple_storage, || { + #scrate::BasicExternalities::execute_with_storage(tuple_storage, || { #( #builder_blocks )* Ok(()) }) diff --git a/srml/support/src/lib.rs b/srml/support/src/lib.rs index 2a9f66bd526e2..bc5899e5bb316 100644 --- a/srml/support/src/lib.rs +++ b/srml/support/src/lib.rs @@ -37,7 +37,7 @@ pub use once_cell; pub use paste; #[cfg(feature = "std")] #[doc(hidden)] -pub use runtime_io::with_storage; +pub use state_machine::BasicExternalities; #[doc(hidden)] pub use runtime_io::storage_root; #[doc(hidden)] diff --git a/srml/support/src/storage/storage_items.rs b/srml/support/src/storage/storage_items.rs index 1edf9c03db61a..bcac87df2f97e 100644 --- a/srml/support/src/storage/storage_items.rs +++ b/srml/support/src/storage/storage_items.rs @@ -254,9 +254,12 @@ macro_rules! __handle_wrap_internal { #[allow(dead_code)] mod tests { use crate::metadata::*; - use crate::metadata::StorageHasher; - use crate::rstd::marker::PhantomData; - use crate::codec::{Encode, Decode, EncodeLike}; + + use std::marker::PhantomData; + + use codec::{Encode, Decode, EncodeLike}; + + use state_machine::testing::TestExternalities; storage_items! { Value: b"a" => u32; @@ -265,7 +268,7 @@ mod tests { #[test] fn value() { - runtime_io::with_storage(&mut Default::default(), || { + TestExternalities::default().execute_with(|| { assert!(Value::get().is_none()); Value::put(&100_000); assert_eq!(Value::get(), Some(100_000)); @@ -276,7 +279,7 @@ mod tests { #[test] fn map() { - runtime_io::with_storage(&mut Default::default(), || { + TestExternalities::default().execute_with(|| { assert!(Map::get(&5).is_none()); Map::insert(&5, &[1; 32]); assert_eq!(Map::get(&5), Some([1; 32])); diff --git a/srml/support/test/tests/final_keys.rs b/srml/support/test/tests/final_keys.rs index 44a6b540a7a0e..099be25296f86 100644 --- a/srml/support/test/tests/final_keys.rs +++ b/srml/support/test/tests/final_keys.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use runtime_io::with_storage; use support::storage::unhashed; use codec::Encode; use support::{StorageDoubleMap, StorageLinkedMap, StorageMap, StorageValue}; +use state_machine::testing::TestExternalities; mod no_instance { use codec::{Encode, Decode, EncodeLike}; @@ -87,7 +87,7 @@ mod instance { #[test] fn final_keys_no_instance() { - with_storage(&mut Default::default(), || { + TestExternalities::default().execute_with(|| { no_instance::Value::put(1); assert_eq!(unhashed::get::(&runtime_io::twox_128(b"FinalKeysNone Value")), Some(1u32)); @@ -133,7 +133,7 @@ fn final_keys_no_instance() { #[test] fn final_keys_default_instance() { - with_storage(&mut Default::default(), || { + TestExternalities::default().execute_with(|| { >::put(1); assert_eq!(unhashed::get::(&runtime_io::twox_128(b"FinalKeysSome Value")), Some(1u32)); @@ -179,7 +179,7 @@ fn final_keys_default_instance() { #[test] fn final_keys_instance_2() { - with_storage(&mut Default::default(), || { + TestExternalities::default().execute_with(|| { >::put(1); assert_eq!( unhashed::get::(&runtime_io::twox_128(b"Instance2FinalKeysSome Value")), From 1c818ddd7bb63eedc64d0aaa874b6491ea815192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 27 Oct 2019 19:09:31 +0100 Subject: [PATCH 47/76] Make `srml-support` tests run again --- core/sr-io/src/lib.rs | 30 ++++++++++++----------- core/state-machine/src/lib.rs | 4 +-- srml/support/src/debug.rs | 6 ++--- srml/support/src/hash.rs | 2 +- srml/support/src/lib.rs | 2 +- srml/support/src/storage/child.rs | 26 +++++++++++++------- srml/support/src/storage/storage_items.rs | 2 +- srml/support/src/storage/unhashed.rs | 14 +++++------ srml/system/src/lib.rs | 6 ++--- srml/system/src/offchain.rs | 4 +-- 10 files changed, 53 insertions(+), 43 deletions(-) diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 1ad22f2e561b4..91b8fd9b2639f 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -380,27 +380,27 @@ pub trait Hashing { /// Conduct a 128-bit Blake2 hash. fn blake2_128(data: &[u8]) -> [u8; 16] { - blake2_128(data) + primitives::hashing::blake2_128(data) } /// Conduct a 256-bit Blake2 hash. fn blake2_256(data: &[u8]) -> [u8; 32] { - blake2_256(data) + primitives::hashing::blake2_256(data) } /// Conduct four XX hashes to give a 256-bit result. fn twox_256(data: &[u8]) -> [u8; 32] { - twox_256(data) + primitives::hashing::twox_256(data) } /// Conduct two XX hashes to give a 128-bit result. fn twox_128(data: &[u8]) -> [u8; 16] { - twox_128(data) + primitives::hashing::twox_128(data) } /// Conduct two XX hashes to give a 64-bit result. fn twox_64(data: &[u8]) -> [u8; 8] { - twox_64(data) + primitives::hashing::twox_64(data) } } @@ -607,20 +607,22 @@ trait Allocator { /// Interface that provides functions for logging from within the runtime. #[runtime_interface] -pub trait Log { +pub trait Logging { /// Request to print a log message on the host. /// /// Note that this will be only displayed if the host is enabled to display log messages with /// given level and target. /// /// Instead of using directly, prefer setting up `RuntimeLogger` and using `log` macros. - fn log(level: LogLevel, target: &str, message: &str) { - ::log::log!( - target: target, - ::log::Level::from(level), - "{}", - message, - ) + fn log(level: LogLevel, target: &str, message: &[u8]) { + if let Ok(message) = std::str::from_utf8(message) { + log::log!( + target: target, + log::Level::from(level), + "{}", + message, + ) + } } } @@ -685,7 +687,7 @@ pub type SubstrateHostFunctions = ( crypto::HostFunctions, hashing::HostFunctions, allocator::HostFunctions, - log::HostFunctions, + logging::HostFunctions, ); #[cfg(test)] diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index c3092367f0646..de1e666bc5cf8 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -83,7 +83,7 @@ pub enum ExecutionStrategy { NativeWhenPossible, /// Use the given wasm module. AlwaysWasm, - /// Run with both the wasm and the native variant (if compatible). Report any discrepency as an error. + /// Run with both the wasm and the native variant (if compatible). Report any discrepancy as an error. Both, /// First native, then if that fails or is not possible, wasm. NativeElseWasm, @@ -109,7 +109,7 @@ pub enum ExecutionManager { /// trusted to provide all storage or not (i.e. the light client cannot be trusted to provide /// for all storage queries since the storage entries it has come from an external node). AlwaysWasm(BackendTrustLevel), - /// Run with both the wasm and the native variant (if compatible). Call `F` in the case of any discrepency. + /// Run with both the wasm and the native variant (if compatible). Call `F` in the case of any discrepancy. Both(F), /// First native, then if that fails or is not possible, wasm. NativeElseWasm, diff --git a/srml/support/src/debug.rs b/srml/support/src/debug.rs index 1c4e463bf1234..2f4d923f99446 100644 --- a/srml/support/src/debug.rs +++ b/srml/support/src/debug.rs @@ -149,7 +149,7 @@ impl fmt::Write for Writer { impl Writer { /// Print the content of this `Writer` out. pub fn print(&self) { - runtime_io::print_utf8(&self.0) + runtime_io::misc::print_utf8(&self.0) } } @@ -198,9 +198,9 @@ impl log::Log for RuntimeLogger { let mut w = Writer::default(); let _ = core::write!(&mut w, "{}", record.args()); - runtime_io::log( + runtime_io::logging::log( record.level().into(), - record.target().as_bytes(), + record.target(), &w.0, ); } diff --git a/srml/support/src/hash.rs b/srml/support/src/hash.rs index cbd78f603241e..c2b63a84db346 100644 --- a/srml/support/src/hash.rs +++ b/srml/support/src/hash.rs @@ -18,7 +18,7 @@ use codec::Codec; use rstd::prelude::Vec; -use runtime_io::{blake2_128, blake2_256, twox_64, twox_128, twox_256}; +use runtime_io::hashing::{blake2_128, blake2_256, twox_64, twox_128, twox_256}; // This trait must be kept coherent with srml-support-procedural HasherKind usage pub trait Hashable: Sized { diff --git a/srml/support/src/lib.rs b/srml/support/src/lib.rs index bc5899e5bb316..97fbaf47aa221 100644 --- a/srml/support/src/lib.rs +++ b/srml/support/src/lib.rs @@ -39,7 +39,7 @@ pub use paste; #[doc(hidden)] pub use state_machine::BasicExternalities; #[doc(hidden)] -pub use runtime_io::storage_root; +pub use runtime_io::storage::root as storage_root; #[doc(hidden)] pub use sr_primitives::RuntimeDebug; diff --git a/srml/support/src/storage/child.rs b/srml/support/src/storage/child.rs index 1d6ee7a6f1a61..d43c2e896f3d8 100644 --- a/srml/support/src/storage/child.rs +++ b/srml/support/src/storage/child.rs @@ -26,7 +26,7 @@ use codec::{Codec, Encode, Decode}; /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. pub fn get(storage_key: &[u8], key: &[u8]) -> Option { - runtime_io::child_storage(storage_key, key).map(|v| { + runtime_io::storage::child_get(storage_key, key).map(|v| { Decode::decode(&mut &v[..]).expect("storage is not null, therefore must be a valid type") }) } @@ -45,13 +45,17 @@ pub fn get_or(storage_key: &[u8], key: &[u8], default_value: /// Return the value of the item in storage under `key`, or `default_value()` if there is no /// explicit entry. -pub fn get_or_else T>(storage_key: &[u8], key: &[u8], default_value: F) -> T { +pub fn get_or_else T>( + storage_key: &[u8], + key: &[u8], + default_value: F, +) -> T { get(storage_key, key).unwrap_or_else(default_value) } /// Put `value` in storage under `key`. pub fn put(storage_key: &[u8], key: &[u8], value: &T) { - value.using_encoded(|slice| runtime_io::set_child_storage(storage_key, key, slice)); + value.using_encoded(|slice| runtime_io::storage::child_set(storage_key, key, slice)); } /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise. @@ -77,31 +81,35 @@ pub fn take_or(storage_key: &[u8],key: &[u8], default_value: T /// Return the value of the item in storage under `key`, or `default_value()` if there is no /// explicit entry. Ensure there is no explicit entry on return. -pub fn take_or_else T>(storage_key: &[u8], key: &[u8], default_value: F) -> T { +pub fn take_or_else T>( + storage_key: &[u8], + key: &[u8], + default_value: F, +) -> T { take(storage_key, key).unwrap_or_else(default_value) } /// Check to see if `key` has an explicit entry in storage. pub fn exists(storage_key: &[u8], key: &[u8]) -> bool { - runtime_io::read_child_storage(storage_key, key, &mut [0;0][..], 0).is_some() + runtime_io::storage::child_read(storage_key, key, &mut [0;0][..], 0).is_some() } /// Remove all `storage_key` key/values pub fn kill_storage(storage_key: &[u8]) { - runtime_io::kill_child_storage(storage_key) + runtime_io::storage::child_storage_kill(storage_key) } /// Ensure `key` has no explicit entry in storage. pub fn kill(storage_key: &[u8], key: &[u8]) { - runtime_io::clear_child_storage(storage_key, key); + runtime_io::storage::child_clear(storage_key, key); } /// Get a Vec of bytes from storage. pub fn get_raw(storage_key: &[u8], key: &[u8]) -> Option> { - runtime_io::child_storage(storage_key, key) + runtime_io::storage::child_get(storage_key, key) } /// Put a raw byte slice into storage. pub fn put_raw(storage_key: &[u8], key: &[u8], value: &[u8]) { - runtime_io::set_child_storage(storage_key, key, value) + runtime_io::storage::child_set(storage_key, key, value) } diff --git a/srml/support/src/storage/storage_items.rs b/srml/support/src/storage/storage_items.rs index bcac87df2f97e..7a5b326d29c39 100644 --- a/srml/support/src/storage/storage_items.rs +++ b/srml/support/src/storage/storage_items.rs @@ -259,7 +259,7 @@ mod tests { use codec::{Encode, Decode, EncodeLike}; - use state_machine::testing::TestExternalities; + use runtime_io::TestExternalities; storage_items! { Value: b"a" => u32; diff --git a/srml/support/src/storage/unhashed.rs b/srml/support/src/storage/unhashed.rs index 6397fd39fcd2d..803a512b2a2c4 100644 --- a/srml/support/src/storage/unhashed.rs +++ b/srml/support/src/storage/unhashed.rs @@ -21,7 +21,7 @@ use codec::{Encode, Decode}; /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. pub fn get(key: &[u8]) -> Option { - runtime_io::storage(key).map(|val| { + runtime_io::storage::get(key).map(|val| { Decode::decode(&mut &val[..]).expect("storage is not null, therefore must be a valid type") }) } @@ -46,7 +46,7 @@ pub fn get_or_else T>(key: &[u8], default_valu /// Put `value` in storage under `key`. pub fn put(key: &[u8], value: &T) { - value.using_encoded(|slice| runtime_io::set_storage(key, slice)); + value.using_encoded(|slice| runtime_io::storage::set(key, slice)); } /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise. @@ -78,25 +78,25 @@ pub fn take_or_else T>(key: &[u8], default_val /// Check to see if `key` has an explicit entry in storage. pub fn exists(key: &[u8]) -> bool { - runtime_io::read_storage(key, &mut [0;0][..], 0).is_some() + runtime_io::storage::read(key, &mut [0;0][..], 0).is_some() } /// Ensure `key` has no explicit entry in storage. pub fn kill(key: &[u8]) { - runtime_io::clear_storage(key); + runtime_io::storage::clear(key); } /// Ensure keys with the given `prefix` have no entries in storage. pub fn kill_prefix(prefix: &[u8]) { - runtime_io::clear_prefix(prefix); + runtime_io::storage::clear_prefix(prefix); } /// Get a Vec of bytes from storage. pub fn get_raw(key: &[u8]) -> Option> { - runtime_io::storage(key) + runtime_io::storage::get(key) } /// Put a raw byte slice into storage. pub fn put_raw(key: &[u8], value: &[u8]) { - runtime_io::set_storage(key, value) + runtime_io::storage::set(key, value) } diff --git a/srml/system/src/lib.rs b/srml/system/src/lib.rs index 87c38096ab934..c81cb7f70cd5b 100644 --- a/srml/system/src/lib.rs +++ b/srml/system/src/lib.rs @@ -424,11 +424,11 @@ decl_storage! { build(|config: &GenesisConfig| { use codec::Encode; - runtime_io::set_storage(well_known_keys::CODE, &config.code); - runtime_io::set_storage(well_known_keys::EXTRINSIC_INDEX, &0u32.encode()); + runtime_io::storage::set(well_known_keys::CODE, &config.code); + runtime_io::storage::set(well_known_keys::EXTRINSIC_INDEX, &0u32.encode()); if let Some(ref changes_trie_config) = config.changes_trie_config { - runtime_io::set_storage( + runtime_io::storage::set( well_known_keys::CHANGES_TRIE_CONFIG, &changes_trie_config.encode(), ); diff --git a/srml/system/src/offchain.rs b/srml/system/src/offchain.rs index 11f7e234f7197..3d44746bfd1f4 100644 --- a/srml/system/src/offchain.rs +++ b/srml/system/src/offchain.rs @@ -114,7 +114,7 @@ where ::create_transaction::(call, public, id, expected) .ok_or(())?; let xt = Self::Extrinsic::new(call, Some(signature_data)).ok_or(())?; - runtime_io::submit_transaction(xt.encode()) + runtime_io::offchain::submit_transaction(xt.encode()) } } @@ -129,7 +129,7 @@ pub trait SubmitUnsignedTransaction { /// and `Err` if transaction was rejected from the pool. fn submit_unsigned(call: impl Into) -> Result<(), ()> { let xt = Self::Extrinsic::new(call.into(), None).ok_or(())?; - runtime_io::submit_transaction(xt.encode()) + runtime_io::offchain::submit_transaction(xt.encode()) } } From 60b1fff03d3e624ea0f3cd807f2fa7d0e817a2ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 27 Oct 2019 21:08:04 +0100 Subject: [PATCH 48/76] More compilation error fixes --- Cargo.lock | 1 + core/application-crypto/src/traits.rs | 2 +- core/executor/src/host_interface.rs | 4 +- core/primitives/src/sr25519.rs | 3 -- core/sr-primitives/src/traits.rs | 2 +- core/test-runtime/src/lib.rs | 10 +--- core/test-runtime/src/system.rs | 5 +- srml/babe/src/lib.rs | 2 +- srml/contracts/src/account_db.rs | 2 +- srml/contracts/src/lib.rs | 4 +- srml/contracts/src/rent.rs | 6 +-- srml/im-online/src/lib.rs | 14 +++--- srml/support/test/Cargo.toml | 4 +- srml/support/test/tests/final_keys.rs | 68 +++++++++++++-------------- srml/support/test/tests/instance.rs | 2 +- 15 files changed, 61 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index abefb21410ac2..43112bb6f91b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4537,6 +4537,7 @@ dependencies = [ "srml-support 2.0.0", "substrate-inherents 2.0.0", "substrate-primitives 2.0.0", + "substrate-state-machine 2.0.0", "trybuild 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/core/application-crypto/src/traits.rs b/core/application-crypto/src/traits.rs index 2196eb6b16526..6969d6d8389f4 100644 --- a/core/application-crypto/src/traits.rs +++ b/core/application-crypto/src/traits.rs @@ -19,7 +19,7 @@ use primitives::crypto::Pair; use codec::Codec; use primitives::crypto::{KeyTypeId, CryptoType, IsWrappedBy, Public}; -use rstd::fmt::Debug; +use rstd::{fmt::Debug, vec::Vec}; /// An application-specific key. pub trait AppKey: 'static + Send + Sync + Sized + CryptoType + Clone { diff --git a/core/executor/src/host_interface.rs b/core/executor/src/host_interface.rs index 9f209abba39bf..b4de06c2616cd 100644 --- a/core/executor/src/host_interface.rs +++ b/core/executor/src/host_interface.rs @@ -201,10 +201,8 @@ impl_wasm_host_interface! { let target_str = std::str::from_utf8(&target) .map_err(|_| "Target invalid utf8 in ext_log")?; - let message_str = std::str::from_utf8(&message) - .map_err(|_| "Message invalid utf8 in ext_log")?; - runtime_io::log::log(level.into(), &target_str, &message_str); + runtime_io::logging::log(level.into(), &target_str, &message); Ok(()) } diff --git a/core/primitives/src/sr25519.rs b/core/primitives/src/sr25519.rs index 6a721aec9c81e..1c82de3d22a86 100644 --- a/core/primitives/src/sr25519.rs +++ b/core/primitives/src/sr25519.rs @@ -130,9 +130,6 @@ impl std::fmt::Display for Public { } } -#[cfg(not(feature = "std"))] -use core as std; - impl rstd::fmt::Debug for Public { #[cfg(feature = "std")] fn fmt(&self, f: &mut rstd::fmt::Formatter) -> rstd::fmt::Result { diff --git a/core/sr-primitives/src/traits.rs b/core/sr-primitives/src/traits.rs index afffee7f09a74..ae0dbb54cf8ca 100644 --- a/core/sr-primitives/src/traits.rs +++ b/core/sr-primitives/src/traits.rs @@ -1137,7 +1137,7 @@ macro_rules! impl_opaque_keys { /// The generated key pairs are stored in the keystore. /// /// Returns the concatenated SCALE encoded public keys. - pub fn generate(seed: Option<&str>) -> $crate::rstd::vec::Vec { + pub fn generate(seed: Option<$crate::rstd::vec::Vec>) -> $crate::rstd::vec::Vec { let keys = Self{ $( $field: <$type as $crate::app_crypto::RuntimeAppPublic>::generate_pair(seed), diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 45d86f891c594..af8880f559591 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -25,15 +25,7 @@ pub mod system; use rstd::{prelude::*, marker::PhantomData}; use codec::{Encode, Decode, Input, Error}; -use primitives::{ - Blake2Hasher, - OpaqueMetadata, - RuntimeDebug, - testing::{ - ED25519, - SR25519, - } -}; +use primitives::{Blake2Hasher, OpaqueMetadata, RuntimeDebug, testing::{ED25519, SR25519}}; use app_crypto::{ed25519, sr25519, RuntimeAppPublic}; pub use app_crypto; use trie_db::{TrieMut, Trie}; diff --git a/core/test-runtime/src/system.rs b/core/test-runtime/src/system.rs index fc4de0ce4b911..b959c0d70f107 100644 --- a/core/test-runtime/src/system.rs +++ b/core/test-runtime/src/system.rs @@ -18,7 +18,10 @@ //! and depositing logs. use rstd::prelude::*; -use runtime_io::{storage_root, storage_changes_root, blake2_256}; +use runtime_io::{ + storage::root as storage_root, storage::changes_root as storage_changes_root, + hashing::blake2_256, +}; use runtime_support::storage::{self, StorageValue, StorageMap}; use runtime_support::storage_items; use sr_primitives::{ diff --git a/srml/babe/src/lib.rs b/srml/babe/src/lib.rs index ee2d66fbe690f..41c9eee06c19c 100644 --- a/srml/babe/src/lib.rs +++ b/srml/babe/src/lib.rs @@ -593,7 +593,7 @@ fn compute_randomness( s.extend_from_slice(&vrf_output[..]); } - runtime_io::blake2_256(&s) + runtime_io::hashing::blake2_256(&s) } impl ProvideInherent for Module { diff --git a/srml/contracts/src/account_db.rs b/srml/contracts/src/account_db.rs index 50bd1fd40e97d..5aa3a64fd9bc8 100644 --- a/srml/contracts/src/account_db.rs +++ b/srml/contracts/src/account_db.rs @@ -24,7 +24,7 @@ use crate::exec::StorageKey; use rstd::cell::RefCell; use rstd::collections::btree_map::{BTreeMap, Entry}; use rstd::prelude::*; -use runtime_io::blake2_256; +use runtime_io::hashing::blake2_256; use sr_primitives::traits::{Bounded, Zero}; use support::traits::{Currency, Get, Imbalance, SignedImbalance, UpdateBalanceOutcome}; use support::{storage::child, StorageMap}; diff --git a/srml/contracts/src/lib.rs b/srml/contracts/src/lib.rs index 8eaef951bae46..5586a245b81aa 100644 --- a/srml/contracts/src/lib.rs +++ b/srml/contracts/src/lib.rs @@ -111,7 +111,7 @@ use serde::{Serialize, Deserialize}; use primitives::crypto::UncheckedFrom; use rstd::{prelude::*, marker::PhantomData, fmt::Debug}; use codec::{Codec, Encode, Decode}; -use runtime_io::blake2_256; +use runtime_io::hashing::blake2_256; use sr_primitives::{ traits::{Hash, StaticLookup, Zero, MaybeSerializeDeserialize, Member, SignedExtension}, weights::DispatchInfo, @@ -773,7 +773,7 @@ impl Module { let tombstone = >::new( // This operation is cheap enough because last_write (delta not included) // is not this block as it has been checked earlier. - &runtime_io::child_storage_root(&origin_contract.trie_id)[..], + &runtime_io::storage::child_root(&origin_contract.trie_id)[..], code_hash, ); diff --git a/srml/contracts/src/rent.rs b/srml/contracts/src/rent.rs index ecc5f610313c0..9f907d8393987 100644 --- a/srml/contracts/src/rent.rs +++ b/srml/contracts/src/rent.rs @@ -99,7 +99,7 @@ fn try_evict_or_and_pay_rent( if balance < subsistence_threshold { // The contract cannot afford to leave a tombstone, so remove the contract info altogether. >::remove(account); - runtime_io::kill_child_storage(&contract.trie_id); + runtime_io::storage::child_storage_kill(&contract.trie_id); return (RentOutcome::Evicted, None); } @@ -146,7 +146,7 @@ fn try_evict_or_and_pay_rent( // threshold, so it leaves a tombstone. // Note: this operation is heavy. - let child_storage_root = runtime_io::child_storage_root(&contract.trie_id); + let child_storage_root = runtime_io::storage::child_root(&contract.trie_id); let tombstone = >::new( &child_storage_root[..], @@ -155,7 +155,7 @@ fn try_evict_or_and_pay_rent( let tombstone_info = ContractInfo::Tombstone(tombstone); >::insert(account, &tombstone_info); - runtime_io::kill_child_storage(&contract.trie_id); + runtime_io::storage::child_storage_kill(&contract.trie_id); return (RentOutcome::Evicted, Some(tombstone_info)); } diff --git a/srml/im-online/src/lib.rs b/srml/im-online/src/lib.rs index 82b866d32bc86..a05d304a103b1 100644 --- a/srml/im-online/src/lib.rs +++ b/srml/im-online/src/lib.rs @@ -270,7 +270,7 @@ decl_module! { debug::RuntimeLogger::init(); // Only send messages if we are a potential validator. - if runtime_io::is_validator() { + if runtime_io::offchain::is_validator() { Self::offchain(now); } } @@ -333,7 +333,8 @@ impl Module { .map(|location| (index as u32, &local_keys[location])) }) { - let network_state = runtime_io::network_state().map_err(|_| OffchainErr::NetworkState)?; + let network_state = runtime_io::offchain::network_state() + .map_err(|_| OffchainErr::NetworkState)?; let heartbeat_data = Heartbeat { block_number, network_state, @@ -370,10 +371,10 @@ impl Module { done, gossipping_at, }; - runtime_io::local_storage_compare_and_set( + runtime_io::offchain::local_storage_compare_and_set( StorageKind::PERSISTENT, DB_KEY, - curr_worker_status.as_ref().map(Vec::as_slice), + curr_worker_status, &enc.encode() ) } @@ -386,8 +387,7 @@ impl Module { done, gossipping_at, }; - runtime_io::local_storage_set( - StorageKind::PERSISTENT, DB_KEY, &enc.encode()); + runtime_io::offchain::local_storage_set(StorageKind::PERSISTENT, DB_KEY, &enc.encode()); } // Checks if a heartbeat gossip already occurred at this block number. @@ -397,7 +397,7 @@ impl Module { now: T::BlockNumber, next_gossip: T::BlockNumber, ) -> Result<(Option>, bool), OffchainErr> { - let last_gossip = runtime_io::local_storage_get(StorageKind::PERSISTENT, DB_KEY); + let last_gossip = runtime_io::offchain::local_storage_get(StorageKind::PERSISTENT, DB_KEY); match last_gossip { Some(last) => { let worker_status: WorkerStatus = Decode::decode(&mut &last[..]) diff --git a/srml/support/test/Cargo.toml b/srml/support/test/Cargo.toml index f40fef9eade60..cbca8a859c187 100644 --- a/srml/support/test/Cargo.toml +++ b/srml/support/test/Cargo.toml @@ -8,11 +8,12 @@ edition = "2018" serde = { version = "1.0.101", default-features = false, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } runtime-io ={ package = "sr-io", path = "../../../core/sr-io", default-features = false } +state-machine ={ package = "substrate-state-machine", path = "../../../core/state-machine", optional = true } support = { package = "srml-support", version = "2", path = "../", default-features = false } inherents = { package = "substrate-inherents", path = "../../../core/inherents", default-features = false } sr-primitives = { package = "sr-primitives", path = "../../../core/sr-primitives", default-features = false } primitives = { package = "substrate-primitives", path = "../../../core/primitives", default-features = false } -trybuild = "1.0.14" +trybuild = "1.0.17" pretty_assertions = "0.6.1" [features] @@ -25,4 +26,5 @@ std = [ "inherents/std", "primitives/std", "sr-primitives/std", + "state-machine", ] diff --git a/srml/support/test/tests/final_keys.rs b/srml/support/test/tests/final_keys.rs index 099be25296f86..56049ecf378cd 100644 --- a/srml/support/test/tests/final_keys.rs +++ b/srml/support/test/tests/final_keys.rs @@ -17,7 +17,7 @@ use support::storage::unhashed; use codec::Encode; use support::{StorageDoubleMap, StorageLinkedMap, StorageMap, StorageValue}; -use state_machine::testing::TestExternalities; +use runtime_io::{TestExternalities, hashing}; mod no_instance { use codec::{Encode, Decode, EncodeLike}; @@ -89,44 +89,44 @@ mod instance { fn final_keys_no_instance() { TestExternalities::default().execute_with(|| { no_instance::Value::put(1); - assert_eq!(unhashed::get::(&runtime_io::twox_128(b"FinalKeysNone Value")), Some(1u32)); + assert_eq!(unhashed::get::(&hashing::twox_128(b"FinalKeysNone Value")), Some(1u32)); no_instance::Map::insert(1, 2); let mut k = b"FinalKeysNone Map".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::blake2_256(&k)), Some(2u32)); no_instance::Map2::insert(1, 2); let mut k = b"FinalKeysNone Map2".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::twox_128(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::twox_128(&k)), Some(2u32)); let head = b"head of FinalKeysNone LinkedMap".to_vec(); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&head)), None); + assert_eq!(unhashed::get::(&hashing::blake2_256(&head)), None); no_instance::LinkedMap::insert(1, 2); let mut k = b"FinalKeysNone LinkedMap".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&k)), Some(2u32)); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&head)), Some(1u32)); + assert_eq!(unhashed::get::(&hashing::blake2_256(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::blake2_256(&head)), Some(1u32)); no_instance::LinkedMap2::insert(1, 2); let mut k = b"FinalKeysNone LinkedMap2".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::twox_128(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::twox_128(&k)), Some(2u32)); no_instance::DoubleMap::insert(&1, &2, &3); let mut k = b"FinalKeysNone DoubleMap".to_vec(); k.extend(1u32.encode()); - let mut k = runtime_io::blake2_256(&k).to_vec(); - k.extend(&runtime_io::blake2_256(&2u32.encode())); + let mut k = hashing::blake2_256(&k).to_vec(); + k.extend(&hashing::blake2_256(&2u32.encode())); assert_eq!(unhashed::get::(&k), Some(3u32)); no_instance::DoubleMap2::insert(&1, &2, &3); let mut k = b"FinalKeysNone DoubleMap2".to_vec(); k.extend(1u32.encode()); - let mut k = runtime_io::twox_128(&k).to_vec(); - k.extend(&runtime_io::blake2_128(&2u32.encode())); + let mut k = hashing::twox_128(&k).to_vec(); + k.extend(&hashing::blake2_128(&2u32.encode())); assert_eq!(unhashed::get::(&k), Some(3u32)); }); } @@ -135,44 +135,44 @@ fn final_keys_no_instance() { fn final_keys_default_instance() { TestExternalities::default().execute_with(|| { >::put(1); - assert_eq!(unhashed::get::(&runtime_io::twox_128(b"FinalKeysSome Value")), Some(1u32)); + assert_eq!(unhashed::get::(&hashing::twox_128(b"FinalKeysSome Value")), Some(1u32)); >::insert(1, 2); let mut k = b"FinalKeysSome Map".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::blake2_256(&k)), Some(2u32)); >::insert(1, 2); let mut k = b"FinalKeysSome Map2".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::twox_128(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::twox_128(&k)), Some(2u32)); let head = b"head of FinalKeysSome LinkedMap".to_vec(); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&head)), None); + assert_eq!(unhashed::get::(&hashing::blake2_256(&head)), None); >::insert(1, 2); let mut k = b"FinalKeysSome LinkedMap".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&k)), Some(2u32)); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&head)), Some(1u32)); + assert_eq!(unhashed::get::(&hashing::blake2_256(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::blake2_256(&head)), Some(1u32)); < instance::LinkedMap2>::insert(1, 2); let mut k = b"FinalKeysSome LinkedMap2".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::twox_128(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::twox_128(&k)), Some(2u32)); >::insert(&1, &2, &3); let mut k = b"FinalKeysSome DoubleMap".to_vec(); k.extend(1u32.encode()); - let mut k = runtime_io::blake2_256(&k).to_vec(); - k.extend(&runtime_io::blake2_256(&2u32.encode())); + let mut k = hashing::blake2_256(&k).to_vec(); + k.extend(&hashing::blake2_256(&2u32.encode())); assert_eq!(unhashed::get::(&k), Some(3u32)); >::insert(&1, &2, &3); let mut k = b"FinalKeysSome DoubleMap2".to_vec(); k.extend(1u32.encode()); - let mut k = runtime_io::twox_128(&k).to_vec(); - k.extend(&runtime_io::blake2_128(&2u32.encode())); + let mut k = hashing::twox_128(&k).to_vec(); + k.extend(&hashing::blake2_128(&2u32.encode())); assert_eq!(unhashed::get::(&k), Some(3u32)); }); } @@ -182,46 +182,46 @@ fn final_keys_instance_2() { TestExternalities::default().execute_with(|| { >::put(1); assert_eq!( - unhashed::get::(&runtime_io::twox_128(b"Instance2FinalKeysSome Value")), + unhashed::get::(&hashing::twox_128(b"Instance2FinalKeysSome Value")), Some(1u32) ); >::insert(1, 2); let mut k = b"Instance2FinalKeysSome Map".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::blake2_256(&k)), Some(2u32)); >::insert(1, 2); let mut k = b"Instance2FinalKeysSome Map2".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::twox_128(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::twox_128(&k)), Some(2u32)); let head = b"head of Instance2FinalKeysSome LinkedMap".to_vec(); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&head)), None); + assert_eq!(unhashed::get::(&hashing::blake2_256(&head)), None); >::insert(1, 2); let mut k = b"Instance2FinalKeysSome LinkedMap".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&k)), Some(2u32)); - assert_eq!(unhashed::get::(&runtime_io::blake2_256(&head)), Some(1u32)); + assert_eq!(unhashed::get::(&hashing::blake2_256(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::blake2_256(&head)), Some(1u32)); >::insert(1, 2); let mut k = b"Instance2FinalKeysSome LinkedMap2".to_vec(); k.extend(1u32.encode()); - assert_eq!(unhashed::get::(&runtime_io::twox_128(&k)), Some(2u32)); + assert_eq!(unhashed::get::(&hashing::twox_128(&k)), Some(2u32)); >::insert(&1, &2, &3); let mut k = b"Instance2FinalKeysSome DoubleMap".to_vec(); k.extend(1u32.encode()); - let mut k = runtime_io::blake2_256(&k).to_vec(); - k.extend(&runtime_io::blake2_256(&2u32.encode())); + let mut k = hashing::blake2_256(&k).to_vec(); + k.extend(&hashing::blake2_256(&2u32.encode())); assert_eq!(unhashed::get::(&k), Some(3u32)); >::insert(&1, &2, &3); let mut k = b"Instance2FinalKeysSome DoubleMap2".to_vec(); k.extend(1u32.encode()); - let mut k = runtime_io::twox_128(&k).to_vec(); - k.extend(&runtime_io::blake2_128(&2u32.encode())); + let mut k = hashing::twox_128(&k).to_vec(); + k.extend(&hashing::blake2_128(&2u32.encode())); assert_eq!(unhashed::get::(&k), Some(3u32)); }); } diff --git a/srml/support/test/tests/instance.rs b/srml/support/test/tests/instance.rs index dd05cb1d8c6e3..54ad1db0417c1 100644 --- a/srml/support/test/tests/instance.rs +++ b/srml/support/test/tests/instance.rs @@ -302,7 +302,7 @@ fn new_test_ext() -> runtime_io::TestExternalities { #[test] fn storage_instance_independance() { let mut storage = (std::collections::HashMap::new(), std::collections::HashMap::new()); - runtime_io::with_storage(&mut storage, || { + state_machine::BasicExternalities::execute_with_storage(&mut storage, || { module2::Value::::put(0); module2::Value::::put(0); module2::Value::::put(0); From a6c6791d97f2f9bc7b91bd0ad7bfd0ee4d11a95c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 28 Oct 2019 09:37:47 +0100 Subject: [PATCH 49/76] Use correct doc syntax --- .../proc-macro/src/bare_function_interface.rs | 2 ++ .../proc-macro/src/host_function_interface.rs | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index 53068ad53822f..8587932c9cc55 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -87,10 +87,12 @@ fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result< <#ty as #crate_::wasm::FromFFIValue>::from_ffi_value(result) } }; + let attrs = &method.attrs; Ok( quote! { #[cfg(not(feature = "std"))] + #( #attrs )* pub fn #function_name( #( #args, )* ) #return_value { // Generate all wrapped ffi values. #( diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/host_function_interface.rs index 95f1afd9d7a42..d2621f237f02f 100644 --- a/core/runtime-interface/proc-macro/src/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/host_function_interface.rs @@ -94,7 +94,7 @@ fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) - Ok( quote! { - #[doc(#doc_string)] + #[doc = #doc_string] pub unsafe fn #function ( #( #arg_names: <#arg_types as #crate_::RIType>::FFIType ),* ) #output { @@ -103,6 +103,7 @@ fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) - use super::*; extern "C" { + /// The extern function. pub fn #function ( #( #arg_names2: <#arg_types2 as #crate_::RIType>::FFIType ),* ) #output; @@ -136,7 +137,7 @@ fn generate_extern_host_exchangeable_function( quote! { #[cfg(not(feature = "std"))] #[allow(non_upper_case_globals)] - #[doc(#doc_string)] + #[doc = #doc_string] pub static #function : #crate_::wasm::ExchangeableFunction< unsafe fn ( #( <#arg_types as #crate_::RIType>::FFIType ),* ) #output > = #crate_::wasm::ExchangeableFunction::new(extern_host_function_impls::#function); From 2d6b05ea975e10f38c5ca7770ea110644a921d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 28 Oct 2019 09:46:09 +0100 Subject: [PATCH 50/76] Fix test-runtime --- Cargo.lock | 1 + core/test-runtime/Cargo.toml | 2 ++ core/test-runtime/src/lib.rs | 17 +++++++++-------- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43112bb6f91b3..f7a3be3097622 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5779,6 +5779,7 @@ dependencies = [ "substrate-keyring 2.0.0", "substrate-offchain-primitives 2.0.0", "substrate-primitives 2.0.0", + "substrate-runtime-interface 2.0.0", "substrate-session 2.0.0", "substrate-state-machine 2.0.0", "substrate-test-runtime-client 2.0.0", diff --git a/core/test-runtime/Cargo.toml b/core/test-runtime/Cargo.toml index 4e7c3f8bca450..189a46eb7eebe 100644 --- a/core/test-runtime/Cargo.toml +++ b/core/test-runtime/Cargo.toml @@ -26,6 +26,7 @@ substrate-trie = { path = "../trie", default-features = false } trie-db = { version = "0.15.2", default-features = false } memory-db = { version = "0.15.2", default-features = false } offchain-primitives = { package = "substrate-offchain-primitives", path = "../offchain/primitives", default-features = false} +runtime-interface = { package = "substrate-runtime-interface", path = "../runtime-interface", default-features = false} executive = { package = "srml-executive", path = "../../srml/executive", default-features = false } cfg-if = "0.1.10" srml-babe = { path = "../../srml/babe", default-features = false } @@ -72,4 +73,5 @@ std = [ "srml-system-rpc-runtime-api/std", "app-crypto/std", "session/std", + "runtime-interface/std", ] diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index af8880f559591..7f7368efd8592 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -408,7 +408,8 @@ fn benchmark_add_one(i: u64) -> u64 { /// The `benchmark_add_one` function as function pointer. #[cfg(not(feature = "std"))] -static BENCHMARK_ADD_ONE: runtime_io::ExchangeableFunction u64> = runtime_io::ExchangeableFunction::new(benchmark_add_one); +static BENCHMARK_ADD_ONE: runtime_interface::wasm::ExchangeableFunction u64> = + runtime_interface::wasm::ExchangeableFunction::new(benchmark_add_one); fn code_using_trie() -> u64 { let pairs = [ @@ -844,7 +845,7 @@ cfg_if! { impl offchain_primitives::OffchainWorkerApi for Runtime { fn offchain_worker(block: u64) { let ex = Extrinsic::IncludeData(block.encode()); - runtime_io::submit_transaction(ex.encode()).unwrap() + runtime_io::offchain::submit_transaction(ex.encode()).unwrap() } } @@ -895,10 +896,10 @@ fn test_sr25519_crypto() -> (sr25519::AppSignature, sr25519::AppPublic) { fn test_read_storage() { const KEY: &[u8] = b":read_storage"; - runtime_io::set_storage(KEY, b"test"); + runtime_io::storage::set(KEY, b"test"); let mut v = [0u8; 4]; - let r = runtime_io::read_storage( + let r = runtime_io::storage::read( KEY, &mut v, 0 @@ -907,7 +908,7 @@ fn test_read_storage() { assert_eq!(&v, b"test"); let mut v = [0u8; 4]; - let r = runtime_io::read_storage(KEY, &mut v, 8); + let r = runtime_io::storage::read(KEY, &mut v, 8); assert_eq!(r, Some(4)); assert_eq!(&v, &[0, 0, 0, 0]); } @@ -915,10 +916,10 @@ fn test_read_storage() { fn test_read_child_storage() { const CHILD_KEY: &[u8] = b":child_storage:default:read_child_storage"; const KEY: &[u8] = b":read_child_storage"; - runtime_io::set_child_storage(CHILD_KEY, KEY, b"test"); + runtime_io::storage::child_set(CHILD_KEY, KEY, b"test"); let mut v = [0u8; 4]; - let r = runtime_io::read_child_storage( + let r = runtime_io::storage::child_read( CHILD_KEY, KEY, &mut v, @@ -928,7 +929,7 @@ fn test_read_child_storage() { assert_eq!(&v, b"test"); let mut v = [0u8; 4]; - let r = runtime_io::read_child_storage(CHILD_KEY, KEY, &mut v, 8); + let r = runtime_io::storage::child_read(CHILD_KEY, KEY, &mut v, 8); assert_eq!(r, Some(4)); assert_eq!(&v, &[0, 0, 0, 0]); } From 9328a2da956765f28b86e7a2320a5928b49b784f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 29 Oct 2019 09:26:24 +0100 Subject: [PATCH 51/76] Fix compilation --- core/executor/runtime-test/src/lib.rs | 57 +++++++++++-------- core/executor/src/lib.rs | 2 +- core/executor/src/sandbox.rs | 9 ++- core/executor/src/wasmi_execution.rs | 9 ++- core/rpc/src/state/tests.rs | 2 +- core/sr-io/src/lib.rs | 14 ++++- .../src/generic/unchecked_extrinsic.rs | 2 +- core/sr-primitives/src/traits.rs | 4 +- core/test-runtime/src/genesismap.rs | 2 +- core/test-runtime/src/lib.rs | 2 +- core/test-runtime/src/system.rs | 2 +- node-template/runtime/src/lib.rs | 3 +- node/cli/Cargo.toml | 2 +- node/cli/src/factory_impl.rs | 2 +- node/runtime/src/lib.rs | 1 - node/testing/Cargo.toml | 2 +- node/testing/src/keyring.rs | 2 +- srml/im-online/src/tests.rs | 2 +- 18 files changed, 73 insertions(+), 46 deletions(-) diff --git a/core/executor/runtime-test/src/lib.rs b/core/executor/runtime-test/src/lib.rs index 61eca8dd4e23f..086481e8170e6 100644 --- a/core/executor/runtime-test/src/lib.rs +++ b/core/executor/runtime-test/src/lib.rs @@ -10,8 +10,8 @@ use rstd::{vec::Vec, vec}; #[cfg(not(feature = "std"))] use runtime_io::{ - set_storage, storage, clear_prefix, blake2_128, blake2_256, - twox_128, twox_256, ed25519_verify, sr25519_verify, + storage, hashing::{blake2_128, blake2_256, twox_128, twox_256}, + crypto::{ed25519_verify, sr25519_verify}, }; #[cfg(not(feature = "std"))] use sr_primitives::{print, traits::{BlakeTwo256, Hash}}; @@ -21,20 +21,20 @@ use primitives::{ed25519, sr25519}; primitives::wasm_export_functions! { fn test_data_in(input: Vec) -> Vec { print("set_storage"); - set_storage(b"input", &input); + storage::set(b"input", &input); print("storage"); - let foo = storage(b"foo").unwrap(); + let foo = storage::get(b"foo").unwrap(); print("set_storage"); - set_storage(b"baz", &foo); + storage::set(b"baz", &foo); print("finished!"); b"all ok!".to_vec() } fn test_clear_prefix(input: Vec) -> Vec { - clear_prefix(&input); + storage::clear_prefix(&input); b"all ok!".to_vec() } @@ -142,40 +142,49 @@ primitives::wasm_export_functions! { fn test_offchain_local_storage() -> bool { let kind = primitives::offchain::StorageKind::PERSISTENT; - assert_eq!(runtime_io::local_storage_get(kind, b"test"), None); - runtime_io::local_storage_set(kind, b"test", b"asd"); - assert_eq!(runtime_io::local_storage_get(kind, b"test"), Some(b"asd".to_vec())); - - let res = runtime_io::local_storage_compare_and_set(kind, b"test", Some(b"asd"), b""); - assert_eq!(runtime_io::local_storage_get(kind, b"test"), Some(b"".to_vec())); + assert_eq!(runtime_io::offchain::local_storage_get(kind, b"test"), None); + runtime_io::offchain::local_storage_set(kind, b"test", b"asd"); + assert_eq!(runtime_io::offchain::local_storage_get(kind, b"test"), Some(b"asd".to_vec())); + + let res = runtime_io::offchain::local_storage_compare_and_set( + kind, + b"test", + Some(b"asd".to_vec()), + b"", + ); + assert_eq!(runtime_io::offchain::local_storage_get(kind, b"test"), Some(b"".to_vec())); res } fn test_offchain_local_storage_with_none() { let kind = primitives::offchain::StorageKind::PERSISTENT; - assert_eq!(runtime_io::local_storage_get(kind, b"test"), None); + assert_eq!(runtime_io::offchain::local_storage_get(kind, b"test"), None); - let res = runtime_io::local_storage_compare_and_set(kind, b"test", None, b"value"); + let res = runtime_io::offchain::local_storage_compare_and_set(kind, b"test", None, b"value"); assert_eq!(res, true); - assert_eq!(runtime_io::local_storage_get(kind, b"test"), Some(b"value".to_vec())); + assert_eq!(runtime_io::offchain::local_storage_get(kind, b"test"), Some(b"value".to_vec())); } fn test_offchain_http() -> bool { use primitives::offchain::HttpRequestStatus; let run = || -> Option<()> { - let id = runtime_io::http_request_start("POST", "http://localhost:12345", &[]).ok()?; - runtime_io::http_request_add_header(id, "X-Auth", "test").ok()?; - runtime_io::http_request_write_body(id, &[1, 2, 3, 4], None).ok()?; - runtime_io::http_request_write_body(id, &[], None).ok()?; - let status = runtime_io::http_response_wait(&[id], None); + let id = runtime_io::offchain::http_request_start( + "POST", + "http://localhost:12345", + &[], + ).ok()?; + runtime_io::offchain::http_request_add_header(id, "X-Auth", "test").ok()?; + runtime_io::offchain::http_request_write_body(id, &[1, 2, 3, 4], None).ok()?; + runtime_io::offchain::http_request_write_body(id, &[], None).ok()?; + let status = runtime_io::offchain::http_response_wait(&[id], None); assert!(status == vec![HttpRequestStatus::Finished(200)], "Expected Finished(200) status."); - let headers = runtime_io::http_response_headers(id); + let headers = runtime_io::offchain::http_response_headers(id); assert_eq!(headers, vec![(b"X-Auth".to_vec(), b"hello".to_vec())]); let mut buffer = vec![0; 64]; - let read = runtime_io::http_response_read_body(id, &mut buffer, None).ok()?; + let read = runtime_io::offchain::http_response_read_body(id, &mut buffer, None).ok()?; assert_eq!(read, 3); - assert_eq!(&buffer[0..read], &[1, 2, 3]); - let read = runtime_io::http_response_read_body(id, &mut buffer, None).ok()?; + assert_eq!(&buffer[0..read as usize], &[1, 2, 3]); + let read = runtime_io::offchain::http_response_read_body(id, &mut buffer, None).ok()?; assert_eq!(read, 0); Some(()) diff --git a/core/executor/src/lib.rs b/core/executor/src/lib.rs index 9b7ca542c9676..71f8659ccf42d 100644 --- a/core/executor/src/lib.rs +++ b/core/executor/src/lib.rs @@ -100,7 +100,7 @@ mod tests { fn call_in_interpreted_wasm_works() { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let res = call_in_wasm( + let res = call_in_wasm::<_, runtime_io::SubstrateHostFunctions>( "test_empty_return", &[], WasmExecutionMethod::Interpreted, diff --git a/core/executor/src/sandbox.rs b/core/executor/src/sandbox.rs index e1e9e0db95263..e7470e6e77c49 100644 --- a/core/executor/src/sandbox.rs +++ b/core/executor/src/sandbox.rs @@ -592,6 +592,7 @@ mod tests { use state_machine::TestExternalities as CoreTestExternalities; use wabt; use runtime_test::WASM_BINARY; + use wasm_interface::HostFunctions; type TestExternalities = CoreTestExternalities; @@ -602,8 +603,12 @@ mod tests { method: &str, data: &[u8], ) -> Result> { - let mut instance = wasmi_execution::create_instance(ext, code, heap_pages) - .map_err(|err| err.to_string())?; + let mut instance = wasmi_execution::create_instance( + ext, + code, + heap_pages, + runtime_io::SubstrateHostFunctions::host_functions(), + ).map_err(|err| err.to_string())?; instance.call(ext, method, data) } diff --git a/core/executor/src/wasmi_execution.rs b/core/executor/src/wasmi_execution.rs index 16fa6deb2cec0..a7dde5850a899 100644 --- a/core/executor/src/wasmi_execution.rs +++ b/core/executor/src/wasmi_execution.rs @@ -649,6 +649,7 @@ mod tests { use substrate_offchain::testing; use trie::{TrieConfiguration, trie_types::Layout}; use codec::{Encode, Decode}; + use wasm_interface::HostFunctions; type TestExternalities = CoreTestExternalities; @@ -659,8 +660,12 @@ mod tests { method: &str, data: &[u8], ) -> Result, Error> { - let mut instance = create_instance(ext, code, heap_pages) - .map_err(|err| err.to_string())?; + let mut instance = create_instance( + ext, + code, + heap_pages, + runtime_io::SubstrateHostFunctions::host_functions(), + ).map_err(|err| err.to_string())?; instance.call(ext, method, data) } diff --git a/core/rpc/src/state/tests.rs b/core/rpc/src/state/tests.rs index 5dfa234337afa..2e3690f3058a2 100644 --- a/core/rpc/src/state/tests.rs +++ b/core/rpc/src/state/tests.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use assert_matches::assert_matches; use futures::stream::Stream; use primitives::storage::well_known_keys; -use sr_io::blake2_256; +use sr_io::hashing::blake2_256; use test_client::{ prelude::*, consensus::BlockOrigin, diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 91b8fd9b2639f..31a63ea90b209 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -528,7 +528,7 @@ pub trait Offchain { /// Write a chunk of request body. /// - /// Writing an empty chunks finalises the request. + /// Writing an empty chunks finalizes the request. /// Passing `None` as deadline blocks forever. /// /// Returns an error in case deadline is reached or the chunk couldn't be written. @@ -596,12 +596,20 @@ pub trait Offchain { trait Allocator { /// Malloc the given number of bytes and return the pointer to the allocated memory location. fn malloc(&mut self, size: u32) -> Pointer { - self.allocate_memory(size).expect("Failed to allocate memory") + match self.allocate_memory(size) { + Ok(res) => res, + Err(e) => { + log::warn!(target: "runtime-interface", "Failed to allocate memory: {}", e); + Pointer::new(0) + } + } } /// Free the given pointer. fn free(&mut self, ptr: Pointer) { - self.deallocate_memory(ptr).expect("Failed to free allocated memory") + if let Err(e) = self.deallocate_memory(ptr) { + log::warn!(target: "runtime-interface", "Failed to free a given pointer: {}", e); + } } } diff --git a/core/sr-primitives/src/generic/unchecked_extrinsic.rs b/core/sr-primitives/src/generic/unchecked_extrinsic.rs index a66c53cbecb7b..4aa21d85921da 100644 --- a/core/sr-primitives/src/generic/unchecked_extrinsic.rs +++ b/core/sr-primitives/src/generic/unchecked_extrinsic.rs @@ -282,7 +282,7 @@ where #[cfg(test)] mod tests { use super::*; - use runtime_io::blake2_256; + use runtime_io::hashing::blake2_256; use crate::codec::{Encode, Decode}; use crate::traits::{SignedExtension, IdentifyAccount, IdentityLookup}; use serde::{Serialize, Deserialize}; diff --git a/core/sr-primitives/src/traits.rs b/core/sr-primitives/src/traits.rs index ae0dbb54cf8ca..82a4c2c94e6a7 100644 --- a/core/sr-primitives/src/traits.rs +++ b/core/sr-primitives/src/traits.rs @@ -1140,7 +1140,9 @@ macro_rules! impl_opaque_keys { pub fn generate(seed: Option<$crate::rstd::vec::Vec>) -> $crate::rstd::vec::Vec { let keys = Self{ $( - $field: <$type as $crate::app_crypto::RuntimeAppPublic>::generate_pair(seed), + $field: <$type as $crate::app_crypto::RuntimeAppPublic>::generate_pair( + seed.clone(), + ), )* }; $crate::codec::Encode::encode(&keys) diff --git a/core/test-runtime/src/genesismap.rs b/core/test-runtime/src/genesismap.rs index a3abf235ff1ca..46ca645c502e7 100644 --- a/core/test-runtime/src/genesismap.rs +++ b/core/test-runtime/src/genesismap.rs @@ -17,7 +17,7 @@ //! Tool for creating the genesis block. use std::collections::HashMap; -use runtime_io::{blake2_256, twox_128}; +use runtime_io::hashing::{blake2_256, twox_128}; use super::{AuthorityId, AccountId, WASM_BINARY}; use codec::{Encode, KeyedVec, Joiner}; use primitives::{ChangesTrieConfiguration, map, storage::well_known_keys}; diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 7f7368efd8592..e5bd00794a654 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -629,7 +629,7 @@ cfg_if! { impl offchain_primitives::OffchainWorkerApi for Runtime { fn offchain_worker(block: u64) { let ex = Extrinsic::IncludeData(block.encode()); - runtime_io::submit_transaction(ex.encode()).unwrap(); + runtime_io::offchain::submit_transaction(ex.encode()).unwrap(); } } diff --git a/core/test-runtime/src/system.rs b/core/test-runtime/src/system.rs index b959c0d70f107..083d80c0ad595 100644 --- a/core/test-runtime/src/system.rs +++ b/core/test-runtime/src/system.rs @@ -319,7 +319,7 @@ mod tests { use crate::{Header, Transfer, WASM_BINARY}; use primitives::{NeverNativeValue, map, traits::CodeExecutor}; use substrate_executor::{NativeExecutor, WasmExecutionMethod, native_executor_instance}; - use runtime_io::twox_128; + use runtime_io::hashing::twox_128; // Declare an instance of the native executor dispatch for the test runtime. native_executor_instance!( diff --git a/node-template/runtime/src/lib.rs b/node-template/runtime/src/lib.rs index acc5143ffe5d2..a7eae834865db 100644 --- a/node-template/runtime/src/lib.rs +++ b/node-template/runtime/src/lib.rs @@ -346,7 +346,7 @@ impl_runtime_apis! { fn slot_duration() -> u64 { Aura::slot_duration() } - + fn authorities() -> Vec { Aura::authorities() } @@ -354,7 +354,6 @@ impl_runtime_apis! { impl substrate_session::SessionKeys for Runtime { fn generate_session_keys(seed: Option>) -> Vec { - let seed = seed.as_ref().map(|s| rstd::str::from_utf8(&s).expect("Seed is an utf8 string")); opaque::SessionKeys::generate(seed) } } diff --git a/node/cli/Cargo.toml b/node/cli/Cargo.toml index 5fa92360d62cf..1236a4b58794b 100644 --- a/node/cli/Cargo.toml +++ b/node/cli/Cargo.toml @@ -14,7 +14,7 @@ exit-future = "0.1.4" jsonrpc-core = "13.2.0" cli = { package = "substrate-cli", path = "../../core/cli" } codec = { package = "parity-scale-codec", version = "1.0.0" } -sr-io = { path = "../../core/sr-io" } +runtime-io = { package = "sr-io", path = "../../core/sr-io" } client = { package = "substrate-client", path = "../../core/client" } primitives = { package = "substrate-primitives", path = "../../core/primitives" } inherents = { package = "substrate-inherents", path = "../../core/inherents" } diff --git a/node/cli/src/factory_impl.rs b/node/cli/src/factory_impl.rs index 48fb7b237f1e6..aaf74a0602082 100644 --- a/node/cli/src/factory_impl.rs +++ b/node/cli/src/factory_impl.rs @@ -247,7 +247,7 @@ fn sign( let payload = (xt.function, extra.clone(), additional_signed); let signature = payload.using_encoded(|b| { if b.len() > 256 { - key.sign(&sr_io::blake2_256(b)) + key.sign(&runtime_io::hashing::blake2_256(b)) } else { key.sign(b) } diff --git a/node/runtime/src/lib.rs b/node/runtime/src/lib.rs index 2c02a8be179d6..60f678f9310f1 100644 --- a/node/runtime/src/lib.rs +++ b/node/runtime/src/lib.rs @@ -693,7 +693,6 @@ impl_runtime_apis! { impl substrate_session::SessionKeys for Runtime { fn generate_session_keys(seed: Option>) -> Vec { - let seed = seed.as_ref().map(|s| rstd::str::from_utf8(&s).expect("Seed is an utf8 string")); SessionKeys::generate(seed) } } diff --git a/node/testing/Cargo.toml b/node/testing/Cargo.toml index 8a4c08ed11bd2..a3382cd53540e 100644 --- a/node/testing/Cargo.toml +++ b/node/testing/Cargo.toml @@ -17,7 +17,7 @@ node-primitives = { path = "../primitives" } node-runtime = { path = "../runtime" } codec = { package = "parity-scale-codec", version = "1.0.0" } primitives = { package = "substrate-primitives", path = "../../core/primitives" } -sr-io = { path = "../../core/sr-io" } +runtime-io = { package = "sr-io", path = "../../core/sr-io" } sr-primitives = { path = "../../core/sr-primitives" } runtime_support = { package = "srml-support", path = "../../srml/support" } session = { package = "srml-session", path = "../../srml/session" } diff --git a/node/testing/src/keyring.rs b/node/testing/src/keyring.rs index ca44a53880fc1..618c813fb529a 100644 --- a/node/testing/src/keyring.rs +++ b/node/testing/src/keyring.rs @@ -83,7 +83,7 @@ pub fn sign(xt: CheckedExtrinsic, version: u32, genesis_hash: [u8; 32]) -> Unche let key = AccountKeyring::from_account_id(&signed).unwrap(); let signature = payload.using_encoded(|b| { if b.len() > 256 { - key.sign(&sr_io::blake2_256(b)) + key.sign(&runtime_io::hashing::blake2_256(b)) } else { key.sign(b) } diff --git a/srml/im-online/src/tests.rs b/srml/im-online/src/tests.rs index 652d751281294..5721fdf28ca01 100644 --- a/srml/im-online/src/tests.rs +++ b/srml/im-online/src/tests.rs @@ -208,7 +208,7 @@ fn should_generate_heartbeats() { assert_eq!(heartbeat, Heartbeat { block_number: 2, - network_state: runtime_io::network_state().unwrap(), + network_state: runtime_io::offchain::network_state().unwrap(), session_index: 2, authority_index: 2, }); From 57c4ed98d8faff39bf467ff923351996838c5565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 29 Oct 2019 14:12:50 +0100 Subject: [PATCH 52/76] Catch native panics when executing the wasm runtime As with the native runtime, we now catch all native panics when we execute the wasm runtime. The panics inside the wasm runtime were already catched before by the wasm executor automatically, but any panic in the host functions could bring down the node. The recent switch to execute the native counterpart of the host function in `sr-io`, makes this change required. The native `sr-io` functions just `panic` when something is not provided or any other error occured. --- core/executor/src/host_interface.rs | 107 +++++---------------------- core/executor/src/native_executor.rs | 69 +++++++++++++---- core/executor/src/wasm_runtime.rs | 27 +++++-- core/executor/src/wasmi_execution.rs | 13 +++- core/wasm-interface/src/lib.rs | 2 +- 5 files changed, 102 insertions(+), 116 deletions(-) diff --git a/core/executor/src/host_interface.rs b/core/executor/src/host_interface.rs index 7d87d93333fd2..4d1515d76999b 100644 --- a/core/executor/src/host_interface.rs +++ b/core/executor/src/host_interface.rs @@ -18,10 +18,8 @@ //! //! These are the host functions callable from within the Substrate runtime. -use crate::error::{Error, Result}; - use codec::Encode; -use std::{convert::TryFrom, str, panic}; +use std::{convert::TryFrom, str}; use primitives::{ blake2_128, blake2_256, twox_64, twox_128, twox_256, ed25519, sr25519, Blake2Hasher, Pair, crypto::KeyTypeId, offchain, @@ -211,10 +209,7 @@ impl_wasm_host_interface! { .map_err(|_| "Invalid attempt to determine key in ext_set_storage")?; let value = context.read_memory(value_data, value_len) .map_err(|_| "Invalid attempt to determine value in ext_set_storage")?; - with_external_storage(move || - Ok(runtime_io::set_storage(&key, &value)) - )?; - Ok(()) + Ok(runtime_io::set_storage(&key, &value)) } ext_set_child_storage( @@ -232,10 +227,7 @@ impl_wasm_host_interface! { let value = context.read_memory(value_data, value_len) .map_err(|_| "Invalid attempt to determine value in ext_set_child_storage")?; - with_external_storage(move || - Ok(runtime_io::set_child_storage(&storage_key, &key, &value)) - )?; - Ok(()) + Ok(runtime_io::set_child_storage(&storage_key, &key, &value)) } ext_clear_child_storage( @@ -249,27 +241,19 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_clear_child_storage")?; - with_external_storage(move || - Ok(runtime_io::clear_child_storage(&storage_key, &key)) - )?; - Ok(()) + Ok(runtime_io::clear_child_storage(&storage_key, &key)) } ext_clear_storage(key_data: Pointer, key_len: WordSize) { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_clear_storage")?; - with_external_storage(move || - Ok(runtime_io::clear_storage(&key)) - )?; - Ok(()) + Ok(runtime_io::clear_storage(&key)) } ext_exists_storage(key_data: Pointer, key_len: WordSize) -> u32 { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_exists_storage")?; - with_external_storage(move || - Ok(if runtime_io::exists_storage(&key) { 1 } else { 0 }) - ) + Ok(if runtime_io::exists_storage(&key) { 1 } else { 0 }) } ext_exists_child_storage( @@ -283,18 +267,13 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_exists_child_storage")?; - with_external_storage(move || - Ok(if runtime_io::exists_child_storage(&storage_key, &key) { 1 } else { 0 }) - ) + Ok(if runtime_io::exists_child_storage(&storage_key, &key) { 1 } else { 0 }) } ext_clear_prefix(prefix_data: Pointer, prefix_len: WordSize) { let prefix = context.read_memory(prefix_data, prefix_len) .map_err(|_| "Invalid attempt to determine prefix in ext_clear_prefix")?; - with_external_storage(move || - Ok(runtime_io::clear_prefix(&prefix)) - )?; - Ok(()) + Ok(runtime_io::clear_prefix(&prefix)) } ext_clear_child_prefix( @@ -307,21 +286,13 @@ impl_wasm_host_interface! { .map_err(|_| "Invalid attempt to determine storage_key in ext_clear_child_prefix")?; let prefix = context.read_memory(prefix_data, prefix_len) .map_err(|_| "Invalid attempt to determine prefix in ext_clear_child_prefix")?; - with_external_storage(move || - Ok(runtime_io::clear_child_prefix(&storage_key, &prefix)) - )?; - - Ok(()) + Ok(runtime_io::clear_child_prefix(&storage_key, &prefix)) } ext_kill_child_storage(storage_key_data: Pointer, storage_key_len: WordSize) { let storage_key = context.read_memory(storage_key_data, storage_key_len) .map_err(|_| "Invalid attempt to determine storage_key in ext_kill_child_storage")?; - with_external_storage(move || - Ok(runtime_io::kill_child_storage(&storage_key)) - )?; - - Ok(()) + Ok(runtime_io::kill_child_storage(&storage_key)) } ext_get_allocated_storage( @@ -331,11 +302,8 @@ impl_wasm_host_interface! { ) -> Pointer { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_get_allocated_storage")?; - let maybe_value = with_external_storage(move || - Ok(runtime_io::storage(&key)) - )?; - if let Some(value) = maybe_value { + if let Some(value) = runtime_io::storage(&key) { let offset = context.allocate_memory(value.len() as u32)?; context.write_memory(offset, &value) .map_err(|_| "Invalid attempt to set memory in ext_get_allocated_storage")?; @@ -361,11 +329,7 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to determine key in ext_get_allocated_child_storage")?; - let maybe_value = with_external_storage(move || - Ok(runtime_io::child_storage(&storage_key, &key)) - )?; - - if let Some(value) = maybe_value { + if let Some(value) = runtime_io::child_storage(&storage_key, &key) { let offset = context.allocate_memory(value.len() as u32)?; context.write_memory(offset, &value) .map_err(|_| "Invalid attempt to set memory in ext_get_allocated_child_storage")?; @@ -388,11 +352,8 @@ impl_wasm_host_interface! { ) -> WordSize { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to get key in ext_get_storage_into")?; - let maybe_value = with_external_storage(move || - Ok(runtime_io::storage(&key)) - )?; - if let Some(value) = maybe_value { + if let Some(value) = runtime_io::storage(&key) { let data = &value[value.len().min(value_offset as usize)..]; let written = std::cmp::min(value_len as usize, data.len()); context.write_memory(value_data, &data[..written]) @@ -417,11 +378,7 @@ impl_wasm_host_interface! { let key = context.read_memory(key_data, key_len) .map_err(|_| "Invalid attempt to get key in ext_get_child_storage_into")?; - let maybe_value = with_external_storage(move || - Ok(runtime_io::child_storage(&storage_key, &key)) - )?; - - if let Some(value) = maybe_value { + if let Some(value) = runtime_io::child_storage(&storage_key, &key) { let data = &value[value.len().min(value_offset as usize)..]; let written = std::cmp::min(value_len as usize, data.len()); context.write_memory(value_data, &data[..written]) @@ -433,12 +390,8 @@ impl_wasm_host_interface! { } ext_storage_root(result: Pointer) { - let r = with_external_storage(move || - Ok(runtime_io::storage_root()) - )?; - context.write_memory(result, r.as_ref()) - .map_err(|_| "Invalid attempt to set memory in ext_storage_root")?; - Ok(()) + context.write_memory(result, runtime_io::storage_root().as_ref()) + .map_err(|_| "Invalid attempt to set memory in ext_storage_root".into()) } ext_child_storage_root( @@ -448,9 +401,7 @@ impl_wasm_host_interface! { ) -> Pointer { let storage_key = context.read_memory(storage_key_data, storage_key_len) .map_err(|_| "Invalid attempt to determine storage_key in ext_child_storage_root")?; - let value = with_external_storage(move || - Ok(runtime_io::child_storage_root(&storage_key)) - )?; + let value = runtime_io::child_storage_root(&storage_key); let offset = context.allocate_memory(value.len() as u32)?; context.write_memory(offset, &value) @@ -468,11 +419,8 @@ impl_wasm_host_interface! { let mut parent_hash = [0u8; 32]; context.read_memory_into(parent_hash_data, &mut parent_hash[..]) .map_err(|_| "Invalid attempt to get parent_hash in ext_storage_changes_root")?; - let r = with_external_storage(move || - Ok(runtime_io::storage_changes_root(parent_hash)) - )?; - if let Some(r) = r { + if let Some(r) = runtime_io::storage_changes_root(parent_hash) { context.write_memory(result, &r[..]) .map_err(|_| "Invalid attempt to set memory in ext_storage_changes_root")?; Ok(1) @@ -1146,25 +1094,6 @@ impl ReadPrimitive for &mut dyn FunctionContext { } } -/// Execute closure that access external storage. -/// -/// All panics that happen within closure are captured and transformed into -/// runtime error. This requires special panic handler mode to be enabled -/// during the call (see `panic_handler::AbortGuard::never_abort`). -/// If this mode isn't enabled, then all panics within externalities are -/// leading to process abort. -fn with_external_storage(f: F) -> std::result::Result - where - F: panic::UnwindSafe + FnOnce() -> Result -{ - // it is safe beause basic methods of StorageExternalities are guaranteed to touch only - // its internal state + we should discard it on error - panic::catch_unwind(move || f()) - .map_err(|_| Error::Runtime) - .and_then(|result| result) - .map_err(|err| format!("{}", err)) -} - fn deadline_to_timestamp(deadline: u64) -> Option { if deadline == 0 { None diff --git a/core/executor/src/native_executor.rs b/core/executor/src/native_executor.rs index 7323703ed9656..082f0ba2adc33 100644 --- a/core/executor/src/native_executor.rs +++ b/core/executor/src/native_executor.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::{result, cell::RefCell, panic::UnwindSafe}; +use std::{result, cell::RefCell, panic::{UnwindSafe, AssertUnwindSafe}}; use crate::error::{Error, Result}; use crate::wasm_runtime::{RuntimesCache, WasmExecutionMethod, WasmRuntime}; use crate::RuntimeInfo; @@ -30,7 +30,7 @@ thread_local! { /// Default num of pages for the heap const DEFAULT_HEAP_PAGES: u64 = 1024; -fn safe_call(f: F) -> Result +pub(crate) fn safe_call(f: F) -> Result where F: UnwindSafe + FnOnce() -> U { // Substrate uses custom panic hook that terminates process on panic. Disable termination for the native call. @@ -65,7 +65,7 @@ pub trait NativeExecutionDispatch: Send + Sync { #[derive(Debug)] pub struct NativeExecutor { /// Dummy field to avoid the compiler complaining about us not using `D`. - _dummy: ::std::marker::PhantomData, + _dummy: std::marker::PhantomData, /// Method used to execute fallback Wasm code. fallback_method: WasmExecutionMethod, /// Native runtime version info. @@ -92,15 +92,45 @@ impl NativeExecutor { } } + /// Execute the given closure `f` with the latest runtime (based on the `CODE` key in `ext`). + /// + /// The closure `f` is expected to return `Err(_)` when there happened a `panic!` in native code + /// while executing the runtime in Wasm. If a `panic!` occurred, the runtime is invalidated to + /// prevent any poisoned state. Native runtime execution does not need to report back + /// any `panic!`. + /// + /// # Safety + /// + /// `runtime` and `ext` are given as `AssertUnwindSafe` to the closure. As described above, the + /// runtime is invalidated on any `panic!` to prevent a poisoned state. `ext` is already + /// implicitly handled as unwind safe, as we store it in a global variable while executing the + /// native runtime. fn with_runtime( &self, ext: &mut E, - f: impl for <'a> FnOnce(&'a mut dyn WasmRuntime, &'a mut E) -> Result, + f: impl for<'a> FnOnce( + AssertUnwindSafe<&'a mut (dyn WasmRuntime + 'static)>, + AssertUnwindSafe<&'a mut E>, + ) -> Result>, ) -> Result where E: Externalities { RUNTIMES_CACHE.with(|cache| { let mut cache = cache.borrow_mut(); - let runtime = cache.fetch_runtime(ext, self.fallback_method, self.default_heap_pages)?; - f(runtime, ext) + let (runtime, code_hash) = cache.fetch_runtime( + ext, + self.fallback_method, + self.default_heap_pages, + )?; + + let runtime = AssertUnwindSafe(runtime); + let ext = AssertUnwindSafe(ext); + + match f(runtime, ext) { + Ok(res) => res, + Err(e) => { + cache.invalidate_runtime(self.fallback_method, code_hash); + Err(e) + } + } }) } } @@ -125,7 +155,7 @@ impl RuntimeInfo for NativeExecutor { &self, ext: &mut E, ) -> Option { - match self.with_runtime(ext, |runtime, _ext| Ok(runtime.version())) { + match self.with_runtime(ext, |runtime, _ext| Ok(Ok(runtime.version()))) { Ok(version) => version, Err(e) => { warn!(target: "executor", "Failed to fetch runtime: {:?}", e); @@ -141,8 +171,8 @@ impl CodeExecutor for NativeExecutor { fn call < E: Externalities, - R:Decode + Encode + PartialEq, - NC: FnOnce() -> result::Result + UnwindSafe + R: Decode + Encode + PartialEq, + NC: FnOnce() -> result::Result + UnwindSafe, >( &self, ext: &mut E, @@ -152,7 +182,7 @@ impl CodeExecutor for NativeExecutor { native_call: Option, ) -> (Result>, bool){ let mut used_native = false; - let result = self.with_runtime(ext, |runtime, ext| { + let result = self.with_runtime(ext, |mut runtime, mut ext| { let onchain_version = runtime.version(); match ( use_native, @@ -170,9 +200,16 @@ impl CodeExecutor for NativeExecutor { .as_ref() .map_or_else(||"".into(), |v| format!("{}", v)) ); - runtime.call(ext, method, data).map(NativeOrEncoded::Encoded) + + safe_call( + move || runtime.call(&mut **ext, method, data).map(NativeOrEncoded::Encoded) + ) } - (false, _, _) => runtime.call(ext, method, data).map(NativeOrEncoded::Encoded), + (false, _, _) => { + safe_call( + move || runtime.call(&mut **ext, method, data).map(NativeOrEncoded::Encoded) + ) + }, (true, true, Some(call)) => { trace!( target: "executor", @@ -184,11 +221,13 @@ impl CodeExecutor for NativeExecutor { ); used_native = true; - with_native_environment(ext, move || (call)()) + let res = with_native_environment(&mut **ext, move || (call)()) .and_then(|r| r .map(NativeOrEncoded::Native) .map_err(|s| Error::ApiError(s.to_string())) - ) + ); + + Ok(res) } _ => { trace!( @@ -199,7 +238,7 @@ impl CodeExecutor for NativeExecutor { ); used_native = true; - D::dispatch(ext, method, data).map(NativeOrEncoded::Encoded) + Ok(D::dispatch(&mut **ext, method, data).map(NativeOrEncoded::Encoded)) } } }); diff --git a/core/executor/src/wasm_runtime.rs b/core/executor/src/wasm_runtime.rs index 8d2291fe04893..29a6c51e39d3b 100644 --- a/core/executor/src/wasm_runtime.rs +++ b/core/executor/src/wasm_runtime.rs @@ -23,7 +23,7 @@ use crate::error::{Error, WasmError}; use crate::wasmi_execution; use log::{trace, warn}; use codec::Decode; -use primitives::{storage::well_known_keys, traits::Externalities}; +use primitives::{storage::well_known_keys, traits::Externalities, H256}; use runtime_version::RuntimeVersion; use std::{collections::hash_map::{Entry, HashMap}}; @@ -99,9 +99,8 @@ impl RuntimesCache { /// /// # Return value /// - /// If no error occurred a tuple `(wasmi::ModuleRef, Option)` is - /// returned. `RuntimeVersion` is contained if the call to `Core_version` returned - /// a version. + /// If no error occurred a tuple `(&mut WasmRuntime, H256)` is + /// returned. `H256` is the hash of the runtime code. /// /// In case of failure one of two errors can be returned: /// @@ -114,7 +113,7 @@ impl RuntimesCache { ext: &mut E, wasm_method: WasmExecutionMethod, default_heap_pages: u64, - ) -> Result<&mut (dyn WasmRuntime + 'static), Error> { + ) -> Result<(&mut (dyn WasmRuntime + 'static), H256), Error> { let code_hash = ext .original_storage_hash(well_known_keys::CODE) .ok_or(Error::InvalidCode("`CODE` not found in storage.".into()))?; @@ -131,7 +130,7 @@ impl RuntimesCache { if !cached_runtime.update_heap_pages(heap_pages) { trace!( target: "runtimes_cache", - "heap_pages were changed. Reinstantiating the instance" + "heap_pages were changed. Reinstantiating the instance", ); *result = create_wasm_runtime(ext, wasm_method, heap_pages); if let Err(ref err) = result { @@ -152,9 +151,23 @@ impl RuntimesCache { }; result.as_mut() - .map(|runtime| runtime.as_mut()) + .map(|runtime| (runtime.as_mut(), code_hash)) .map_err(|ref e| Error::InvalidCode(format!("{:?}", e))) } + + /// Invalidate the runtime for the given `wasm_method` and `code_hash`. + /// + /// Invalidation of a runtime is useful when there was a `panic!` in native while executing it. + /// The `panic!` maybe have brought the runtime into a poisoned state and so, it is better to + /// invalidate this runtime instance. + pub fn invalidate_runtime( + &mut self, + wasm_method: WasmExecutionMethod, + code_hash: H256, + ) { + // Just remove the instance, it will be re-created the next time it is requested. + self.instances.remove(&(wasm_method, code_hash.into())); + } } /// Create a wasm runtime with the given `code`. diff --git a/core/executor/src/wasmi_execution.rs b/core/executor/src/wasmi_execution.rs index 2b832b490641d..a83729b4b6c8d 100644 --- a/core/executor/src/wasmi_execution.rs +++ b/core/executor/src/wasmi_execution.rs @@ -16,7 +16,7 @@ //! Implementation of a Wasm runtime using the Wasmi interpreter. -use std::{str, mem}; +use std::{str, mem, panic::AssertUnwindSafe}; use wasmi::{ Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder, ModuleRef, memory_units::Pages, RuntimeValue::{I32, I64, self}, @@ -625,9 +625,14 @@ pub fn create_instance(ext: &mut E, code: &[u8], heap_pages: u ", ); - let version = call_in_wasm_module(ext, &instance, "Core_version", &[]) - .ok() - .and_then(|v| RuntimeVersion::decode(&mut v.as_slice()).ok()); + let mut ext = AssertUnwindSafe(ext); + let call_instance = AssertUnwindSafe(&instance); + let version = crate::native_executor::safe_call( + move || call_in_wasm_module(&mut **ext, *call_instance, "Core_version", &[]) + .ok() + .and_then(|v| RuntimeVersion::decode(&mut v.as_slice()).ok()) + ).map_err(WasmError::Instantiation)?; + Ok(WasmiRuntime { instance, version, diff --git a/core/wasm-interface/src/lib.rs b/core/wasm-interface/src/lib.rs index b3cbde556eea0..1e67260638900 100644 --- a/core/wasm-interface/src/lib.rs +++ b/core/wasm-interface/src/lib.rs @@ -180,7 +180,7 @@ pub trait Function { fn execute( &self, context: &mut dyn FunctionContext, - args: &mut dyn Iterator, + args: &mut dyn Iterator, ) -> Result>; } From 52fce3baaa2692280a9d5921f6b08cd799ea8ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 30 Oct 2019 16:51:55 +0100 Subject: [PATCH 53/76] Fix compilation --- core/executor/src/wasmi_execution.rs | 11 ++++++++--- core/wasm-interface/src/lib.rs | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/core/executor/src/wasmi_execution.rs b/core/executor/src/wasmi_execution.rs index 0ff603450f907..b30f8411bd21c 100644 --- a/core/executor/src/wasmi_execution.rs +++ b/core/executor/src/wasmi_execution.rs @@ -612,10 +612,15 @@ pub fn create_instance( let mut ext = AssertUnwindSafe(ext); let call_instance = AssertUnwindSafe(&instance); + let host_functions_slice = host_functions.as_slice(); let version = crate::native_executor::safe_call( - move || call_in_wasm_module(&mut **ext, *call_instance, "Core_version", &[], &host_functions) - .ok() - .and_then(|v| RuntimeVersion::decode(&mut v.as_slice()).ok()) + move || call_in_wasm_module( + &mut **ext, + *call_instance, + "Core_version", + &[], + host_functions_slice, + ).ok().and_then(|v| RuntimeVersion::decode(&mut v.as_slice()).ok()) ).map_err(WasmError::Instantiation)?; Ok(WasmiRuntime { diff --git a/core/wasm-interface/src/lib.rs b/core/wasm-interface/src/lib.rs index 9c6ed76843c9a..8b91facd75608 100644 --- a/core/wasm-interface/src/lib.rs +++ b/core/wasm-interface/src/lib.rs @@ -177,7 +177,7 @@ impl Signature { } /// Something that provides a function implementation on the host for a wasm function. -pub trait Function { +pub trait Function: std::panic::RefUnwindSafe { /// Returns the name of this function. fn name(&self) -> &str; /// Returns the signature of this function. From 33eb1708cb52e4dc7ee596db6cf01642adfca2f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 31 Oct 2019 12:04:41 +0100 Subject: [PATCH 54/76] Don't panic in a panic --- core/offchain/src/testing.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/offchain/src/testing.rs b/core/offchain/src/testing.rs index e1cc7f71a3811..3f4415efa7a1d 100644 --- a/core/offchain/src/testing.rs +++ b/core/offchain/src/testing.rs @@ -119,7 +119,8 @@ impl State { impl Drop for State { fn drop(&mut self) { - if !self.expected_requests.is_empty() { + // If we panic! while we are already in a panic, the test dies with an illegal instruction. + if !self.expected_requests.is_empty() && !std::thread::panicking() { panic!("Unfulfilled expected requests: {:?}", self.expected_requests); } } From 5a67f451a1609fdea7a061ba7fd48a72d15d00f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 31 Oct 2019 17:15:51 +0100 Subject: [PATCH 55/76] Move `sr-sandbox` to new runtime interface --- Cargo.lock | 3 +- core/executor/runtime-test/src/lib.rs | 2 +- core/executor/src/sandbox.rs | 13 +-- core/sr-io/Cargo.toml | 4 - core/sr-io/build.rs | 13 --- core/sr-io/src/lib.rs | 94 ++++++++++++++++-- core/sr-sandbox/Cargo.toml | 7 +- core/sr-sandbox/build.rs | 13 --- core/sr-sandbox/src/lib.rs | 2 +- core/sr-sandbox/with_std.rs | 3 +- core/sr-sandbox/without_std.rs | 138 +++++++++----------------- 11 files changed, 139 insertions(+), 153 deletions(-) delete mode 100644 core/sr-io/build.rs delete mode 100755 core/sr-sandbox/build.rs diff --git a/Cargo.lock b/Cargo.lock index e8f5e9cddd9d7..2432cde99171b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3927,7 +3927,6 @@ dependencies = [ "libsecp256k1 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", "substrate-externalities 2.0.0", "substrate-primitives 2.0.0", @@ -3962,7 +3961,7 @@ version = "2.0.0" dependencies = [ "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "sr-io 2.0.0", "sr-std 2.0.0", "substrate-primitives 2.0.0", "wabt 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/core/executor/runtime-test/src/lib.rs b/core/executor/runtime-test/src/lib.rs index 086481e8170e6..16c6c353d9e37 100644 --- a/core/executor/runtime-test/src/lib.rs +++ b/core/executor/runtime-test/src/lib.rs @@ -248,7 +248,7 @@ fn execute_sandboxed( }; let mut instance = sandbox::Instance::new(code, &env_builder, &mut state)?; - let result = instance.invoke(b"call", args, &mut state); + let result = instance.invoke("call", args, &mut state); result.map_err(|_| sandbox::HostError) } diff --git a/core/executor/src/sandbox.rs b/core/executor/src/sandbox.rs index e7470e6e77c49..9e664783e2f7c 100644 --- a/core/executor/src/sandbox.rs +++ b/core/executor/src/sandbox.rs @@ -668,6 +668,7 @@ mod tests { } #[test] + #[should_panic(expected = "Failed to allocate memory: \"AllocatorOutOfSpace\"")] fn sandbox_should_trap_when_heap_exhausted() { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -683,17 +684,7 @@ mod tests { ) "#).unwrap().encode(); - let res = call_wasm(&mut ext, 8, &test_code[..], "test_exhaust_heap", &code); - assert_eq!(res.is_err(), true); - if let Err(err) = res { - assert_eq!( - format!("{}", err), - format!( - "{}", - wasmi::Error::Trap(Error::FunctionExecution("AllocatorOutOfSpace".into()).into()), - ), - ); - } + call_wasm(&mut ext, 8, &test_code[..], "test_exhaust_heap", &code).unwrap(); } #[test] diff --git a/core/sr-io/Cargo.toml b/core/sr-io/Cargo.toml index f0d568f98d589..4e6d0b652b427 100644 --- a/core/sr-io/Cargo.toml +++ b/core/sr-io/Cargo.toml @@ -2,12 +2,8 @@ name = "sr-io" version = "2.0.0" authors = ["Parity Technologies "] -build = "build.rs" edition = "2018" -[build-dependencies] -rustc_version = "0.2.3" - [dependencies] codec = { package = "parity-scale-codec", version = "1.0.6", default-features = false } hash-db = { version = "0.15.2", default-features = false } diff --git a/core/sr-io/build.rs b/core/sr-io/build.rs deleted file mode 100644 index 5b5d06b65a253..0000000000000 --- a/core/sr-io/build.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Set a nightly feature - -use rustc_version::{version, version_meta, Channel}; - -fn main() { - // Assert we haven't traveled back in time - assert!(version().unwrap().major >= 1); - - // Set cfg flags depending on release channel - if let Channel::Nightly = version_meta().unwrap().channel { - println!("cargo:rustc-cfg=feature=\"nightly\""); - } -} diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 31a63ea90b209..88edb469c5a62 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -70,6 +70,7 @@ pub enum EcdsaVerifyError { /// /// Panicking here is aligned with what the `without_std` environment would do /// in the case of an invalid child storage key. +#[cfg(feature = "std")] fn child_storage_key_or_panic(storage_key: &[u8]) -> ChildStorageKey { match ChildStorageKey::from_slice(storage_key) { Some(storage_key) => storage_key, @@ -596,20 +597,12 @@ pub trait Offchain { trait Allocator { /// Malloc the given number of bytes and return the pointer to the allocated memory location. fn malloc(&mut self, size: u32) -> Pointer { - match self.allocate_memory(size) { - Ok(res) => res, - Err(e) => { - log::warn!(target: "runtime-interface", "Failed to allocate memory: {}", e); - Pointer::new(0) - } - } + self.allocate_memory(size).expect("Failed to allocate memory") } /// Free the given pointer. fn free(&mut self, ptr: Pointer) { - if let Err(e) = self.deallocate_memory(ptr) { - log::warn!(target: "runtime-interface", "Failed to free a given pointer: {}", e); - } + self.deallocate_memory(ptr).expect("Failed to deallocate memory") } } @@ -634,6 +627,86 @@ pub trait Logging { } } +/// Wasm-only interface that provides functions for interacting with the sandbox. +#[runtime_interface(wasm_only)] +pub trait Sandbox { + /// Instantiate a new sandbox instance with the given `wasm_code`. + fn instantiate( + &mut self, + dispatch_thunk: u32, + wasm_code: &[u8], + env_def: &[u8], + state_ptr: Pointer, + ) -> u32 { + self.sandbox() + .instance_new(dispatch_thunk, wasm_code, env_def, state_ptr.into()) + .expect("Failed to instantiate a new sandbox") + } + + /// Invoke `function` in the sandbox with `sandbox_idx`. + fn invoke( + &mut self, + instance_idx: u32, + function: &str, + args: &[u8], + return_val_ptr: Pointer, + return_val_len: u32, + state_ptr: Pointer, + ) -> u32 { + self.sandbox().invoke( + instance_idx, + &function, + &args, + return_val_ptr, + return_val_len, + state_ptr.into(), + ).expect("Failed to invoke function with sandbox") + } + + /// Create a new memory instance with the given `initial` and `maximum` size. + fn memory_new(&mut self, initial: u32, maximum: u32) -> u32 { + self.sandbox() + .memory_new(initial, maximum) + .expect("Failed to create new memory with sandbox") + } + + /// Get the memory starting at `offset` from the instance with `memory_idx` into the buffer. + fn memory_get( + &mut self, + memory_idx: u32, + offset: u32, + buf_ptr: Pointer, + buf_len: u32, + ) -> u32 { + self.sandbox() + .memory_get(memory_idx, offset, buf_ptr, buf_len) + .expect("Failed to get memory with sandbox") + } + + /// Set the memory in the given `memory_idx` to the given value at `offset`. + fn memory_set( + &mut self, + memory_idx: u32, + offset: u32, + val_ptr: Pointer, + val_len: u32, + ) -> u32 { + self.sandbox() + .memory_set(memory_idx, offset, val_ptr, val_len) + .expect("Failed to set memory with sandbox") + } + + /// Teardown the memory instance with the given `memory_idx`. + fn memory_teardown(&mut self, memory_idx: u32) { + self.sandbox().memory_teardown(memory_idx).expect("Failed to teardown memory with sandbox") + } + + /// Teardown the sandbox instance with the given `instance_idx`. + fn instance_teardown(&mut self, instance_idx: u32) { + self.sandbox().instance_teardown(instance_idx).expect("Failed to teardown sandbox instance") + } +} + /// Allocator used by Substrate when executing the Wasm runtime. #[cfg(not(feature = "std"))] struct WasmAllocator; @@ -696,6 +769,7 @@ pub type SubstrateHostFunctions = ( hashing::HostFunctions, allocator::HostFunctions, logging::HostFunctions, + sandbox::HostFunctions, ); #[cfg(test)] diff --git a/core/sr-sandbox/Cargo.toml b/core/sr-sandbox/Cargo.toml index 87b4e742a0413..20d569f043c60 100755 --- a/core/sr-sandbox/Cargo.toml +++ b/core/sr-sandbox/Cargo.toml @@ -2,16 +2,13 @@ name = "sr-sandbox" version = "2.0.0" authors = ["Parity Technologies "] -build = "build.rs" edition = "2018" -[build-dependencies] -rustc_version = "0.2.3" - [dependencies] wasmi = { version = "0.5.1", optional = true } primitives = { package = "substrate-primitives", path = "../primitives", default-features = false } rstd = { package = "sr-std", path = "../sr-std", default-features = false } +runtime-io = { package = "sr-io", path = "../sr-io", default-features = false } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false } [dev-dependencies] @@ -25,6 +22,6 @@ std = [ "primitives/std", "rstd/std", "codec/std", + "runtime-io/std", ] -nightly = [] strict = [] diff --git a/core/sr-sandbox/build.rs b/core/sr-sandbox/build.rs deleted file mode 100755 index 5b5d06b65a253..0000000000000 --- a/core/sr-sandbox/build.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Set a nightly feature - -use rustc_version::{version, version_meta, Channel}; - -fn main() { - // Assert we haven't traveled back in time - assert!(version().unwrap().major >= 1); - - // Set cfg flags depending on release channel - if let Channel::Nightly = version_meta().unwrap().channel { - println!("cargo:rustc-cfg=feature=\"nightly\""); - } -} diff --git a/core/sr-sandbox/src/lib.rs b/core/sr-sandbox/src/lib.rs index c9f9135661586..4639cf983af44 100755 --- a/core/sr-sandbox/src/lib.rs +++ b/core/sr-sandbox/src/lib.rs @@ -197,7 +197,7 @@ impl Instance { /// - Trap occured at the execution time. pub fn invoke( &mut self, - name: &[u8], + name: &str, args: &[TypedValue], state: &mut T, ) -> Result { diff --git a/core/sr-sandbox/with_std.rs b/core/sr-sandbox/with_std.rs index ea7ce818350d0..3f0db8a9b61a5 100755 --- a/core/sr-sandbox/with_std.rs +++ b/core/sr-sandbox/with_std.rs @@ -282,13 +282,12 @@ impl Instance { pub fn invoke( &mut self, - name: &[u8], + name: &str, args: &[TypedValue], state: &mut T, ) -> Result { let args = args.iter().cloned().map(Into::into).collect::>(); - let name = ::std::str::from_utf8(name).map_err(|_| Error::Execution)?; let mut externals = GuestExternals { state, defined_host_functions: &self.defined_host_functions, diff --git a/core/sr-sandbox/without_std.rs b/core/sr-sandbox/without_std.rs index ee5f7697fe71d..d7fffbf88b27f 100755 --- a/core/sr-sandbox/without_std.rs +++ b/core/sr-sandbox/without_std.rs @@ -14,12 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use rstd::prelude::*; -use rstd::{slice, marker, mem, vec}; -use rstd::rc::Rc; +use rstd::{prelude::*, slice, marker, mem, vec, rc::Rc}; use codec::{Decode, Encode}; use primitives::sandbox as sandbox_primitives; use super::{Error, TypedValue, ReturnValue, HostFuncType}; +use runtime_io::sandbox; mod ffi { use rstd::mem; @@ -43,51 +42,6 @@ mod ffi { assert!(mem::size_of::() == mem::size_of::>()); mem::transmute::>(idx) } - - extern "C" { - pub fn ext_sandbox_instantiate( - dispatch_thunk: extern "C" fn( - serialized_args_ptr: *const u8, - serialized_args_len: usize, - state: usize, - f: HostFuncIndex, - ) -> u64, - wasm_ptr: *const u8, - wasm_len: usize, - imports_ptr: *const u8, - imports_len: usize, - state: usize, - ) -> u32; - pub fn ext_sandbox_invoke( - instance_idx: u32, - export_ptr: *const u8, - export_len: usize, - args_ptr: *const u8, - args_len: usize, - return_val_ptr: *mut u8, - return_val_len: usize, - state: usize, - ) -> u32; - pub fn ext_sandbox_memory_new(initial: u32, maximum: u32) -> u32; - pub fn ext_sandbox_memory_get( - memory_idx: u32, - offset: u32, - buf_ptr: *mut u8, - buf_len: usize, - ) -> u32; - pub fn ext_sandbox_memory_set( - memory_idx: u32, - offset: u32, - val_ptr: *const u8, - val_len: usize, - ) -> u32; - pub fn ext_sandbox_memory_teardown( - memory_idx: u32, - ); - pub fn ext_sandbox_instance_teardown( - instance_idx: u32, - ); - } } struct MemoryHandle { @@ -96,9 +50,7 @@ struct MemoryHandle { impl Drop for MemoryHandle { fn drop(&mut self) { - unsafe { - ffi::ext_sandbox_memory_teardown(self.memory_idx); - } + sandbox::memory_teardown(self.memory_idx); } } @@ -111,15 +63,13 @@ pub struct Memory { impl Memory { pub fn new(initial: u32, maximum: Option) -> Result { - let result = unsafe { - let maximum = if let Some(maximum) = maximum { - maximum - } else { - sandbox_primitives::MEM_UNLIMITED - }; - ffi::ext_sandbox_memory_new(initial, maximum) + let maximum = if let Some(maximum) = maximum { + maximum + } else { + sandbox_primitives::MEM_UNLIMITED }; - match result { + + match sandbox::memory_new(initial, maximum) { sandbox_primitives::ERR_MODULE => Err(Error::Module), memory_idx => Ok(Memory { handle: Rc::new(MemoryHandle { memory_idx, }), @@ -128,7 +78,12 @@ impl Memory { } pub fn get(&self, offset: u32, buf: &mut [u8]) -> Result<(), Error> { - let result = unsafe { ffi::ext_sandbox_memory_get(self.handle.memory_idx, offset, buf.as_mut_ptr(), buf.len()) }; + let result = sandbox::memory_get( + self.handle.memory_idx, + offset, + buf.as_mut_ptr(), + buf.len() as u32, + ); match result { sandbox_primitives::ERR_OK => Ok(()), sandbox_primitives::ERR_OUT_OF_BOUNDS => Err(Error::OutOfBounds), @@ -137,7 +92,12 @@ impl Memory { } pub fn set(&self, offset: u32, val: &[u8]) -> Result<(), Error> { - let result = unsafe { ffi::ext_sandbox_memory_set(self.handle.memory_idx, offset, val.as_ptr(), val.len()) }; + let result = sandbox::memory_set( + self.handle.memory_idx, + offset, + val.as_ptr() as _ , + val.len() as u32, + ); match result { sandbox_primitives::ERR_OK => Ok(()), sandbox_primitives::ERR_OUT_OF_BOUNDS => Err(Error::OutOfBounds), @@ -251,26 +211,27 @@ extern "C" fn dispatch_thunk( } impl Instance { - pub fn new(code: &[u8], env_def_builder: &EnvironmentDefinitionBuilder, state: &mut T) -> Result, Error> { + pub fn new( + code: &[u8], + env_def_builder: &EnvironmentDefinitionBuilder, + state: &mut T, + ) -> Result, Error> { let serialized_env_def: Vec = env_def_builder.env_def.encode(); - let result = unsafe { - // It's very important to instantiate thunk with the right type. - let dispatch_thunk = dispatch_thunk::; - - ffi::ext_sandbox_instantiate( - dispatch_thunk, - code.as_ptr(), - code.len(), - serialized_env_def.as_ptr(), - serialized_env_def.len(), - state as *const T as usize, - ) - }; + // It's very important to instantiate thunk with the right type. + let dispatch_thunk = dispatch_thunk::; + let result = sandbox::instantiate( + dispatch_thunk as u32, + code, + &serialized_env_def, + state as *const T as _, + ); + let instance_idx = match result { sandbox_primitives::ERR_MODULE => return Err(Error::Module), sandbox_primitives::ERR_EXECUTION => return Err(Error::Execution), instance_idx => instance_idx, }; + // We need to retain memories to keep them alive while the Instance is alive. let retained_memories = env_def_builder.retained_memories.clone(); Ok(Instance { @@ -282,25 +243,22 @@ impl Instance { pub fn invoke( &mut self, - name: &[u8], + name: &str, args: &[TypedValue], state: &mut T, ) -> Result { let serialized_args = args.to_vec().encode(); let mut return_val = vec![0u8; sandbox_primitives::ReturnValue::ENCODED_MAX_SIZE]; - let result = unsafe { - ffi::ext_sandbox_invoke( - self.instance_idx, - name.as_ptr(), - name.len(), - serialized_args.as_ptr(), - serialized_args.len(), - return_val.as_mut_ptr(), - return_val.len(), - state as *const T as usize, - ) - }; + let result = sandbox::invoke( + self.instance_idx, + name, + &serialized_args, + return_val.as_mut_ptr() as _, + return_val.len() as u32, + state as *const T as _, + ); + match result { sandbox_primitives::ERR_OK => { let return_val = sandbox_primitives::ReturnValue::decode(&mut &return_val[..]) @@ -315,8 +273,6 @@ impl Instance { impl Drop for Instance { fn drop(&mut self) { - unsafe { - ffi::ext_sandbox_instance_teardown(self.instance_idx); - } + sandbox::instance_teardown(self.instance_idx); } } From 1c6874711b6bf57738a3a6e587a53348adda0704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 31 Oct 2019 17:30:28 +0100 Subject: [PATCH 56/76] Fixes tests after sandbox changes --- core/sr-sandbox/with_std.rs | 13 +++++++++---- srml/contracts/src/wasm/mod.rs | 8 ++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/core/sr-sandbox/with_std.rs b/core/sr-sandbox/with_std.rs index 3f0db8a9b61a5..afc092686eeca 100755 --- a/core/sr-sandbox/with_std.rs +++ b/core/sr-sandbox/with_std.rs @@ -257,7 +257,11 @@ pub struct Instance { } impl Instance { - pub fn new(code: &[u8], env_def_builder: &EnvironmentDefinitionBuilder, state: &mut T) -> Result, Error> { + pub fn new( + code: &[u8], + env_def_builder: &EnvironmentDefinitionBuilder, + state: &mut T, + ) -> Result, Error> { let module = Module::from_buffer(code).map_err(|_| Error::Module)?; let not_started_instance = ModuleInstance::new(&module, env_def_builder) .map_err(|_| Error::Module)?; @@ -269,7 +273,8 @@ impl Instance { state, defined_host_functions: &defined_host_functions, }; - let instance = not_started_instance.run_start(&mut externals).map_err(|_| Error::Execution)?; + let instance = not_started_instance.run_start(&mut externals) + .map_err(|_| Error::Execution)?; instance }; @@ -349,7 +354,7 @@ mod tests { env_builder.add_host_func("env", "polymorphic_id", env_polymorphic_id); let mut instance = Instance::new(code, &env_builder, &mut state)?; - let result = instance.invoke(b"call", args, &mut state); + let result = instance.invoke("call", args, &mut state); result.map_err(|_| HostError) } @@ -473,7 +478,7 @@ mod tests { // But this fails since we imported a function that returns i32 as if it returned i64. assert_matches!( - instance.invoke(b"call", &[], &mut ()), + instance.invoke("call", &[], &mut ()), Err(Error::Execution) ); } diff --git a/srml/contracts/src/wasm/mod.rs b/srml/contracts/src/wasm/mod.rs index 4fa9412bc1769..42ee6ee0a2c5a 100644 --- a/srml/contracts/src/wasm/mod.rs +++ b/srml/contracts/src/wasm/mod.rs @@ -58,7 +58,7 @@ pub struct PrefabWasmModule { /// Wasm executable loaded by `WasmLoader` and executed by `WasmVm`. pub struct WasmExecutable { - entrypoint_name: &'static [u8], + entrypoint_name: &'static str, prefab_module: PrefabWasmModule, } @@ -79,14 +79,14 @@ impl<'a, T: Trait> crate::exec::Loader for WasmLoader<'a> { fn load_init(&self, code_hash: &CodeHash) -> Result { let prefab_module = load_code::(code_hash, self.schedule)?; Ok(WasmExecutable { - entrypoint_name: b"deploy", + entrypoint_name: "deploy", prefab_module, }) } fn load_main(&self, code_hash: &CodeHash) -> Result { let prefab_module = load_code::(code_hash, self.schedule)?; Ok(WasmExecutable { - entrypoint_name: b"call", + entrypoint_name: "call", prefab_module, }) } @@ -406,7 +406,7 @@ mod tests { let exec = WasmExecutable { // Use a "call" convention. - entrypoint_name: b"call", + entrypoint_name: "call", prefab_module, }; From a546aab1a46b880424170255a3deab9cad69cc0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 1 Nov 2019 17:15:55 +0100 Subject: [PATCH 57/76] Make sure we detect invalid utf8 --- core/runtime-interface/src/impls.rs | 2 +- core/runtime-interface/src/lib.rs | 6 ++++++ core/runtime-interface/test-wasm/src/lib.rs | 13 +++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index 7a835b1c66fd5..6a1a454340e15 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -400,7 +400,7 @@ impl FromFFIValue for str { let vec = context.read_memory(Pointer::new(ptr), len)?; // The data is valid utf8, as it is stored as `&str` in wasm. - Ok(unsafe { String::from_utf8_unchecked(vec) }) + String::from_utf8(vec).map_err(|_| "Invalid utf8 data provided".into()) } } diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 67fdebbe94a6c..e9aa86443866e 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -136,4 +136,10 @@ mod tests { fn host_function_not_found() { call_wasm_method::<()>("test_return_data"); } + + #[test] + #[should_panic(expected = "Trap { kind: Host(FunctionExecution(\"Invalid utf8 data provided\")")] + fn test_invalid_utf8_data_should_return_an_error() { + call_wasm_method::("test_invalid_utf8_data_should_return_an_error"); + } } diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index 75a789c05ba17..9adaed3137526 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -71,6 +71,11 @@ pub trait TestApi { fn return_input_public_key(key: Public) -> Public { key } + + /// A function that is called with invalid utf8 data from the runtime. + fn invalid_utf8_data(_data: &str) { + + } } /// This function is not used, but we require it for the compiler to include `runtime-io`. @@ -143,4 +148,12 @@ wasm_export_functions! { let ret_key_data: &[u8] = ret_key.as_ref(); assert_eq!(key_data, ret_key_data); } + + fn test_invalid_utf8_data_should_return_an_error() { + let data = vec![0, 159, 146, 150]; + // I'm an evil hacker, trying to hack! + let data_str = unsafe { rstd::str::from_utf8_unchecked(&data) }; + + test_api::invalid_utf8_data(data_str); + } } From 162c62bce90438a1b3ac54fa06309357522c45d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 1 Nov 2019 20:15:26 +0100 Subject: [PATCH 58/76] Fixes after master merge --- core/executor/src/integration_tests/mod.rs | 20 ++++++++++++++++++- .../executor/src/integration_tests/sandbox.rs | 8 ++++---- core/executor/src/wasmi_execution.rs | 4 ++-- core/runtime-interface/src/lib.rs | 5 ++++- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/core/executor/src/integration_tests/mod.rs b/core/executor/src/integration_tests/mod.rs index 640795f8f0d5e..6db9911d44ab4 100644 --- a/core/executor/src/integration_tests/mod.rs +++ b/core/executor/src/integration_tests/mod.rs @@ -28,10 +28,28 @@ use substrate_offchain::testing; use test_case::test_case; use trie::{TrieConfiguration, trie_types::Layout}; -use crate::{WasmExecutionMethod, call_in_wasm}; +use crate::WasmExecutionMethod; pub type TestExternalities = CoreTestExternalities; +fn call_in_wasm( + function: &str, + call_data: &[u8], + execution_method: WasmExecutionMethod, + ext: &mut E, + code: &[u8], + heap_pages: u64, +) -> crate::error::Result> { + crate::call_in_wasm::( + function, + call_data, + execution_method, + ext, + code, + heap_pages, + ) +} + #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn returning_should_work(wasm_method: WasmExecutionMethod) { diff --git a/core/executor/src/integration_tests/sandbox.rs b/core/executor/src/integration_tests/sandbox.rs index 30adcb3e7fdb0..369c9c4a81d1d 100644 --- a/core/executor/src/integration_tests/sandbox.rs +++ b/core/executor/src/integration_tests/sandbox.rs @@ -14,14 +14,14 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use super::{TestExternalities, call_in_wasm}; +use crate::WasmExecutionMethod; + use codec::Encode; use runtime_test::WASM_BINARY; use test_case::test_case; use wabt; -use crate::{WasmExecutionMethod, call_in_wasm}; -use crate::integration_tests::TestExternalities; - #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn sandbox_should_work(wasm_method: WasmExecutionMethod) { @@ -95,7 +95,7 @@ fn sandbox_trap(wasm_method: WasmExecutionMethod) { #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] -#[should_panic(expected = "Failed to allocate memory: \"AllocatorOutOfSpace\"")] +#[should_panic(expected = "Failed to allocate memory: \"Allocator ran out of space\"")] fn sandbox_should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); diff --git a/core/executor/src/wasmi_execution.rs b/core/executor/src/wasmi_execution.rs index e9043d8f8ea36..8ad949ec6a9d0 100644 --- a/core/executor/src/wasmi_execution.rs +++ b/core/executor/src/wasmi_execution.rs @@ -377,7 +377,7 @@ fn call_in_wasm_module( match result { Ok(Some(I64(r))) => { let (ptr, length) = interpret_runtime_api_result(r); - memory.get(offset, length).map_err(|_| Error::Runtime) + memory.get(ptr.into(), length as usize).map_err(|_| Error::Runtime) }, Err(e) => { trace!( @@ -619,7 +619,7 @@ pub fn create_instance( *call_instance, "Core_version", &[], - host_function_slice, + host_functions_slice, ) ).map_err(|_| WasmError::Instantiation("panic in call to get runtime version".to_string()))?; let version = version_result diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index e9aa86443866e..d9c0eece76ee6 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -138,7 +138,10 @@ mod tests { } #[test] - #[should_panic(expected = "Trap { kind: Host(FunctionExecution(\"Invalid utf8 data provided\")")] + #[should_panic( + expected = + "FunctionExecution(\"ext_test_api_invalid_utf8_data\", \"Invalid utf8 data provided\")" + )] fn test_invalid_utf8_data_should_return_an_error() { call_wasm_method::("test_invalid_utf8_data_should_return_an_error"); } From f0c71907d9780fccdcb36e7b03239fcc2aff07c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 1 Nov 2019 23:44:59 +0100 Subject: [PATCH 59/76] Adds pass by enum strategy --- core/primitives/src/lib.rs | 2 +- core/primitives/src/offchain.rs | 6 +- core/runtime-interface/proc-macro/src/lib.rs | 7 + .../proc-macro/src/pass_by_enum.rs | 101 +++++++++++++ core/runtime-interface/src/lib.rs | 3 + core/runtime-interface/src/pass_by.rs | 137 ++++++++++++++---- 6 files changed, 223 insertions(+), 33 deletions(-) create mode 100644 core/runtime-interface/proc-macro/src/pass_by_enum.rs diff --git a/core/primitives/src/lib.rs b/core/primitives/src/lib.rs index e4a98644d9fc4..01a965cd8c3cb 100644 --- a/core/primitives/src/lib.rs +++ b/core/primitives/src/lib.rs @@ -236,7 +236,7 @@ pub trait TypeId { /// A log level matching the one from `log` crate. /// /// Used internally by `runtime_io::log` method. -#[derive(Encode, Decode, runtime_interface::pass_by::PassByCodec)] +#[derive(Encode, Decode, runtime_interface::pass_by::PassByEnum, Copy, Clone)] pub enum LogLevel { /// `Error` log level. Error = 1, diff --git a/core/primitives/src/offchain.rs b/core/primitives/src/offchain.rs index 3de2adfb9617f..e79f09c76dbf4 100644 --- a/core/primitives/src/offchain.rs +++ b/core/primitives/src/offchain.rs @@ -19,12 +19,12 @@ use codec::{Encode, Decode}; use rstd::{prelude::{Vec, Box}, convert::TryFrom}; use crate::RuntimeDebug; -use runtime_interface::pass_by::{PassByCodec, PassByInner}; +use runtime_interface::pass_by::{PassByCodec, PassByInner, PassByEnum}; pub use crate::crypto::KeyTypeId; /// A type of supported crypto. -#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, RuntimeDebug, PassByCodec)] +#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, RuntimeDebug, PassByEnum)] #[repr(C)] pub enum StorageKind { /// Persistent storage is non-revertible and not fork-aware. It means that any value @@ -71,7 +71,7 @@ impl From for u32 { } /// An error enum returned by some http methods. -#[derive(Clone, Copy, PartialEq, Eq, RuntimeDebug, Encode, Decode, PassByCodec)] +#[derive(Clone, Copy, PartialEq, Eq, RuntimeDebug, Encode, Decode, PassByEnum)] #[repr(C)] pub enum HttpError { /// The requested action couldn't been completed within a deadline. diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index 6f37f5f71f534..c0360fe4bd856 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -31,6 +31,7 @@ use utils::generate_runtime_interface_include; mod bare_function_interface; mod host_function_interface; mod pass_by_codec; +mod pass_by_enum; mod pass_by_inner; mod trait_decl_impl; mod utils; @@ -92,3 +93,9 @@ pub fn pass_by_inner(input: proc_macro::TokenStream) -> proc_macro::TokenStream let input = parse_macro_input!(input as DeriveInput); pass_by_inner::derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() } + +#[proc_macro_derive(PassByEnum)] +pub fn pass_by_enum(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = parse_macro_input!(input as DeriveInput); + pass_by_enum::derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() +} diff --git a/core/runtime-interface/proc-macro/src/pass_by_enum.rs b/core/runtime-interface/proc-macro/src/pass_by_enum.rs new file mode 100644 index 0000000000000..ac5e67552083b --- /dev/null +++ b/core/runtime-interface/proc-macro/src/pass_by_enum.rs @@ -0,0 +1,101 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Derive macro implementation of `PassBy` with the associated type set to `Enum`. +//! +//! Besides `PassBy`, `TryFrom` and `From for u8` are implemented for the type. + +use crate::utils::{generate_crate_access, generate_runtime_interface_include}; + +use syn::{DeriveInput, Result, Data, Fields, Error, Ident}; + +use quote::quote; + +use proc_macro2::{TokenStream, Span}; + +/// The derive implementation for `PassBy` with `Enum`. +pub fn derive_impl(input: DeriveInput) -> Result { + let crate_include = generate_runtime_interface_include(); + let crate_ = generate_crate_access(); + let ident = input.ident; + let enum_fields = get_enum_field_idents(&input.data)? + .enumerate() + .map(|(i, v)| { + let i = i as u8; + + v.map(|v| (quote!(#i => Ok(#ident::#v)), quote!(#ident::#v => #i))) + }) + .collect::>>()?; + let try_from_variants = enum_fields.iter().map(|i| &i.0); + let into_variants = enum_fields.iter().map(|i| &i.1); + + let res = quote! { + const _: () = { + #crate_include + + impl #crate_::pass_by::PassBy for #ident { + type PassBy = #crate_::pass_by::Enum<#ident>; + } + + impl #crate_::rstd::convert::TryFrom for #ident { + type Error = (); + + fn try_from(inner: u8) -> #crate_::rstd::result::Result { + match inner { + #( #try_from_variants, )* + _ => Err(()), + } + } + } + + impl From<#ident> for u8 { + fn from(var: #ident) -> u8 { + match var { + #( #into_variants ),* + } + } + } + }; + }; + + Ok(res) +} + +/// Get the enum fields idents of the given `data` object as iterator. +/// +/// Returns an error if the number of variants is greater than `256`, the given `data` is not an +/// enum or a variant is not an unit. +fn get_enum_field_idents<'a>(data: &'a Data) -> Result>> { + match data { + Data::Enum(d) => { + if d.variants.len() <= 256 { + Ok( + d.variants.iter().map(|v| if let Fields::Unit = v.fields { + Ok(&v.ident) + } else { + Err(Error::new( + Span::call_site(), + "`PassByEnum` only supports unit variants.", + )) + }) + ) + } else { + Err(Error::new(Span::call_site(), "`PassByEnum` only supports `256` variants.")) + } + }, + _ => Err(Error::new(Span::call_site(), "`PassByEnum` only supports enums as input type.")) + } +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index d9c0eece76ee6..12ca0ddcbd146 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -30,6 +30,9 @@ #[cfg(feature = "std")] pub use wasm_interface; +#[doc(hidden)] +pub use rstd; + pub use substrate_runtime_interface_proc_macro::runtime_interface; #[doc(hidden)] diff --git a/core/runtime-interface/src/pass_by.rs b/core/runtime-interface/src/pass_by.rs index 53ea6e6bba3bd..8314f32d7a3aa 100644 --- a/core/runtime-interface/src/pass_by.rs +++ b/core/runtime-interface/src/pass_by.rs @@ -27,12 +27,12 @@ use crate::wasm::*; #[cfg(feature = "std")] use wasm_interface::{FunctionContext, Pointer, Result}; -use rstd::{marker::PhantomData}; +use rstd::{marker::PhantomData, convert::TryFrom}; #[cfg(not(feature = "std"))] use rstd::{slice, vec::Vec}; -pub use substrate_runtime_interface_proc_macro::{PassByCodec, PassByInner}; +pub use substrate_runtime_interface_proc_macro::{PassByCodec, PassByInner, PassByEnum}; /// Something that should be passed between wasm and the host using the given strategy. /// @@ -89,6 +89,48 @@ pub trait PassByImpl: RIType { fn from_ffi_value(arg: Self::FFIType) -> T; } +impl RIType for T { + type FFIType = ::FFIType; +} + +#[cfg(feature = "std")] +impl IntoFFIValue for T { + fn into_ffi_value( + self, + context: &mut dyn FunctionContext, + ) -> Result<::FFIType> { + T::PassBy::into_ffi_value(self, context) + } +} + +#[cfg(feature = "std")] +impl FromFFIValue for T { + type SelfInstance = Self; + + fn from_ffi_value( + context: &mut dyn FunctionContext, + arg: ::FFIType, + ) -> Result { + T::PassBy::from_ffi_value(context, arg) + } +} + +#[cfg(not(feature = "std"))] +impl IntoFFIValue for T { + type Owned = >::Owned; + + fn into_ffi_value(&self) -> WrappedFFIValue<::FFIType, Self::Owned> { + T::PassBy::into_ffi_value(self) + } +} + +#[cfg(not(feature = "std"))] +impl FromFFIValue for T { + fn from_ffi_value(arg: ::FFIType) -> Self { + T::PassBy::from_ffi_value(arg) + } +} + /// The implementation of the pass by codec strategy. This strategy uses a SCALE encoded /// representation of the type between wasm and the host. /// @@ -127,7 +169,8 @@ impl PassByImpl for Codec { ) -> Result { let (ptr, len) = pointer_and_len_from_u64(arg); let vec = context.read_memory(Pointer::new(ptr), len)?; - Ok(T::decode(&mut &vec[..]).expect("Wasm to host values are encoded correctly; qed")) + T::decode(&mut &vec[..]) + .map_err(|e| format!("Could not decode value from wasm: {}", e.what())) } } @@ -243,45 +286,81 @@ impl, I: RIType> RIType for Inner { type FFIType = I::FFIType; } -impl RIType for T { - type FFIType = ::FFIType; -} +/// The implementation of the pass by enum strategy. This strategy uses an `u8` internally to pass +/// the enum between wasm and the host. So, this strategy only supports enums with unit variants. +/// +/// Use this type as associated type for [`PassBy`] to implement this strategy for a type. +/// +/// This type expects the type that wants to implement this strategy as generic parameter. Besides +/// that the type needs to implement `TryFrom` and `From for u8`. +/// +/// # Example +/// ``` +/// # use substrate_runtime_interface::pass_by::{PassBy, Enum}; +/// #[derive(Clone, Copy)] +/// enum Test { +/// Test1, +/// Test2, +/// } +/// +/// impl From for u8 { +/// fn from(val: Test) -> u8 { +/// match val { +/// Test::Test1 => 0, +/// Test::Test2 => 1, +/// } +/// } +/// } +/// +/// impl std::convert::TryFrom for Test { +/// type Error = (); +/// +/// fn try_from(val: u8) -> Result { +/// match val { +/// 0 => Ok(Test::Test1), +/// 1 => Ok(Test::Test2), +/// _ => Err(()), +/// } +/// } +/// } +/// +/// impl PassBy for Test { +/// type PassBy = Enum; +/// } +/// ``` +pub struct Enum + TryFrom>(PhantomData); #[cfg(feature = "std")] -impl IntoFFIValue for T { +impl + TryFrom> PassByImpl for Enum { fn into_ffi_value( - self, - context: &mut dyn FunctionContext, - ) -> Result<::FFIType> { - T::PassBy::into_ffi_value(self, context) + instance: T, + _: &mut dyn FunctionContext, + ) -> Result { + Ok(instance.into()) } -} - -#[cfg(feature = "std")] -impl FromFFIValue for T { - type SelfInstance = Self; fn from_ffi_value( - context: &mut dyn FunctionContext, - arg: ::FFIType, - ) -> Result { - T::PassBy::from_ffi_value(context, arg) + _: &mut dyn FunctionContext, + arg: Self::FFIType, + ) -> Result { + T::try_from(arg).map_err(|_| format!("Invalid enum discriminant: {}", arg)) } } #[cfg(not(feature = "std"))] -impl IntoFFIValue for T { - type Owned = >::Owned; +impl + TryFrom> PassByImpl for Enum { + type Owned = (); - fn into_ffi_value(&self) -> WrappedFFIValue<::FFIType, Self::Owned> { - T::PassBy::into_ffi_value(self) + fn into_ffi_value(instance: &T) -> WrappedFFIValue { + let value: u8 = (*instance).into(); + value.into() } -} -#[cfg(not(feature = "std"))] -impl FromFFIValue for T { - fn from_ffi_value(arg: ::FFIType) -> Self { - T::PassBy::from_ffi_value(arg) + fn from_ffi_value(arg: Self::FFIType) -> T { + T::try_from(arg).expect("Host to wasm provides a valid enum discriminant; qed") } } +impl + TryFrom> RIType for Enum { + type FFIType = u8; +} From ac33c027f8ae6aa9a884a18a92e3f794b8730c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 3 Nov 2019 13:57:58 +0100 Subject: [PATCH 60/76] Fix wasmtime integration --- core/executor/Cargo.toml | 16 ++-- .../executor/src/integration_tests/sandbox.rs | 2 +- core/executor/src/wasm_runtime.rs | 2 +- core/executor/src/wasmtime/runtime.rs | 45 ++++++----- core/executor/src/wasmtime/trampoline.rs | 76 ++++++++++++------- .../proc-macro/src/bare_function_interface.rs | 9 ++- core/runtime-interface/src/pass_by.rs | 11 ++- 7 files changed, 98 insertions(+), 63 deletions(-) diff --git a/core/executor/Cargo.toml b/core/executor/Cargo.toml index b44469332dc0f..228ef5e0c311b 100644 --- a/core/executor/Cargo.toml +++ b/core/executor/Cargo.toml @@ -47,12 +47,12 @@ test-case = "0.3.3" default = [] wasm-extern-trace = [] wasmtime = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "cranelift-wasm", - "wasmtime-environ", - "wasmtime-jit", - "wasmtime-runtime", + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "cranelift-wasm", + "wasmtime-environ", + "wasmtime-jit", + "wasmtime-runtime", ] diff --git a/core/executor/src/integration_tests/sandbox.rs b/core/executor/src/integration_tests/sandbox.rs index 369c9c4a81d1d..c18b848acce7e 100644 --- a/core/executor/src/integration_tests/sandbox.rs +++ b/core/executor/src/integration_tests/sandbox.rs @@ -95,7 +95,7 @@ fn sandbox_trap(wasm_method: WasmExecutionMethod) { #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] -#[should_panic(expected = "Failed to allocate memory: \"Allocator ran out of space\"")] +#[should_panic(expected = "Allocator ran out of space")] fn sandbox_should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); diff --git a/core/executor/src/wasm_runtime.rs b/core/executor/src/wasm_runtime.rs index 22c8e3b4b8509..21f36e2784b4f 100644 --- a/core/executor/src/wasm_runtime.rs +++ b/core/executor/src/wasm_runtime.rs @@ -209,7 +209,7 @@ pub fn create_wasm_runtime_with_code( .map(|runtime| -> Box { Box::new(runtime) }), #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => - wasmtime::create_instance(ext, code, heap_pages) + wasmtime::create_instance(ext, code, heap_pages, host_functions) .map(|runtime| -> Box { Box::new(runtime) }), } } diff --git a/core/executor/src/wasmtime/runtime.rs b/core/executor/src/wasmtime/runtime.rs index fa360773fb28c..9063c7bd6a170 100644 --- a/core/executor/src/wasmtime/runtime.rs +++ b/core/executor/src/wasmtime/runtime.rs @@ -17,7 +17,6 @@ //! Defines the compiled Wasm runtime that uses Wasmtime internally. use crate::error::{Error, Result, WasmError}; -use crate::host_interface::SubstrateExternals; use crate::wasm_runtime::WasmRuntime; use crate::wasm_utils::interpret_runtime_api_result; use crate::wasmtime::function_executor::FunctionExecutorState; @@ -36,7 +35,7 @@ use std::collections::HashMap; use std::convert::TryFrom; use std::panic::AssertUnwindSafe; use std::rc::Rc; -use wasm_interface::{HostFunctions, Pointer, WordSize}; +use wasm_interface::{Pointer, WordSize, Function}; use wasmtime_environ::{Module, translate_signature}; use wasmtime_jit::{ ActionOutcome, ActionError, CodeMemory, CompilationStrategy, CompiledModule, Compiler, Context, @@ -52,6 +51,8 @@ pub struct WasmtimeRuntime { max_heap_pages: Option, heap_pages: u32, version: Option, + /// The host functions registered for this instance. + host_functions: Vec<&'static dyn Function>, } impl WasmRuntime for WasmtimeRuntime { @@ -65,6 +66,10 @@ impl WasmRuntime for WasmtimeRuntime { } } + fn host_functions(&self) -> &[&'static dyn Function] { + &self.host_functions + } + fn call(&mut self, ext: &mut dyn Externalities, method: &str, data: &[u8]) -> Result> { call_method( &mut self.context, @@ -83,10 +88,13 @@ impl WasmRuntime for WasmtimeRuntime { /// Create a new `WasmtimeRuntime` given the code. This function performs translation from Wasm to /// machine code, which can be computationally heavy. -pub fn create_instance(ext: &mut E, code: &[u8], heap_pages: u64) - -> std::result::Result -{ - let (mut compiled_module, mut context) = create_compiled_unit(code)?; +pub fn create_instance( + ext: &mut E, + code: &[u8], + heap_pages: u64, + host_functions: Vec<&'static dyn Function>, +) -> std::result::Result { + let (mut compiled_module, mut context) = create_compiled_unit(code, &host_functions)?; // Inspect the module for the min and max memory sizes. let (min_memory_size, max_memory_size) = { @@ -137,12 +145,14 @@ pub fn create_instance(ext: &mut E, code: &[u8], heap_pages: u max_heap_pages, heap_pages, version, + host_functions, }) } -fn create_compiled_unit(code: &[u8]) - -> std::result::Result<(CompiledModule, Context), WasmError> -{ +fn create_compiled_unit( + code: &[u8], + host_functions: &[&'static dyn Function], +) -> std::result::Result<(CompiledModule, Context), WasmError> { let compilation_strategy = CompilationStrategy::Cranelift; let compiler = new_compiler(compilation_strategy)?; @@ -154,7 +164,7 @@ fn create_compiled_unit(code: &[u8]) // Instantiate and link the env module. let global_exports = context.get_global_exports(); let compiler = new_compiler(compilation_strategy)?; - let env_module = instantiate_env_module(global_exports, compiler)?; + let env_module = instantiate_env_module(global_exports, compiler, host_functions)?; context.name_instance("env".to_owned(), env_module); // Compile the wasm module. @@ -208,14 +218,12 @@ fn call_method( let trap_error = reset_env_state_and_take_trap(context, None)?; let (output_ptr, output_len) = match outcome { ActionOutcome::Returned { values } => match values.as_slice() { - [RuntimeValue::I64(retval)] => - interpret_runtime_api_result(*retval), + [RuntimeValue::I64(retval)] => interpret_runtime_api_result(*retval), _ => return Err(Error::InvalidReturn), } - ActionOutcome::Trapped { message } => - return Err(trap_error.unwrap_or_else(|| - format!("Wasm execution trapped: {}", message).into() - )), + ActionOutcome::Trapped { message } => return Err(trap_error.unwrap_or_else( + || format!("Wasm execution trapped: {}", message).into() + )), }; // Read the output data from guest memory. @@ -229,6 +237,7 @@ fn call_method( fn instantiate_env_module( global_exports: Rc>>>, compiler: Compiler, + host_functions: &[&'static dyn Function], ) -> std::result::Result { let isa = target_isa()?; @@ -240,7 +249,7 @@ fn instantiate_env_module( let mut finished_functions = >::new(); let mut code_memory = CodeMemory::new(); - for function in SubstrateExternals::functions().iter() { + for function in host_functions { let sig = translate_signature( cranelift_ir_signature(function.signature(), &call_conv), pointer_type @@ -266,7 +275,7 @@ fn instantiate_env_module( let imports = Imports::none(); let data_initializers = Vec::new(); let signatures = PrimaryMap::new(); - let env_state = EnvState::new::(code_memory, compiler); + let env_state = EnvState::new(code_memory, compiler, host_functions); let result = InstanceHandle::new( Rc::new(module), diff --git a/core/executor/src/wasmtime/trampoline.rs b/core/executor/src/wasmtime/trampoline.rs index 7abc59faa5ef5..f6561f485d8e9 100644 --- a/core/executor/src/wasmtime/trampoline.rs +++ b/core/executor/src/wasmtime/trampoline.rs @@ -24,8 +24,8 @@ use cranelift_codegen::ir::{InstBuilder, StackSlotData, StackSlotKind, TrapCode} use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use wasmtime_jit::{CodeMemory, Compiler}; use wasmtime_runtime::{VMContext, VMFunctionBody}; -use wasm_interface::{HostFunctions, Function, Value, ValueType}; -use std::{cmp, panic, ptr}; +use wasm_interface::{Function, Value, ValueType}; +use std::{cmp, panic::{self, AssertUnwindSafe}, ptr}; use crate::error::{Error, WasmError}; use crate::wasmtime::function_executor::{FunctionExecutorState, FunctionExecutor}; @@ -33,7 +33,6 @@ use crate::wasmtime::function_executor::{FunctionExecutorState, FunctionExecutor const CALL_SUCCESS: u32 = 0; const CALL_FAILED_WITH_ERROR: u32 = 1; const CALL_WITH_BAD_HOST_STATE: u32 = 2; -const CALL_PANICKED: u32 = 3; /// A code to trap with that indicates a host call error. const TRAP_USER_CODE: u16 = 0; @@ -45,7 +44,7 @@ const MAX_WASM_TYPE_SIZE: usize = 8; /// The top-level host state of the "env" module. This state is used by the trampoline function to /// construct a `FunctionExecutor` which can execute the host call. pub struct EnvState { - externals: &'static [&'static dyn Function], + host_functions: Vec<&'static dyn Function>, compiler: Compiler, // The code memory must be kept around on the state to prevent it from being dropped. #[allow(dead_code)] @@ -58,13 +57,17 @@ pub struct EnvState { impl EnvState { /// Construct a new `EnvState` which owns the given code memory. - pub fn new(code_memory: CodeMemory, compiler: Compiler) -> Self { + pub fn new( + code_memory: CodeMemory, + compiler: Compiler, + host_functions: &[&'static dyn Function], + ) -> Self { EnvState { - externals: HF::functions(), trap: None, compiler, code_memory, executor_state: None, + host_functions: host_functions.to_vec(), } } @@ -78,11 +81,10 @@ impl EnvState { /// to the call arguments on the stack as arguments. Returns zero on success and a non-zero value /// on failure. unsafe extern "C" fn stub_fn(vmctx: *mut VMContext, func_index: u32, values_vec: *mut i64) -> u32 { - let result = panic::catch_unwind(|| { - if let Some(state) = (*vmctx).host_state().downcast_mut::() { + if let Some(state) = (*vmctx).host_state().downcast_mut::() { match stub_fn_inner( vmctx, - state.externals, + &state.host_functions, &mut state.compiler, state.executor_state.as_mut(), func_index, @@ -94,12 +96,10 @@ unsafe extern "C" fn stub_fn(vmctx: *mut VMContext, func_index: u32, values_vec: CALL_FAILED_WITH_ERROR } } - } else { - // Well, we can't even set a trap message, so we'll just exit without one. - CALL_WITH_BAD_HOST_STATE - } - }); - result.unwrap_or(CALL_PANICKED) + } else { + // Well, we can't even set a trap message, so we'll just exit without one. + CALL_WITH_BAD_HOST_STATE + } } /// Implements most of the logic in `stub_fn` but returning a `Result` instead of an integer error @@ -111,8 +111,7 @@ unsafe fn stub_fn_inner( executor_state: Option<&mut FunctionExecutorState>, func_index: u32, values_vec: *mut i64, -) -> Result<(), Error> -{ +) -> Result<(), Error> { let func = externals.get(func_index as usize) .ok_or_else(|| format!("call to undefined external function with index {}", func_index))?; let executor_state = executor_state @@ -120,22 +119,41 @@ unsafe fn stub_fn_inner( // Build the external function context. let mut context = FunctionExecutor::new(vmctx, compiler, executor_state)?; + let mut context = AssertUnwindSafe(&mut context); - let signature = func.signature(); + // Execute and write output back to the stack. + let return_val = panic::catch_unwind(move || { + let signature = func.signature(); - // Read the arguments from the stack. - let mut args = signature.args.iter() - .enumerate() - .map(|(i, ¶m_type)| read_value_from(values_vec.offset(i as isize), param_type)); + // Read the arguments from the stack. + let mut args = signature.args.iter() + .enumerate() + .map(|(i, ¶m_type)| read_value_from(values_vec.offset(i as isize), param_type)); - // Execute and write output back to the stack. - let return_val = func.execute(&mut context, &mut args) - .map_err(|e| Error::FunctionExecution(func.name().to_string(), e))?; - if let Some(val) = return_val { - write_value_to(values_vec, val); - } + func.execute(&mut **context, &mut args) + }); + + match return_val { + Ok(ret_val) => { + if let Some(val) = ret_val + .map_err(|e| Error::FunctionExecution(func.name().to_string(), e))? { + write_value_to(values_vec, val); + } - Ok(()) + Ok(()) + }, + Err(e) => { + let message = if let Some(err) = e.downcast_ref::() { + err.to_string() + } else if let Some(err) = e.downcast_ref::<&str>() { + err.to_string() + } else { + "Panicked without any further information!".into() + }; + + Err(Error::FunctionExecution(func.name().to_string(), message)) + } + } } /// Create a trampoline for invoking a host function. diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/bare_function_interface.rs index 8587932c9cc55..300cd89e73a0a 100644 --- a/core/runtime-interface/proc-macro/src/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/bare_function_interface.rs @@ -21,11 +21,12 @@ //! functions will prepare the parameters for the FFI boundary, call the external host function //! exported into wasm and convert back the result. //! -//! [`generate`] is the entry point for generating for each trait method one bare function. +//! [`generate`](bare_function_interface::generate) is the entry point for generating for each +//! trait method one bare function. //! -//! [`function_for_method`] generates the bare function per trait method. Each bare function -//! contains both implementations. The implementations are feature-gated, so that one is compiled -//! for the native and the other for the wasm side. +//! [`function_for_method`](bare_function_interface::function_for_method) generates the bare +//! function per trait method. Each bare function contains both implementations. The implementations +//! are feature-gated, so that one is compiled for the native and the other for the wasm side. use crate::utils::{ generate_crate_access, create_host_function_ident, get_function_arguments, diff --git a/core/runtime-interface/src/pass_by.rs b/core/runtime-interface/src/pass_by.rs index 8314f32d7a3aa..10ed924f90f4c 100644 --- a/core/runtime-interface/src/pass_by.rs +++ b/core/runtime-interface/src/pass_by.rs @@ -14,8 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Provides the [`PassBy`] trait to simplify the implementation of the runtime interface traits -//! for custom types. [`Codec`] and [`Inner`] are the two provided strategy implementations. +//! Provides the [`PassBy`](pass_by::PassBy) trait to simplify the implementation of the +//! runtime interface traits for custom types. [`Codec`](pass_by::Codec), [`Inner`](pass_by::Inner) +//! and [`Enum`](pass_by::Enum) are the provided strategy implementations. use crate::{RIType, impls::{pointer_and_len_from_u64, pointer_and_len_to_u64}}; @@ -138,6 +139,8 @@ impl FromFFIValue for T { /// /// This type expects the type that wants to implement this strategy as generic parameter. /// +/// [`PassByCodec`](derive.PassByCodec.html) is a derive macro to implement this strategy. +/// /// # Example /// ``` /// # use substrate_runtime_interface::pass_by::{PassBy, Codec}; @@ -223,6 +226,8 @@ pub trait PassByInner: Sized { /// This type expects the type that wants to use this strategy as generic parameter `T` and the /// inner type as generic parameter `I`. /// +/// [`PassByInner`](derive.PassByInner.html) is a derive macro to implement this strategy. +/// /// # Example /// ``` /// # use substrate_runtime_interface::pass_by::{PassBy, Inner, PassByInner}; @@ -294,6 +299,8 @@ impl, I: RIType> RIType for Inner { /// This type expects the type that wants to implement this strategy as generic parameter. Besides /// that the type needs to implement `TryFrom` and `From for u8`. /// +/// [`PassByEnum`](derive.PassByEnum.html) is a derive macro to implement this strategy. +/// /// # Example /// ``` /// # use substrate_runtime_interface::pass_by::{PassBy, Enum}; From 959ae7dc3d0519150107c0cdc456f0a63efb2744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 4 Nov 2019 10:52:21 +0100 Subject: [PATCH 61/76] Some macro structure clean up --- core/runtime-interface/proc-macro/src/lib.rs | 61 +++-------------- .../{pass_by_codec.rs => pass_by/codec.rs} | 0 .../src/{pass_by_enum.rs => pass_by/enum_.rs} | 0 .../{pass_by_inner.rs => pass_by/inner.rs} | 0 .../proc-macro/src/pass_by/mod.rs | 25 +++++++ .../bare_function_interface.rs | 0 .../host_function_interface.rs | 0 .../proc-macro/src/runtime_interface/mod.rs | 65 +++++++++++++++++++ .../trait_decl_impl.rs | 0 9 files changed, 98 insertions(+), 53 deletions(-) rename core/runtime-interface/proc-macro/src/{pass_by_codec.rs => pass_by/codec.rs} (100%) rename core/runtime-interface/proc-macro/src/{pass_by_enum.rs => pass_by/enum_.rs} (100%) rename core/runtime-interface/proc-macro/src/{pass_by_inner.rs => pass_by/inner.rs} (100%) create mode 100644 core/runtime-interface/proc-macro/src/pass_by/mod.rs rename core/runtime-interface/proc-macro/src/{ => runtime_interface}/bare_function_interface.rs (100%) rename core/runtime-interface/proc-macro/src/{ => runtime_interface}/host_function_interface.rs (100%) create mode 100644 core/runtime-interface/proc-macro/src/runtime_interface/mod.rs rename core/runtime-interface/proc-macro/src/{ => runtime_interface}/trait_decl_impl.rs (100%) diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index c0360fe4bd856..eebd4cb57e43e 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -18,84 +18,39 @@ extern crate proc_macro; -use proc_macro2::{Span, TokenStream}; +use syn::{parse_macro_input, ItemTrait, DeriveInput}; -use syn::{parse_macro_input, Ident, ItemTrait, Result, DeriveInput}; - -use quote::quote; - -use inflector::Inflector; - -use utils::generate_runtime_interface_include; - -mod bare_function_interface; -mod host_function_interface; -mod pass_by_codec; -mod pass_by_enum; -mod pass_by_inner; -mod trait_decl_impl; +mod pass_by; +mod runtime_interface; mod utils; -mod kw { - // Custom keyword `wasm_only` that can be given as attribute to [`runtime_interface`]. - syn::custom_keyword!(wasm_only); -} - #[proc_macro_attribute] pub fn runtime_interface( attrs: proc_macro::TokenStream, input: proc_macro::TokenStream, ) -> proc_macro::TokenStream { let trait_def = parse_macro_input!(input as ItemTrait); - let wasm_only = parse_macro_input!(attrs as Option); + let wasm_only = parse_macro_input!(attrs as Option); - runtime_interface_impl(trait_def, wasm_only.is_some()) + runtime_interface::runtime_interface_impl(trait_def, wasm_only.is_some()) .unwrap_or_else(|e| e.to_compile_error()) .into() } -fn runtime_interface_impl(trait_def: ItemTrait, is_wasm_only: bool) -> Result { - let bare_functions = bare_function_interface::generate(&trait_def, is_wasm_only)?; - let crate_include = generate_runtime_interface_include(); - let mod_name = Ident::new(&trait_def.ident.to_string().to_snake_case(), Span::call_site()); - let trait_decl_impl = trait_decl_impl::process(&trait_def, is_wasm_only)?; - let host_functions = host_function_interface::generate(&trait_def, is_wasm_only)?; - let vis = trait_def.vis; - let attrs = &trait_def.attrs; - - let res = quote! { - #( #attrs )* - #vis mod #mod_name { - use super::*; - #crate_include - - #bare_functions - - #trait_decl_impl - - #host_functions - } - }; - - // println!("{}", res); - - Ok(res) -} - #[proc_macro_derive(PassByCodec)] pub fn pass_by_codec(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let input = parse_macro_input!(input as DeriveInput); - pass_by_codec::derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() + pass_by::codec_derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() } #[proc_macro_derive(PassByInner)] pub fn pass_by_inner(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let input = parse_macro_input!(input as DeriveInput); - pass_by_inner::derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() + pass_by::inner_derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() } #[proc_macro_derive(PassByEnum)] pub fn pass_by_enum(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let input = parse_macro_input!(input as DeriveInput); - pass_by_enum::derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() + pass_by::enum_derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() } diff --git a/core/runtime-interface/proc-macro/src/pass_by_codec.rs b/core/runtime-interface/proc-macro/src/pass_by/codec.rs similarity index 100% rename from core/runtime-interface/proc-macro/src/pass_by_codec.rs rename to core/runtime-interface/proc-macro/src/pass_by/codec.rs diff --git a/core/runtime-interface/proc-macro/src/pass_by_enum.rs b/core/runtime-interface/proc-macro/src/pass_by/enum_.rs similarity index 100% rename from core/runtime-interface/proc-macro/src/pass_by_enum.rs rename to core/runtime-interface/proc-macro/src/pass_by/enum_.rs diff --git a/core/runtime-interface/proc-macro/src/pass_by_inner.rs b/core/runtime-interface/proc-macro/src/pass_by/inner.rs similarity index 100% rename from core/runtime-interface/proc-macro/src/pass_by_inner.rs rename to core/runtime-interface/proc-macro/src/pass_by/inner.rs diff --git a/core/runtime-interface/proc-macro/src/pass_by/mod.rs b/core/runtime-interface/proc-macro/src/pass_by/mod.rs new file mode 100644 index 0000000000000..cfcb7d6c6aab5 --- /dev/null +++ b/core/runtime-interface/proc-macro/src/pass_by/mod.rs @@ -0,0 +1,25 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! All the `PassBy*` derive implementations. + +mod codec; +mod enum_; +mod inner; + +pub use codec::derive_impl as codec_derive_impl; +pub use enum_::derive_impl as enum_derive_impl; +pub use inner::derive_impl as inner_derive_impl; diff --git a/core/runtime-interface/proc-macro/src/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs similarity index 100% rename from core/runtime-interface/proc-macro/src/bare_function_interface.rs rename to core/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs diff --git a/core/runtime-interface/proc-macro/src/host_function_interface.rs b/core/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs similarity index 100% rename from core/runtime-interface/proc-macro/src/host_function_interface.rs rename to core/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs diff --git a/core/runtime-interface/proc-macro/src/runtime_interface/mod.rs b/core/runtime-interface/proc-macro/src/runtime_interface/mod.rs new file mode 100644 index 0000000000000..142fff646ed18 --- /dev/null +++ b/core/runtime-interface/proc-macro/src/runtime_interface/mod.rs @@ -0,0 +1,65 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::utils::generate_runtime_interface_include; + +use proc_macro2::{Span, TokenStream}; + +use syn::{Ident, ItemTrait, Result}; + +use inflector::Inflector; + +use quote::quote; + +mod bare_function_interface; +mod host_function_interface; +mod trait_decl_impl; + +/// Custom keywords supported by the `runtime_interface` attribute. +pub mod keywords { + // Custom keyword `wasm_only` that can be given as attribute to [`runtime_interface`]. + syn::custom_keyword!(wasm_only); +} + +/// Implementation of the `runtime_interface` attribute. +/// +/// It expects the trait definition the attribute was put above and if this should be an wasm only +/// interface. +pub fn runtime_interface_impl(trait_def: ItemTrait, is_wasm_only: bool) -> Result { + let bare_functions = bare_function_interface::generate(&trait_def, is_wasm_only)?; + let crate_include = generate_runtime_interface_include(); + let mod_name = Ident::new(&trait_def.ident.to_string().to_snake_case(), Span::call_site()); + let trait_decl_impl = trait_decl_impl::process(&trait_def, is_wasm_only)?; + let host_functions = host_function_interface::generate(&trait_def, is_wasm_only)?; + let vis = trait_def.vis; + let attrs = &trait_def.attrs; + + let res = quote! { + #( #attrs )* + #vis mod #mod_name { + use super::*; + #crate_include + + #bare_functions + + #trait_decl_impl + + #host_functions + } + }; + + Ok(res) +} diff --git a/core/runtime-interface/proc-macro/src/trait_decl_impl.rs b/core/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs similarity index 100% rename from core/runtime-interface/proc-macro/src/trait_decl_impl.rs rename to core/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs From f6da18e738aba7ef1edb327ee6b0c33ddb4fe6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 4 Nov 2019 13:29:48 +0100 Subject: [PATCH 62/76] Rework and test exchangebale host functions --- .../bare_function_interface.rs | 33 ++------ .../host_function_interface.rs | 84 ++++++++++--------- .../runtime-interface/proc-macro/src/utils.rs | 10 +++ core/runtime-interface/src/lib.rs | 5 ++ core/runtime-interface/test-wasm/src/lib.rs | 20 ++++- core/sr-io/src/lib.rs | 4 +- 6 files changed, 89 insertions(+), 67 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs index 300cd89e73a0a..7b67a9e4ebe7f 100644 --- a/core/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs @@ -29,13 +29,12 @@ //! are feature-gated, so that one is compiled for the native and the other for the wasm side. use crate::utils::{ - generate_crate_access, create_host_function_ident, get_function_arguments, - get_function_argument_names, get_function_argument_types_without_ref, get_trait_methods, + generate_crate_access, create_exchangeable_host_function_ident, get_function_arguments, + get_function_argument_names, get_trait_methods, }; use syn::{ - Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Result, ReturnType, spanned::Spanned, - parse_quote, + Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Result, spanned::Spanned, parse_quote, }; use proc_macro2::{TokenStream, Span}; @@ -61,7 +60,7 @@ fn function_for_method( is_wasm_only: bool, ) -> Result { let std_impl = function_std_impl(trait_name, method, is_wasm_only)?; - let no_std_impl = function_no_std_impl(trait_name, method)?; + let no_std_impl = function_no_std_impl(method)?; Ok( quote! { @@ -73,21 +72,12 @@ fn function_for_method( } /// Generates the bare function implementation for `cfg(not(feature = "std"))`. -fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result { +fn function_no_std_impl(method: &TraitItemMethod) -> Result { let function_name = &method.sig.ident; - let host_function_name = create_host_function_ident(&method.sig.ident, trait_name); + let host_function_name = create_exchangeable_host_function_ident(&method.sig.ident); let args = get_function_arguments(&method.sig); let arg_names = get_function_argument_names(&method.sig); - let arg_names2 = get_function_argument_names(&method.sig); - let arg_types = get_function_argument_types_without_ref(&method.sig); - let crate_ = generate_crate_access(); let return_value = &method.sig.output; - let convert_return_value = match return_value { - ReturnType::Default => quote!(), - ReturnType::Type(_, ref ty) => quote! { - <#ty as #crate_::wasm::FromFFIValue>::from_ffi_value(result) - } - }; let attrs = &method.attrs; Ok( @@ -95,17 +85,8 @@ fn function_no_std_impl(trait_name: &Ident, method: &TraitItemMethod) -> Result< #[cfg(not(feature = "std"))] #( #attrs )* pub fn #function_name( #( #args, )* ) #return_value { - // Generate all wrapped ffi values. - #( - let #arg_names = <#arg_types as #crate_::wasm::IntoFFIValue>::into_ffi_value( - &#arg_names, - ); - )* - // Call the host function - let result = unsafe { #host_function_name.get()( #( #arg_names2.get(), )* ) }; - - #convert_return_value + #host_function_name.get()( #( #arg_names, )* ) } } ) diff --git a/core/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs b/core/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs index d2621f237f02f..a489cb8ba1404 100644 --- a/core/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs @@ -23,7 +23,8 @@ use crate::utils::{ generate_crate_access, create_host_function_ident, get_function_argument_names, get_function_argument_types_without_ref, get_function_argument_types_ref_and_mut, - get_function_argument_names_and_types_without_ref, get_trait_methods, + get_function_argument_names_and_types_without_ref, get_trait_methods, get_function_arguments, + get_function_argument_types, create_exchangeable_host_function_ident, }; use syn::{ @@ -48,9 +49,9 @@ pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result(t) })?; - let extern_host_exchangeable_functions = get_trait_methods(trait_def) + let exchangeable_host_functions = get_trait_methods(trait_def) .try_fold(TokenStream::new(), |mut t, m| { - t.extend(generate_extern_host_exchangeable_function(m, trait_name)?); + t.extend(generate_exchangeable_host_function(m)?); Ok::<_, Error>(t) })?; let host_functions_struct = generate_host_functions_struct(trait_def, is_wasm_only)?; @@ -67,7 +68,7 @@ pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result Result Result { let crate_ = generate_crate_access(); + let args = get_function_arguments(&method.sig); let arg_types = get_function_argument_types_without_ref(&method.sig); let arg_types2 = get_function_argument_types_without_ref(&method.sig); let arg_names = get_function_argument_names(&method.sig); let arg_names2 = get_function_argument_names(&method.sig); let arg_names3 = get_function_argument_names(&method.sig); - let function = create_host_function_ident(&method.sig.ident, trait_name); - let doc_string = format!(" Default extern host function implementation for [`../{}`].", function); + let function = &method.sig.ident; + let ext_function = create_host_function_ident(&method.sig.ident, trait_name); + let doc_string = format!( + " Default extern host function implementation for [`super::{}`].", + method.sig.ident, + ); + let return_value = &method.sig.output; - let output = match method.sig.output { + let ffi_return_value = match method.sig.output { ReturnType::Default => quote!(), ReturnType::Type(_, ref ty) => quote! { -> <#ty as #crate_::RIType>::FFIType }, }; + let convert_return_value = match return_value { + ReturnType::Default => quote!(), + ReturnType::Type(_, ref ty) => quote! { + <#ty as #crate_::wasm::FromFFIValue>::from_ffi_value(result) + } + }; + Ok( quote! { #[doc = #doc_string] - pub unsafe fn #function ( - #( #arg_names: <#arg_types as #crate_::RIType>::FFIType ),* - ) #output { - /// Contains the actual extern function. - mod implementation { - use super::*; - - extern "C" { - /// The extern function. - pub fn #function ( - #( #arg_names2: <#arg_types2 as #crate_::RIType>::FFIType ),* - ) #output; - } + pub fn #function ( #( #args ),* ) #return_value { + extern "C" { + /// The extern function. + pub fn #ext_function ( + #( #arg_names: <#arg_types as #crate_::RIType>::FFIType ),* + ) #ffi_return_value; } - implementation::#function( #( #arg_names3 ),* ) + // Generate all wrapped ffi values. + #( + let #arg_names2 = <#arg_types2 as #crate_::wasm::IntoFFIValue>::into_ffi_value( + &#arg_names2, + ); + )* + + let result = unsafe { #ext_function( #( #arg_names3.get() ),* ) }; + + #convert_return_value } } ) } -/// Generate the extern host exchangeable function for the given method. -fn generate_extern_host_exchangeable_function( - method: &TraitItemMethod, - trait_name: &Ident, -) -> Result { +/// Generate the host exchangeable function for the given method. +fn generate_exchangeable_host_function(method: &TraitItemMethod) -> Result { let crate_ = generate_crate_access(); - let arg_types = get_function_argument_types_without_ref(&method.sig); - let function = create_host_function_ident(&method.sig.ident, trait_name); - let doc_string = format!(" Exchangeable extern host function used by [`{}`].", method.sig.ident); - - let output = match method.sig.output { - ReturnType::Default => quote!(), - ReturnType::Type(_, ref ty) => quote! { - -> <#ty as #crate_::RIType>::FFIType - } - }; + let arg_types = get_function_argument_types(&method.sig); + let function = &method.sig.ident; + let exchangeable_function = create_exchangeable_host_function_ident(&method.sig.ident); + let doc_string = format!(" Exchangeable host function used by [`{}`].", method.sig.ident); + let output = &method.sig.output; Ok( quote! { #[cfg(not(feature = "std"))] #[allow(non_upper_case_globals)] #[doc = #doc_string] - pub static #function : #crate_::wasm::ExchangeableFunction< - unsafe fn ( #( <#arg_types as #crate_::RIType>::FFIType ),* ) #output + pub static #exchangeable_function : #crate_::wasm::ExchangeableFunction< + fn ( #( #arg_types ),* ) #output > = #crate_::wasm::ExchangeableFunction::new(extern_host_function_impls::#function); } ) diff --git a/core/runtime-interface/proc-macro/src/utils.rs b/core/runtime-interface/proc-macro/src/utils.rs index 6f54572e1c615..e7e780dc107f3 100644 --- a/core/runtime-interface/proc-macro/src/utils.rs +++ b/core/runtime-interface/proc-macro/src/utils.rs @@ -60,6 +60,11 @@ pub fn generate_crate_access() -> TokenStream { } } +/// Create the exchangeable host function identifier for the given function name. +pub fn create_exchangeable_host_function_ident(name: &Ident) -> Ident { + Ident::new(&format!("host_{}", name), Span::call_site()) +} + /// Create the host function identifier for the given function name. pub fn create_host_function_ident(name: &Ident, trait_name: &Ident) -> Ident { Ident::new( @@ -87,6 +92,11 @@ pub fn get_function_argument_names<'a>(sig: &'a Signature) -> impl Iterator(sig: &'a Signature) -> impl Iterator> { + get_function_arguments(sig).map(|pt| &pt.ty) +} + /// Returns the function argument types, minus any `Self` type. If any of the arguments /// is a reference, the underlying type without the ref is returned. pub fn get_function_argument_types_without_ref<'a>( diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 12ca0ddcbd146..30a9962031ce0 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -148,4 +148,9 @@ mod tests { fn test_invalid_utf8_data_should_return_an_error() { call_wasm_method::("test_invalid_utf8_data_should_return_an_error"); } + + #[test] + fn test_overwrite_native_function_implementation() { + call_wasm_method::("test_overwrite_native_function_implementation"); + } } diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index 9adaed3137526..d0d256e633758 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -73,8 +73,12 @@ pub trait TestApi { } /// A function that is called with invalid utf8 data from the runtime. - fn invalid_utf8_data(_data: &str) { + fn invalid_utf8_data(_data: &str) {} + /// Overwrite the native implementation in wasm. The native implementation always returns + /// `false` and the replacement function will return always `true`. + fn overwrite_native_function_implementation() -> bool { + false } } @@ -156,4 +160,18 @@ wasm_export_functions! { test_api::invalid_utf8_data(data_str); } + + fn test_overwrite_native_function_implementation() { + fn new_implementation() -> bool { + true + } + + // Check native implementation + assert!(!test_api::overwrite_native_function_implementation()); + + let _guard = test_api::host_overwrite_native_function_implementation + .replace_implementation(new_implementation); + + assert!(test_api::overwrite_native_function_implementation()); + } } diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 88edb469c5a62..ca922e1c245ad 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -34,14 +34,14 @@ use rstd::ops::Deref; #[cfg(feature = "std")] use primitives::{ crypto::Pair, traits::KeystoreExt, offchain::OffchainExt, hexdisplay::HexDisplay, + storage::ChildStorageKey, }; use primitives::{ - crypto::KeyTypeId, ed25519, sr25519, H256, storage::ChildStorageKey, + crypto::KeyTypeId, ed25519, sr25519, H256, LogLevel, offchain::{ Timestamp, HttpRequestId, HttpRequestStatus, HttpError, StorageKind, OpaqueNetworkState, }, - LogLevel, }; #[cfg(feature = "std")] From 573cec0128497d2e12f80970c8bfa6f10531f11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 4 Nov 2019 14:33:28 +0100 Subject: [PATCH 63/76] PassBy derive macros documentation --- Cargo.lock | 2 + core/runtime-interface/proc-macro/Cargo.toml | 4 ++ core/runtime-interface/proc-macro/src/lib.rs | 58 +++++++++++++++++++ .../proc-macro/src/pass_by/mod.rs | 2 +- core/runtime-interface/src/pass_by.rs | 2 +- 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 232a0db10b04a..67d635f7fcb0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5919,9 +5919,11 @@ name = "substrate-runtime-interface-proc-macro" version = "2.0.0" dependencies = [ "Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-runtime-interface 2.0.0", "syn 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/core/runtime-interface/proc-macro/Cargo.toml b/core/runtime-interface/proc-macro/Cargo.toml index 2d4c4bf1148ba..b6c3c9355624a 100644 --- a/core/runtime-interface/proc-macro/Cargo.toml +++ b/core/runtime-interface/proc-macro/Cargo.toml @@ -13,3 +13,7 @@ quote = "1.0.2" proc-macro2 = "1.0.3" Inflector = "0.11.4" proc-macro-crate = "0.1.4" + +[dev-dependencies] +substrate-runtime-interface = { path = ".." } +codec = { package = "parity-scale-codec", version = "1.0.6", features = [ "derive" ] } diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index eebd4cb57e43e..a39ce3235fa8c 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -37,18 +37,76 @@ pub fn runtime_interface( .into() } +/// Derive macro for implementing `PassBy` with the `Codec` strategy. +/// +/// This requires that the type implements `Encode` and `Decode` from `parity-scale-codec`. +/// +/// # Example +/// +/// ``` +/// # use substrate_runtime_interface::pass_by::PassByCodec; +/// # use codec::{Encode, Decode}; +/// #[derive(PassByCodec, Encode, Decode)] +/// struct EncodableType { +/// name: Vec, +/// param: u32, +/// } +/// ``` #[proc_macro_derive(PassByCodec)] pub fn pass_by_codec(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let input = parse_macro_input!(input as DeriveInput); pass_by::codec_derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() } +/// Derive macro for implementing `PassBy` with the `Inner` strategy. +/// +/// Besides implementing `PassBy`, this derive also implements the helper trait `PassByInner`. +/// +/// The type is required to be a struct with just one field. The field type needs to implement +/// the required traits to pass it between the wasm and the native side. (See the runtime interface +/// crate for more information about these traits.) +/// +/// # Example +/// +/// ``` +/// # use substrate_runtime_interface::pass_by::PassByInner; +/// #[derive(PassByInner)] +/// struct Data([u8; 32]); +/// ``` +/// +/// ``` +/// # use substrate_runtime_interface::pass_by::PassByInner; +/// #[derive(PassByInner)] +/// struct Data { +/// data: [u8; 32], +/// } +/// ``` #[proc_macro_derive(PassByInner)] pub fn pass_by_inner(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let input = parse_macro_input!(input as DeriveInput); pass_by::inner_derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into() } +/// Derive macro for implementing `PassBy` with the `Enum` strategy. +/// +/// Besides implementing `PassBy`, this derive also implements `TryFrom` and `From for u8` +/// for the type. +/// +/// The type is required to be an enum with only unit variants and at maximum `256` variants. Also +/// it is required that the type implements `Copy`. +/// +/// # Example +/// +/// ``` +/// # use substrate_runtime_interface::pass_by::PassByEnum; +/// #[derive(PassByEnum, Copy, Clone)] +/// enum Data { +/// Okay, +/// NotOkay, +/// // This will not work with the derive. +/// //Why(u32), +/// } +/// ``` #[proc_macro_derive(PassByEnum)] pub fn pass_by_enum(input: proc_macro::TokenStream) -> proc_macro::TokenStream { let input = parse_macro_input!(input as DeriveInput); diff --git a/core/runtime-interface/proc-macro/src/pass_by/mod.rs b/core/runtime-interface/proc-macro/src/pass_by/mod.rs index cfcb7d6c6aab5..23d511183293c 100644 --- a/core/runtime-interface/proc-macro/src/pass_by/mod.rs +++ b/core/runtime-interface/proc-macro/src/pass_by/mod.rs @@ -20,6 +20,6 @@ mod codec; mod enum_; mod inner; -pub use codec::derive_impl as codec_derive_impl; +pub use self::codec::derive_impl as codec_derive_impl; pub use enum_::derive_impl as enum_derive_impl; pub use inner::derive_impl as inner_derive_impl; diff --git a/core/runtime-interface/src/pass_by.rs b/core/runtime-interface/src/pass_by.rs index 10ed924f90f4c..d1ce20740b4c7 100644 --- a/core/runtime-interface/src/pass_by.rs +++ b/core/runtime-interface/src/pass_by.rs @@ -37,7 +37,7 @@ pub use substrate_runtime_interface_proc_macro::{PassByCodec, PassByInner, PassB /// Something that should be passed between wasm and the host using the given strategy. /// -/// See [`Codec`] or [`Inner`] for more information about the provided strategies. +/// See [`Codec`], [`Inner`] or [`Enum`] for more information about the provided strategies. pub trait PassBy: Sized { /// The strategy that should be used to pass the type. type PassBy: PassByImpl; From 1e0f2ebf9dc931b4bb1d0a08b1be0d28c74afb5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 6 Nov 2019 15:44:12 +0100 Subject: [PATCH 64/76] Docs for `runtime_interface` macro --- Cargo.lock | 1 + core/runtime-interface/proc-macro/Cargo.toml | 6 + core/runtime-interface/proc-macro/src/lib.rs | 148 ++++++++++++++++++- 3 files changed, 154 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 67d635f7fcb0e..84814472b67e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5923,6 +5923,7 @@ dependencies = [ "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "substrate-externalities 2.0.0", "substrate-runtime-interface 2.0.0", "syn 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/core/runtime-interface/proc-macro/Cargo.toml b/core/runtime-interface/proc-macro/Cargo.toml index b6c3c9355624a..4c39c8282ba62 100644 --- a/core/runtime-interface/proc-macro/Cargo.toml +++ b/core/runtime-interface/proc-macro/Cargo.toml @@ -17,3 +17,9 @@ proc-macro-crate = "0.1.4" [dev-dependencies] substrate-runtime-interface = { path = ".." } codec = { package = "parity-scale-codec", version = "1.0.6", features = [ "derive" ] } +externalities = { package = "substrate-externalities", path = "../../externalities" } + +# We actually don't need the `std` feature in this crate, but the tests require it. +[features] +default = [ "std" ] +std = [] diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index a39ce3235fa8c..a07df0e99af56 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -14,7 +14,16 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Procedural macros for generating runtime interfaces. +//! This crate provides procedural macros for usage within the context of the Substrate runtime +//! interface. +//! +//! The following macros are provided: +//! +//! 1. The [`#[runtime_interface]`](attr.runtime_interface.html) attribute macro for generating the +//! runtime interfaces. +//! 2. The [`PassByCodec`](derive.PassByCodec.html) derive macro for implementing `PassBy` with `Codec`. +//! 3. The [`PassByEnum`](derive.PassByInner.html) derive macro for implementing `PassBy` with `Enum`. +//! 4. The [`PassByInner`](derive.PassByInner.html) derive macro for implementing `PassBy` with `Inner`. extern crate proc_macro; @@ -24,6 +33,143 @@ mod pass_by; mod runtime_interface; mod utils; +/// Attribute macro for transforming a trait declaration into a runtime interface. +/// +/// A runtime interface is a fixed interface between a Substrate compatible runtime and the native +/// node. This interface is callable from a native and a wasm runtime. The macro will generate the +/// corresponding code for the native implementation and the code for calling from the wasm +/// side to the native implementation. +/// +/// The macro expects the runtime interface declaration as trait declaration: +/// +/// ``` +/// # use substrate_runtime_interface::runtime_interface; +/// +/// #[runtime_interface] +/// trait Interface { +/// /// A function that can be called from native/wasm. +/// /// +/// /// The implementation given to this function is only compiled on native. +/// fn call_some_complex_code(data: &[u8]) -> Vec { +/// // Here you could call some rather complex code that only compiles on native or +/// // is way faster in native than executing it in wasm. +/// Vec::new() +/// } +/// +/// /// A function can take a `&self` or `&mut self` argument to get access to the +/// /// `Externalities`. (The generated method does not require +/// /// this argument, so the function can be called just with the `optional` argument) +/// fn set_or_clear(&mut self, optional: Option>) { +/// match optional { +/// Some(value) => self.set_storage([1, 2, 3, 4].to_vec(), value), +/// None => self.clear_storage(&[1, 2, 3, 4]), +/// } +/// } +/// } +/// ``` +/// +/// +/// The given example will generate roughly the following code for native: +/// +/// ``` +/// // The name of the trait is converted to snake case and used as mod name. +/// // +/// // Be aware that this module is not `public`, the visibility of the module is determined based +/// // on the visibility of the trait declaration. +/// mod interface { +/// trait Interface { +/// fn call_some_complex_code(data: &[u8]) -> Vec; +/// fn set_or_clear(&mut self, optional: Option>); +/// } +/// +/// impl Interface for &mut dyn externalities::Externalities { +/// fn call_some_complex_code(data: &[u8]) -> Vec { Vec::new() } +/// fn set_or_clear(&mut self, optional: Option>) { +/// match optional { +/// Some(value) => self.set_storage([1, 2, 3, 4].to_vec(), value), +/// None => self.clear_storage(&[1, 2, 3, 4]), +/// } +/// } +/// } +/// +/// pub fn call_some_complex_code(data: &[u8]) -> Vec { +/// <&mut dyn externalities::Externalities as Interface>::call_some_complex_code(data) +/// } +/// +/// pub fn set_or_clear(optional: Option>) { +/// externalities::with_externalities(|mut ext| Interface::set_or_clear(&mut ext, optional)) +/// .expect("`set_or_clear` called outside of an Externalities-provided environment.") +/// } +/// +/// /// This type implements the `HostFunctions` trait (from `substrate-wasm-interface`) and +/// /// provides the host implementation for the wasm side. The host implementation converts the +/// /// arguments from wasm to native and calls the corresponding native function. +/// /// +/// /// This type needs to be passed to the wasm executor, so that the host functions will be +/// /// registered in the executor. +/// pub struct HostFunctions; +/// } +/// ``` +/// +/// +/// The given example will generate roughly the following code for wasm: +/// +/// ``` +/// mod interface { +/// mod extern_host_functions_impls { +/// extern "C" { +/// /// Every function is exported as `ext_TRAIT_NAME_FUNCTION_NAME`. +/// /// +/// /// The type for each argument of the exported function depends on +/// /// `::FFIType`. +/// pub fn ext_Interface_call_some_complex_code(data: u64); +/// pub fn ext_Interface_set_or_clear(optional: u64); +/// } +/// } +/// +/// /// The type is actually `ExchangeableFunction` (from `substrate-runtime-interface`). +/// /// +/// /// This can be used to replace the implementation of the `call_some_complex_code` function. +/// /// Instead of calling into the host, the callee will automatically call the other +/// /// implementation. +/// /// +/// /// To replace the implementation: +/// /// +/// /// `host_call_some_complex_code.replace_implementation(some_other_impl)` +/// pub static host_call_some_complex_code: () = (); +/// pub static host_set_or_clear: () = (); +/// +/// pub fn call_some_complex_code(data: &[u8]) -> Vec { +/// // This is the actual call: `host_call_some_complex_code.get()(data)` +/// // +/// // But that does not work for several reasons in this example, so we just return an +/// // empty vector. +/// Vec::new() +/// } +/// +/// pub fn set_or_clear(optional: Option>) { +/// // Same as above +/// } +/// } +/// ``` +/// +/// # Argument types +/// +/// The macro supports any kind of argument type, as long as it implements `RIType` and the required +/// `FromFFIValue`/`IntoFFIValue` from `substrate-runtime-interface`. The macro will convert each +/// argument to the corresponding FFI representation and will call into the host using this FFI +/// representation. On the host each argument is converted back to the native representation and +/// the native implementation is called. Any return value is handled in the same way. +/// +/// # Wasm only interfaces +/// +/// Some interfaces are only required from within the wasm runtime e.g. the allocator interface. +/// To support this, the macro can be called like `#[runtime_interface(wasm_only)]`. This instructs +/// the macro to make two significant changes to the generated code: +/// +/// 1. The generated functions are not callable from the native side. +/// 2. The trait as shown above is not implemented for `Externalities` and is instead implemented +/// for `FunctionExecutor` (from `substrate-wasm-interface`). #[proc_macro_attribute] pub fn runtime_interface( attrs: proc_macro::TokenStream, From 987bb1b431be05cd28c9e1250f96e10f22faefbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 8 Nov 2019 06:54:39 +0100 Subject: [PATCH 65/76] Support wild card argument names --- .../bare_function_interface.rs | 2 +- .../host_function_interface.rs | 6 +-- .../src/runtime_interface/trait_decl_impl.rs | 4 +- .../runtime-interface/proc-macro/src/utils.rs | 51 ++++++++++++------- core/runtime-interface/test-wasm/src/lib.rs | 4 +- 5 files changed, 42 insertions(+), 25 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs b/core/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs index 7b67a9e4ebe7f..dbedba000e262 100644 --- a/core/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs @@ -100,7 +100,7 @@ fn function_std_impl( ) -> Result { let function_name = &method.sig.ident; let crate_ = generate_crate_access(); - let args = get_function_arguments(&method.sig).cloned().map(FnArg::Typed).chain( + let args = get_function_arguments(&method.sig).map(FnArg::Typed).chain( // Add the function context as last parameter when this is a wasm only interface. iter::from_fn(|| if is_wasm_only { diff --git a/core/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs b/core/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs index a489cb8ba1404..f710e9b60cbf4 100644 --- a/core/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs +++ b/core/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs @@ -287,7 +287,7 @@ fn generate_wasm_to_ffi_values<'a>( trait_name, ); - let var_name = generate_ffi_value_var_name(name)?; + let var_name = generate_ffi_value_var_name(&name)?; Ok(quote! { let val = args.next().ok_or_else(|| #error_message)?; @@ -308,7 +308,7 @@ fn generate_ffi_to_host_value<'a>( get_function_argument_names_and_types_without_ref(sig) .zip(mut_access.map(|v| v.and_then(|m| m.1))) .map(move |((name, ty), mut_access)| { - let ffi_value_var_name = generate_ffi_value_var_name(name)?; + let ffi_value_var_name = generate_ffi_value_var_name(&name)?; Ok( quote! { @@ -380,7 +380,7 @@ fn generate_into_preallocated_ffi_value(sig: &Signature) -> Result ref_and_mut.zip(names_and_types) .filter_map(|(ram, (name, ty))| ram.map(|_| (name, ty))) .map(|(name, ty)| { - let ffi_var_name = generate_ffi_value_var_name(name)?; + let ffi_var_name = generate_ffi_value_var_name(&name)?; Ok( quote! { diff --git a/core/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs b/core/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs index 29f94c4556a88..49d07a52fc92a 100644 --- a/core/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs +++ b/core/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs @@ -84,11 +84,11 @@ impl Fold for ToEssentialTraitDef { let arg_types = get_function_argument_types_without_ref(&method.sig); arg_types.filter_map(|ty| - match &**ty { + match *ty { Type::ImplTrait(impl_trait) => Some(impl_trait), _ => None } - ).for_each(|invalid| self.push_error(invalid, "`impl Trait` syntax not supported.")); + ).for_each(|invalid| self.push_error(&invalid, "`impl Trait` syntax not supported.")); fold::fold_trait_item_method(self, method) } diff --git a/core/runtime-interface/proc-macro/src/utils.rs b/core/runtime-interface/proc-macro/src/utils.rs index e7e780dc107f3..9d484e1aba0e8 100644 --- a/core/runtime-interface/proc-macro/src/utils.rs +++ b/core/runtime-interface/proc-macro/src/utils.rs @@ -19,7 +19,8 @@ use proc_macro2::{TokenStream, Span}; use syn::{ - Ident, Error, Signature, Pat, PatType, FnArg, Type, token, TraitItemMethod, ItemTrait, TraitItem + Ident, Error, Signature, Pat, PatType, FnArg, Type, token, TraitItemMethod, ItemTrait, + TraitItem, parse_quote, spanned::Spanned, }; use proc_macro_crate::crate_name; @@ -78,34 +79,48 @@ pub fn create_host_function_ident(name: &Ident, trait_name: &Ident) -> Ident { } /// Returns the function arguments of the given `Signature`, minus any `self` arguments. -pub fn get_function_arguments<'a>(sig: &'a Signature) -> impl Iterator { +pub fn get_function_arguments<'a>(sig: &'a Signature) -> impl Iterator + 'a { sig.inputs .iter() .filter_map(|a| match a { FnArg::Receiver(_) => None, FnArg::Typed(pat_type) => Some(pat_type), }) + .enumerate() + .map(|(i, arg)| { + let mut res = arg.clone(); + if let Pat::Wild(wild) = &*arg.pat { + let ident = Ident::new( + &format!("__runtime_interface_generated_{}_", i), + wild.span(), + ); + + res.pat = Box::new(parse_quote!( #ident )) + } + + res + }) } /// Returns the function argument names of the given `Signature`, minus any `self`. -pub fn get_function_argument_names<'a>(sig: &'a Signature) -> impl Iterator> { - get_function_arguments(sig).map(|pt| &pt.pat) +pub fn get_function_argument_names<'a>(sig: &'a Signature) -> impl Iterator> + 'a { + get_function_arguments(sig).map(|pt| pt.pat) } /// Returns the function argument types of the given `Signature`, minus any `Self` type. -pub fn get_function_argument_types<'a>(sig: &'a Signature) -> impl Iterator> { - get_function_arguments(sig).map(|pt| &pt.ty) +pub fn get_function_argument_types<'a>(sig: &'a Signature) -> impl Iterator> + 'a { + get_function_arguments(sig).map(|pt| pt.ty) } /// Returns the function argument types, minus any `Self` type. If any of the arguments /// is a reference, the underlying type without the ref is returned. pub fn get_function_argument_types_without_ref<'a>( sig: &'a Signature, -) -> impl Iterator> { +) -> impl Iterator> + 'a { get_function_arguments(sig) - .map(|pt| &pt.ty) - .map(|ty| match &**ty { - Type::Reference(type_ref) => &type_ref.elem, + .map(|pt| pt.ty) + .map(|ty| match *ty { + Type::Reference(type_ref) => type_ref.elem, _ => ty, }) } @@ -114,11 +129,11 @@ pub fn get_function_argument_types_without_ref<'a>( /// is a reference, the underlying type without the ref is returned. pub fn get_function_argument_names_and_types_without_ref<'a>( sig: &'a Signature, -) -> impl Iterator, &'a Box)> { +) -> impl Iterator, Box)> + 'a { get_function_arguments(sig) - .map(|pt| match &*pt.ty { - Type::Reference(type_ref) => (&pt.pat, &type_ref.elem), - _ => (&pt.pat, &pt.ty), + .map(|pt| match *pt.ty { + Type::Reference(type_ref) => (pt.pat, type_ref.elem), + _ => (pt.pat, pt.ty), }) } @@ -126,11 +141,11 @@ pub fn get_function_argument_names_and_types_without_ref<'a>( /// argument is not a reference, `None` is returned. pub fn get_function_argument_types_ref_and_mut<'a>( sig: &'a Signature, -) -> impl Iterator)>> { +) -> impl Iterator)>> + 'a { get_function_arguments(sig) - .map(|pt| &pt.ty) - .map(|ty| match &**ty { - Type::Reference(type_ref) => Some((&type_ref.and_token, type_ref.mutability.as_ref())), + .map(|pt| pt.ty) + .map(|ty| match *ty { + Type::Reference(type_ref) => Some((type_ref.and_token, type_ref.mutability)), _ => None, }) } diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index d0d256e633758..c364e669014e4 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -73,7 +73,9 @@ pub trait TestApi { } /// A function that is called with invalid utf8 data from the runtime. - fn invalid_utf8_data(_data: &str) {} + /// + /// This also checks that we accept `_` (wild card) argument names. + fn invalid_utf8_data(_: &str) {} /// Overwrite the native implementation in wasm. The native implementation always returns /// `false` and the replacement function will return always `true`. From a5ff11bbaa3c5ac9bf274b2ec7054b19e5929307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 8 Nov 2019 08:45:39 +0100 Subject: [PATCH 66/76] Adds ui tests --- Cargo.lock | 2 ++ core/runtime-interface/proc-macro/Cargo.toml | 4 ++- core/runtime-interface/proc-macro/src/lib.rs | 10 +++---- .../proc-macro/src/pass_by/inner.rs | 7 ++++- .../src/runtime_interface/trait_decl_impl.rs | 20 +++++++++----- core/runtime-interface/proc-macro/tests/ui.rs | 27 +++++++++++++++++++ .../tests/ui/no_generic_parameters.rs | 8 ++++++ .../tests/ui/no_generic_parameters.stderr | 11 ++++++++ .../tests/ui/no_method_implementation.rs | 8 ++++++ .../tests/ui/no_method_implementation.stderr | 5 ++++ .../tests/ui/pass_by_enum_with_struct.rs | 6 +++++ .../tests/ui/pass_by_enum_with_struct.stderr | 5 ++++ .../ui/pass_by_enum_with_value_variant.rs | 8 ++++++ .../ui/pass_by_enum_with_value_variant.stderr | 5 ++++ .../tests/ui/pass_by_inner_with_two_fields.rs | 9 +++++++ .../ui/pass_by_inner_with_two_fields.stderr | 5 ++++ .../proc-macro/tests/ui/take_self_by_value.rs | 8 ++++++ .../tests/ui/take_self_by_value.stderr | 5 ++++ core/sr-api-macros/tests/trybuild.rs | 16 +++++++++++ 19 files changed, 155 insertions(+), 14 deletions(-) create mode 100644 core/runtime-interface/proc-macro/tests/ui.rs create mode 100644 core/runtime-interface/proc-macro/tests/ui/no_generic_parameters.rs create mode 100644 core/runtime-interface/proc-macro/tests/ui/no_generic_parameters.stderr create mode 100644 core/runtime-interface/proc-macro/tests/ui/no_method_implementation.rs create mode 100644 core/runtime-interface/proc-macro/tests/ui/no_method_implementation.stderr create mode 100644 core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_struct.rs create mode 100644 core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_struct.stderr create mode 100644 core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_value_variant.rs create mode 100644 core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_value_variant.stderr create mode 100644 core/runtime-interface/proc-macro/tests/ui/pass_by_inner_with_two_fields.rs create mode 100644 core/runtime-interface/proc-macro/tests/ui/pass_by_inner_with_two_fields.stderr create mode 100644 core/runtime-interface/proc-macro/tests/ui/take_self_by_value.rs create mode 100644 core/runtime-interface/proc-macro/tests/ui/take_self_by_value.stderr diff --git a/Cargo.lock b/Cargo.lock index 84814472b67e5..618acacc87bb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5923,9 +5923,11 @@ dependencies = [ "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustversion 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-externalities 2.0.0", "substrate-runtime-interface 2.0.0", "syn 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "trybuild 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/core/runtime-interface/proc-macro/Cargo.toml b/core/runtime-interface/proc-macro/Cargo.toml index 4c39c8282ba62..0b073b854749d 100644 --- a/core/runtime-interface/proc-macro/Cargo.toml +++ b/core/runtime-interface/proc-macro/Cargo.toml @@ -15,9 +15,11 @@ Inflector = "0.11.4" proc-macro-crate = "0.1.4" [dev-dependencies] -substrate-runtime-interface = { path = ".." } +runtime-interface = { package = "substrate-runtime-interface", path = ".." } codec = { package = "parity-scale-codec", version = "1.0.6", features = [ "derive" ] } externalities = { package = "substrate-externalities", path = "../../externalities" } +rustversion = "1.0.0" +trybuild = "1.0.17" # We actually don't need the `std` feature in this crate, but the tests require it. [features] diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index a07df0e99af56..f97de6a023634 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -43,7 +43,7 @@ mod utils; /// The macro expects the runtime interface declaration as trait declaration: /// /// ``` -/// # use substrate_runtime_interface::runtime_interface; +/// # use runtime_interface::runtime_interface; /// /// #[runtime_interface] /// trait Interface { @@ -190,7 +190,7 @@ pub fn runtime_interface( /// # Example /// /// ``` -/// # use substrate_runtime_interface::pass_by::PassByCodec; +/// # use runtime_interface::pass_by::PassByCodec; /// # use codec::{Encode, Decode}; /// #[derive(PassByCodec, Encode, Decode)] /// struct EncodableType { @@ -215,13 +215,13 @@ pub fn pass_by_codec(input: proc_macro::TokenStream) -> proc_macro::TokenStream /// # Example /// /// ``` -/// # use substrate_runtime_interface::pass_by::PassByInner; +/// # use runtime_interface::pass_by::PassByInner; /// #[derive(PassByInner)] /// struct Data([u8; 32]); /// ``` /// /// ``` -/// # use substrate_runtime_interface::pass_by::PassByInner; +/// # use runtime_interface::pass_by::PassByInner; /// #[derive(PassByInner)] /// struct Data { /// data: [u8; 32], @@ -244,7 +244,7 @@ pub fn pass_by_inner(input: proc_macro::TokenStream) -> proc_macro::TokenStream /// # Example /// /// ``` -/// # use substrate_runtime_interface::pass_by::PassByEnum; +/// # use runtime_interface::pass_by::PassByEnum; /// #[derive(PassByEnum, Copy, Clone)] /// enum Data { /// Okay, diff --git a/core/runtime-interface/proc-macro/src/pass_by/inner.rs b/core/runtime-interface/proc-macro/src/pass_by/inner.rs index 5fa9577ca3104..0ca8cd0b5cd91 100644 --- a/core/runtime-interface/proc-macro/src/pass_by/inner.rs +++ b/core/runtime-interface/proc-macro/src/pass_by/inner.rs @@ -101,5 +101,10 @@ fn extract_inner_ty_and_name(data: &Data) -> Result<(Type, Option)> { } } - Err(Error::new(Span::call_site(), "Only newtype structs are supported by `PassByInner`!")) + Err( + Error::new( + Span::call_site(), + "Only newtype/one field structs are supported by `PassByInner`!", + ) + ) } diff --git a/core/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs b/core/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs index 49d07a52fc92a..0e5ae906ab680 100644 --- a/core/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs +++ b/core/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs @@ -21,7 +21,7 @@ use crate::utils::{generate_crate_access, get_function_argument_types_without_re use syn::{ ItemTrait, TraitItemMethod, Result, TraitItem, Error, fold::{self, Fold}, spanned::Spanned, - Visibility, Receiver, Type, + Visibility, Receiver, Type, Generics, }; use proc_macro2::TokenStream; @@ -57,7 +57,7 @@ impl ToEssentialTraitDef { errors: Vec::new(), }; - let res = fold::fold_item_trait(&mut folder, trait_def); + let res = folder.fold_item_trait(trait_def); if let Some(first_error) = folder.errors.pop() { Err( @@ -74,12 +74,18 @@ impl ToEssentialTraitDef { fn push_error(&mut self, span: &S, msg: &str) { self.errors.push(Error::new(span.span(), msg)); } + + fn error_on_generic_parameters(&mut self, generics: &Generics) { + if let Some(param) = generics.params.first() { + self.push_error(param, "Generic parameters not supported."); + } + } } impl Fold for ToEssentialTraitDef { fn fold_trait_item_method(&mut self, mut method: TraitItemMethod) -> TraitItemMethod { if method.default.take().is_none() { - self.push_error(&method, "Methods need to have an implementation"); + self.push_error(&method, "Methods need to have an implementation."); } let arg_types = get_function_argument_types_without_ref(&method.sig); @@ -90,13 +96,13 @@ impl Fold for ToEssentialTraitDef { } ).for_each(|invalid| self.push_error(&invalid, "`impl Trait` syntax not supported.")); + self.error_on_generic_parameters(&method.sig.generics); + fold::fold_trait_item_method(self, method) } fn fold_item_trait(&mut self, mut trait_def: ItemTrait) -> ItemTrait { - if let Some(first_param) = trait_def.generics.params.first() { - self.push_error(first_param, "Generic parameters are not supported"); - } + self.error_on_generic_parameters(&trait_def.generics); trait_def.vis = Visibility::Inherited; fold::fold_item_trait(self, trait_def) @@ -104,7 +110,7 @@ impl Fold for ToEssentialTraitDef { fn fold_receiver(&mut self, receiver: Receiver) -> Receiver { if receiver.reference.is_none() { - self.push_error(&receiver, "Taking `Self` by value is not allowed"); + self.push_error(&receiver, "Taking `Self` by value is not allowed."); } fold::fold_receiver(self, receiver) diff --git a/core/runtime-interface/proc-macro/tests/ui.rs b/core/runtime-interface/proc-macro/tests/ui.rs new file mode 100644 index 0000000000000..5b14ee81e8e8a --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui.rs @@ -0,0 +1,27 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::env; + +#[rustversion::attr(not(stable), ignore)] +#[test] +fn ui() { + // As trybuild is using `cargo check`, we don't need the real WASM binaries. + env::set_var("BUILD_DUMMY_WASM_BINARY", "1"); + + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/*.rs"); +} diff --git a/core/runtime-interface/proc-macro/tests/ui/no_generic_parameters.rs b/core/runtime-interface/proc-macro/tests/ui/no_generic_parameters.rs new file mode 100644 index 0000000000000..489fe5d9b4f91 --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/no_generic_parameters.rs @@ -0,0 +1,8 @@ +use runtime_interface::runtime_interface; + +#[runtime_interface] +trait Test { + fn test() {} +} + +fn main() {} diff --git a/core/runtime-interface/proc-macro/tests/ui/no_generic_parameters.stderr b/core/runtime-interface/proc-macro/tests/ui/no_generic_parameters.stderr new file mode 100644 index 0000000000000..c3e46655e5be7 --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/no_generic_parameters.stderr @@ -0,0 +1,11 @@ +error: Generic parameters not supported. + --> $DIR/no_generic_parameters.rs:5:10 + | +5 | fn test() {} + | ^ + +error: Generic parameters not supported. + --> $DIR/no_generic_parameters.rs:4:12 + | +4 | trait Test { + | ^ diff --git a/core/runtime-interface/proc-macro/tests/ui/no_method_implementation.rs b/core/runtime-interface/proc-macro/tests/ui/no_method_implementation.rs new file mode 100644 index 0000000000000..5291942420fd5 --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/no_method_implementation.rs @@ -0,0 +1,8 @@ +use runtime_interface::runtime_interface; + +#[runtime_interface] +trait Test { + fn test(); +} + +fn main() {} diff --git a/core/runtime-interface/proc-macro/tests/ui/no_method_implementation.stderr b/core/runtime-interface/proc-macro/tests/ui/no_method_implementation.stderr new file mode 100644 index 0000000000000..31b2d39762340 --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/no_method_implementation.stderr @@ -0,0 +1,5 @@ +error: Methods need to have an implementation. + --> $DIR/no_method_implementation.rs:5:2 + | +5 | fn test(); + | ^^ diff --git a/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_struct.rs b/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_struct.rs new file mode 100644 index 0000000000000..a729e0a99adca --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_struct.rs @@ -0,0 +1,6 @@ +use runtime_interface::pass_by::PassByEnum; + +#[derive(PassByEnum)] +struct Test; + +fn main() {} diff --git a/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_struct.stderr b/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_struct.stderr new file mode 100644 index 0000000000000..6502a36fc18b7 --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_struct.stderr @@ -0,0 +1,5 @@ +error: `PassByEnum` only supports enums as input type. + --> $DIR/pass_by_enum_with_struct.rs:3:10 + | +3 | #[derive(PassByEnum)] + | ^^^^^^^^^^ diff --git a/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_value_variant.rs b/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_value_variant.rs new file mode 100644 index 0000000000000..d2558e797770b --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_value_variant.rs @@ -0,0 +1,8 @@ +use runtime_interface::pass_by::PassByEnum; + +#[derive(PassByEnum)] +enum Test { + Var0(u32), +} + +fn main() {} diff --git a/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_value_variant.stderr b/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_value_variant.stderr new file mode 100644 index 0000000000000..1f03436d4e007 --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_value_variant.stderr @@ -0,0 +1,5 @@ +error: `PassByEnum` only supports unit variants. + --> $DIR/pass_by_enum_with_value_variant.rs:3:10 + | +3 | #[derive(PassByEnum)] + | ^^^^^^^^^^ diff --git a/core/runtime-interface/proc-macro/tests/ui/pass_by_inner_with_two_fields.rs b/core/runtime-interface/proc-macro/tests/ui/pass_by_inner_with_two_fields.rs new file mode 100644 index 0000000000000..eab79eae1910c --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/pass_by_inner_with_two_fields.rs @@ -0,0 +1,9 @@ +use runtime_interface::pass_by::PassByInner; + +#[derive(PassByInner)] +struct Test { + data: u32, + data2: u32, +} + +fn main() {} diff --git a/core/runtime-interface/proc-macro/tests/ui/pass_by_inner_with_two_fields.stderr b/core/runtime-interface/proc-macro/tests/ui/pass_by_inner_with_two_fields.stderr new file mode 100644 index 0000000000000..7f576a69f0e50 --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/pass_by_inner_with_two_fields.stderr @@ -0,0 +1,5 @@ +error: Only newtype/one field structs are supported by `PassByInner`! + --> $DIR/pass_by_inner_with_two_fields.rs:3:10 + | +3 | #[derive(PassByInner)] + | ^^^^^^^^^^^ diff --git a/core/runtime-interface/proc-macro/tests/ui/take_self_by_value.rs b/core/runtime-interface/proc-macro/tests/ui/take_self_by_value.rs new file mode 100644 index 0000000000000..f01c2de21ef61 --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/take_self_by_value.rs @@ -0,0 +1,8 @@ +use runtime_interface::runtime_interface; + +#[runtime_interface] +trait Test { + fn test(self) {} +} + +fn main() {} diff --git a/core/runtime-interface/proc-macro/tests/ui/take_self_by_value.stderr b/core/runtime-interface/proc-macro/tests/ui/take_self_by_value.stderr new file mode 100644 index 0000000000000..9b17a63a35cbc --- /dev/null +++ b/core/runtime-interface/proc-macro/tests/ui/take_self_by_value.stderr @@ -0,0 +1,5 @@ +error: Taking `Self` by value is not allowed. + --> $DIR/take_self_by_value.rs:5:10 + | +5 | fn test(self) {} + | ^^^^ diff --git a/core/sr-api-macros/tests/trybuild.rs b/core/sr-api-macros/tests/trybuild.rs index 9baea83196e9b..5b14ee81e8e8a 100644 --- a/core/sr-api-macros/tests/trybuild.rs +++ b/core/sr-api-macros/tests/trybuild.rs @@ -1,3 +1,19 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + use std::env; #[rustversion::attr(not(stable), ignore)] From dad538fdbe5e7005318bc7ac69940c589bde63c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 8 Nov 2019 09:17:42 +0100 Subject: [PATCH 67/76] Make sure that we are backwards compatible to the old runtime interfaces --- core/executor/src/host_interface.rs | 4 +--- core/executor/src/native_executor.rs | 5 ++--- core/executor/src/wasm_runtime.rs | 7 +++++-- core/runtime-interface/test-wasm/src/lib.rs | 15 +++++++++++++++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/core/executor/src/host_interface.rs b/core/executor/src/host_interface.rs index 5ff8cd8990ea6..08c56e13a2ed6 100644 --- a/core/executor/src/host_interface.rs +++ b/core/executor/src/host_interface.rs @@ -14,9 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Definition and implementation of the Substrate Wasm host interface. -//! -//! These are the host functions callable from within the Substrate runtime. +//! Definition and implementation of the old and deprecated Substrate runtime interface for the host. use codec::Encode; use std::{convert::TryFrom, str}; diff --git a/core/executor/src/native_executor.rs b/core/executor/src/native_executor.rs index 206740d419a8a..a2ff2e216fe0d 100644 --- a/core/executor/src/native_executor.rs +++ b/core/executor/src/native_executor.rs @@ -15,7 +15,7 @@ // along with Substrate. If not, see . use crate::{ - RuntimeInfo, error::{Error, Result}, host_interface::SubstrateExternals, + RuntimeInfo, error::{Error, Result}, wasm_runtime::{RuntimesCache, WasmExecutionMethod, WasmRuntime}, }; @@ -127,8 +127,7 @@ impl NativeExecutor { ext, self.fallback_method, self.default_heap_pages, - // Use the `SubstrateExternals` as well, to be backwards compatible. - <(runtime_io::SubstrateHostFunctions, SubstrateExternals)>::host_functions(), + runtime_io::SubstrateHostFunctions::host_functions(), )?; let runtime = AssertUnwindSafe(runtime); diff --git a/core/executor/src/wasm_runtime.rs b/core/executor/src/wasm_runtime.rs index 21f36e2784b4f..88738910fedd0 100644 --- a/core/executor/src/wasm_runtime.rs +++ b/core/executor/src/wasm_runtime.rs @@ -32,7 +32,7 @@ use runtime_version::RuntimeVersion; use std::{collections::hash_map::{Entry, HashMap}}; -use wasm_interface::Function; +use wasm_interface::{Function, HostFunctions}; /// The Substrate Wasm runtime. pub trait WasmRuntime { @@ -201,8 +201,11 @@ pub fn create_wasm_runtime_with_code( wasm_method: WasmExecutionMethod, heap_pages: u64, code: &[u8], - host_functions: Vec<&'static dyn Function>, + mut host_functions: Vec<&'static dyn Function>, ) -> Result, WasmError> { + // Add the old and deprecated host functions as well, so that we support old wasm runtimes. + host_functions.extend(crate::host_interface::SubstrateExternals::host_functions()); + match wasm_method { WasmExecutionMethod::Interpreted => wasmi_execution::create_instance(ext, code, heap_pages, host_functions) diff --git a/core/runtime-interface/test-wasm/src/lib.rs b/core/runtime-interface/test-wasm/src/lib.rs index c364e669014e4..d61315c521b68 100644 --- a/core/runtime-interface/test-wasm/src/lib.rs +++ b/core/runtime-interface/test-wasm/src/lib.rs @@ -84,6 +84,21 @@ pub trait TestApi { } } +/// Two random external functions from the old runtime interface. +/// This ensures that we still inherently export these functions from the host and that we are still +/// compatible with old wasm runtimes. +extern "C" { + pub fn ext_clear_storage(key_data: *const u8, key_len: u32); + pub fn ext_keccak_256(data: *const u8, len: u32, out: *mut u8); +} + +/// Make sure the old runtime interface needs to be imported. +#[no_mangle] +pub fn force_old_runtime_interface_import() { + unsafe { ext_clear_storage(rstd::ptr::null(), 0); } + unsafe { ext_keccak_256(rstd::ptr::null(), 0, rstd::ptr::null_mut()); } +} + /// This function is not used, but we require it for the compiler to include `runtime-io`. /// `runtime-io` is required for its panic and oom handler. #[no_mangle] From 70149b632014d2aad377118b5acdbb59d149cf78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 8 Nov 2019 16:08:52 +0100 Subject: [PATCH 68/76] Documentation --- core/runtime-interface/src/impls.rs | 26 +++++++++++ core/runtime-interface/src/lib.rs | 62 +++++++++++++++++++++++---- core/runtime-interface/src/pass_by.rs | 15 ++++++- core/wasm-interface/src/lib.rs | 6 ++- 4 files changed, 98 insertions(+), 11 deletions(-) diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index 6a1a454340e15..3380c471a8b99 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -65,6 +65,7 @@ macro_rules! impl_traits_for_primitives { )* ) => { $( + /// The type is passed directly. impl RIType for $rty { type FFIType = $fty; } @@ -115,6 +116,10 @@ impl_traits_for_primitives! { i64, i64, } +/// `bool` is passed as `u8`. +/// +/// - `1`: true +/// - `0`: false impl RIType for bool { type FFIType = u8; } @@ -151,6 +156,12 @@ impl IntoFFIValue for bool { } } +/// The type is passed as `u64`. +/// +/// The `u64` value is build by `length 32bit << 32 | pointer 32bit` +/// +/// If `T == u8` the length and the pointer are taken directly from the `Self`. +/// Otherwise `Self` is encoded and the length and the pointer are taken from the encoded vector. impl RIType for Vec { type FFIType = u64; } @@ -204,6 +215,12 @@ impl FromFFIValue for Vec { } } +/// The type is passed as `u64`. +/// +/// The `u64` value is build by `length 32bit << 32 | pointer 32bit` +/// +/// If `T == u8` the length and the pointer are taken directly from the `Self`. +/// Otherwise `Self` is encoded and the length and the pointer are taken from the encoded vector. impl RIType for [T] { type FFIType = u64; } @@ -275,6 +292,9 @@ macro_rules! impl_traits_for_arrays { $(,)? ) => { $( + /// The type is passed as `u32`. + /// + /// The `u32` is the pointer to the array. impl RIType for [u8; $n] { type FFIType = u32; } @@ -386,6 +406,11 @@ for_primitive_types! { H512 64, } +/// The type is passed as `u64`. +/// +/// The `u64` value is build by `length 32bit << 32 | pointer 32bit` +/// +/// The length and the pointer are taken directly from the `Self`. impl RIType for str { type FFIType = u64; } @@ -419,6 +444,7 @@ impl RIType for Pointer { type FFIType = u32; } +/// The type is passed as `u32`. #[cfg(not(feature = "std"))] impl RIType for Pointer { type FFIType = u32; diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 30a9962031ce0..dfabc2e925c0d 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -14,15 +14,61 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Traits and macros for creating interfaces between the runtime and the node. +//! Substrate runtime interface //! -//! To make the communication between wasm and the host as fast as possible, we use a two-way -//! conversion strategy. First, we convert the value into a [`WrappedFFIValue`]. -//! This value stores a reference or an owned value that are convertible into the actual ffi value. -//! So, for values like `Vec` we just store a reference to the underlying slice and don't need -//! to allocate any memory. In the second step we get the actual ffi value from this -//! [`WrappedFFIValue`] to call into the host. The created [`WrappedFFIValue`] will remain on -//! the stack while we call into the host. +//! This crate provides types, traits and macros around runtime interfaces. A runtime interface is +//! a fixed interface between a Substrate runtime and a Substrate node. For a native runtime the +//! interface maps to a direct function call of the implementation. For a wasm runtime the interface +//! maps to an external function call. These external functions are exported by the wasm executor +//! and they map to the same implementation as the native calls. +//! +//! # Using a type in a runtime interface +//! +//! Any type that should be used in a runtime interface as argument or return value needs to +//! implement [`RIType`]. The associated type `FFIType` is the type that is used in the FFI +//! function to represent the actual type. For example `[T]` is represented by an `u64`. The slice +//! pointer and the length will be mapped to an `u64` value. For more information, see the +//! implementation of [`RIType`] for [`T`]. The FFI function definition is used when calling from +//! the wasm runtime into the node. +//! +//! Traits are used to convert from a type to the corresponding [`RIType::FFIType`]. +//! Depending on where and how a type should be used in a function signature, a combination of the +//! following traits need to be implemented: +//! +//! 1. Pass as function argument: [`wasm::IntoFFIValue`] and [`host::FromFFIValue`] +//! 2. As function return value: [`wasm::FromFFIValue`] and [`host::IntoFFIValue`] +//! 3. Pass as mutable function argument: [`host::IntoPreallocatedFFIValue`] +//! +//! The traits are implemented for most of the common types like `[T]`, `Vec`, arrays and +//! primitive types. +//! +//! For custom types, we provide the [`PassBy`](pass_by::PassBy) trait and strategies that define +//! how a type is passed between the wasm runtime and the node. Each strategy also provides a derive +//! macro to simplify the implementation. +//! +//! # Performance +//! +//! To not waste any more performance when calling into the node, not all types are SCALE encoded +//! when being passed as arguments between the wasm runtime and the node. For most types that +//! are raw bytes like `Vec`, `[u8]` or `[u8; N]` we pass them directly, without SCALE encoding +//! them in front of. The implementation of [`RIType`] each type provides more information on how +//! the data is passed. +//! +//! # Declaring a runtime interface +//! +//! Declaring a runtime interface is similar to declaring a trait in Rust: +//! +//! ``` +//! #[substrate_runtime_interface::runtime_interface] +//! trait RuntimeInterface { +//! fn some_function(value: &[u8]) -> bool { +//! value.iter().all(|v| v > 125) +//! } +//! } +//! ``` +//! +//! For more information on declaring a runtime interface, see +//! [`#[runtime_interface]`](attr.runtime_interface.html). #![cfg_attr(not(feature = "std"), no_std)] diff --git a/core/runtime-interface/src/pass_by.rs b/core/runtime-interface/src/pass_by.rs index d1ce20740b4c7..46265237c0c47 100644 --- a/core/runtime-interface/src/pass_by.rs +++ b/core/runtime-interface/src/pass_by.rs @@ -15,8 +15,10 @@ // along with Substrate. If not, see . //! Provides the [`PassBy`](pass_by::PassBy) trait to simplify the implementation of the -//! runtime interface traits for custom types. [`Codec`](pass_by::Codec), [`Inner`](pass_by::Inner) -//! and [`Enum`](pass_by::Enum) are the provided strategy implementations. +//! runtime interface traits for custom types. +//! +//! [`Codec`](pass_by::Codec), [`Inner`](pass_by::Inner) and [`Enum`](pass_by::Enum) are the +//! provided strategy implementations. use crate::{RIType, impls::{pointer_and_len_from_u64, pointer_and_len_to_u64}}; @@ -196,6 +198,11 @@ impl PassByImpl for Codec { } } +/// The type is passed as `u64`. +/// +/// The `u64` value is build by `length 32bit << 32 | pointer 32bit` +/// +/// `Self` is encoded and the length and the pointer are taken from the encoded vector. impl RIType for Codec { type FFIType = u64; } @@ -287,6 +294,7 @@ impl, I: RIType> PassByImpl for Inner } } +/// The type is passed as the inner type. impl, I: RIType> RIType for Inner { type FFIType = I::FFIType; } @@ -368,6 +376,9 @@ impl + TryFrom> PassByImpl for Enum { } } +/// The type is passed as `u8`. +/// +/// The value is corresponds to the discriminant of the variant. impl + TryFrom> RIType for Enum { type FFIType = u8; } diff --git a/core/wasm-interface/src/lib.rs b/core/wasm-interface/src/lib.rs index d941a83e168ba..726d2e7de0cc5 100644 --- a/core/wasm-interface/src/lib.rs +++ b/core/wasm-interface/src/lib.rs @@ -344,7 +344,9 @@ impl_into_and_from_value! { i64, I64, } +/// Something that can write a primitive to wasm memory location. pub trait WritePrimitive { + /// Write the given value `t` to the given memory location `ptr`. fn write_primitive(&mut self, ptr: Pointer, t: T) -> Result<()>; } @@ -362,8 +364,10 @@ impl WritePrimitive for &mut dyn FunctionContext { } } +/// Something that can read a primitive from a wasm memory location. pub trait ReadPrimitive { - fn read_primitive(&self, offset: Pointer) -> Result; + /// Read a primitive from the given memory location `ptr`. + fn read_primitive(&self, ptr: Pointer) -> Result; } impl ReadPrimitive for &mut dyn FunctionContext { From 113d47352f93e7b7030f3c69d4a50729ac607651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 8 Nov 2019 17:28:11 +0100 Subject: [PATCH 69/76] Fixes after latest master merge --- Cargo.lock | 28 ++++++++----------- core/executor/src/wasm_runtime.rs | 8 +++--- core/primitives/src/ecdsa.rs | 1 + core/primitives/src/ed25519.rs | 1 + core/primitives/src/sr25519.rs | 1 + core/runtime-interface/Cargo.toml | 11 ++++++-- .../runtime-interface/proc-macro/src/utils.rs | 2 +- core/runtime-interface/src/lib.rs | 2 +- core/test-runtime/src/system.rs | 2 +- node/cli/Cargo.toml | 2 +- srml/evm/src/backend.rs | 2 +- srml/im-online/src/tests.rs | 2 +- 12 files changed, 33 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee6392e852ee2..01c348025b170 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1002,7 +1002,7 @@ dependencies = [ "evm-core 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "evm-gasometer 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "evm-runtime 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1012,7 +1012,7 @@ name = "evm-core" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "primitive-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1022,7 +1022,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "evm-core 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "evm-runtime 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1031,7 +1031,7 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "evm-core 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3250,7 +3250,7 @@ dependencies = [ [[package]] name = "primitive-types" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fixed-hash 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4155,7 +4155,7 @@ dependencies = [ "integer-sqrt 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", @@ -4469,7 +4469,7 @@ version = "2.0.0" dependencies = [ "evm 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4971,11 +4971,6 @@ name = "static_assertions" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "static_slice" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "stream-cipher" version = "0.3.2" @@ -5523,7 +5518,7 @@ name = "substrate-externalities" version = "2.0.0" dependencies = [ "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "sr-std 2.0.0", "substrate-primitives-storage 2.0.0", ] @@ -5761,7 +5756,7 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5873,7 +5868,7 @@ version = "2.0.0" dependencies = [ "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "sr-io 2.0.0", "sr-std 2.0.0", "static_assertions 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -7652,7 +7647,7 @@ dependencies = [ "checksum plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" "checksum ppv-lite86 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e3cbf9f658cdb5000fcf6f362b8ea2ba154b9f146a61c7a20d647034c6b6561b" "checksum pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427" -"checksum primitive-types 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97b5a08dda18910f056e5c2060c034e77cab18e0bd7d895e44f03207af4c71d5" +"checksum primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a0253db64c26d8b4e7896dd2063b516d2a1b9e0a5da26b5b78335f236d1e9522" "checksum proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" "checksum proc-macro-error 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "aeccfe4d5d8ea175d5f0e4a2ad0637e0f4121d63bd99d356fb1f39ab2e7c6097" "checksum proc-macro-hack 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "114cdf1f426eb7f550f01af5f53a33c0946156f6814aec939b3bd77e844f9a9d" @@ -7752,7 +7747,6 @@ dependencies = [ "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" "checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5" "checksum static_assertions 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0fa13613355688665b68639b1c378a62dbedea78aff0fc59a4fa656cbbdec657" -"checksum static_slice 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "92a7e0c5e3dfb52e8fbe0e63a1b947bbb17b4036408b151353c4491374931362" "checksum stream-cipher 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8131256a5896cabcf5eb04f4d6dacbe1aefda854b0d9896e09cb58829ec5638c" "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" "checksum string-interner 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd710eadff449a1531351b0e43eb81ea404336fa2f56c777427ab0e32a4cf183" diff --git a/core/executor/src/wasm_runtime.rs b/core/executor/src/wasm_runtime.rs index 94d0c679bac34..5242931cfa063 100644 --- a/core/executor/src/wasm_runtime.rs +++ b/core/executor/src/wasm_runtime.rs @@ -142,8 +142,9 @@ impl RuntimesCache { Entry::Occupied(o) => { let result = o.into_mut(); if let Ok(ref mut cached_runtime) = result { - let heap_pages_changed = !cached_runtime.update_heap_pages(heap_pages); - let host_functions_changed = cached_runtime.host_functions() != &host_functions[..]; + let heap_pages_changed = !cached_runtime.runtime.update_heap_pages(heap_pages); + let host_functions_changed = cached_runtime.runtime.host_functions() + != &host_functions[..]; if heap_pages_changed || host_functions_changed { let changed = if heap_pages_changed { "heap_pages" @@ -240,13 +241,12 @@ fn create_versioned_wasm_runtime( let version_result = { // `ext` is already implicitly handled as unwind safe, as we store it in a global variable. let mut ext = AssertUnwindSafe(ext); - let host_functions = runtime.host_functions(); // The following unwind safety assertion is OK because if the method call panics, the // runtime will be dropped. let mut runtime = AssertUnwindSafe(runtime.as_mut()); crate::native_executor::safe_call( - move || runtime.call(&mut **ext, "Core_version", &[], host_functions) + move || runtime.call(&mut **ext, "Core_version", &[]) ).map_err(|_| WasmError::Instantiation("panic in call to get runtime version".into()))? }; let encoded_version = version_result diff --git a/core/primitives/src/ecdsa.rs b/core/primitives/src/ecdsa.rs index 691e9fba5e184..f1d4d2446aa13 100644 --- a/core/primitives/src/ecdsa.rs +++ b/core/primitives/src/ecdsa.rs @@ -18,6 +18,7 @@ //! Simple ECDSA API. // end::description[] +#[cfg(feature = "full_crypto")] use rstd::vec::Vec; use rstd::cmp::Ordering; diff --git a/core/primitives/src/ed25519.rs b/core/primitives/src/ed25519.rs index 661190accd9e2..c0894b8782d49 100644 --- a/core/primitives/src/ed25519.rs +++ b/core/primitives/src/ed25519.rs @@ -18,6 +18,7 @@ //! Simple Ed25519 API. // end::description[] +#[cfg(feature = "full_crypto")] use rstd::vec::Vec; use crate::{hash::H256, hash::H512}; diff --git a/core/primitives/src/sr25519.rs b/core/primitives/src/sr25519.rs index 9046dcea7a01a..eed2b70830889 100644 --- a/core/primitives/src/sr25519.rs +++ b/core/primitives/src/sr25519.rs @@ -20,6 +20,7 @@ //! Note: `CHAIN_CODE_LENGTH` must be equal to `crate::crypto::JUNCTION_ID_LEN` //! for this to work. // end::description[] +#[cfg(feature = "full_crypto")] use rstd::vec::Vec; #[cfg(feature = "full_crypto")] use schnorrkel::{signing_context, ExpansionMode, Keypair, SecretKey, MiniSecretKey, PublicKey, diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index da211de0df554..7b0b649450631 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -12,7 +12,7 @@ externalities = { package = "substrate-externalities", path = "../externalities" codec = { package = "parity-scale-codec", version = "1.0.6", default-features = false } environmental = { version = "1.0.2", optional = true } static_assertions = "1.0.0" -primitive-types = { version = "0.5.1", default-features = false } +primitive-types = { version = "0.6.1", default-features = false } [dev-dependencies] executor = { package = "substrate-executor", path = "../executor" } @@ -23,4 +23,11 @@ runtime-io = { package = "sr-io", path = "../sr-io" } [features] default = [ "std" ] -std = [ "wasm-interface", "rstd/std", "codec/std", "externalities", "environmental", "primitive-types/std" ] +std = [ + "wasm-interface", + "rstd/std", + "codec/std", + "externalities", + "environmental", + "primitive-types/std", +] diff --git a/core/runtime-interface/proc-macro/src/utils.rs b/core/runtime-interface/proc-macro/src/utils.rs index 9d484e1aba0e8..d225d643a82af 100644 --- a/core/runtime-interface/proc-macro/src/utils.rs +++ b/core/runtime-interface/proc-macro/src/utils.rs @@ -55,7 +55,7 @@ pub fn generate_runtime_interface_include() -> TokenStream { /// Generates the access to the `substrate-runtime-interface` crate. pub fn generate_crate_access() -> TokenStream { if env::var("CARGO_PKG_NAME").unwrap() == "substrate-runtime-interface" { - quote!( crate ) + quote!( substrate_runtime_interface ) } else { quote!( proc_macro_runtime_interface ) } diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index dfabc2e925c0d..f7ef7a11e85b3 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -62,7 +62,7 @@ //! #[substrate_runtime_interface::runtime_interface] //! trait RuntimeInterface { //! fn some_function(value: &[u8]) -> bool { -//! value.iter().all(|v| v > 125) +//! value.iter().all(|v| *v > 125) //! } //! } //! ``` diff --git a/core/test-runtime/src/system.rs b/core/test-runtime/src/system.rs index b801467d19b9f..dcb9aa4f32583 100644 --- a/core/test-runtime/src/system.rs +++ b/core/test-runtime/src/system.rs @@ -22,7 +22,7 @@ use runtime_io::{ storage::root as storage_root, storage::changes_root as storage_changes_root, hashing::blake2_256, }; -use runtime_support::storage::{self, StorageMap}; +use runtime_support::storage; use runtime_support::{decl_storage, decl_module}; use sr_primitives::{ traits::{Hash as HashT, BlakeTwo256, Header as _}, generic, ApplyError, ApplyResult, diff --git a/node/cli/Cargo.toml b/node/cli/Cargo.toml index 931488d3bcaeb..20379d52e768d 100644 --- a/node/cli/Cargo.toml +++ b/node/cli/Cargo.toml @@ -39,7 +39,7 @@ babe-primitives = { package = "substrate-consensus-babe-primitives", path = "../ grandpa_primitives = { package = "substrate-finality-grandpa-primitives", path = "../../core/finality-grandpa/primitives" } # core dependencies -sr-io = { path = "../../core/sr-io" } +runtime-io = { package = "sr-io", path = "../../core/sr-io" } client = { package = "substrate-client", path = "../../core/client" } inherents = { package = "substrate-inherents", path = "../../core/inherents" } chain-spec = { package = "substrate-chain-spec", path = "../../core/chain-spec" } diff --git a/srml/evm/src/backend.rs b/srml/evm/src/backend.rs index 6de5429dde478..1f3dfe309b4ee 100644 --- a/srml/evm/src/backend.rs +++ b/srml/evm/src/backend.rs @@ -96,7 +96,7 @@ impl<'vicinity, T: Trait> BackendT for Backend<'vicinity, T> { } fn chain_id(&self) -> U256 { - U256::from(runtime_io::chain_id()) + U256::from(runtime_io::misc::chain_id()) } fn exists(&self, _address: H160) -> bool { diff --git a/srml/im-online/src/tests.rs b/srml/im-online/src/tests.rs index 13b57668876da..382eb4f1d1f04 100644 --- a/srml/im-online/src/tests.rs +++ b/srml/im-online/src/tests.rs @@ -316,7 +316,7 @@ fn should_not_send_a_report_if_already_online() { assert_eq!(heartbeat, Heartbeat { block_number: 4, - network_state: runtime_io::network_state().unwrap(), + network_state: runtime_io::offchain::network_state().unwrap(), session_index: 2, authority_index: 0, }); From e5faddfa989f06036895681f7ce6b4f49ccb369b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 8 Nov 2019 23:44:36 +0100 Subject: [PATCH 70/76] Make `wasmtime` happy --- core/sr-io/src/lib.rs | 4 ++-- core/sr-sandbox/without_std.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 896cde804fa3f..12b51ddefd8d5 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -629,7 +629,7 @@ pub trait Logging { /// Wasm-only interface that provides functions for interacting with the sandbox. #[runtime_interface(wasm_only)] -pub trait Sandbox { +pub trait Sandboxing { /// Instantiate a new sandbox instance with the given `wasm_code`. fn instantiate( &mut self, @@ -769,7 +769,7 @@ pub type SubstrateHostFunctions = ( hashing::HostFunctions, allocator::HostFunctions, logging::HostFunctions, - sandbox::HostFunctions, + sandboxing::HostFunctions, ); #[cfg(test)] diff --git a/core/sr-sandbox/without_std.rs b/core/sr-sandbox/without_std.rs index d7fffbf88b27f..cbeca7360d317 100755 --- a/core/sr-sandbox/without_std.rs +++ b/core/sr-sandbox/without_std.rs @@ -18,7 +18,7 @@ use rstd::{prelude::*, slice, marker, mem, vec, rc::Rc}; use codec::{Decode, Encode}; use primitives::sandbox as sandbox_primitives; use super::{Error, TypedValue, ReturnValue, HostFuncType}; -use runtime_io::sandbox; +use runtime_io::sandboxing as sandbox; mod ffi { use rstd::mem; From c06174aa63cc8163ba2a35cfe9ce7dcbaa70df33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 9 Nov 2019 19:27:18 +0100 Subject: [PATCH 71/76] Make `full_crypto` work --- core/primitives/Cargo.toml | 7 ++++--- core/runtime-interface/Cargo.toml | 8 ++++++++ core/runtime-interface/src/impls.rs | 14 +++++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/core/primitives/Cargo.toml b/core/primitives/Cargo.toml index 3cbf837a7efd6..20c7e7c9caa2f 100644 --- a/core/primitives/Cargo.toml +++ b/core/primitives/Cargo.toml @@ -94,8 +94,8 @@ std = [ "runtime-interface/std", ] -# This feature enables all crypto primitives for `no_std` builds like microcontrollers -# or Intel SGX. +# This feature enables all crypto primitives for `no_std` builds like microcontrollers +# or Intel SGX. # For the regular wasm runtime builds this should not be used. full_crypto = [ "ed25519-dalek", @@ -104,5 +104,6 @@ full_crypto = [ "libsecp256k1", "hex", "sha2", - "twox-hash" + "twox-hash", + "runtime-interface/disable_target_static_assertions", ] diff --git a/core/runtime-interface/Cargo.toml b/core/runtime-interface/Cargo.toml index 7b0b649450631..b809e0ccbe1ac 100644 --- a/core/runtime-interface/Cargo.toml +++ b/core/runtime-interface/Cargo.toml @@ -31,3 +31,11 @@ std = [ "environmental", "primitive-types/std", ] + +# ATTENTION +# +# Only use when you know what you are doing. +# +# Disables static assertions in `impls.rs` that checks the word size. To prevent any footgun, the +# check is changed into a runtime check. +disable_target_static_assertions = [] diff --git a/core/runtime-interface/src/impls.rs b/core/runtime-interface/src/impls.rs index 3380c471a8b99..7a6adc90c96ea 100644 --- a/core/runtime-interface/src/impls.rs +++ b/core/runtime-interface/src/impls.rs @@ -22,7 +22,7 @@ use crate::host::*; #[cfg(not(feature = "std"))] use crate::wasm::*; -#[cfg(not(feature = "std"))] +#[cfg(all(not(feature = "std"), not(feature = "disable_target_static_assertions")))] use static_assertions::assert_eq_size; #[cfg(feature = "std")] @@ -39,18 +39,26 @@ use rstd::borrow::Cow; use rstd::{slice, boxed::Box}; // Make sure that our assumptions for storing a pointer + its size in `u64` is valid. -#[cfg(not(feature = "std"))] +#[cfg(all(not(feature = "std"), not(feature = "disable_target_static_assertions")))] assert_eq_size!(usize, u32); -#[cfg(not(feature = "std"))] +#[cfg(all(not(feature = "std"), not(feature = "disable_target_static_assertions")))] assert_eq_size!(*const u8, u32); /// Converts a pointer and length into an `u64`. pub fn pointer_and_len_to_u64(ptr: u32, len: u32) -> u64 { + // The static assertions from above are changed into a runtime check. + #[cfg(all(feature = "std", not(feature = "disable_target_static_assertions")))] + assert_eq!(4, rstd::mem::size_of::()); + (u64::from(len) << 32) | u64::from(ptr) } /// Splits an `u64` into the pointer and length. pub fn pointer_and_len_from_u64(val: u64) -> (u32, u32) { + // The static assertions from above are changed into a runtime check. + #[cfg(all(feature = "std", not(feature = "disable_target_static_assertions")))] + assert_eq!(4, rstd::mem::size_of::()); + let ptr = (val & (!0u32 as u64)) as u32; let len = (val >> 32) as u32; From 44333d2f54c6b004e8b7cf443418d255eaf76c9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 10 Nov 2019 00:20:21 +0100 Subject: [PATCH 72/76] Make the new interface versionable --- core/runtime-interface/proc-macro/src/lib.rs | 6 +++--- core/runtime-interface/proc-macro/src/utils.rs | 2 +- core/runtime-interface/src/lib.rs | 7 +++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/runtime-interface/proc-macro/src/lib.rs b/core/runtime-interface/proc-macro/src/lib.rs index f97de6a023634..febc388c77b17 100644 --- a/core/runtime-interface/proc-macro/src/lib.rs +++ b/core/runtime-interface/proc-macro/src/lib.rs @@ -118,12 +118,12 @@ mod utils; /// mod interface { /// mod extern_host_functions_impls { /// extern "C" { -/// /// Every function is exported as `ext_TRAIT_NAME_FUNCTION_NAME`. +/// /// Every function is exported as `ext_TRAIT_NAME_FUNCTION_NAME_version_VERSION`. /// /// /// /// The type for each argument of the exported function depends on /// /// `::FFIType`. -/// pub fn ext_Interface_call_some_complex_code(data: u64); -/// pub fn ext_Interface_set_or_clear(optional: u64); +/// pub fn ext_Interface_call_some_complex_code_version_1(data: u64); +/// pub fn ext_Interface_set_or_clear_version_1(optional: u64); /// } /// } /// diff --git a/core/runtime-interface/proc-macro/src/utils.rs b/core/runtime-interface/proc-macro/src/utils.rs index d225d643a82af..d868452dcec2a 100644 --- a/core/runtime-interface/proc-macro/src/utils.rs +++ b/core/runtime-interface/proc-macro/src/utils.rs @@ -70,7 +70,7 @@ pub fn create_exchangeable_host_function_ident(name: &Ident) -> Ident { pub fn create_host_function_ident(name: &Ident, trait_name: &Ident) -> Ident { Ident::new( &format!( - "ext_{}_{}", + "ext_{}_{}_version_1", trait_name.to_string().to_snake_case(), name, ), diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index f7ef7a11e85b3..3f8a753a691ee 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -181,7 +181,9 @@ mod tests { } #[test] - #[should_panic(expected = "Other(\"Instantiation: Export ext_test_api_return_input not found\")")] + #[should_panic( + expected = "Other(\"Instantiation: Export ext_test_api_return_input_version_1 not found\")" + )] fn host_function_not_found() { call_wasm_method::<()>("test_return_data"); } @@ -189,7 +191,8 @@ mod tests { #[test] #[should_panic( expected = - "FunctionExecution(\"ext_test_api_invalid_utf8_data\", \"Invalid utf8 data provided\")" + "FunctionExecution(\"ext_test_api_invalid_utf8_data_version_1\", \ + \"Invalid utf8 data provided\")" )] fn test_invalid_utf8_data_should_return_an_error() { call_wasm_method::("test_invalid_utf8_data_should_return_an_error"); From cd2b226edd90043970d32eef65726db02a67436a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 10 Nov 2019 14:58:24 +0100 Subject: [PATCH 73/76] Rename `Sanboxing` to `Sandbox` --- core/executor/src/wasmtime/runtime.rs | 2 +- core/sr-io/src/lib.rs | 4 ++-- core/sr-sandbox/without_std.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/executor/src/wasmtime/runtime.rs b/core/executor/src/wasmtime/runtime.rs index 489b09bc5a276..1a7385cc46025 100644 --- a/core/executor/src/wasmtime/runtime.rs +++ b/core/executor/src/wasmtime/runtime.rs @@ -86,7 +86,7 @@ pub fn create_instance( heap_pages: u64, host_functions: Vec<&'static dyn Function>, ) -> std::result::Result { - let (mut compiled_module, mut context) = create_compiled_unit(code, &host_functions)?; + let (compiled_module, context) = create_compiled_unit(code, &host_functions)?; // Inspect the module for the min and max memory sizes. let (min_memory_size, max_memory_size) = { diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 12b51ddefd8d5..896cde804fa3f 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -629,7 +629,7 @@ pub trait Logging { /// Wasm-only interface that provides functions for interacting with the sandbox. #[runtime_interface(wasm_only)] -pub trait Sandboxing { +pub trait Sandbox { /// Instantiate a new sandbox instance with the given `wasm_code`. fn instantiate( &mut self, @@ -769,7 +769,7 @@ pub type SubstrateHostFunctions = ( hashing::HostFunctions, allocator::HostFunctions, logging::HostFunctions, - sandboxing::HostFunctions, + sandbox::HostFunctions, ); #[cfg(test)] diff --git a/core/sr-sandbox/without_std.rs b/core/sr-sandbox/without_std.rs index cbeca7360d317..d7fffbf88b27f 100755 --- a/core/sr-sandbox/without_std.rs +++ b/core/sr-sandbox/without_std.rs @@ -18,7 +18,7 @@ use rstd::{prelude::*, slice, marker, mem, vec, rc::Rc}; use codec::{Decode, Encode}; use primitives::sandbox as sandbox_primitives; use super::{Error, TypedValue, ReturnValue, HostFuncType}; -use runtime_io::sandboxing as sandbox; +use runtime_io::sandbox; mod ffi { use rstd::mem; From 4ce996ccb11780984b73729bd6619dfa03c9b5cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 10 Nov 2019 16:35:58 +0100 Subject: [PATCH 74/76] Don't finalize in test while importing --- node/cli/src/service.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/cli/src/service.rs b/node/cli/src/service.rs index c5780d9f3556c..52ad35fe7b315 100644 --- a/node/cli/src/service.rs +++ b/node/cli/src/service.rs @@ -390,7 +390,7 @@ mod tests { origin: BlockOrigin::File, justification: Vec::new(), internal_justification: Vec::new(), - finalized: true, + finalized: false, body: Some(block.extrinsics), header: block.header, auxiliary: Vec::new(), @@ -520,7 +520,7 @@ mod tests { justification: None, post_digests: vec![item], body: Some(new_body), - finalized: true, + finalized: false, auxiliary: Vec::new(), fork_choice: ForkChoiceStrategy::LongestChain, allow_missing_state: false, From d23587fb94f115d5daeb2a9f48c0243a3d0b8e28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 10 Nov 2019 17:47:17 +0100 Subject: [PATCH 75/76] Fix Performance regression --- ...erface.rs => deprecated_host_interface.rs} | 2 ++ core/executor/src/lib.rs | 2 +- core/executor/src/native_executor.rs | 19 +++++++++---- core/executor/src/wasm_runtime.rs | 28 +++++++++++++------ core/runtime-interface/src/lib.rs | 9 +++++- core/wasm-interface/src/lib.rs | 2 +- 6 files changed, 45 insertions(+), 17 deletions(-) rename core/executor/src/{host_interface.rs => deprecated_host_interface.rs} (99%) diff --git a/core/executor/src/host_interface.rs b/core/executor/src/deprecated_host_interface.rs similarity index 99% rename from core/executor/src/host_interface.rs rename to core/executor/src/deprecated_host_interface.rs index 08c56e13a2ed6..0499cad5663ce 100644 --- a/core/executor/src/host_interface.rs +++ b/core/executor/src/deprecated_host_interface.rs @@ -37,6 +37,8 @@ macro_rules! debug_trace { ( $( $x:tt )* ) => () } +/// The old and deprecated Substrate externals. These are still required for backwards compatibility +/// reasons. pub struct SubstrateExternals; enum RecoverResult { diff --git a/core/executor/src/lib.rs b/core/executor/src/lib.rs index c9cf9d1ead4c7..0638a71d1c873 100644 --- a/core/executor/src/lib.rs +++ b/core/executor/src/lib.rs @@ -36,7 +36,7 @@ mod wasmi_execution; mod native_executor; mod sandbox; mod allocator; -mod host_interface; +pub mod deprecated_host_interface; mod wasm_runtime; #[cfg(feature = "wasmtime")] mod wasmtime; diff --git a/core/executor/src/native_executor.rs b/core/executor/src/native_executor.rs index 134bab3f15e45..3b33ee514dd39 100644 --- a/core/executor/src/native_executor.rs +++ b/core/executor/src/native_executor.rs @@ -29,7 +29,7 @@ use log::{trace, warn}; use std::{result, cell::RefCell, panic::{UnwindSafe, AssertUnwindSafe}}; -use wasm_interface::HostFunctions; +use wasm_interface::{HostFunctions, Function}; thread_local! { static RUNTIMES_CACHE: RefCell = RefCell::new(RuntimesCache::new()); @@ -70,7 +70,6 @@ pub trait NativeExecutionDispatch: Send + Sync { /// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence /// and dispatch to native code when possible, falling back on `WasmExecutor` when not. -#[derive(Debug)] pub struct NativeExecutor { /// Dummy field to avoid the compiler complaining about us not using `D`. _dummy: std::marker::PhantomData, @@ -80,6 +79,8 @@ pub struct NativeExecutor { native_version: NativeVersion, /// The number of 64KB pages to allocate for Wasm execution. default_heap_pages: u64, + /// The host functions registered with this instance. + host_functions: Vec<&'static dyn Function>, } impl NativeExecutor { @@ -92,11 +93,18 @@ impl NativeExecutor { /// `default_heap_pages` - Number of 64KB pages to allocate for Wasm execution. /// Defaults to `DEFAULT_HEAP_PAGES` if `None` is provided. pub fn new(fallback_method: WasmExecutionMethod, default_heap_pages: Option) -> Self { + let mut host_functions = runtime_io::SubstrateHostFunctions::host_functions(); + // Add the old and deprecated host functions as well, so that we support old wasm runtimes. + host_functions.extend( + crate::deprecated_host_interface::SubstrateExternals::host_functions(), + ); + NativeExecutor { _dummy: Default::default(), fallback_method, native_version: D::native_version(), default_heap_pages: default_heap_pages.unwrap_or(DEFAULT_HEAP_PAGES), + host_functions, } } @@ -128,7 +136,7 @@ impl NativeExecutor { ext, self.fallback_method, self.default_heap_pages, - runtime_io::SubstrateHostFunctions::host_functions(), + &self.host_functions, )?; let runtime = AssertUnwindSafe(runtime); @@ -152,6 +160,7 @@ impl Clone for NativeExecutor { fallback_method: self.fallback_method, native_version: D::native_version(), default_heap_pages: self.default_heap_pages, + host_functions: self.host_functions.clone(), } } } @@ -203,7 +212,7 @@ impl CodeExecutor for NativeExecutor { target: "executor", "Request for native execution failed (native: {}, chain: {})", self.native_version.runtime_version, - onchain_version + onchain_version, ); safe_call( @@ -220,7 +229,7 @@ impl CodeExecutor for NativeExecutor { target: "executor", "Request for native execution with native call succeeded (native: {}, chain: {}).", self.native_version.runtime_version, - onchain_version + onchain_version, ); used_native = true; diff --git a/core/executor/src/wasm_runtime.rs b/core/executor/src/wasm_runtime.rs index 5242931cfa063..384eb07bff2c4 100644 --- a/core/executor/src/wasm_runtime.rs +++ b/core/executor/src/wasm_runtime.rs @@ -31,7 +31,7 @@ use primitives::{storage::well_known_keys, traits::Externalities, H256}; use runtime_version::RuntimeVersion; use std::{collections::hash_map::{Entry, HashMap}, panic::AssertUnwindSafe}; -use wasm_interface::{Function, HostFunctions}; +use wasm_interface::Function; /// The Substrate Wasm runtime. pub trait WasmRuntime { @@ -127,7 +127,7 @@ impl RuntimesCache { ext: &mut E, wasm_method: WasmExecutionMethod, default_heap_pages: u64, - host_functions: Vec<&'static dyn Function>, + host_functions: &[&'static dyn Function], ) -> Result<(&mut (dyn WasmRuntime + 'static), &RuntimeVersion, H256), Error> { let code_hash = ext .original_storage_hash(well_known_keys::CODE) @@ -144,7 +144,7 @@ impl RuntimesCache { if let Ok(ref mut cached_runtime) = result { let heap_pages_changed = !cached_runtime.runtime.update_heap_pages(heap_pages); let host_functions_changed = cached_runtime.runtime.host_functions() - != &host_functions[..]; + != host_functions; if heap_pages_changed || host_functions_changed { let changed = if heap_pages_changed { "heap_pages" @@ -161,7 +161,7 @@ impl RuntimesCache { ext, wasm_method, heap_pages, - host_functions, + host_functions.into(), ); if let Err(ref err) = result { warn!(target: "runtimes_cache", "cannot create a runtime: {:?}", err); @@ -176,7 +176,7 @@ impl RuntimesCache { ext, wasm_method, heap_pages, - host_functions, + host_functions.into(), ); if let Err(ref err) = result { warn!(target: "runtimes_cache", "cannot create a runtime: {:?}", err); @@ -210,11 +210,8 @@ pub fn create_wasm_runtime_with_code( wasm_method: WasmExecutionMethod, heap_pages: u64, code: &[u8], - mut host_functions: Vec<&'static dyn Function>, + host_functions: Vec<&'static dyn Function>, ) -> Result, WasmError> { - // Add the old and deprecated host functions as well, so that we support old wasm runtimes. - host_functions.extend(crate::host_interface::SubstrateExternals::host_functions()); - match wasm_method { WasmExecutionMethod::Interpreted => wasmi_execution::create_instance(code, heap_pages, host_functions) @@ -259,3 +256,16 @@ fn create_versioned_wasm_runtime( version, }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn host_functions_are_equal() { + let host_functions = runtime_io::SubstrateHostFunctions::host_functions(); + + let equal = &host_functions[..] == &host_functions[..]; + assert!(equal, "Host functions are not equal"); + } +} diff --git a/core/runtime-interface/src/lib.rs b/core/runtime-interface/src/lib.rs index 3f8a753a691ee..fb70d252a6772 100644 --- a/core/runtime-interface/src/lib.rs +++ b/core/runtime-interface/src/lib.rs @@ -130,7 +130,14 @@ mod tests { let mut ext = TestExternalities::default(); let mut ext_ext = ext.ext(); - executor::call_in_wasm::<_, (HF, runtime_io::SubstrateHostFunctions)>( + executor::call_in_wasm::< + _, + ( + HF, + runtime_io::SubstrateHostFunctions, + executor::deprecated_host_interface::SubstrateExternals + ) + >( method, &[], executor::WasmExecutionMethod::Interpreted, diff --git a/core/wasm-interface/src/lib.rs b/core/wasm-interface/src/lib.rs index 726d2e7de0cc5..b2d57d080d533 100644 --- a/core/wasm-interface/src/lib.rs +++ b/core/wasm-interface/src/lib.rs @@ -189,7 +189,7 @@ impl Signature { } /// Something that provides a function implementation on the host for a wasm function. -pub trait Function: std::panic::RefUnwindSafe { +pub trait Function: std::panic::RefUnwindSafe + Send + Sync { /// Returns the name of this function. fn name(&self) -> &str; /// Returns the signature of this function. From 696ed43a7adcdcc3ac7d5e9e0318a5167daeef72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 10 Nov 2019 18:16:43 +0100 Subject: [PATCH 76/76] Fix test --- core/executor/src/wasm_runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/executor/src/wasm_runtime.rs b/core/executor/src/wasm_runtime.rs index 384eb07bff2c4..c58f63a1e03de 100644 --- a/core/executor/src/wasm_runtime.rs +++ b/core/executor/src/wasm_runtime.rs @@ -259,7 +259,7 @@ fn create_versioned_wasm_runtime( #[cfg(test)] mod tests { - use super::*; + use wasm_interface::HostFunctions; #[test] fn host_functions_are_equal() {