From 04f21672a36007eb0df029f1e760fb71f102d02b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 13 Mar 2023 15:04:02 +0100 Subject: [PATCH 01/55] Start --- Cargo.lock | 1 + client/api/src/call_executor.rs | 7 +- client/api/src/execution_extensions.rs | 81 +------ client/api/src/lib.rs | 2 +- client/block-builder/src/lib.rs | 34 +-- client/offchain/src/lib.rs | 9 +- .../rpc-spec-v2/src/chain_head/chain_head.rs | 1 - client/service/src/client/call_executor.rs | 31 +-- client/service/src/client/client.rs | 12 +- primitives/api/Cargo.toml | 2 + .../api/proc-macro/src/decl_runtime_apis.rs | 36 +--- .../api/proc-macro/src/impl_runtime_apis.rs | 15 +- .../proc-macro/src/mock_impl_runtime_apis.rs | 49 +---- primitives/api/src/lib.rs | 18 +- primitives/state-machine/src/lib.rs | 198 +----------------- 15 files changed, 91 insertions(+), 405 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a970be356b2dc..2f310a7843f66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9885,6 +9885,7 @@ dependencies = [ "parity-scale-codec", "sp-api-proc-macro", "sp-core", + "sp-externalities", "sp-runtime", "sp-state-machine", "sp-std", diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index db8e4d8495af2..6f25b2b1ede72 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -21,11 +21,11 @@ use sc_executor::{RuntimeVersion, RuntimeVersionOf}; use sp_core::traits::CallContext; use sp_runtime::traits::Block as BlockT; -use sp_state_machine::{ExecutionStrategy, OverlayedChanges, StorageProof}; +use sp_state_machine::{OverlayedChanges, StorageProof}; use std::cell::RefCell; use crate::execution_extensions::ExecutionExtensions; -use sp_api::{ExecutionContext, ProofRecorder, StorageTransactionCache}; +use sp_api::{ProofRecorder, StorageTransactionCache}; /// Executor Provider pub trait ExecutorProvider { @@ -58,7 +58,6 @@ pub trait CallExecutor: RuntimeVersionOf { at_hash: B::Hash, method: &str, call_data: &[u8], - strategy: ExecutionStrategy, context: CallContext, ) -> Result, sp_blockchain::Error>; @@ -79,7 +78,7 @@ pub trait CallExecutor: RuntimeVersionOf { >, >, proof_recorder: &Option>, - context: ExecutionContext, + call_context: CallContext, ) -> sp_blockchain::Result>; /// Extract RuntimeVersion of given block diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index b491d7672e8f0..6dc518ab00e7c 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -25,50 +25,19 @@ use codec::Decode; use parking_lot::RwLock; use sc_transaction_pool_api::OffchainSubmitTransaction; -use sp_core::{ - offchain::{self, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt}, - ExecutionContext, -}; +use sp_core::offchain::{self, Capabilities, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt}; use sp_externalities::{Extension, Extensions}; use sp_keystore::{KeystoreExt, SyncCryptoStorePtr}; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, NumberFor}, }; -pub use sp_state_machine::ExecutionStrategy; -use sp_state_machine::{DefaultHandler, ExecutionManager}; +use sp_state_machine::DefaultHandler; use std::{ marker::PhantomData, sync::{Arc, Weak}, }; -/// Execution strategies settings. -#[derive(Debug, Clone)] -pub struct ExecutionStrategies { - /// Execution strategy used when syncing. - pub syncing: ExecutionStrategy, - /// Execution strategy used when importing blocks. - pub importing: ExecutionStrategy, - /// Execution strategy used when constructing blocks. - pub block_construction: ExecutionStrategy, - /// Execution strategy used for offchain workers. - pub offchain_worker: ExecutionStrategy, - /// Execution strategy used in other cases. - pub other: ExecutionStrategy, -} - -impl Default for ExecutionStrategies { - fn default() -> ExecutionStrategies { - ExecutionStrategies { - syncing: ExecutionStrategy::NativeElseWasm, - importing: ExecutionStrategy::NativeElseWasm, - block_construction: ExecutionStrategy::AlwaysWasm, - offchain_worker: ExecutionStrategy::NativeWhenPossible, - other: ExecutionStrategy::NativeElseWasm, - } - } -} - /// Generate the starting set of [`Extensions`]. /// /// These [`Extensions`] are passed to the environment a runtime is executed in. @@ -162,7 +131,6 @@ impl DbExternaliti /// and is responsible for producing a correct `Extensions` object. /// for each call, based on required `Capabilities`. pub struct ExecutionExtensions { - strategies: ExecutionStrategies, keystore: Option, offchain_db: Option>, // FIXME: these two are only RwLock because of https://github.com/paritytech/substrate/issues/4587 @@ -178,7 +146,6 @@ pub struct ExecutionExtensions { impl Default for ExecutionExtensions { fn default() -> Self { Self { - strategies: Default::default(), keystore: None, offchain_db: None, transaction_pool: RwLock::new(None), @@ -190,14 +157,12 @@ impl Default for ExecutionExtensions { impl ExecutionExtensions { /// Create new `ExecutionExtensions` given a `keystore` and `ExecutionStrategies`. pub fn new( - strategies: ExecutionStrategies, keystore: Option, offchain_db: Option>, ) -> Self { let transaction_pool = RwLock::new(None); let extensions_factory = Box::new(()); Self { - strategies, keystore, offchain_db, extensions_factory: RwLock::new(extensions_factory), @@ -205,11 +170,6 @@ impl ExecutionExtensions { } } - /// Get a reference to the execution strategies. - pub fn strategies(&self) -> &ExecutionStrategies { - &self.strategies - } - /// Set the new extensions_factory pub fn set_extensions_factory(&self, maker: impl ExtensionsFactory + 'static) { *self.extensions_factory.write() = Box::new(maker); @@ -229,9 +189,8 @@ impl ExecutionExtensions { &self, block_hash: Block::Hash, block_number: NumberFor, - context: ExecutionContext, ) -> Extensions { - let capabilities = context.capabilities(); + let capabilities = Capabilities::empty(); let mut extensions = self.extensions_factory @@ -264,37 +223,15 @@ impl ExecutionExtensions { } } - if let ExecutionContext::OffchainCall(Some(ext)) = context { - extensions.register(OffchainWorkerExt::new(offchain::LimitedExternalities::new( - capabilities, - ext.0, - ))); - } + // if let ExecutionContext::OffchainCall(Some(ext)) = context { + // extensions.register(OffchainWorkerExt::new(offchain::LimitedExternalities::new( + // capabilities, + // ext.0, + // ))); + // } extensions } - - /// Create `ExecutionManager` and `Extensions` for given offchain call. - /// - /// Based on the execution context and capabilities it produces - /// the right manager and extensions object to support desired set of APIs. - pub fn manager_and_extensions( - &self, - block_hash: Block::Hash, - block_number: NumberFor, - context: ExecutionContext, - ) -> (ExecutionManager>, Extensions) { - let manager = match context { - ExecutionContext::BlockConstruction => self.strategies.block_construction.get_manager(), - ExecutionContext::Syncing => self.strategies.syncing.get_manager(), - ExecutionContext::Importing => self.strategies.importing.get_manager(), - ExecutionContext::OffchainCall(Some((_, capabilities))) if capabilities.is_all() => - self.strategies.offchain_worker.get_manager(), - ExecutionContext::OffchainCall(_) => self.strategies.other.get_manager(), - }; - - (manager, self.extensions(block_hash, block_number, context)) - } } /// A wrapper type to pass `BlockId` to the actual transaction pool. diff --git a/client/api/src/lib.rs b/client/api/src/lib.rs index 0faddc10fe016..faadf3663a59d 100644 --- a/client/api/src/lib.rs +++ b/client/api/src/lib.rs @@ -36,7 +36,7 @@ pub use proof_provider::*; pub use sp_blockchain as blockchain; pub use sp_blockchain::HeaderBackend; -pub use sp_state_machine::{CompactProof, ExecutionStrategy, StorageProof}; +pub use sp_state_machine::{CompactProof, StorageProof}; pub use sp_storage::{ChildInfo, PrefixedStorageKey, StorageData, StorageKey}; /// Usage Information Provider interface diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index d97afadd40156..6812df99fe6a3 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -32,7 +32,7 @@ use sp_api::{ ApiExt, ApiRef, Core, ProvideRuntimeApi, StorageChanges, StorageProof, TransactionOutcome, }; use sp_blockchain::{ApplyExtrinsicFailed, Error}; -use sp_core::ExecutionContext; +use sp_core::traits::CallContext; use sp_runtime::{ legacy, traits::{Block as BlockT, Hash, HashFor, Header as HeaderT, NumberFor, One}, @@ -178,11 +178,9 @@ where api.record_proof(); } - api.initialize_block_with_context( - parent_hash, - ExecutionContext::BlockConstruction, - &header, - )?; + api.set_call_context(CallContext::Onchain); + + api.initialize_block(parent_hash, &header)?; let version = api .api_version::>(parent_hash)? @@ -209,18 +207,10 @@ where self.api.execute_in_transaction(|api| { let res = if version < 6 { #[allow(deprecated)] - api.apply_extrinsic_before_version_6_with_context( - parent_hash, - ExecutionContext::BlockConstruction, - xt.clone(), - ) - .map(legacy::byte_sized_error::convert_to_latest) + api.apply_extrinsic_before_version_6(parent_hash, xt.clone()) + .map(legacy::byte_sized_error::convert_to_latest) } else { - api.apply_extrinsic_with_context( - parent_hash, - ExecutionContext::BlockConstruction, - xt.clone(), - ) + api.apply_extrinsic(parent_hash, xt.clone()) }; match res { @@ -242,9 +232,7 @@ where /// supplied by `self.api`, combined as [`BuiltBlock`]. /// The storage proof will be `Some(_)` when proof recording was enabled. pub fn build(mut self) -> Result>, Error> { - let header = self - .api - .finalize_block_with_context(self.parent_hash, ExecutionContext::BlockConstruction)?; + let header = self.api.finalize_block(self.parent_hash)?; debug_assert_eq!( header.extrinsics_root().clone(), @@ -282,11 +270,7 @@ where .execute_in_transaction(move |api| { // `create_inherents` should not change any state, to ensure this we always rollback // the transaction. - TransactionOutcome::Rollback(api.inherent_extrinsics_with_context( - parent_hash, - ExecutionContext::BlockConstruction, - inherent_data, - )) + TransactionOutcome::Rollback(api.inherent_extrinsics(parent_hash, inherent_data)) }) .map_err(|e| Error::Application(Box::new(e))) } diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index c69f01de19bf6..928803082c748 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -155,18 +155,19 @@ where capabilities.set(offchain::Capabilities::HTTP, self.enable_http); self.spawn_worker(move || { - let runtime = client.runtime_api(); + let mut runtime = client.runtime_api(); let api = Box::new(api); tracing::debug!(target: LOG_TARGET, "Running offchain workers at {:?}", hash); + // runtime.register_extension(): + let context = ExecutionContext::OffchainCall(Some((api, capabilities))); let run = if version == 2 { - runtime.offchain_worker_with_context(hash, context, &header) + runtime.offchain_worker(hash, &header) } else { #[allow(deprecated)] - runtime.offchain_worker_before_version_2_with_context( + runtime.offchain_worker_before_version_2( hash, - context, *header.number(), ) }; diff --git a/client/rpc-spec-v2/src/chain_head/chain_head.rs b/client/rpc-spec-v2/src/chain_head/chain_head.rs index c63e874c1bc15..59eb90529b91b 100644 --- a/client/rpc-spec-v2/src/chain_head/chain_head.rs +++ b/client/rpc-spec-v2/src/chain_head/chain_head.rs @@ -394,7 +394,6 @@ where hash, &function, &call_parameters, - client.execution_extensions().strategies().other, CallContext::Offchain, ) .map(|result| { diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 82d3e36ac80ea..8f47560342f3c 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -28,7 +28,7 @@ use sp_core::{ }; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; use sp_state_machine::{ - backend::AsTrieBackend, ExecutionStrategy, Ext, OverlayedChanges, StateMachine, StorageProof, + backend::AsTrieBackend, Ext, OverlayedChanges, StateMachine, StorageProof, }; use std::{cell::RefCell, sync::Arc}; @@ -182,11 +182,7 @@ where let runtime_code = self.check_override(runtime_code, at_hash)?.0; - let extensions = self.execution_extensions.extensions( - at_hash, - at_number, - ExecutionContext::OffchainCall(None), - ); + let extensions = self.execution_extensions.extensions(at_hash, at_number); let mut sm = StateMachine::new( &state, @@ -201,8 +197,7 @@ where ) .set_parent_hash(at_hash); - sm.execute_using_consensus_failure_handler(strategy.get_manager()) - .map_err(Into::into) + sm.execute().map_err(Into::into) } fn contextual_call( @@ -213,7 +208,7 @@ where changes: &RefCell, storage_transaction_cache: Option<&RefCell>>, recorder: &Option>, - context: ExecutionContext, + call_context: CallContext, ) -> Result, sp_blockchain::Error> { let mut storage_transaction_cache = storage_transaction_cache.map(|c| c.borrow_mut()); @@ -221,13 +216,7 @@ where self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(at_hash))?; let state = self.backend.state_at(at_hash)?; - let call_context = match context { - ExecutionContext::OffchainCall(_) => CallContext::Offchain, - _ => CallContext::Onchain, - }; - - let (execution_manager, extensions) = - self.execution_extensions.manager_and_extensions(at_hash, at_number, context); + let extensions = self.execution_extensions.extensions(at_hash, at_number); let changes = &mut *changes.borrow_mut(); @@ -261,7 +250,7 @@ where ) .with_storage_transaction_cache(storage_transaction_cache.as_deref_mut()) .set_parent_hash(at_hash); - state_machine.execute_using_consensus_failure_handler(execution_manager) + state_machine.execute() }, None => { let mut state_machine = StateMachine::new( @@ -277,7 +266,7 @@ where ) .with_storage_transaction_cache(storage_transaction_cache.as_deref_mut()) .set_parent_hash(at_hash); - state_machine.execute_using_consensus_failure_handler(execution_manager) + state_machine.execute() }, } .map_err(Into::into) @@ -317,11 +306,7 @@ where method, call_data, &runtime_code, - self.execution_extensions.extensions( - at_hash, - at_number, - ExecutionContext::OffchainCall(None), - ), + self.execution_extensions.extensions(at_hash, at_number), ) .map_err(Into::into) } diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 3613cc760f569..17674c05574c5 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -62,7 +62,7 @@ use sp_core::{ well_known_keys, ChildInfo, ChildType, PrefixedStorageKey, StorageChild, StorageData, StorageKey, }, - traits::SpawnNamed, + traits::{CallContext, SpawnNamed}, }; #[cfg(feature = "test-helpers")] use sp_keystore::SyncCryptoStorePtr; @@ -879,12 +879,12 @@ where // We should enact state, but don't have any storage changes, so we need to execute the // block. (true, None, Some(ref body)) => { - let runtime_api = self.runtime_api(); - let execution_context = import_block.origin.into(); + let mut runtime_api = self.runtime_api(); - runtime_api.execute_block_with_context( + runtime_api.set_call_context(CallContext::Onchain); + + runtime_api.execute_block( *parent_hash, - execution_context, Block::new(import_block.header.clone(), body.clone()), )?; @@ -1722,7 +1722,7 @@ where params.overlayed_changes, Some(params.storage_transaction_cache), params.recorder, - params.context, + params.call_context, ) .map_err(Into::into) } diff --git a/primitives/api/Cargo.toml b/primitives/api/Cargo.toml index 75197bcaea900..c4e67f882d466 100644 --- a/primitives/api/Cargo.toml +++ b/primitives/api/Cargo.toml @@ -16,6 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false } sp-api-proc-macro = { version = "4.0.0-dev", path = "proc-macro" } sp-core = { version = "7.0.0", default-features = false, path = "../core" } +sp-externalities = { version = "0.13.0", optional = true, path = "../externalities" } sp-std = { version = "5.0.0", default-features = false, path = "../std" } sp-runtime = { version = "7.0.0", default-features = false, path = "../runtime" } sp-version = { version = "5.0.0", default-features = false, path = "../version" } @@ -34,6 +35,7 @@ default = ["std"] std = [ "codec/std", "sp-core/std", + "sp-externalities", "sp-std/std", "sp-runtime/std", "sp-state-machine/std", diff --git a/primitives/api/proc-macro/src/decl_runtime_apis.rs b/primitives/api/proc-macro/src/decl_runtime_apis.rs index d9a72adadec98..ba91efa2464ea 100644 --- a/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -340,7 +340,6 @@ impl<'a> ToClientSideDecl<'a> { fn __runtime_api_internal_call_api_at( &self, at: #block_hash, - context: #crate_::ExecutionContext, params: std::vec::Vec, fn_name: &dyn Fn(#crate_::RuntimeVersion) -> &'static str, ) -> std::result::Result, #crate_::ApiError>; @@ -360,10 +359,9 @@ impl<'a> ToClientSideDecl<'a> { items.into_iter().for_each(|i| match i { TraitItem::Method(method) => { - let (fn_decl, fn_decl_ctx) = - self.fold_trait_item_method(method, trait_generics_num); + let fn_decl = + self.create_method_decl(method, trait_generics_num); result.push(fn_decl.into()); - result.push(fn_decl_ctx.into()); }, r => result.push(r), }); @@ -371,41 +369,12 @@ impl<'a> ToClientSideDecl<'a> { result } - fn fold_trait_item_method( - &mut self, - method: TraitItemMethod, - trait_generics_num: usize, - ) -> (TraitItemMethod, TraitItemMethod) { - let crate_ = self.crate_; - let context = quote!( #crate_::ExecutionContext::OffchainCall(None) ); - let fn_decl = self.create_method_decl(method.clone(), context, trait_generics_num); - let fn_decl_ctx = self.create_method_decl_with_context(method, trait_generics_num); - - (fn_decl, fn_decl_ctx) - } - - fn create_method_decl_with_context( - &mut self, - method: TraitItemMethod, - trait_generics_num: usize, - ) -> TraitItemMethod { - let crate_ = self.crate_; - let context_arg: syn::FnArg = parse_quote!( context: #crate_::ExecutionContext ); - let mut fn_decl_ctx = self.create_method_decl(method, quote!(context), trait_generics_num); - fn_decl_ctx.sig.ident = - Ident::new(&format!("{}_with_context", &fn_decl_ctx.sig.ident), Span::call_site()); - fn_decl_ctx.sig.inputs.insert(2, context_arg); - - fn_decl_ctx - } - /// Takes the method declared by the user and creates the declaration we require for the runtime /// api client side. This method will call by default the `method_runtime_api_impl` for doing /// the actual call into the runtime. fn create_method_decl( &mut self, mut method: TraitItemMethod, - context: TokenStream, trait_generics_num: usize, ) -> TraitItemMethod { let params = match extract_parameter_names_types_and_borrows( @@ -493,7 +462,6 @@ impl<'a> ToClientSideDecl<'a> { >::__runtime_api_internal_call_api_at( self, __runtime_api_at_param__, - #context, __runtime_api_impl_params_encoded__, &|version| { #( diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index f32961f892bba..b62fd601068ad 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -209,6 +209,8 @@ fn generate_runtime_api_base_structures() -> Result { #crate_::StorageTransactionCache >, recorder: std::option::Option<#crate_::ProofRecorder>, + call_context: #crate_::CallContext, + extensions: std::cell::RefCell<#crate_::Extensions>, } #[cfg(any(feature = "std", test))] @@ -292,6 +294,14 @@ fn generate_runtime_api_base_structures() -> Result { state_version, ) } + + fn set_call_context(&mut self, call_context: #crate_::CallContext) { + self.call_context = call_context; + } + + fn register_extension(&mut self, extension: E) { + std::cell::RefCell::borrow_mut(&self.extensions).register(extension); + } } #[cfg(any(feature = "std", test))] @@ -311,6 +321,8 @@ fn generate_runtime_api_base_structures() -> Result { changes: std::default::Default::default(), recorder: std::default::Default::default(), storage_transaction_cache: std::default::Default::default(), + call_context: #crate_::CallContext::Offchain, + extensions: std::default::Default::default(), }.into() } } @@ -424,7 +436,6 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> { fn __runtime_api_internal_call_api_at( &self, at: <__SR_API_BLOCK__ as #crate_::BlockT>::Hash, - context: #crate_::ExecutionContext, params: std::vec::Vec, fn_name: &dyn Fn(#crate_::RuntimeVersion) -> &'static str, ) -> std::result::Result, #crate_::ApiError> { @@ -446,7 +457,7 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> { arguments: params, overlayed_changes: &self.changes, storage_transaction_cache: &self.storage_transaction_cache, - context, + call_context: self.call_context, recorder: &self.recorder, }; diff --git a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs index 4c79787a1e8df..f3a6ac7baf6df 100644 --- a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs @@ -124,13 +124,20 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result where Self: Sized { unimplemented!("`into_storage_changes` not implemented for runtime api mocks") } + + fn set_call_context(&mut self, _: #crate_::CallContext) { + unimplemented!("`set_call_context` not implemented for runtime api mocks") + } + + fn register_extension(&mut self, _: E) { + unimplemented!("`register_extension` not implemented for runtime api mocks") + } } impl #crate_::Core<#block_type> for #self_ty { fn __runtime_api_internal_call_api_at( &self, _: <#block_type as #crate_::BlockT>::Hash, - _: #crate_::ExecutionContext, _: std::vec::Vec, _: &dyn Fn(#crate_::RuntimeVersion) -> &'static str, ) -> std::result::Result, #crate_::ApiError> { @@ -144,14 +151,6 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result::Hash, - _: #crate_::ExecutionContext, - ) -> std::result::Result<#crate_::RuntimeVersion, #crate_::ApiError> { - unimplemented!("`Core::version` not implemented for runtime api mocks") - } - fn execute_block( &self, _: <#block_type as #crate_::BlockT>::Hash, @@ -160,15 +159,6 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result::Hash, - _: #crate_::ExecutionContext, - _: #block_type, - ) -> std::result::Result<(), #crate_::ApiError> { - unimplemented!("`Core::execute_block` not implemented for runtime api mocks") - } - fn initialize_block( &self, _: <#block_type as #crate_::BlockT>::Hash, @@ -176,15 +166,6 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result std::result::Result<(), #crate_::ApiError> { unimplemented!("`Core::initialize_block` not implemented for runtime api mocks") } - - fn initialize_block_with_context( - &self, - _: <#block_type as #crate_::BlockT>::Hash, - _: #crate_::ExecutionContext, - _: &<#block_type as #crate_::BlockT>::Header, - ) -> std::result::Result<(), #crate_::ApiError> { - unimplemented!("`Core::initialize_block` not implemented for runtime api mocks") - } } )) } @@ -258,26 +239,12 @@ impl<'a> FoldRuntimeApiImpl<'a> { let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID); - // We also need to overwrite all the `_with_context` methods. To do this, - // we clone all methods and add them again with the new name plus one more argument. - impl_item.items.extend(impl_item.items.clone().into_iter().filter_map(|i| { - if let syn::ImplItem::Method(mut m) = i { - m.sig.ident = quote::format_ident!("{}_with_context", m.sig.ident); - m.sig.inputs.insert(2, parse_quote!( _: #crate_::ExecutionContext )); - - Some(m.into()) - } else { - None - } - })); - let block_type = self.block_type; impl_item.items.push(parse_quote! { fn __runtime_api_internal_call_api_at( &self, _: <#block_type as #crate_::BlockT>::Hash, - _: #crate_::ExecutionContext, _: std::vec::Vec, _: &dyn Fn(#crate_::RuntimeVersion) -> &'static str, ) -> std::result::Result, #crate_::ApiError> { diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index 7542ca3f20ccd..e6a3d967a8730 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -76,11 +76,17 @@ pub use codec::{self, Decode, DecodeLimit, Encode}; #[cfg(feature = "std")] pub use hash_db::Hasher; #[doc(hidden)] +pub use sp_core::offchain; +#[doc(hidden)] #[cfg(not(feature = "std"))] pub use sp_core::to_substrate_wasm_fn_return_value; +#[doc(hidden)] +#[cfg(feature = "std")] +pub use sp_core::traits::CallContext; use sp_core::OpaqueMetadata; #[doc(hidden)] -pub use sp_core::{offchain, ExecutionContext}; +#[cfg(feature = "std")] +pub use sp_externalities::{Extension, Extensions}; #[doc(hidden)] #[cfg(feature = "std")] pub use sp_runtime::StateVersion; @@ -582,6 +588,12 @@ pub trait ApiExt { ) -> Result, String> where Self: Sized; + + /// Set the [`CallContext`] to be used by the runtime api calls done by this instance. + fn set_call_context(&mut self, call_context: CallContext); + + /// Register an [`Extension`] that will be accessible while executing a runtime api call. + fn register_extension(&mut self, extension: E); } /// Parameters for [`CallApiAt::call_api_at`]. @@ -597,8 +609,8 @@ pub struct CallApiAtParams<'a, Block: BlockT, Backend: StateBackend, /// The cache for storage transactions. pub storage_transaction_cache: &'a RefCell>, - /// The context this function is executed in. - pub context: ExecutionContext, + /// The call context of this call. + pub call_context: CallContext, /// The optional proof recorder for recording storage accesses. pub recorder: &'a Option>, } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 90ee962dafa9e..25e0beb5404ba 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -166,14 +166,7 @@ mod execution { traits::{CallContext, CodeExecutor, ReadRuntimeVersionExt, RuntimeCode, SpawnNamed}, }; use sp_externalities::Extensions; - use std::{ - collections::{HashMap, HashSet}, - fmt, - }; - - const PROOF_CLOSE_TRANSACTION: &str = "\ - Closing a transaction that was started in this function. Client initiated transactions - are protected from being closed by the runtime. qed"; + use std::collections::{HashMap, HashSet}; pub(crate) type CallResult = Result, E>; @@ -183,21 +176,6 @@ mod execution { /// Trie backend with in-memory storage. pub type InMemoryBackend = TrieBackend, H>; - /// Strategy for executing a call into the runtime. - #[derive(Copy, Clone, Eq, PartialEq, Debug)] - pub enum ExecutionStrategy { - /// Execute with the native equivalent if it is compatible with the given wasm module; - /// otherwise fall back to the wasm. - NativeWhenPossible, - /// Use the given wasm module. - AlwaysWasm, - /// Run with both the wasm and the native variant (if compatible). Report any discrepancy - /// as an error. - Both, - /// First native, then if that fails or is not possible, wasm. - NativeElseWasm, - } - /// Storage backend trust level. #[derive(Debug, Clone)] pub enum BackendTrustLevel { @@ -209,73 +187,6 @@ mod execution { Untrusted, } - /// Like `ExecutionStrategy` only it also stores a handler in case of consensus failure. - #[derive(Clone)] - pub enum ExecutionManager { - /// Execute with the native equivalent if it is compatible with the given wasm module; - /// otherwise fall back to the wasm. - NativeWhenPossible, - /// Use the given wasm module. The backend on which code is executed code could be - /// trusted to provide all storage or not (i.e. the light client cannot be trusted to - /// provide for all storage queries since the storage entries it has come from an external - /// node). - AlwaysWasm(BackendTrustLevel), - /// Run with both the wasm and the native variant (if compatible). Call `F` in the case of - /// any discrepancy. - Both(F), - /// First native, then if that fails or is not possible, wasm. - NativeElseWasm, - } - - impl<'a, F> From<&'a ExecutionManager> for ExecutionStrategy { - fn from(s: &'a ExecutionManager) -> Self { - match *s { - ExecutionManager::NativeWhenPossible => ExecutionStrategy::NativeWhenPossible, - ExecutionManager::AlwaysWasm(_) => ExecutionStrategy::AlwaysWasm, - ExecutionManager::NativeElseWasm => ExecutionStrategy::NativeElseWasm, - ExecutionManager::Both(_) => ExecutionStrategy::Both, - } - } - } - - impl ExecutionStrategy { - /// Gets the corresponding manager for the execution strategy. - pub fn get_manager(self) -> ExecutionManager> { - match self { - ExecutionStrategy::AlwaysWasm => - ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted), - ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible, - ExecutionStrategy::NativeElseWasm => ExecutionManager::NativeElseWasm, - ExecutionStrategy::Both => ExecutionManager::Both(|wasm_result, native_result| { - warn!( - "Consensus error between wasm {:?} and native {:?}. Using wasm.", - wasm_result, native_result, - ); - warn!(" Native result {:?}", native_result); - warn!(" Wasm result {:?}", wasm_result); - wasm_result - }), - } - } - } - - /// Evaluate to ExecutionManager::NativeElseWasm, without having to figure out the type. - pub fn native_else_wasm() -> ExecutionManager> { - ExecutionManager::NativeElseWasm - } - - /// Evaluate to ExecutionManager::AlwaysWasm with trusted backend, without having to figure out - /// the type. - fn always_wasm() -> ExecutionManager> { - ExecutionManager::AlwaysWasm(BackendTrustLevel::Trusted) - } - - /// Evaluate ExecutionManager::AlwaysWasm with untrusted backend, without having to figure out - /// the type. - fn always_untrusted_wasm() -> ExecutionManager> { - ExecutionManager::AlwaysWasm(BackendTrustLevel::Untrusted) - } - /// The substrate state machine. pub struct StateMachine<'a, B, H, Exec> where @@ -374,13 +285,7 @@ mod execution { /// blocks (e.g. a transaction at a time), ensure a different method is used. /// /// Returns the SCALE encoded result of the executed function. - pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result, Box> { - // We are not giving a native call and thus we are sure that the result can never be a - // native value. - self.execute_using_consensus_failure_handler(strategy.get_manager()) - } - - fn execute_aux(&mut self, use_native: bool) -> (CallResult, bool) { + pub fn execute(&mut self) -> Result, Box> { let mut cache = StorageTransactionCache::default(); let cache = match self.storage_transaction_cache.as_mut() { @@ -405,14 +310,10 @@ mod execution { "Call", ); - let (result, was_native) = self.exec.call( - &mut ext, - self.runtime_code, - self.method, - self.call_data, - use_native, - self.context, - ); + let result = self + .exec + .call(&mut ext, self.runtime_code, self.method, self.call_data, false, self.context) + .0; self.overlay .exit_runtime() @@ -421,92 +322,11 @@ mod execution { trace!( target: "state", ext_id = %HexDisplay::from(&ext_id.to_le_bytes()), - ?was_native, ?result, "Return", ); - (result, was_native) - } - - fn execute_call_with_both_strategy( - &mut self, - on_consensus_failure: Handler, - ) -> CallResult - where - Handler: - FnOnce(CallResult, CallResult) -> CallResult, - { - self.overlay.start_transaction(); - let (result, was_native) = self.execute_aux(true); - - if was_native { - self.overlay.rollback_transaction().expect(PROOF_CLOSE_TRANSACTION); - let (wasm_result, _) = self.execute_aux(false); - - if (result.is_ok() && - wasm_result.is_ok() && result.as_ref().ok() == wasm_result.as_ref().ok()) || - result.is_err() && wasm_result.is_err() - { - result - } else { - on_consensus_failure(wasm_result, result) - } - } else { - self.overlay.commit_transaction().expect(PROOF_CLOSE_TRANSACTION); - result - } - } - - fn execute_call_with_native_else_wasm_strategy(&mut self) -> CallResult { - self.overlay.start_transaction(); - let (result, was_native) = self.execute_aux(true); - - if !was_native || result.is_ok() { - self.overlay.commit_transaction().expect(PROOF_CLOSE_TRANSACTION); - result - } else { - self.overlay.rollback_transaction().expect(PROOF_CLOSE_TRANSACTION); - self.execute_aux(false).0 - } - } - - /// Execute a call using the given state backend, overlayed changes, and call executor. - /// - /// On an error, no prospective changes are written to the overlay. - /// - /// Note: changes to code will be in place if this call is made again. For running partial - /// blocks (e.g. a transaction at a time), ensure a different method is used. - /// - /// Returns the result of the executed function either in native representation `R` or - /// in SCALE encoded representation. - pub fn execute_using_consensus_failure_handler( - &mut self, - manager: ExecutionManager, - ) -> Result, Box> - where - Handler: - FnOnce(CallResult, CallResult) -> CallResult, - { - let result = { - match manager { - ExecutionManager::Both(on_consensus_failure) => - self.execute_call_with_both_strategy(on_consensus_failure), - ExecutionManager::NativeElseWasm => - self.execute_call_with_native_else_wasm_strategy(), - ExecutionManager::AlwaysWasm(trust_level) => { - let _abort_guard = match trust_level { - BackendTrustLevel::Trusted => None, - BackendTrustLevel::Untrusted => - Some(sp_panic_handler::AbortGuard::never_abort()), - }; - self.execute_aux(false).0 - }, - ExecutionManager::NativeWhenPossible => self.execute_aux(true).0, - } - }; - - result.map_err(|e| Box::new(e) as _) + result.map_err(|e| Box::new(e) as Box<_>) } } @@ -580,7 +400,7 @@ mod execution { spawn_handle, CallContext::Offchain, ) - .execute_using_consensus_failure_handler::<_>(always_wasm())?; + .execute()?; let proof = proving_backend .extract_proof() @@ -645,7 +465,7 @@ mod execution { spawn_handle, CallContext::Offchain, ) - .execute_using_consensus_failure_handler(always_untrusted_wasm()) + .execute() } /// Generate storage read proof. From 08c0b326006fd70b44737b9ee5ff0b2c50413fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 13 Mar 2023 17:31:17 +0100 Subject: [PATCH 02/55] More work! --- Cargo.lock | 2 + client/api/src/execution_extensions.rs | 140 ++----------------------- client/offchain/Cargo.toml | 2 + client/offchain/src/lib.rs | 87 ++++++++++++--- 4 files changed, 88 insertions(+), 143 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f310a7843f66..d5ee896eeeec0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8998,6 +8998,7 @@ dependencies = [ "hyper-rustls", "lazy_static", "libp2p", + "log", "num_cpus", "once_cell", "parity-scale-codec", @@ -9014,6 +9015,7 @@ dependencies = [ "sp-api", "sp-consensus", "sp-core", + "sp-keystore", "sp-offchain", "sp-runtime", "sp-tracing", diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 6dc518ab00e7c..96000f13f31fe 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -47,21 +47,12 @@ pub trait ExtensionsFactory: Send + Sync { /// - `block_hash`: The hash of the block in the context that extensions will be used. /// - `block_number`: The number of the block in the context that extensions will be used. /// - `capabilities`: The capabilities - fn extensions_for( - &self, - block_hash: Block::Hash, - block_number: NumberFor, - capabilities: offchain::Capabilities, - ) -> Extensions; + fn extensions_for(&self, block_hash: Block::Hash, block_number: NumberFor) + -> Extensions; } impl ExtensionsFactory for () { - fn extensions_for( - &self, - _: Block::Hash, - _: NumberFor, - _capabilities: offchain::Capabilities, - ) -> Extensions { + fn extensions_for(&self, _: Block::Hash, _: NumberFor) -> Extensions { Extensions::new() } } @@ -71,10 +62,9 @@ impl> ExtensionsFactory for Ve &self, block_hash: Block::Hash, block_number: NumberFor, - capabilities: offchain::Capabilities, ) -> Extensions { let mut exts = Extensions::new(); - exts.extend(self.iter().map(|e| e.extensions_for(block_hash, block_number, capabilities))); + exts.extend(self.iter().map(|e| e.extensions_for(block_hash, block_number))); exts } } @@ -97,12 +87,7 @@ impl ExtensionBeforeBlock { impl ExtensionsFactory for ExtensionBeforeBlock { - fn extensions_for( - &self, - _: Block::Hash, - block_number: NumberFor, - _: offchain::Capabilities, - ) -> Extensions { + fn extensions_for(&self, _: Block::Hash, block_number: NumberFor) -> Extensions { let mut exts = Extensions::new(); if block_number < self.before { @@ -113,74 +98,25 @@ impl ExtensionsFactory } } -/// Create a Offchain DB accessor object. -pub trait DbExternalitiesFactory: Send + Sync { - /// Create [`offchain::DbExternalities`] instance. - fn create(&self) -> Box; -} - -impl DbExternalitiesFactory for T { - fn create(&self) -> Box { - Box::new(self.clone()) - } -} - /// A producer of execution extensions for offchain calls. /// /// This crate aggregates extensions available for the offchain calls /// and is responsible for producing a correct `Extensions` object. /// for each call, based on required `Capabilities`. pub struct ExecutionExtensions { - keystore: Option, - offchain_db: Option>, - // FIXME: these two are only RwLock because of https://github.com/paritytech/substrate/issues/4587 - // remove when fixed. - // To break retain cycle between `Client` and `TransactionPool` we require this - // extension to be a `Weak` reference. - // That's also the reason why it's being registered lazily instead of - // during initialization. - transaction_pool: RwLock>>>, extensions_factory: RwLock>>, } impl Default for ExecutionExtensions { fn default() -> Self { - Self { - keystore: None, - offchain_db: None, - transaction_pool: RwLock::new(None), - extensions_factory: RwLock::new(Box::new(())), - } + Self { extensions_factory: RwLock::new(Box::new(())) } } } impl ExecutionExtensions { /// Create new `ExecutionExtensions` given a `keystore` and `ExecutionStrategies`. - pub fn new( - keystore: Option, - offchain_db: Option>, - ) -> Self { - let transaction_pool = RwLock::new(None); - let extensions_factory = Box::new(()); - Self { - keystore, - offchain_db, - extensions_factory: RwLock::new(extensions_factory), - transaction_pool, - } - } - - /// Set the new extensions_factory - pub fn set_extensions_factory(&self, maker: impl ExtensionsFactory + 'static) { - *self.extensions_factory.write() = Box::new(maker); - } - - /// Register transaction pool extension. - pub fn register_transaction_pool(&self, pool: &Arc) - where - T: OffchainSubmitTransaction + 'static, - { - *self.transaction_pool.write() = Some(Arc::downgrade(pool) as _); + pub fn new() -> Self { + Self::default() } /// Based on the execution context and capabilities it produces @@ -190,66 +126,8 @@ impl ExecutionExtensions { block_hash: Block::Hash, block_number: NumberFor, ) -> Extensions { - let capabilities = Capabilities::empty(); - - let mut extensions = - self.extensions_factory - .read() - .extensions_for(block_hash, block_number, capabilities); - - if capabilities.contains(offchain::Capabilities::KEYSTORE) { - if let Some(ref keystore) = self.keystore { - extensions.register(KeystoreExt(keystore.clone())); - } - } - - if capabilities.contains(offchain::Capabilities::TRANSACTION_POOL) { - if let Some(pool) = self.transaction_pool.read().as_ref().and_then(|x| x.upgrade()) { - extensions.register(TransactionPoolExt(Box::new(TransactionPoolAdapter { - at: BlockId::Hash(block_hash), - pool, - }) as _)); - } - } - - if capabilities.contains(offchain::Capabilities::OFFCHAIN_DB_READ) || - capabilities.contains(offchain::Capabilities::OFFCHAIN_DB_WRITE) - { - if let Some(offchain_db) = self.offchain_db.as_ref() { - extensions.register(OffchainDbExt::new(offchain::LimitedExternalities::new( - capabilities, - offchain_db.create(), - ))); - } - } - - // if let ExecutionContext::OffchainCall(Some(ext)) = context { - // extensions.register(OffchainWorkerExt::new(offchain::LimitedExternalities::new( - // capabilities, - // ext.0, - // ))); - // } + let extensions = self.extensions_factory.read().extensions_for(block_hash, block_number); extensions } } - -/// A wrapper type to pass `BlockId` to the actual transaction pool. -struct TransactionPoolAdapter { - at: BlockId, - pool: Arc>, -} - -impl offchain::TransactionPool for TransactionPoolAdapter { - fn submit_transaction(&mut self, data: Vec) -> Result<(), ()> { - let xt = match Block::Extrinsic::decode(&mut &*data) { - Ok(xt) => xt, - Err(e) => { - log::warn!("Unable to decode extrinsic: {:?}: {}", data, e); - return Err(()) - }, - }; - - self.pool.submit_at(&self.at, xt) - } -} diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index dd6e2e44caf55..30ab0a2a6fe52 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -36,6 +36,8 @@ sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } sp-core = { version = "7.0.0", path = "../../primitives/core" } sp-offchain = { version = "4.0.0-dev", path = "../../primitives/offchain" } sp-runtime = { version = "7.0.0", path = "../../primitives/runtime" } +sp-keystore = { version = "0.13.0", path = "../../primitives/keystore" } +log = "0.4.17" [dev-dependencies] lazy_static = "1.4.0" diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 928803082c748..2ee9dec986c91 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -43,9 +43,14 @@ use futures::{ }; use parking_lot::Mutex; use sc_network_common::service::{NetworkPeers, NetworkStateInfo}; +use sc_transaction_pool_api::OffchainSubmitTransaction; use sp_api::{ApiExt, ProvideRuntimeApi}; -use sp_core::{offchain, traits::SpawnNamed, ExecutionContext}; -use sp_runtime::traits::{self, Header}; +use sp_core::{offchain, traits::SpawnNamed}; +use sp_keystore::{KeystoreExt, SyncCryptoStorePtr}; +use sp_runtime::{ + generic::BlockId, + traits::{self, Block, Header}, +}; use threadpool::ThreadPool; mod api; @@ -76,16 +81,36 @@ pub struct OffchainWorkers { thread_pool: Mutex, shared_http_client: api::SharedClient, enable_http: bool, + keystore: Option, + offchain_db: Option>, + transaction_pool: Option>>, } impl OffchainWorkers { /// Creates new [`OffchainWorkers`]. - pub fn new(client: Arc) -> Self { - Self::new_with_options(client, OffchainWorkerOptions { enable_http_requests: true }) + pub fn new( + client: Arc, + keystore: Option, + offchain_db: Option>, + transaction_pool: Option>>, + ) -> Self { + Self::new_with_options( + client, + keystore, + offchain_db, + transaction_pool, + OffchainWorkerOptions { enable_http_requests: true }, + ) } /// Creates new [`OffchainWorkers`] using the given `options`. - pub fn new_with_options(client: Arc, options: OffchainWorkerOptions) -> Self { + pub fn new_with_options( + client: Arc, + keystore: Option, + offchain_db: Option>, + transaction_pool: Option>>, + options: OffchainWorkerOptions, + ) -> Self { Self { client, _block: PhantomData, @@ -95,6 +120,9 @@ impl OffchainWorkers { )), shared_http_client: api::SharedClient::new(), enable_http: options.enable_http_requests, + keystore, + offchain_db, + transaction_pool, } } } @@ -152,25 +180,40 @@ where let client = self.client.clone(); let mut capabilities = offchain::Capabilities::all(); - capabilities.set(offchain::Capabilities::HTTP, self.enable_http); + self.spawn_worker(move || { let mut runtime = client.runtime_api(); let api = Box::new(api); tracing::debug!(target: LOG_TARGET, "Running offchain workers at {:?}", hash); - // runtime.register_extension(): + if let Some(keystore) = self.keystore.as_ref() { + runtime.register_extension(KeystoreExt(keystore.clone())); + } + + if let Some(pool) = self.transaction_pool.as_ref() { + runtime.register_extension(offchain::TransactionPoolExt(Box::new( + TransactionPoolAdapter { hash, pool }, + ) as _)); + } + + if let Some(offchain_db) = self.offchain_db.as_ref() { + runtime.register_extension(offchain::OffchainDbExt::new( + offchain::LimitedExternalities::new(capabilities, offchain_db.create()), + )); + } + + runtime.register_extension(offchain::OffchainWorkerExt::new( + offchain::LimitedExternalities::new(capabilities, api), + )); - let context = ExecutionContext::OffchainCall(Some((api, capabilities))); let run = if version == 2 { runtime.offchain_worker(hash, &header) } else { #[allow(deprecated)] - runtime.offchain_worker_before_version_2( - hash, - *header.number(), - ) + runtime.offchain_worker_before_version_2(hash, *header.number()) }; + if let Err(e) = run { tracing::error!( target: LOG_TARGET, @@ -240,6 +283,26 @@ pub async fn notification_future( .await; } +/// A wrapper type to pass `BlockId` to the actual transaction pool. +struct TransactionPoolAdapter { + hash: Block::Hash, + pool: Arc>, +} + +impl offchain::TransactionPool for TransactionPoolAdapter { + fn submit_transaction(&mut self, data: Vec) -> Result<(), ()> { + let xt = match Block::Extrinsic::decode(&mut &*data) { + Ok(xt) => xt, + Err(e) => { + log::warn!("Unable to decode extrinsic: {:?}: {}", data, e); + return Err(()) + }, + }; + + self.pool.submit_at(&BlockId::Hash(self.hash), xt) + } +} + #[cfg(test)] mod tests { use super::*; From d1d5d4c2ae7de134de8f955e9d880ee914fd5802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 14 Mar 2023 23:46:36 +0100 Subject: [PATCH 03/55] Moar --- client/api/src/execution_extensions.rs | 4 +- client/offchain/Cargo.toml | 1 + client/offchain/src/api.rs | 17 ++++----- client/offchain/src/lib.rs | 43 +++++++++++++--------- client/rpc/src/state/state_full.rs | 1 - client/service/src/client/call_executor.rs | 10 +---- client/service/src/config.rs | 1 - client/transaction-pool/src/lib.rs | 3 -- 8 files changed, 38 insertions(+), 42 deletions(-) diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 96000f13f31fe..79408d987bbfc 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -115,8 +115,8 @@ impl Default for ExecutionExtensions { impl ExecutionExtensions { /// Create new `ExecutionExtensions` given a `keystore` and `ExecutionStrategies`. - pub fn new() -> Self { - Self::default() + pub fn new(extensions_factory: impl ExtensionsFactory + 'static) -> Self { + Self { extensions_factory: RwLock::new(Box::new(extensions_factory)) } } /// Based on the execution context and capabilities it produces diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index 30ab0a2a6fe52..b7d165be4a5f7 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -31,6 +31,7 @@ tracing = "0.1.29" sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-network-common = { version = "0.10.0-dev", path = "../network/common" } sc-peerset = { version = "4.0.0-dev", path = "../peerset" } +sc-transaction-pool-api = { version = "4.0.0-dev", path = "../transaction-pool/api" } sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } sp-core = { version = "7.0.0", path = "../../primitives/core" } diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index 6fdd91cda2390..cc5bd5e9878a0 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -25,8 +25,8 @@ pub use http::SharedClient; use libp2p::{Multiaddr, PeerId}; use sp_core::{ offchain::{ - self, HttpError, HttpRequestId, HttpRequestStatus, OffchainStorage, OpaqueMultiaddr, - OpaqueNetworkState, StorageKind, Timestamp, + self, DbExternalities, HttpError, HttpRequestId, HttpRequestStatus, OffchainStorage, + OpaqueMultiaddr, OpaqueNetworkState, StorageKind, Timestamp, }, OpaquePeerId, }; @@ -55,22 +55,19 @@ pub struct Db { persistent: Storage, } -impl Db { +impl Db { /// Create new instance of Offchain DB. pub fn new(persistent: Storage) -> Self { Self { persistent } } - /// Create new instance of Offchain DB, backed by given backend. - pub fn factory_from_backend( - backend: &Backend, - ) -> Option> + /// Create new instance of Offchain DB, backed by the given backend. + pub fn from_backend(backend: &Backend) -> Option> where - Backend: sc_client_api::Backend, + Backend: sc_client_api::Backend, Block: sp_runtime::traits::Block, - Storage: 'static, { - sc_client_api::Backend::offchain_storage(backend).map(|db| Box::new(Self::new(db)) as _) + sc_client_api::Backend::offchain_storage(backend).map(|db| Db::new(db)) } } diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 2ee9dec986c91..08a6ae02dae51 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -37,6 +37,7 @@ use std::{fmt, marker::PhantomData, sync::Arc}; +use codec::Decode; use futures::{ future::{ready, Future}, prelude::*, @@ -75,24 +76,24 @@ pub struct OffchainWorkerOptions { } /// An offchain workers manager. -pub struct OffchainWorkers { +pub struct OffchainWorkers { client: Arc, _block: PhantomData, thread_pool: Mutex, shared_http_client: api::SharedClient, enable_http: bool, keystore: Option, - offchain_db: Option>, + offchain_db: Option>, transaction_pool: Option>>, } -impl OffchainWorkers { +impl OffchainWorkers { /// Creates new [`OffchainWorkers`]. pub fn new( client: Arc, keystore: Option, - offchain_db: Option>, - transaction_pool: Option>>, + offchain_db: Option, + transaction_pool: Option>>, ) -> Self { Self::new_with_options( client, @@ -107,8 +108,8 @@ impl OffchainWorkers { pub fn new_with_options( client: Arc, keystore: Option, - offchain_db: Option>, - transaction_pool: Option>>, + offchain_db: Option, + transaction_pool: Option>>, options: OffchainWorkerOptions, ) -> Self { Self { @@ -121,23 +122,26 @@ impl OffchainWorkers { shared_http_client: api::SharedClient::new(), enable_http: options.enable_http_requests, keystore, - offchain_db, + offchain_db: offchain_db.map(|d| api::Db::new(d)), transaction_pool, } } } -impl fmt::Debug for OffchainWorkers { +impl fmt::Debug + for OffchainWorkers +{ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_tuple("OffchainWorkers").finish() } } -impl OffchainWorkers +impl OffchainWorkers where Block: traits::Block, Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: OffchainWorkerApi, + Storage: offchain::OffchainStorage + 'static, { /// Start the offchain workers after given block. #[must_use] @@ -182,24 +186,28 @@ where let mut capabilities = offchain::Capabilities::all(); capabilities.set(offchain::Capabilities::HTTP, self.enable_http); + let keystore = self.keystore.clone(); + let db = self.offchain_db.clone(); + let tx_pool = self.transaction_pool.clone(); + self.spawn_worker(move || { let mut runtime = client.runtime_api(); let api = Box::new(api); tracing::debug!(target: LOG_TARGET, "Running offchain workers at {:?}", hash); - if let Some(keystore) = self.keystore.as_ref() { + if let Some(keystore) = keystore { runtime.register_extension(KeystoreExt(keystore.clone())); } - if let Some(pool) = self.transaction_pool.as_ref() { + if let Some(pool) = tx_pool { runtime.register_extension(offchain::TransactionPoolExt(Box::new( - TransactionPoolAdapter { hash, pool }, + TransactionPoolAdapter { hash, pool: pool.clone() }, ) as _)); } - if let Some(offchain_db) = self.offchain_db.as_ref() { + if let Some(offchain_db) = db { runtime.register_extension(offchain::OffchainDbExt::new( - offchain::LimitedExternalities::new(capabilities, offchain_db.create()), + offchain::LimitedExternalities::new(capabilities, offchain_db.clone()), )); } @@ -246,10 +254,10 @@ where } /// Inform the offchain worker about new imported blocks -pub async fn notification_future( +pub async fn notification_future( is_validator: bool, client: Arc, - offchain: Arc>, + offchain: Arc>, spawner: Spawner, network_provider: Arc, ) where @@ -258,6 +266,7 @@ pub async fn notification_future( ProvideRuntimeApi + sc_client_api::BlockchainEvents + Send + Sync + 'static, Client::Api: OffchainWorkerApi, Spawner: SpawnNamed, + Storage: offchain::OffchainStorage + 'static, { client .import_notification_stream() diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index f26d42484f24f..112bfefb05d07 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -207,7 +207,6 @@ where block, &method, &call_data, - self.client.execution_extensions().strategies().other, CallContext::Offchain, ) .map(Into::into) diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 8f47560342f3c..26ba683545776 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -22,14 +22,9 @@ use sc_client_api::{ }; use sc_executor::{RuntimeVersion, RuntimeVersionOf}; use sp_api::{ProofRecorder, StorageTransactionCache}; -use sp_core::{ - traits::{CallContext, CodeExecutor, RuntimeCode, SpawnNamed}, - ExecutionContext, -}; +use sp_core::traits::{CallContext, CodeExecutor, RuntimeCode, SpawnNamed}; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; -use sp_state_machine::{ - backend::AsTrieBackend, Ext, OverlayedChanges, StateMachine, StorageProof, -}; +use sp_state_machine::{backend::AsTrieBackend, Ext, OverlayedChanges, StateMachine, StorageProof}; use std::{cell::RefCell, sync::Arc}; /// Call executor that executes methods locally, querying all required @@ -168,7 +163,6 @@ where at_hash: Block::Hash, method: &str, call_data: &[u8], - strategy: ExecutionStrategy, context: CallContext, ) -> sp_blockchain::Result> { let mut changes = OverlayedChanges::default(); diff --git a/client/service/src/config.rs b/client/service/src/config.rs index 8e843b58f285e..b1383ffef5bc3 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -18,7 +18,6 @@ //! Service configuration. -pub use sc_client_api::execution_extensions::{ExecutionStrategies, ExecutionStrategy}; pub use sc_client_db::{BlocksPruning, Database, DatabaseSource, PruningMode}; pub use sc_executor::{WasmExecutionMethod, WasmtimeInstantiationStrategy}; pub use sc_network::{ diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index c3a85a373ba31..b3e13a33a8383 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -396,9 +396,6 @@ where client.usage_info().chain.finalized_hash, )); - // make transaction pool available for off-chain runtime calls. - client.execution_extensions().register_transaction_pool(&pool); - pool } } From 55af8691da45276e5756da9547b2f5e0d1453fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 15 Mar 2023 17:24:56 +0100 Subject: [PATCH 04/55] More changes --- client/offchain/src/lib.rs | 163 ++++++++++----------- client/service/src/builder.rs | 38 +---- client/service/src/client/call_executor.rs | 2 - client/service/src/client/client.rs | 9 +- client/service/src/config.rs | 2 - client/service/src/lib.rs | 7 +- client/service/test/src/client/mod.rs | 1 - primitives/api/test/tests/runtime_calls.rs | 79 ++-------- test-utils/client/src/lib.rs | 26 +--- test-utils/runtime/src/lib.rs | 19 --- 10 files changed, 92 insertions(+), 254 deletions(-) diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 08a6ae02dae51..e538dbbcbc7b1 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -43,6 +43,7 @@ use futures::{ prelude::*, }; use parking_lot::Mutex; +use sc_client_api::BlockchainEvents; use sc_network_common::service::{NetworkPeers, NetworkStateInfo}; use sc_transaction_pool_api::OffchainSubmitTransaction; use sp_api::{ApiExt, ProvideRuntimeApi}; @@ -68,7 +69,19 @@ pub trait NetworkProvider: NetworkStateInfo + NetworkPeers {} impl NetworkProvider for T where T: NetworkStateInfo + NetworkPeers {} /// Options for [`OffchainWorkers`] -pub struct OffchainWorkerOptions { +pub struct OffchainWorkerOptions { + /// Provides access to the runtime api. + pub runtime_api_provider: Arc, + /// Provides access to the keystore. + pub keystore: Option, + /// Provides access to the offchain database. + pub offchain_db: Option, + /// Provides access to the transaction pool. + pub transaction_pool: Option>>, + /// Provides access to network information. + pub network_provider: Arc, + /// Is the node running as validator? + pub is_validator: bool, /// Enable http requests from offchain workers? /// /// If not enabled, any http request will panic. @@ -76,82 +89,95 @@ pub struct OffchainWorkerOptions { } /// An offchain workers manager. -pub struct OffchainWorkers { - client: Arc, - _block: PhantomData, +pub struct OffchainWorkers { + runtime_api_provider: Arc, thread_pool: Mutex, shared_http_client: api::SharedClient, - enable_http: bool, + enable_http_requests: bool, keystore: Option, offchain_db: Option>, transaction_pool: Option>>, + network_provider: Arc, + is_validator: bool, } -impl OffchainWorkers { +impl OffchainWorkers { /// Creates new [`OffchainWorkers`]. pub fn new( - client: Arc, - keystore: Option, - offchain_db: Option, - transaction_pool: Option>>, - ) -> Self { - Self::new_with_options( - client, + OffchainWorkerOptions { + runtime_api_provider, keystore, offchain_db, transaction_pool, - OffchainWorkerOptions { enable_http_requests: true }, - ) - } - - /// Creates new [`OffchainWorkers`] using the given `options`. - pub fn new_with_options( - client: Arc, - keystore: Option, - offchain_db: Option, - transaction_pool: Option>>, - options: OffchainWorkerOptions, + network_provider, + is_validator, + enable_http_requests, + }: OffchainWorkerOptions, ) -> Self { Self { - client, - _block: PhantomData, + runtime_api_provider, thread_pool: Mutex::new(ThreadPool::with_name( "offchain-worker".into(), num_cpus::get(), )), shared_http_client: api::SharedClient::new(), - enable_http: options.enable_http_requests, + enable_http_requests, keystore, offchain_db: offchain_db.map(|d| api::Db::new(d)), transaction_pool, + is_validator, + network_provider, } } } -impl fmt::Debug - for OffchainWorkers +impl fmt::Debug + for OffchainWorkers { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_tuple("OffchainWorkers").finish() } } -impl OffchainWorkers +impl OffchainWorkers where Block: traits::Block, - Client: ProvideRuntimeApi + Send + Sync + 'static, - Client::Api: OffchainWorkerApi, + RA: ProvideRuntimeApi + Send + Sync + 'static, + RA::Api: OffchainWorkerApi, Storage: offchain::OffchainStorage + 'static, { + /// Run the offchain workers on every block import. + pub async fn run>( + self, + import_events: Arc, + spawner: impl SpawnNamed, + ) { + import_events + .import_notification_stream() + .for_each(move |n| { + if n.is_new_best { + spawner.spawn( + "offchain-on-block", + Some("offchain-worker"), + self.on_block_imported(&n.header).boxed(), + ); + } else { + tracing::debug!( + target: LOG_TARGET, + "Skipping offchain workers for non-canon block: {:?}", + n.header, + ) + } + + ready(()) + }) + .await; + } + /// Start the offchain workers after given block. #[must_use] - pub fn on_block_imported( - &self, - header: &Block::Header, - network_provider: Arc, - is_validator: bool, - ) -> impl Future { - let runtime = self.client.runtime_api(); + fn on_block_imported(&self, header: &Block::Header) -> impl Future { + let runtime = self.runtime_api_provider.runtime_api(); let hash = header.hash(); let has_api_v1 = runtime.has_api_with::, _>(hash, |v| v == 1); let has_api_v2 = runtime.has_api_with::, _>(hash, |v| v == 2); @@ -172,19 +198,21 @@ where }; tracing::debug!( target: LOG_TARGET, - "Checking offchain workers at {:?}: version:{}", - hash, - version + "Checking offchain workers at {hash:?}: version: {version}", ); + let process = (version > 0).then(|| { - let (api, runner) = - api::AsyncApi::new(network_provider, is_validator, self.shared_http_client.clone()); - tracing::debug!(target: LOG_TARGET, "Spawning offchain workers at {:?}", hash); + let (api, runner) = api::AsyncApi::new( + self.network_provider.clone(), + self.is_validator, + self.shared_http_client.clone(), + ); + tracing::debug!(target: LOG_TARGET, "Spawning offchain workers at {hash:?}"); let header = header.clone(); - let client = self.client.clone(); + let client = self.runtime_api_provider.clone(); let mut capabilities = offchain::Capabilities::all(); - capabilities.set(offchain::Capabilities::HTTP, self.enable_http); + capabilities.set(offchain::Capabilities::HTTP, self.enable_http_requests); let keystore = self.keystore.clone(); let db = self.offchain_db.clone(); @@ -193,7 +221,7 @@ where self.spawn_worker(move || { let mut runtime = client.runtime_api(); let api = Box::new(api); - tracing::debug!(target: LOG_TARGET, "Running offchain workers at {:?}", hash); + tracing::debug!(target: LOG_TARGET, "Running offchain workers at {hash:?}"); if let Some(keystore) = keystore { runtime.register_extension(KeystoreExt(keystore.clone())); @@ -253,45 +281,6 @@ where } } -/// Inform the offchain worker about new imported blocks -pub async fn notification_future( - is_validator: bool, - client: Arc, - offchain: Arc>, - spawner: Spawner, - network_provider: Arc, -) where - Block: traits::Block, - Client: - ProvideRuntimeApi + sc_client_api::BlockchainEvents + Send + Sync + 'static, - Client::Api: OffchainWorkerApi, - Spawner: SpawnNamed, - Storage: offchain::OffchainStorage + 'static, -{ - client - .import_notification_stream() - .for_each(move |n| { - if n.is_new_best { - spawner.spawn( - "offchain-on-block", - Some("offchain-worker"), - offchain - .on_block_imported(&n.header, network_provider.clone(), is_validator) - .boxed(), - ); - } else { - tracing::debug!( - target: LOG_TARGET, - "Skipping offchain workers for non-canon block: {:?}", - n.header, - ) - } - - ready(()) - }) - .await; -} - /// A wrapper type to pass `BlockId` to the actual transaction pool. struct TransactionPoolAdapter { hash: Block::Hash, diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index ea4b630003123..e2e9e6f8655b2 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -235,11 +235,7 @@ where .unwrap_or_default(); let client = { - let extensions = sc_client_api::execution_extensions::ExecutionExtensions::new( - config.execution_strategies.clone(), - Some(keystore_container.sync_keystore()), - sc_offchain::OffchainDb::factory_from_backend(&*backend), - ); + let extensions = sc_client_api::execution_extensions::ExecutionExtensions::default(); let wasm_runtime_substitutes = config .chain_spec @@ -396,38 +392,6 @@ pub struct SpawnTasksParams<'a, TBl: BlockT, TCl, TExPool, TRpc, Backend> { pub telemetry: Option<&'a mut Telemetry>, } -/// Build a shared offchain workers instance. -pub fn build_offchain_workers( - config: &Configuration, - spawn_handle: SpawnTaskHandle, - client: Arc, - network: Arc, -) -> Option>> -where - TBl: BlockT, - TCl: Send + Sync + ProvideRuntimeApi + BlockchainEvents + 'static, - >::Api: sc_offchain::OffchainWorkerApi, -{ - let offchain_workers = Some(Arc::new(sc_offchain::OffchainWorkers::new(client.clone()))); - - // Inform the offchain worker about new imported blocks - if let Some(offchain) = offchain_workers.clone() { - spawn_handle.spawn( - "offchain-notifications", - Some("offchain-worker"), - sc_offchain::notification_future( - config.role.is_authority(), - client, - offchain, - Clone::clone(&spawn_handle), - network, - ), - ); - } - - offchain_workers -} - /// Spawn the tasks that are required to run a node. pub fn spawn_tasks( params: SpawnTasksParams, diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 26ba683545776..88a59a1a83dd2 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -392,7 +392,6 @@ mod tests { backend.clone(), executor.clone(), genesis_block_builder, - None, Box::new(TaskExecutor::new()), None, None, @@ -468,7 +467,6 @@ mod tests { backend.clone(), executor.clone(), genesis_block_builder, - None, Box::new(TaskExecutor::new()), None, None, diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 17674c05574c5..df403f0a119cd 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -161,7 +161,6 @@ pub fn new_in_mem( backend: Arc>, executor: E, genesis_block_builder: G, - keystore: Option, prometheus_registry: Option, telemetry: Option, spawn_handle: Box, @@ -181,7 +180,6 @@ where backend, executor, genesis_block_builder, - keystore, spawn_handle, prometheus_registry, telemetry, @@ -224,7 +222,6 @@ pub fn new_with_backend( backend: Arc, executor: E, genesis_block_builder: G, - keystore: Option, spawn_handle: Box, prometheus_registry: Option, telemetry: Option, @@ -239,11 +236,7 @@ where Block: BlockT, B: backend::LocalBackend + 'static, { - let extensions = ExecutionExtensions::new( - Default::default(), - keystore, - sc_offchain::OffchainDb::factory_from_backend(&*backend), - ); + let extensions = ExecutionExtensions::default(); let call_executor = LocalCallExecutor::new( backend.clone(), diff --git a/client/service/src/config.rs b/client/service/src/config.rs index b1383ffef5bc3..2edccf3c8fe05 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -86,8 +86,6 @@ pub struct Configuration { /// over on-chain runtimes when the spec version matches. Set to `None` to /// disable overrides (default). pub wasm_runtime_overrides: Option, - /// Execution strategies. - pub execution_strategies: ExecutionStrategies, /// RPC over HTTP binding address. `None` if disabled. pub rpc_http: Option, /// RPC over Websockets binding address. `None` if disabled. diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 9dab81a5b4ed9..22d0a6c301983 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -57,10 +57,9 @@ use sp_runtime::{ pub use self::{ builder::{ - build_network, build_offchain_workers, new_client, new_db_backend, new_full_client, - new_full_parts, new_full_parts_with_genesis_builder, spawn_tasks, BuildNetworkParams, - KeystoreContainer, NetworkStarter, SpawnTasksParams, TFullBackend, TFullCallExecutor, - TFullClient, + build_network, new_client, new_db_backend, new_full_client, new_full_parts, + new_full_parts_with_genesis_builder, spawn_tasks, BuildNetworkParams, KeystoreContainer, + NetworkStarter, SpawnTasksParams, TFullBackend, TFullCallExecutor, TFullClient, }, client::{ClientConfig, LocalCallExecutor}, error::Error, diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index cae69413c7a02..d74e7e1f2ff9b 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -1933,7 +1933,6 @@ fn cleans_up_closed_notification_sinks_on_block_import() { genesis_block_builder, None, None, - None, Box::new(TaskExecutor::new()), client_config, ) diff --git a/primitives/api/test/tests/runtime_calls.rs b/primitives/api/test/tests/runtime_calls.rs index 69f9a88ffadb3..c27af57dc5d13 100644 --- a/primitives/api/test/tests/runtime_calls.rs +++ b/primitives/api/test/tests/runtime_calls.rs @@ -18,7 +18,7 @@ use sp_api::{Core, ProvideRuntimeApi}; use sp_runtime::traits::{HashFor, Header as HeaderT}; use sp_state_machine::{ - create_proof_check_backend, execution_proof_check_on_trie_backend, ExecutionStrategy, + create_proof_check_backend, execution_proof_check_on_trie_backend, }; use substrate_test_runtime_client::{ prelude::*, @@ -30,40 +30,18 @@ use codec::Encode; use sc_block_builder::BlockBuilderProvider; use sp_consensus::SelectChain; -fn calling_function_with_strat(strat: ExecutionStrategy) { - let client = TestClientBuilder::new().set_execution_strategy(strat).build(); - let runtime_api = client.runtime_api(); - let best_hash = client.chain_info().best_hash; - - assert_eq!(runtime_api.benchmark_add_one(best_hash, &1).unwrap(), 2); -} - #[test] -fn calling_native_runtime_function() { - calling_function_with_strat(ExecutionStrategy::NativeWhenPossible); -} - -#[test] -fn calling_wasm_runtime_function() { - calling_function_with_strat(ExecutionStrategy::AlwaysWasm); -} - -#[test] -fn calling_native_runtime_signature_changed_function() { - let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::NativeWhenPossible) - .build(); +fn calling_runtime_function() { + let client = TestClientBuilder::new().build(); let runtime_api = client.runtime_api(); let best_hash = client.chain_info().best_hash; - assert_eq!(runtime_api.function_signature_changed(best_hash).unwrap(), 1); + assert_eq!(runtime_api.benchmark_add_one(best_hash, &1).unwrap(), 2); } #[test] -fn calling_wasm_runtime_signature_changed_old_function() { - let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::AlwaysWasm) - .build(); +fn calling_runtime_signature_changed_old_function() { + let client = TestClientBuilder::new().build(); let runtime_api = client.runtime_api(); let best_hash = client.chain_info().best_hash; @@ -72,46 +50,9 @@ fn calling_wasm_runtime_signature_changed_old_function() { assert_eq!(&res, &[1, 2]); } -#[test] -fn calling_with_both_strategy_and_fail_on_wasm_should_return_error() { - let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); - let runtime_api = client.runtime_api(); - let best_hash = client.chain_info().best_hash; - assert!(runtime_api.fail_on_wasm(best_hash).is_err()); -} - -#[test] -fn calling_with_both_strategy_and_fail_on_native_should_work() { - let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); - let runtime_api = client.runtime_api(); - let best_hash = client.chain_info().best_hash; - assert_eq!(runtime_api.fail_on_native(best_hash).unwrap(), 1); -} - -#[test] -fn calling_with_native_else_wasm_and_fail_on_wasm_should_work() { - let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::NativeElseWasm) - .build(); - let runtime_api = client.runtime_api(); - let best_hash = client.chain_info().best_hash; - assert_eq!(runtime_api.fail_on_wasm(best_hash).unwrap(), 1); -} - -#[test] -fn calling_with_native_else_wasm_and_fail_on_native_should_work() { - let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::NativeElseWasm) - .build(); - let runtime_api = client.runtime_api(); - let best_hash = client.chain_info().best_hash; - assert_eq!(runtime_api.fail_on_native(best_hash).unwrap(), 1); -} - #[test] fn use_trie_function() { let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::AlwaysWasm) .build(); let runtime_api = client.runtime_api(); let best_hash = client.chain_info().best_hash; @@ -120,7 +61,7 @@ fn use_trie_function() { #[test] fn initialize_block_works() { - let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); + let client = TestClientBuilder::new().build(); let runtime_api = client.runtime_api(); let best_hash = client.chain_info().best_hash; runtime_api @@ -141,7 +82,6 @@ fn initialize_block_works() { #[test] fn record_proof_works() { let (client, longest_chain) = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::Both) .build_with_longest_chain(); let storage_root = @@ -198,7 +138,7 @@ fn record_proof_works() { #[test] fn call_runtime_api_with_multiple_arguments() { - let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); + let client = TestClientBuilder::new().build(); let data = vec![1, 2, 4, 5, 6, 7, 8, 8, 10, 12]; let best_hash = client.chain_info().best_hash; @@ -213,8 +153,7 @@ fn disable_logging_works() { if std::env::var("RUN_TEST").is_ok() { sp_tracing::try_init_simple(); - let mut builder = - TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::AlwaysWasm); + let mut builder = TestClientBuilder::new(); builder.genesis_init_mut().set_wasm_code( substrate_test_runtime_client::runtime::wasm_binary_logging_disabled_unwrap().to_vec(), ); diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index a27178792579a..efeb591b4fcd7 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -22,10 +22,7 @@ pub mod client_ext; pub use self::client_ext::{ClientBlockImportExt, ClientExt}; -pub use sc_client_api::{ - execution_extensions::{ExecutionExtensions, ExecutionStrategies}, - BadBlocks, ForkBlocks, -}; +pub use sc_client_api::{execution_extensions::ExecutionExtensions, BadBlocks, ForkBlocks}; pub use sc_client_db::{self, Backend, BlocksPruning}; pub use sc_executor::{self, NativeElseWasmExecutor, WasmExecutionMethod}; pub use sc_service::{client, RpcHandlers}; @@ -35,7 +32,6 @@ pub use sp_keyring::{ }; pub use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; pub use sp_runtime::{Storage, StorageChild}; -pub use sp_state_machine::ExecutionStrategy; use futures::{future::Future, stream::StreamExt}; use sc_client_api::BlockchainEvents; @@ -63,7 +59,6 @@ impl GenesisInit for () { /// A builder for creating a test client instance. pub struct TestClientBuilder { - execution_strategies: ExecutionStrategies, genesis_init: G, /// The key is an unprefixed storage key, this only contains /// default child trie content. @@ -115,7 +110,6 @@ impl pub fn with_backend(backend: Arc) -> Self { TestClientBuilder { backend, - execution_strategies: ExecutionStrategies::default(), child_storage_extension: Default::default(), genesis_init: Default::default(), _executor: Default::default(), @@ -158,18 +152,6 @@ impl self } - /// Set the execution strategy that should be used by all contexts. - pub fn set_execution_strategy(mut self, execution_strategy: ExecutionStrategy) -> Self { - self.execution_strategies = ExecutionStrategies { - syncing: execution_strategy, - importing: execution_strategy, - block_construction: execution_strategy, - offchain_worker: execution_strategy, - other: execution_strategy, - }; - self - } - /// Sets custom block rules. pub fn set_block_rules( mut self, @@ -293,11 +275,7 @@ impl executor, Box::new(sp_core::testing::TaskExecutor::new()), Default::default(), - ExecutionExtensions::new( - self.execution_strategies.clone(), - self.keystore.clone(), - sc_offchain::OffchainDb::factory_from_backend(&*self.backend), - ), + ExecutionExtensions::default(), ) .expect("Creates LocalCallExecutor"); diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 83b4e2977b5cd..b20847b8fb7c2 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -344,8 +344,6 @@ cfg_if! { fn function_signature_changed() -> Vec; /// The new signature. fn function_signature_changed() -> u64; - fn fail_on_native() -> u64; - fn fail_on_wasm() -> u64; /// trie no_std testing fn use_trie() -> u64; fn benchmark_indirect_call() -> u64; @@ -396,8 +394,6 @@ cfg_if! { fn fail_convert_return_value() -> DecodeFails; /// In wasm we just emulate the old behavior. fn function_signature_changed() -> Vec; - fn fail_on_native() -> u64; - fn fail_on_wasm() -> u64; /// trie no_std testing fn use_trie() -> u64; fn benchmark_indirect_call() -> u64; @@ -794,13 +790,6 @@ cfg_if! { 1 } - fn fail_on_native() -> u64 { - panic!("Failing because we are on native") - } - fn fail_on_wasm() -> u64 { - 1 - } - fn use_trie() -> u64 { code_using_trie() } @@ -1089,14 +1078,6 @@ cfg_if! { vec } - fn fail_on_native() -> u64 { - 1 - } - - fn fail_on_wasm() -> u64 { - panic!("Failing because we are on wasm") - } - fn use_trie() -> u64 { code_using_trie() } From 4374d0165a368d9373b7298de799a490edb24f10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 16 Mar 2023 17:19:12 +0100 Subject: [PATCH 05/55] More fixes --- client/cli/src/arg_enums.rs | 11 -------- client/cli/src/config.rs | 17 ----------- client/cli/src/params/import_params.rs | 31 --------------------- client/consensus/aura/src/import_queue.rs | 4 +-- client/consensus/babe/src/lib.rs | 4 +-- client/consensus/grandpa/src/lib.rs | 3 +- client/merkle-mountain-range/rpc/src/lib.rs | 18 +++--------- client/offchain/src/api.rs | 2 +- client/offchain/src/lib.rs | 4 +-- client/service/test/src/lib.rs | 1 - 10 files changed, 10 insertions(+), 85 deletions(-) diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs index 472c1722f9ed8..0caa954d0ed23 100644 --- a/client/cli/src/arg_enums.rs +++ b/client/cli/src/arg_enums.rs @@ -158,17 +158,6 @@ pub enum ExecutionStrategy { NativeElseWasm, } -impl Into for ExecutionStrategy { - fn into(self) -> sc_client_api::ExecutionStrategy { - match self { - ExecutionStrategy::Native => sc_client_api::ExecutionStrategy::NativeWhenPossible, - ExecutionStrategy::Wasm => sc_client_api::ExecutionStrategy::AlwaysWasm, - ExecutionStrategy::Both => sc_client_api::ExecutionStrategy::Both, - ExecutionStrategy::NativeElseWasm => sc_client_api::ExecutionStrategy::NativeElseWasm, - } - } -} - /// Available RPC methods. #[allow(missing_docs)] #[derive(Debug, Copy, Clone, PartialEq, ValueEnum)] diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index 5fedcf99a12ef..7a2a6992873b2 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -24,7 +24,6 @@ use crate::{ }; use log::warn; use names::{Generator, Name}; -use sc_client_api::execution_extensions::ExecutionStrategies; use sc_service::{ config::{ BasePath, Configuration, DatabaseSource, KeystoreConfig, NetworkConfiguration, @@ -287,21 +286,6 @@ pub trait CliConfiguration: Sized { self.import_params().map(|x| x.wasm_runtime_overrides()).unwrap_or_default() } - /// Get the execution strategies. - /// - /// By default this is retrieved from `ImportParams` if it is available. Otherwise its - /// `ExecutionStrategies::default()`. - fn execution_strategies( - &self, - is_dev: bool, - is_validator: bool, - ) -> Result { - Ok(self - .import_params() - .map(|x| x.execution_strategies(is_dev, is_validator)) - .unwrap_or_default()) - } - /// Get the RPC HTTP address (`None` if disabled). /// /// By default this is `None`. @@ -532,7 +516,6 @@ pub trait CliConfiguration: Sized { blocks_pruning: self.blocks_pruning()?, wasm_method: self.wasm_method()?, wasm_runtime_overrides: self.wasm_runtime_overrides(), - execution_strategies: self.execution_strategies(is_dev, is_validator)?, rpc_http: self.rpc_http(DCV::rpc_http_listen_port())?, rpc_ws: self.rpc_ws(DCV::rpc_ws_listen_port())?, rpc_ipc: self.rpc_ipc()?, diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs index e36fbb5ffd91c..a555786c37a05 100644 --- a/client/cli/src/params/import_params.rs +++ b/client/cli/src/params/import_params.rs @@ -27,7 +27,6 @@ use crate::{ params::{DatabaseParams, PruningParams}, }; use clap::Args; -use sc_client_api::execution_extensions::ExecutionStrategies; use std::path::PathBuf; /// Parameters for block import. @@ -119,36 +118,6 @@ impl ImportParams { pub fn wasm_runtime_overrides(&self) -> Option { self.wasm_runtime_overrides.clone() } - - /// Get execution strategies for the parameters - pub fn execution_strategies(&self, is_dev: bool, is_validator: bool) -> ExecutionStrategies { - let exec = &self.execution_strategies; - let exec_all_or = |strat: Option, default: ExecutionStrategy| { - let default = if is_dev { ExecutionStrategy::Native } else { default }; - - exec.execution.unwrap_or_else(|| strat.unwrap_or(default)).into() - }; - - let default_execution_import_block = if is_validator { - DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR - } else { - DEFAULT_EXECUTION_IMPORT_BLOCK - }; - - ExecutionStrategies { - syncing: exec_all_or(exec.execution_syncing, DEFAULT_EXECUTION_SYNCING), - importing: exec_all_or(exec.execution_import_block, default_execution_import_block), - block_construction: exec_all_or( - exec.execution_block_construction, - DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, - ), - offchain_worker: exec_all_or( - exec.execution_offchain_worker, - DEFAULT_EXECUTION_OFFCHAIN_WORKER, - ), - other: exec_all_or(exec.execution_other, DEFAULT_EXECUTION_OTHER), - } - } } /// Execution strategies parameters. diff --git a/client/consensus/aura/src/import_queue.rs b/client/consensus/aura/src/import_queue.rs index 46e0ccb4e302a..177039dce6eff 100644 --- a/client/consensus/aura/src/import_queue.rs +++ b/client/consensus/aura/src/import_queue.rs @@ -144,7 +144,6 @@ where at_hash: B::Hash, inherent_data: sp_inherents::InherentData, create_inherent_data_providers: CIDP::InherentDataProviders, - execution_context: ExecutionContext, ) -> Result<(), Error> where C: ProvideRuntimeApi, @@ -154,7 +153,7 @@ where let inherent_res = self .client .runtime_api() - .check_inherents_with_context(at_hash, execution_context, block, inherent_data) + .check_inherents(at_hash, block, inherent_data) .map_err(|e| Error::Client(e.into()))?; if !inherent_res.ok() { @@ -255,7 +254,6 @@ where parent_hash, inherent_data, create_inherent_data_providers, - block.origin.into(), ) .await .map_err(|e| e.to_string())?; diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index 1d42057a36b23..9a5b236b08efa 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -1024,12 +1024,11 @@ where at_hash: Block::Hash, inherent_data: InherentData, create_inherent_data_providers: CIDP::InherentDataProviders, - execution_context: ExecutionContext, ) -> Result<(), Error> { let inherent_res = self .client .runtime_api() - .check_inherents_with_context(at_hash, execution_context, block, inherent_data) + .check_inherents(at_hash, block, inherent_data) .map_err(Error::RuntimeApi)?; if !inherent_res.ok() { @@ -1266,7 +1265,6 @@ where parent_hash, inherent_data, create_inherent_data_providers, - block.origin.into(), ) .await?; } diff --git a/client/consensus/grandpa/src/lib.rs b/client/consensus/grandpa/src/lib.rs index 7bf48a498fabb..ba972c0edfdc2 100644 --- a/client/consensus/grandpa/src/lib.rs +++ b/client/consensus/grandpa/src/lib.rs @@ -64,7 +64,7 @@ use prometheus_endpoint::{PrometheusError, Registry}; use sc_client_api::{ backend::{AuxStore, Backend}, utils::is_descendent_of, - BlockchainEvents, CallExecutor, ExecutionStrategy, ExecutorProvider, Finalizer, LockImportRun, + BlockchainEvents, CallExecutor, ExecutorProvider, Finalizer, LockImportRun, StorageProvider, TransactionFor, }; use sc_consensus::BlockImport; @@ -479,7 +479,6 @@ where self.expect_block_hash_from_id(&BlockId::Number(Zero::zero()))?, "GrandpaApi_grandpa_authorities", &[], - ExecutionStrategy::NativeElseWasm, CallContext::Offchain, ) .and_then(|call_result| { diff --git a/client/merkle-mountain-range/rpc/src/lib.rs b/client/merkle-mountain-range/rpc/src/lib.rs index daf2cd1ec298b..024023f57c7a5 100644 --- a/client/merkle-mountain-range/rpc/src/lib.rs +++ b/client/merkle-mountain-range/rpc/src/lib.rs @@ -172,12 +172,7 @@ where self.client.info().best_hash); let (leaves, proof) = api - .generate_proof_with_context( - block_hash, - sp_core::ExecutionContext::OffchainCall(None), - block_numbers, - best_known_block_number, - ) + .generate_proof(block_hash, block_numbers, best_known_block_number) .map_err(runtime_error_into_rpc_error)? .map_err(mmr_error_into_rpc_error)?; @@ -193,14 +188,9 @@ where let decoded_proof = Decode::decode(&mut &proof.proof.0[..]) .map_err(|e| CallError::InvalidParams(anyhow::Error::new(e)))?; - api.verify_proof_with_context( - proof.block_hash, - sp_core::ExecutionContext::OffchainCall(None), - leaves, - decoded_proof, - ) - .map_err(runtime_error_into_rpc_error)? - .map_err(mmr_error_into_rpc_error)?; + api.verify_proof(proof.block_hash, leaves, decoded_proof) + .map_err(runtime_error_into_rpc_error)? + .map_err(mmr_error_into_rpc_error)?; Ok(true) } diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index cc5bd5e9878a0..0a877ae501130 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -25,7 +25,7 @@ pub use http::SharedClient; use libp2p::{Multiaddr, PeerId}; use sp_core::{ offchain::{ - self, DbExternalities, HttpError, HttpRequestId, HttpRequestStatus, OffchainStorage, + self, HttpError, HttpRequestId, HttpRequestStatus, OffchainStorage, OpaqueMultiaddr, OpaqueNetworkState, StorageKind, Timestamp, }, OpaquePeerId, diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index e538dbbcbc7b1..7657818e66857 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -35,7 +35,7 @@ #![warn(missing_docs)] -use std::{fmt, marker::PhantomData, sync::Arc}; +use std::{fmt, sync::Arc}; use codec::Decode; use futures::{ @@ -51,7 +51,7 @@ use sp_core::{offchain, traits::SpawnNamed}; use sp_keystore::{KeystoreExt, SyncCryptoStorePtr}; use sp_runtime::{ generic::BlockId, - traits::{self, Block, Header}, + traits::{self, Header}, }; use threadpool::ThreadPool; diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index b1a09a0620fcd..dd6c56c06ea83 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -252,7 +252,6 @@ fn node_config< chain_spec: Box::new((*spec).clone()), wasm_method: sc_service::config::WasmExecutionMethod::Interpreted, wasm_runtime_overrides: Default::default(), - execution_strategies: Default::default(), rpc_http: None, rpc_ipc: None, rpc_ws: None, From cc3c0305925924c8017543181c0beae446bb8668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 17 Mar 2023 16:40:44 +0100 Subject: [PATCH 06/55] More worrk --- Cargo.lock | 1 + bin/node/cli/Cargo.toml | 1 + client/cli/src/arg_enums.rs | 13 ------ client/cli/src/params/import_params.rs | 3 -- client/cli/src/runner.rs | 1 - client/consensus/pow/src/lib.rs | 4 +- client/service/src/client/client.rs | 2 - primitives/runtime/src/runtime_logger.rs | 4 +- primitives/state-machine/src/lib.rs | 40 +------------------ .../benchmarking-cli/src/pallet/command.rs | 9 ++--- .../frame/benchmarking-cli/src/pallet/mod.rs | 4 -- .../benchmarking-cli/src/pallet/writer.rs | 2 - 12 files changed, 10 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5ee896eeeec0..f2f50c72aaafc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4895,6 +4895,7 @@ dependencies = [ "sc-network", "sc-network-common", "sc-network-sync", + "sc-offchain", "sc-rpc", "sc-service", "sc-service-test", diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 4451935c36035..c1cf4741c15aa 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -81,6 +81,7 @@ sc-authority-discovery = { version = "0.10.0-dev", path = "../../../client/autho sc-sync-state-rpc = { version = "0.10.0-dev", path = "../../../client/sync-state-rpc" } sc-sysinfo = { version = "6.0.0-dev", path = "../../../client/sysinfo" } sc-storage-monitor = { version = "0.1.0", path = "../../../client/storage-monitor" } +sc-offchain = { version = "4.0.0-dev", path = "../../../client/offchain" } # frame dependencies frame-system = { version = "4.0.0-dev", path = "../../../frame/system" } diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs index 0caa954d0ed23..541c584a0437c 100644 --- a/client/cli/src/arg_enums.rs +++ b/client/cli/src/arg_enums.rs @@ -256,16 +256,3 @@ impl Into for SyncMode { } } } - -/// Default value for the `--execution-syncing` parameter. -pub const DEFAULT_EXECUTION_SYNCING: ExecutionStrategy = ExecutionStrategy::Wasm; -/// Default value for the `--execution-import-block` parameter. -pub const DEFAULT_EXECUTION_IMPORT_BLOCK: ExecutionStrategy = ExecutionStrategy::Wasm; -/// Default value for the `--execution-import-block` parameter when the node is a validator. -pub const DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR: ExecutionStrategy = ExecutionStrategy::Wasm; -/// Default value for the `--execution-block-construction` parameter. -pub const DEFAULT_EXECUTION_BLOCK_CONSTRUCTION: ExecutionStrategy = ExecutionStrategy::Wasm; -/// Default value for the `--execution-offchain-worker` parameter. -pub const DEFAULT_EXECUTION_OFFCHAIN_WORKER: ExecutionStrategy = ExecutionStrategy::Wasm; -/// Default value for the `--execution-other` parameter. -pub const DEFAULT_EXECUTION_OTHER: ExecutionStrategy = ExecutionStrategy::Wasm; diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs index a555786c37a05..414304bdfff1d 100644 --- a/client/cli/src/params/import_params.rs +++ b/client/cli/src/params/import_params.rs @@ -19,9 +19,6 @@ use crate::{ arg_enums::{ ExecutionStrategy, WasmExecutionMethod, WasmtimeInstantiationStrategy, - DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, DEFAULT_EXECUTION_IMPORT_BLOCK, - DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR, DEFAULT_EXECUTION_OFFCHAIN_WORKER, - DEFAULT_EXECUTION_OTHER, DEFAULT_EXECUTION_SYNCING, DEFAULT_WASMTIME_INSTANTIATION_STRATEGY, DEFAULT_WASM_EXECUTION_METHOD, }, params::{DatabaseParams, PruningParams}, diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index 8917a37c499c0..1f97d604789f6 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -340,7 +340,6 @@ mod tests { )), wasm_method: Default::default(), wasm_runtime_overrides: None, - execution_strategies: Default::default(), rpc_http: None, rpc_ws: None, rpc_ipc: None, diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs index 913686b7bf36d..f5e372ce3119a 100644 --- a/client/consensus/pow/src/lib.rs +++ b/client/consensus/pow/src/lib.rs @@ -269,7 +269,6 @@ where block: B, at_hash: B::Hash, inherent_data_providers: CIDP::InherentDataProviders, - execution_context: ExecutionContext, ) -> Result<(), Error> { if *block.header().number() < self.check_inherents_after { return Ok(()) @@ -283,7 +282,7 @@ where let inherent_res = self .client .runtime_api() - .check_inherents_with_context(at_hash, execution_context, block, inherent_data) + .check_inherents(at_hash, block, inherent_data) .map_err(|e| Error::Client(e.into()))?; if !inherent_res.ok() { @@ -348,7 +347,6 @@ where self.create_inherent_data_providers .create_inherent_data_providers(parent_hash, ()) .await?, - block.origin.into(), ) .await?; } diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index df403f0a119cd..60156b4932dbd 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -64,8 +64,6 @@ use sp_core::{ }, traits::{CallContext, SpawnNamed}, }; -#[cfg(feature = "test-helpers")] -use sp_keystore::SyncCryptoStorePtr; use sp_runtime::{ generic::{BlockId, SignedBlock}, traits::{ diff --git a/primitives/runtime/src/runtime_logger.rs b/primitives/runtime/src/runtime_logger.rs index 63e96a52a527f..eb3769b7344e2 100644 --- a/primitives/runtime/src/runtime_logger.rs +++ b/primitives/runtime/src/runtime_logger.rs @@ -78,9 +78,7 @@ mod tests { sp_tracing::try_init_simple(); log::set_max_level(log::LevelFilter::from_str(&env::var("RUST_LOG").unwrap()).unwrap()); - let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::AlwaysWasm) - .build(); + let client = TestClientBuilder::new().build(); let runtime_api = client.runtime_api(); runtime_api .do_trace_log(client.chain_info().genesis_hash) diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 25e0beb5404ba..35d4053a226db 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1208,7 +1208,7 @@ mod tests { CallContext::Offchain, ); - assert_eq!(state_machine.execute(ExecutionStrategy::NativeWhenPossible).unwrap(), vec![66]); + assert_eq!(state_machine.execute().unwrap(), vec![66]); } #[test] @@ -1237,43 +1237,7 @@ mod tests { CallContext::Offchain, ); - assert_eq!(state_machine.execute(ExecutionStrategy::NativeElseWasm).unwrap(), vec![66]); - } - - #[test] - fn dual_execution_strategy_detects_consensus_failure() { - dual_execution_strategy_detects_consensus_failure_inner(StateVersion::V0); - dual_execution_strategy_detects_consensus_failure_inner(StateVersion::V1); - } - fn dual_execution_strategy_detects_consensus_failure_inner(state_version: StateVersion) { - let mut consensus_failed = false; - let backend = trie_backend::tests::test_trie(state_version, None, None); - let mut overlayed_changes = Default::default(); - let wasm_code = RuntimeCode::empty(); - - let mut state_machine = StateMachine::new( - &backend, - &mut overlayed_changes, - &DummyCodeExecutor { - native_available: true, - native_succeeds: true, - fallback_succeeds: false, - }, - "test", - &[], - Default::default(), - &wasm_code, - TaskExecutor::new(), - CallContext::Offchain, - ); - - assert!(state_machine - .execute_using_consensus_failure_handler(ExecutionManager::Both(|we, _ne| { - consensus_failed = true; - we - }),) - .is_err()); - assert!(consensus_failed); + assert_eq!(state_machine.execute().unwrap(), vec![66]); } #[test] diff --git a/utils/frame/benchmarking-cli/src/pallet/command.rs b/utils/frame/benchmarking-cli/src/pallet/command.rs index 60f078142e17f..dd815c3c6a839 100644 --- a/utils/frame/benchmarking-cli/src/pallet/command.rs +++ b/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -183,7 +183,6 @@ impl PalletCmd { } let spec = config.chain_spec; - let strategy = self.execution.unwrap_or(ExecutionStrategy::Native); let pallet = self.pallet.clone().unwrap_or_default(); let pallet = pallet.as_bytes(); let extrinsic = self.extrinsic.clone().unwrap_or_default(); @@ -240,7 +239,7 @@ impl PalletCmd { sp_core::testing::TaskExecutor::new(), CallContext::Offchain, ) - .execute(strategy.into()) + .execute() .map_err(|e| format!("{}: {}", ERROR_METADATA_NOT_FOUND, e))?; let (list, storage_info) = @@ -378,7 +377,7 @@ impl PalletCmd { sp_core::testing::TaskExecutor::new(), CallContext::Offchain, ) - .execute(strategy.into()) + .execute() .map_err(|e| { format!("Error executing and verifying runtime benchmark: {}", e) })?; @@ -419,7 +418,7 @@ impl PalletCmd { sp_core::testing::TaskExecutor::new(), CallContext::Offchain, ) - .execute(strategy.into()) + .execute() .map_err(|e| format!("Error executing runtime benchmark: {}", e))?; let batch = @@ -452,7 +451,7 @@ impl PalletCmd { sp_core::testing::TaskExecutor::new(), CallContext::Offchain, ) - .execute(strategy.into()) + .execute() .map_err(|e| format!("Error executing runtime benchmark: {}", e))?; let batch = diff --git a/utils/frame/benchmarking-cli/src/pallet/mod.rs b/utils/frame/benchmarking-cli/src/pallet/mod.rs index f214569051d45..c94db41c72428 100644 --- a/utils/frame/benchmarking-cli/src/pallet/mod.rs +++ b/utils/frame/benchmarking-cli/src/pallet/mod.rs @@ -129,10 +129,6 @@ pub struct PalletCmd { #[clap(flatten)] pub shared_params: sc_cli::SharedParams, - /// The execution strategy that should be used for benchmarks. - #[arg(long, value_name = "STRATEGY", value_enum, ignore_case = true)] - pub execution: Option, - /// Method for executing Wasm runtime code. #[arg( long = "wasm-execution", diff --git a/utils/frame/benchmarking-cli/src/pallet/writer.rs b/utils/frame/benchmarking-cli/src/pallet/writer.rs index a609f9e38c0ce..236c9c57a3916 100644 --- a/utils/frame/benchmarking-cli/src/pallet/writer.rs +++ b/utils/frame/benchmarking-cli/src/pallet/writer.rs @@ -90,7 +90,6 @@ struct CmdData { repeat: u32, lowest_range_values: Vec, highest_range_values: Vec, - execution: String, wasm_execution: String, chain: String, db_cache: u32, @@ -423,7 +422,6 @@ pub(crate) fn write_results( repeat: cmd.repeat, lowest_range_values: cmd.lowest_range_values.clone(), highest_range_values: cmd.highest_range_values.clone(), - execution: format!("{:?}", cmd.execution), wasm_execution: cmd.wasm_method.to_string(), chain: format!("{:?}", cmd.shared_params.chain), db_cache: cmd.database_cache_size, From 0cac727f6826d99dc05141c864e4354b087b7e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 20 Mar 2023 09:20:35 +0100 Subject: [PATCH 07/55] More fixes --- Cargo.lock | 2 +- bin/node-template/node/Cargo.toml | 1 + bin/node-template/node/src/service.rs | 7 +-- bin/node/bench/src/construct.rs | 10 +--- bin/node/bench/src/import.rs | 8 +-- bin/node/bench/src/main.rs | 34 ++++++------ bin/node/bench/src/txpool.rs | 4 +- bin/node/testing/src/bench.rs | 53 +++---------------- client/api/Cargo.toml | 1 - client/api/src/execution_extensions.rs | 15 +----- client/consensus/aura/src/import_queue.rs | 2 +- client/consensus/babe/src/lib.rs | 2 +- client/consensus/pow/src/lib.rs | 1 - .../frame/benchmarking-cli/src/pallet/mod.rs | 4 +- utils/frame/try-runtime/cli/src/lib.rs | 4 +- 15 files changed, 37 insertions(+), 111 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2f50c72aaafc..6d143e61ca1d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5060,6 +5060,7 @@ dependencies = [ "sc-consensus-grandpa", "sc-executor", "sc-keystore", + "sc-offchain", "sc-rpc", "sc-rpc-api", "sc-service", @@ -8224,7 +8225,6 @@ dependencies = [ "sp-core", "sp-database", "sp-externalities", - "sp-keystore", "sp-runtime", "sp-state-machine", "sp-storage", diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 979c00120ddb7..2aff8771d2482 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -28,6 +28,7 @@ sc-telemetry = { version = "4.0.0-dev", path = "../../../client/telemetry" } sc-keystore = { version = "4.0.0-dev", path = "../../../client/keystore" } sc-transaction-pool = { version = "4.0.0-dev", path = "../../../client/transaction-pool" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../client/transaction-pool/api" } +sc-offchain = { version = "4.0.0-dev", path = "../../../client/offchain" } sc-consensus-aura = { version = "0.10.0-dev", path = "../../../client/consensus/aura" } sp-consensus-aura = { version = "0.10.0-dev", path = "../../../primitives/consensus/aura" } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 34e4e566d92fc..9ddb19d523fd6 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -204,12 +204,7 @@ pub fn new_full(mut config: Configuration) -> Result })?; if config.offchain_worker.enabled { - sc_service::build_offchain_workers( - &config, - task_manager.spawn_handle(), - client.clone(), - network.clone(), - ); + sc_offchain::OffchainWorkers::new() } let role = config.role.clone(); diff --git a/bin/node/bench/src/construct.rs b/bin/node/bench/src/construct.rs index ec2a829f692a6..0aaf5837cba8d 100644 --- a/bin/node/bench/src/construct.rs +++ b/bin/node/bench/src/construct.rs @@ -43,7 +43,6 @@ use crate::{ }; pub struct ConstructionBenchmarkDescription { - pub profile: Profile, pub key_types: KeyTypes, pub block_type: BlockType, pub size: SizeType, @@ -51,7 +50,6 @@ pub struct ConstructionBenchmarkDescription { } pub struct ConstructionBenchmark { - profile: Profile, database: BenchDb, transactions: Transactions, } @@ -60,11 +58,6 @@ impl core::BenchmarkDescription for ConstructionBenchmarkDescription { fn path(&self) -> Path { let mut path = Path::new(&["node", "proposer"]); - match self.profile { - Profile::Wasm => path.push("wasm"), - Profile::Native => path.push("native"), - } - match self.key_types { KeyTypes::Sr25519 => path.push("sr25519"), KeyTypes::Ed25519 => path.push("ed25519"), @@ -99,7 +92,6 @@ impl core::BenchmarkDescription for ConstructionBenchmarkDescription { } Box::new(ConstructionBenchmark { - profile: self.profile, database: bench_db, transactions: Transactions(extrinsics), }) @@ -116,7 +108,7 @@ impl core::BenchmarkDescription for ConstructionBenchmarkDescription { impl core::Benchmark for ConstructionBenchmark { fn run(&mut self, mode: Mode) -> std::time::Duration { - let context = self.database.create_context(self.profile); + let context = self.database.create_context(); let _ = context .client diff --git a/bin/node/bench/src/import.rs b/bin/node/bench/src/import.rs index 167377ea9a220..dc3210e447ff6 100644 --- a/bin/node/bench/src/import.rs +++ b/bin/node/bench/src/import.rs @@ -51,7 +51,6 @@ pub struct ImportBenchmarkDescription { } pub struct ImportBenchmark { - profile: Profile, database: BenchDb, block: Block, block_type: BlockType, @@ -61,11 +60,6 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription { fn path(&self) -> Path { let mut path = Path::new(&["node", "import"]); - match self.profile { - Profile::Wasm => path.push("wasm"), - Profile::Native => path.push("native"), - } - match self.key_types { KeyTypes::Sr25519 => path.push("sr25519"), KeyTypes::Ed25519 => path.push("ed25519"), @@ -110,7 +104,7 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription { impl core::Benchmark for ImportBenchmark { fn run(&mut self, mode: Mode) -> std::time::Duration { - let mut context = self.database.create_context(self.profile); + let mut context = self.database.create_context(); let _ = context .client diff --git a/bin/node/bench/src/main.rs b/bin/node/bench/src/main.rs index 051d8ddb9bf55..39db4e4d942b5 100644 --- a/bin/node/bench/src/main.rs +++ b/bin/node/bench/src/main.rs @@ -30,7 +30,7 @@ mod txpool; use clap::Parser; -use node_testing::bench::{BlockType, DatabaseType as BenchDataBaseType, KeyTypes, Profile}; +use node_testing::bench::{BlockType, DatabaseType as BenchDataBaseType, KeyTypes}; use crate::{ common::SizeType, @@ -85,23 +85,21 @@ fn main() { let mut import_benchmarks = Vec::new(); - for profile in [Profile::Wasm, Profile::Native] { - for size in [ - SizeType::Empty, - SizeType::Small, - SizeType::Medium, - SizeType::Large, - SizeType::Full, - SizeType::Custom(opt.transactions.unwrap_or(0)), + for size in [ + SizeType::Empty, + SizeType::Small, + SizeType::Medium, + SizeType::Large, + SizeType::Full, + SizeType::Custom(opt.transactions.unwrap_or(0)), + ] { + for block_type in [ + BlockType::RandomTransfersKeepAlive, + BlockType::RandomTransfersReaping, + BlockType::Noop, ] { - for block_type in [ - BlockType::RandomTransfersKeepAlive, - BlockType::RandomTransfersReaping, - BlockType::Noop, - ] { - for database_type in [BenchDataBaseType::RocksDb, BenchDataBaseType::ParityDb] { - import_benchmarks.push((profile, size, block_type, database_type)); - } + for database_type in [BenchDataBaseType::RocksDb, BenchDataBaseType::ParityDb] { + import_benchmarks.push((size, block_type, database_type)); } } } @@ -138,14 +136,12 @@ fn main() { .iter().map(move |db_type| (size, db_type))) => TrieWriteBenchmarkDescription { database_size: *size, database_type: *db_type }, ConstructionBenchmarkDescription { - profile: Profile::Wasm, key_types: KeyTypes::Sr25519, block_type: BlockType::RandomTransfersKeepAlive, size: SizeType::Medium, database_type: BenchDataBaseType::RocksDb, }, ConstructionBenchmarkDescription { - profile: Profile::Wasm, key_types: KeyTypes::Sr25519, block_type: BlockType::RandomTransfersKeepAlive, size: SizeType::Large, diff --git a/bin/node/bench/src/txpool.rs b/bin/node/bench/src/txpool.rs index 4e8e5c0d9a4fd..a3524ac5bc890 100644 --- a/bin/node/bench/src/txpool.rs +++ b/bin/node/bench/src/txpool.rs @@ -23,7 +23,7 @@ use std::borrow::Cow; -use node_testing::bench::{BenchDb, BlockType, DatabaseType, KeyTypes, Profile}; +use node_testing::bench::{BenchDb, BlockType, DatabaseType, KeyTypes}; use sc_transaction_pool::BasicPool; use sc_transaction_pool_api::{TransactionPool, TransactionSource}; @@ -57,7 +57,7 @@ impl core::BenchmarkDescription for PoolBenchmarkDescription { impl core::Benchmark for PoolBenchmark { fn run(&mut self, mode: Mode) -> std::time::Duration { - let context = self.database.create_context(Profile::Wasm); + let context = self.database.create_context(); let _ = context .client diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 1a9af13028483..aae2eba27c409 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -354,7 +354,7 @@ impl BenchDb { dir.path().to_string_lossy(), ); let (_client, _backend, _task_executor) = - Self::bench_client(database_type, dir.path(), Profile::Native, &keyring); + Self::bench_client(database_type, dir.path(), &keyring); let directory_guard = Guard(dir); BenchDb { keyring, directory_guard, database_type } @@ -380,7 +380,6 @@ impl BenchDb { fn bench_client( database_type: DatabaseType, dir: &std::path::Path, - profile: Profile, keyring: &BenchKeyring, ) -> (Client, std::sync::Arc, TaskExecutor) { let db_config = sc_client_db::DatabaseSettings { @@ -415,7 +414,7 @@ impl BenchDb { genesis_block_builder, None, None, - ExecutionExtensions::new(profile.into_execution_strategies(), None, None), + Default::default(), Box::new(task_executor.clone()), None, None, @@ -439,11 +438,7 @@ impl BenchDb { client .runtime_api() - .inherent_extrinsics_with_context( - client.chain_info().genesis_hash, - ExecutionContext::BlockConstruction, - inherent_data, - ) + .inherent_extrinsics(client.chain_info().genesis_hash, inherent_data) .expect("Get inherents failed") } @@ -454,12 +449,8 @@ impl BenchDb { /// Get cliet for this database operations. pub fn client(&mut self) -> Client { - let (client, _backend, _task_executor) = Self::bench_client( - self.database_type, - self.directory_guard.path(), - Profile::Wasm, - &self.keyring, - ); + let (client, _backend, _task_executor) = + Self::bench_client(self.database_type, self.directory_guard.path(), &self.keyring); client } @@ -502,10 +493,10 @@ impl BenchDb { } /// Clone this database and create context for testing/benchmarking. - pub fn create_context(&self, profile: Profile) -> BenchContext { + pub fn create_context(&self) -> BenchContext { let BenchDb { directory_guard, keyring, database_type } = self.clone(); let (client, backend, task_executor) = - Self::bench_client(database_type, directory_guard.path(), profile, &keyring); + Self::bench_client(database_type, directory_guard.path(), &keyring); BenchContext { client: Arc::new(client), @@ -606,36 +597,6 @@ impl BenchKeyring { } } -/// Profile for exetion strategies. -#[derive(Clone, Copy, Debug)] -pub enum Profile { - /// As native as possible. - Native, - /// As wasm as possible. - Wasm, -} - -impl Profile { - fn into_execution_strategies(self) -> ExecutionStrategies { - match self { - Profile::Wasm => ExecutionStrategies { - syncing: ExecutionStrategy::AlwaysWasm, - importing: ExecutionStrategy::AlwaysWasm, - block_construction: ExecutionStrategy::AlwaysWasm, - offchain_worker: ExecutionStrategy::AlwaysWasm, - other: ExecutionStrategy::AlwaysWasm, - }, - Profile::Native => ExecutionStrategies { - syncing: ExecutionStrategy::NativeElseWasm, - importing: ExecutionStrategy::NativeElseWasm, - block_construction: ExecutionStrategy::NativeElseWasm, - offchain_worker: ExecutionStrategy::NativeElseWasm, - other: ExecutionStrategy::NativeElseWasm, - }, - } - } -} - struct Guard(tempfile::TempDir); impl Guard { diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml index f494200852729..049ce5569240d 100644 --- a/client/api/Cargo.toml +++ b/client/api/Cargo.toml @@ -31,7 +31,6 @@ sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/comm sp-core = { version = "7.0.0", default-features = false, path = "../../primitives/core" } sp-database = { version = "4.0.0-dev", path = "../../primitives/database" } sp-externalities = { version = "0.13.0", path = "../../primitives/externalities" } -sp-keystore = { version = "0.13.0", default-features = false, path = "../../primitives/keystore" } sp-runtime = { version = "7.0.0", default-features = false, path = "../../primitives/runtime" } sp-state-machine = { version = "0.13.0", path = "../../primitives/state-machine" } sp-storage = { version = "7.0.0", path = "../../primitives/storage" } diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 79408d987bbfc..8e24c9ffc51fa 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -22,21 +22,10 @@ //! strategy for the runtime calls and provide the right `Externalities` //! extensions to support APIs for particular execution context & capabilities. -use codec::Decode; use parking_lot::RwLock; -use sc_transaction_pool_api::OffchainSubmitTransaction; -use sp_core::offchain::{self, Capabilities, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt}; use sp_externalities::{Extension, Extensions}; -use sp_keystore::{KeystoreExt, SyncCryptoStorePtr}; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, NumberFor}, -}; -use sp_state_machine::DefaultHandler; -use std::{ - marker::PhantomData, - sync::{Arc, Weak}, -}; +use sp_runtime::traits::{Block as BlockT, NumberFor}; +use std::marker::PhantomData; /// Generate the starting set of [`Extensions`]. /// diff --git a/client/consensus/aura/src/import_queue.rs b/client/consensus/aura/src/import_queue.rs index 177039dce6eff..6f1c1c7f4c540 100644 --- a/client/consensus/aura/src/import_queue.rs +++ b/client/consensus/aura/src/import_queue.rs @@ -38,7 +38,7 @@ use sp_blockchain::HeaderBackend; use sp_consensus::Error as ConsensusError; use sp_consensus_aura::{digests::CompatibleDigestItem, inherents::AuraInherentData, AuraApi}; use sp_consensus_slots::Slot; -use sp_core::{crypto::Pair, ExecutionContext}; +use sp_core::crypto::Pair; use sp_inherents::{CreateInherentDataProviders, InherentDataProvider as _}; use sp_runtime::{ traits::{Block as BlockT, Header, NumberFor}, diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index 9a5b236b08efa..c2ee872f791aa 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -117,7 +117,7 @@ use sp_blockchain::{ use sp_consensus::{BlockOrigin, Environment, Error as ConsensusError, Proposer, SelectChain}; use sp_consensus_babe::inherents::BabeInherentData; use sp_consensus_slots::Slot; -use sp_core::{crypto::ByteArray, ExecutionContext}; +use sp_core::crypto::ByteArray; use sp_inherents::{CreateInherentDataProviders, InherentData, InherentDataProvider}; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; use sp_runtime::{ diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs index f5e372ce3119a..763cf10e6cd4f 100644 --- a/client/consensus/pow/src/lib.rs +++ b/client/consensus/pow/src/lib.rs @@ -58,7 +58,6 @@ use sp_block_builder::BlockBuilder as BlockBuilderApi; use sp_blockchain::HeaderBackend; use sp_consensus::{Environment, Error as ConsensusError, Proposer, SelectChain, SyncOracle}; use sp_consensus_pow::{Seal, TotalDifficulty, POW_ENGINE_ID}; -use sp_core::ExecutionContext; use sp_inherents::{CreateInherentDataProviders, InherentDataProvider}; use sp_runtime::{ generic::{BlockId, Digest, DigestItem}, diff --git a/utils/frame/benchmarking-cli/src/pallet/mod.rs b/utils/frame/benchmarking-cli/src/pallet/mod.rs index c94db41c72428..f205fd0bcd432 100644 --- a/utils/frame/benchmarking-cli/src/pallet/mod.rs +++ b/utils/frame/benchmarking-cli/src/pallet/mod.rs @@ -20,8 +20,8 @@ mod writer; use crate::shared::HostInfoParams; use sc_cli::{ - ExecutionStrategy, WasmExecutionMethod, WasmtimeInstantiationStrategy, - DEFAULT_WASMTIME_INSTANTIATION_STRATEGY, DEFAULT_WASM_EXECUTION_METHOD, + WasmExecutionMethod, WasmtimeInstantiationStrategy, DEFAULT_WASMTIME_INSTANTIATION_STRATEGY, + DEFAULT_WASM_EXECUTION_METHOD, }; use std::{fmt::Debug, path::PathBuf}; diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 6fb4b44392546..7a41b3ed372df 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -879,7 +879,7 @@ pub(crate) fn state_machine_call( sp_core::testing::TaskExecutor::new(), CallContext::Offchain, ) - .execute(sp_state_machine::ExecutionStrategy::AlwaysWasm) + .execute() .map_err(|e| format!("failed to execute '{}': {}", method, e)) .map_err::(Into::into)?; @@ -919,7 +919,7 @@ pub(crate) fn state_machine_call_with_proof(Into::into)?; From 7dbaa81ed59ecf7b38041bedf5a3463891164913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 20 Mar 2023 17:31:36 +0100 Subject: [PATCH 08/55] More fixes to make it compile --- bin/node-template/node/src/service.rs | 19 ++++++++++++-- bin/node/bench/src/construct.rs | 8 +++--- bin/node/bench/src/import.rs | 9 +++---- bin/node/bench/src/main.rs | 3 +-- bin/node/cli/src/service.rs | 21 ++++++++++----- bin/node/testing/src/bench.rs | 6 +---- client/api/src/execution_extensions.rs | 5 ++++ client/offchain/src/lib.rs | 14 ++++++++-- client/service/src/client/call_executor.rs | 6 +---- client/service/test/src/client/mod.rs | 16 +++++------- primitives/api/test/benches/bench.rs | 11 +++----- primitives/runtime/src/runtime_logger.rs | 2 +- test-utils/runtime/src/lib.rs | 26 +++++++------------ .../benchmarking-cli/src/pallet/command.rs | 4 +-- 14 files changed, 81 insertions(+), 69 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 9ddb19d523fd6..7061ea7a65b32 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -1,7 +1,8 @@ //! Service and ServiceFactory implementation. Specialized wrapper over substrate service. +use futures::FutureExt; use node_template_runtime::{self, opaque::Block, RuntimeApi}; -use sc_client_api::BlockBackend; +use sc_client_api::{Backend, BlockBackend}; use sc_consensus_aura::{ImportQueueParams, SlotProportion, StartAuraParams}; use sc_consensus_grandpa::SharedVoterState; pub use sc_executor::NativeElseWasmExecutor; @@ -204,7 +205,21 @@ pub fn new_full(mut config: Configuration) -> Result })?; if config.offchain_worker.enabled { - sc_offchain::OffchainWorkers::new() + task_manager.spawn_handle().spawn( + "offchain-workers-runner", + "offchain-worker", + sc_offchain::OffchainWorkers::new(sc_offchain::OffchainWorkerOptions { + runtime_api_provider: client.clone(), + is_validator: config.role.is_authority(), + keystore: Some(keystore_container.sync_keystore()), + offchain_db: backend.offchain_storage(), + transaction_pool: Some(transaction_pool.clone()), + network_provider: network.clone(), + enable_http_requests: true, + }) + .run(client.clone(), task_manager.spawn_handle()) + .boxed(), + ); } let role = config.role.clone(); diff --git a/bin/node/bench/src/construct.rs b/bin/node/bench/src/construct.rs index 0aaf5837cba8d..67c37bc323a77 100644 --- a/bin/node/bench/src/construct.rs +++ b/bin/node/bench/src/construct.rs @@ -28,7 +28,7 @@ use futures::Future; use std::{borrow::Cow, collections::HashMap, pin::Pin, sync::Arc}; use node_primitives::Block; -use node_testing::bench::{BenchDb, BlockType, DatabaseType, KeyTypes, Profile}; +use node_testing::bench::{BenchDb, BlockType, DatabaseType, KeyTypes}; use sc_transaction_pool_api::{ ImportNotificationStream, PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, TransactionSource, TransactionStatusStreamFor, TxHash, @@ -99,8 +99,10 @@ impl core::BenchmarkDescription for ConstructionBenchmarkDescription { fn name(&self) -> Cow<'static, str> { format!( - "Block construction ({:?}/{}, {:?}, {:?} backend)", - self.block_type, self.size, self.profile, self.database_type, + "Block construction ({:?}/{}, {:?} backend)", + self.block_type, + self.size, + self.database_type, ) .into() } diff --git a/bin/node/bench/src/import.rs b/bin/node/bench/src/import.rs index dc3210e447ff6..e1882f1ed6eb5 100644 --- a/bin/node/bench/src/import.rs +++ b/bin/node/bench/src/import.rs @@ -33,7 +33,7 @@ use std::borrow::Cow; use node_primitives::Block; -use node_testing::bench::{BenchDb, BlockType, DatabaseType, KeyTypes, Profile}; +use node_testing::bench::{BenchDb, BlockType, DatabaseType, KeyTypes}; use sc_client_api::backend::Backend; use sp_state_machine::InspectState; @@ -43,7 +43,6 @@ use crate::{ }; pub struct ImportBenchmarkDescription { - pub profile: Profile, pub key_types: KeyTypes, pub block_type: BlockType, pub size: SizeType, @@ -82,21 +81,19 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription { } fn setup(self: Box) -> Box { - let profile = self.profile; let mut bench_db = BenchDb::with_key_types(self.database_type, 50_000, self.key_types); let block = bench_db.generate_block(self.block_type.to_content(self.size.transactions())); Box::new(ImportBenchmark { database: bench_db, block_type: self.block_type, block, - profile, }) } fn name(&self) -> Cow<'static, str> { format!( - "Block import ({:?}/{}, {:?}, {:?} backend)", - self.block_type, self.size, self.profile, self.database_type, + "Block import ({:?}/{}, {:?} backend)", + self.block_type, self.size, self.database_type, ) .into() } diff --git a/bin/node/bench/src/main.rs b/bin/node/bench/src/main.rs index 39db4e4d942b5..1f69c97695801 100644 --- a/bin/node/bench/src/main.rs +++ b/bin/node/bench/src/main.rs @@ -105,9 +105,8 @@ fn main() { } let benchmarks = matrix!( - (profile, size, block_type, database_type) in import_benchmarks.into_iter() => + (size, block_type, database_type) in import_benchmarks.into_iter() => ImportBenchmarkDescription { - profile, key_types: KeyTypes::Sr25519, size, block_type, diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index e9a34b2a5c728..f3dd75a6a65ad 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -28,7 +28,7 @@ use futures::prelude::*; use kitchensink_runtime::RuntimeApi; use node_executor::ExecutorDispatch; use node_primitives::Block; -use sc_client_api::BlockBackend; +use sc_client_api::{Backend, BlockBackend}; use sc_consensus_babe::{self, SlotProportion}; use sc_executor::NativeElseWasmExecutor; use sc_network::NetworkService; @@ -368,11 +368,20 @@ pub fn new_full_base( })?; if config.offchain_worker.enabled { - sc_service::build_offchain_workers( - &config, - task_manager.spawn_handle(), - client.clone(), - network.clone(), + task_manager.spawn_handle().spawn( + "offchain-workers-runner", + "offchain-work", + sc_offchain::OffchainWorkers::new(sc_offchain::OffchainWorkerOptions { + runtime_api_provider: client.clone(), + keystore: Some(keystore_container.sync_keystore()), + offchain_db: backend.offchain_storage(), + transaction_pool: Some(transaction_pool.clone()), + network_provider: network.clone(), + is_validator: config.role.is_authority(), + enable_http_requests: true, + }) + .run(client.clone(), task_manager.spawn_handle()) + .boxed(), ); } diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index aae2eba27c409..8f695351b7b49 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -40,17 +40,13 @@ use kitchensink_runtime::{ }; use node_primitives::Block; use sc_block_builder::BlockBuilderProvider; -use sc_client_api::{ - execution_extensions::{ExecutionExtensions, ExecutionStrategies}, - ExecutionStrategy, -}; use sc_client_db::PruningMode; use sc_consensus::{BlockImport, BlockImportParams, ForkChoiceStrategy, ImportResult, ImportedAux}; use sc_executor::{NativeElseWasmExecutor, WasmExecutionMethod, WasmtimeInstantiationStrategy}; use sp_api::ProvideRuntimeApi; use sp_block_builder::BlockBuilder; use sp_consensus::BlockOrigin; -use sp_core::{blake2_256, ed25519, sr25519, traits::SpawnNamed, ExecutionContext, Pair, Public}; +use sp_core::{blake2_256, ed25519, sr25519, traits::SpawnNamed, Pair, Public}; use sp_inherents::InherentData; use sp_runtime::{ traits::{Block as BlockT, IdentifyAccount, Verify}, diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 8e24c9ffc51fa..c4c3732c96a16 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -108,6 +108,11 @@ impl ExecutionExtensions { Self { extensions_factory: RwLock::new(Box::new(extensions_factory)) } } + /// Set the new extensions_factory + pub fn set_extensions_factory(&self, maker: impl ExtensionsFactory + 'static) { + *self.extensions_factory.write() = Box::new(maker); + } + /// Based on the execution context and capabilities it produces /// the extensions object to support desired set of APIs. pub fn extensions( diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 7657818e66857..e0bd02854174c 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -312,6 +312,7 @@ mod tests { use sc_peerset::ReputationChange; use sc_transaction_pool::{BasicPool, FullChainApi}; use sc_transaction_pool_api::{InPoolTransaction, TransactionPool}; + use sp_api::offchain::testing::TestPersistentOffchainDB; use sp_consensus::BlockOrigin; use sp_runtime::generic::BlockId; use std::{collections::HashSet, sync::Arc}; @@ -410,6 +411,7 @@ mod tests { } } + #[derive(Clone)] struct TestPool(Arc, Block>>); impl sc_transaction_pool_api::OffchainSubmitTransaction for TestPool { @@ -442,8 +444,16 @@ mod tests { let header = client.header(client.chain_info().genesis_hash).unwrap().unwrap(); // when - let offchain = OffchainWorkers::new(client); - futures::executor::block_on(offchain.on_block_imported(&header, network, false)); + let offchain = OffchainWorkers::new(OffchainWorkerOptions { + runtime_api_provider: client, + keystore: None, + offchain_db: None::, + transaction_pool: Some(Arc::new(pool.clone())), + network_provider: network, + is_validator: false, + enable_http_requests: false, + }); + futures::executor::block_on(offchain.on_block_imported(&header)); // then assert_eq!(pool.0.status().ready, 1); diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 88a59a1a83dd2..53fd7913d7bfd 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -410,11 +410,7 @@ mod tests { backend.clone(), ) .unwrap(), - execution_extensions: Arc::new(ExecutionExtensions::new( - Default::default(), - None, - None, - )), + execution_extensions: Arc::new(ExecutionExtensions::default()), }; let check = call_executor diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index d74e7e1f2ff9b..313989fcdef9b 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -37,9 +37,7 @@ use sp_runtime::{ traits::{BlakeTwo256, Block as BlockT, Header as HeaderT}, ConsensusEngineId, Justifications, StateVersion, }; -use sp_state_machine::{ - backend::Backend as _, ExecutionStrategy, InMemoryBackend, OverlayedChanges, StateMachine, -}; +use sp_state_machine::{backend::Backend as _, InMemoryBackend, OverlayedChanges, StateMachine}; use sp_storage::{ChildInfo, StorageKey}; use sp_trie::{LayoutV0, TrieConfiguration}; use std::{collections::HashSet, sync::Arc}; @@ -117,7 +115,7 @@ fn construct_block( task_executor.clone() as Box<_>, CallContext::Onchain, ) - .execute(ExecutionStrategy::NativeElseWasm) + .execute() .unwrap(); for tx in transactions.iter() { @@ -132,7 +130,7 @@ fn construct_block( task_executor.clone() as Box<_>, CallContext::Onchain, ) - .execute(ExecutionStrategy::NativeElseWasm) + .execute() .unwrap(); } @@ -147,7 +145,7 @@ fn construct_block( task_executor.clone() as Box<_>, CallContext::Onchain, ) - .execute(ExecutionStrategy::NativeElseWasm) + .execute() .unwrap(); header = Header::decode(&mut &ret_data[..]).unwrap(); @@ -220,7 +218,7 @@ fn construct_genesis_should_work_with_native() { TaskExecutor::new(), CallContext::Onchain, ) - .execute(ExecutionStrategy::NativeElseWasm) + .execute() .unwrap(); } @@ -254,7 +252,7 @@ fn construct_genesis_should_work_with_wasm() { TaskExecutor::new(), CallContext::Onchain, ) - .execute(ExecutionStrategy::AlwaysWasm) + .execute() .unwrap(); } @@ -288,7 +286,7 @@ fn construct_genesis_with_bad_transaction_should_panic() { TaskExecutor::new(), CallContext::Onchain, ) - .execute(ExecutionStrategy::NativeElseWasm); + .execute(); assert!(r.is_err()); } diff --git a/primitives/api/test/benches/bench.rs b/primitives/api/test/benches/bench.rs index 88ebdbc6134aa..45bea08af6ded 100644 --- a/primitives/api/test/benches/bench.rs +++ b/primitives/api/test/benches/bench.rs @@ -17,7 +17,6 @@ use criterion::{criterion_group, criterion_main, Criterion}; use sp_api::ProvideRuntimeApi; -use sp_state_machine::ExecutionStrategy; use substrate_test_runtime_client::{ runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, }; @@ -56,17 +55,13 @@ fn sp_api_benchmark(c: &mut Criterion) { }); c.bench_function("calling function by function pointer in wasm", |b| { - let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::AlwaysWasm) - .build(); + let client = TestClientBuilder::new().build(); let best_hash = client.chain_info().best_hash; b.iter(|| client.runtime_api().benchmark_indirect_call(best_hash).unwrap()) }); - c.bench_function("calling function in wasm", |b| { - let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::AlwaysWasm) - .build(); + c.bench_function("calling function", |b| { + let client = TestClientBuilder::new().build(); let best_hash = client.chain_info().best_hash; b.iter(|| client.runtime_api().benchmark_direct_call(best_hash).unwrap()) }); diff --git a/primitives/runtime/src/runtime_logger.rs b/primitives/runtime/src/runtime_logger.rs index eb3769b7344e2..851e5e3c37d53 100644 --- a/primitives/runtime/src/runtime_logger.rs +++ b/primitives/runtime/src/runtime_logger.rs @@ -68,7 +68,7 @@ mod tests { use sp_api::ProvideRuntimeApi; use std::{env, str::FromStr}; use substrate_test_runtime_client::{ - runtime::TestAPI, DefaultTestClientBuilderExt, ExecutionStrategy, TestClientBuilder, + runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, }; diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index b20847b8fb7c2..371d5e993e64f 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1320,10 +1320,9 @@ fn test_witness(proof: StorageProof, root: crate::Hash) { mod tests { use codec::Encode; use sc_block_builder::BlockBuilderProvider; - use sp_api::ProvideRuntimeApi; + use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_consensus::BlockOrigin; - use sp_core::{storage::well_known_keys::HEAP_PAGES, ExecutionContext}; - use sp_state_machine::ExecutionStrategy; + use sp_core::{storage::well_known_keys::HEAP_PAGES, traits::CallContext}; use substrate_test_runtime_client::{ prelude::*, runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, }; @@ -1333,20 +1332,15 @@ mod tests { // This tests that the on-chain HEAP_PAGES parameter is respected. // Create a client devoting only 8 pages of wasm memory. This gives us ~512k of heap memory. - let mut client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::AlwaysWasm) - .set_heap_pages(8) - .build(); + let mut client = TestClientBuilder::new().set_heap_pages(8).build(); let best_hash = client.chain_info().best_hash; // Try to allocate 1024k of memory on heap. This is going to fail since it is twice larger // than the heap. - let ret = client.runtime_api().vec_with_capacity_with_context( - best_hash, - // Use `BlockImport` to ensure we use the on chain heap pages as configured above. - ExecutionContext::Importing, - 1048576, - ); + let mut runtime_api = client.runtime_api(); + // This is currently required to allocate the 1024k of memory as configured above. + runtime_api.set_call_context(CallContext::Onchain); + let ret = runtime_api.vec_with_capacity(best_hash, 1048576); assert!(ret.is_err()); // Create a block that sets the `:heap_pages` to 32 pages of memory which corresponds to @@ -1368,8 +1362,7 @@ mod tests { #[test] fn test_storage() { - let client = - TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); + let client = TestClientBuilder::new().build(); let runtime_api = client.runtime_api(); let best_hash = client.chain_info().best_hash; @@ -1395,8 +1388,7 @@ mod tests { let backend = sp_state_machine::TrieBackendBuilder::<_, crate::Hashing>::new(db, root).build(); let proof = sp_state_machine::prove_read(backend, vec![b"value3"]).unwrap(); - let client = - TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); + let client = TestClientBuilder::new().build(); let runtime_api = client.runtime_api(); let best_hash = client.chain_info().best_hash; diff --git a/utils/frame/benchmarking-cli/src/pallet/command.rs b/utils/frame/benchmarking-cli/src/pallet/command.rs index dd815c3c6a839..6fb57ac77e529 100644 --- a/utils/frame/benchmarking-cli/src/pallet/command.rs +++ b/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -23,9 +23,7 @@ use frame_benchmarking::{ }; use frame_support::traits::StorageInfo; use linked_hash_map::LinkedHashMap; -use sc_cli::{ - execution_method_from_cli, CliConfiguration, ExecutionStrategy, Result, SharedParams, -}; +use sc_cli::{execution_method_from_cli, CliConfiguration, Result, SharedParams}; use sc_client_db::BenchmarkingState; use sc_executor::NativeElseWasmExecutor; use sc_service::{Configuration, NativeExecutionDispatch}; From 5b2fc2320b5813d4c3e5720203faabde21c9e98f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 21 Mar 2023 13:29:34 +0100 Subject: [PATCH 09/55] Adds `NoOffchainStorage` --- client/offchain/src/lib.rs | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index e0bd02854174c..a9457810d69e3 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -68,6 +68,34 @@ pub trait NetworkProvider: NetworkStateInfo + NetworkPeers {} impl NetworkProvider for T where T: NetworkStateInfo + NetworkPeers {} +/// Special type that implements [`OffchainStorage`]. +/// +/// This type can not be constructed and should only be used when passing `None` as `offchain_db` to +/// [`OffchainWorkerOptions`] to make the compiler happy. +#[derive(Clone)] +pub struct NoOffchainStorage { + // Ensure no one can construct this type + _priv: (), +} + +impl offchain::OffchainStorage for NoOffchainStorage { + fn set(&mut self, _: &[u8], _: &[u8], _: &[u8]) { + unimplemented!("`NoOffchainStorage` can not be constructed!") + } + + fn remove(&mut self, _: &[u8], _: &[u8]) { + unimplemented!("`NoOffchainStorage` can not be constructed!") + } + + fn get(&self, _: &[u8], _: &[u8]) -> Option> { + unimplemented!("`NoOffchainStorage` can not be constructed!") + } + + fn compare_and_set(&mut self, _: &[u8], _: &[u8], _: Option<&[u8]>, _: &[u8]) -> bool { + unimplemented!("`NoOffchainStorage` can not be constructed!") + } +} + /// Options for [`OffchainWorkers`] pub struct OffchainWorkerOptions { /// Provides access to the runtime api. @@ -75,6 +103,8 @@ pub struct OffchainWorkerOptions { /// Provides access to the keystore. pub keystore: Option, /// Provides access to the offchain database. + /// + /// Use [`NoOffchainStorage`] as type when passing `None` to have some type that works. pub offchain_db: Option, /// Provides access to the transaction pool. pub transaction_pool: Option>>, @@ -312,7 +342,6 @@ mod tests { use sc_peerset::ReputationChange; use sc_transaction_pool::{BasicPool, FullChainApi}; use sc_transaction_pool_api::{InPoolTransaction, TransactionPool}; - use sp_api::offchain::testing::TestPersistentOffchainDB; use sp_consensus::BlockOrigin; use sp_runtime::generic::BlockId; use std::{collections::HashSet, sync::Arc}; @@ -447,7 +476,7 @@ mod tests { let offchain = OffchainWorkers::new(OffchainWorkerOptions { runtime_api_provider: client, keystore: None, - offchain_db: None::, + offchain_db: None::, transaction_pool: Some(Arc::new(pool.clone())), network_provider: network, is_validator: false, From 9fed302219f566c9c82f61fb899bf63fd5168bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 21 Mar 2023 21:51:02 +0100 Subject: [PATCH 10/55] Pass the extensions --- primitives/api/proc-macro/src/impl_runtime_apis.rs | 1 + primitives/api/src/lib.rs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index b62fd601068ad..6cbb4ad443674 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -459,6 +459,7 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> { storage_transaction_cache: &self.storage_transaction_cache, call_context: self.call_context, recorder: &self.recorder, + extensions: &self.extensions, }; #crate_::CallApiAt::<__SR_API_BLOCK__>::call_api_at( diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index e6a3d967a8730..37a8117e1d6c6 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -613,6 +613,8 @@ pub struct CallApiAtParams<'a, Block: BlockT, Backend: StateBackend>, + /// The extensions that should be used for this call. + pub extensions: &'a RefCell, } /// Something that can call into the an api at a given block. From 673021236e27cc8d6c48252c0bbae63209c16239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 22 Mar 2023 21:18:43 +0100 Subject: [PATCH 11/55] Small basti making small progress --- client/api/src/call_executor.rs | 2 ++ client/service/src/client/call_executor.rs | 15 +++++++++------ client/service/src/client/client.rs | 1 + primitives/state-machine/src/lib.rs | 12 ++++++------ 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index 6f25b2b1ede72..8152f6e7b68fc 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -23,6 +23,7 @@ use sp_core::traits::CallContext; use sp_runtime::traits::Block as BlockT; use sp_state_machine::{OverlayedChanges, StorageProof}; use std::cell::RefCell; +use sp_externalities::Extensions; use crate::execution_extensions::ExecutionExtensions; use sp_api::{ProofRecorder, StorageTransactionCache}; @@ -79,6 +80,7 @@ pub trait CallExecutor: RuntimeVersionOf { >, proof_recorder: &Option>, call_context: CallContext, + extensions: &RefCell, ) -> sp_blockchain::Result>; /// Extract RuntimeVersion of given block diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 53fd7913d7bfd..771500730deaf 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -23,6 +23,7 @@ use sc_client_api::{ use sc_executor::{RuntimeVersion, RuntimeVersionOf}; use sp_api::{ProofRecorder, StorageTransactionCache}; use sp_core::traits::{CallContext, CodeExecutor, RuntimeCode, SpawnNamed}; +use sp_externalities::Extensions; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; use sp_state_machine::{backend::AsTrieBackend, Ext, OverlayedChanges, StateMachine, StorageProof}; use std::{cell::RefCell, sync::Arc}; @@ -176,7 +177,7 @@ where let runtime_code = self.check_override(runtime_code, at_hash)?.0; - let extensions = self.execution_extensions.extensions(at_hash, at_number); + let mut extensions = self.execution_extensions.extensions(at_hash, at_number); let mut sm = StateMachine::new( &state, @@ -184,7 +185,7 @@ where &self.executor, method, call_data, - extensions, + &mut extensions, &runtime_code, self.spawn_handle.clone(), context, @@ -203,6 +204,7 @@ where storage_transaction_cache: Option<&RefCell>>, recorder: &Option>, call_context: CallContext, + extensions: &RefCell, ) -> Result, sp_blockchain::Error> { let mut storage_transaction_cache = storage_transaction_cache.map(|c| c.borrow_mut()); @@ -210,7 +212,7 @@ where self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(at_hash))?; let state = self.backend.state_at(at_hash)?; - let extensions = self.execution_extensions.extensions(at_hash, at_number); + // let extensions = self.execution_extensions.extensions(at_hash, at_number); let changes = &mut *changes.borrow_mut(); @@ -222,6 +224,7 @@ where let runtime_code = state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; let runtime_code = self.check_override(runtime_code, at_hash)?.0; + let mut extensions = extensions.borrow_mut(); match recorder { Some(recorder) => { @@ -237,7 +240,7 @@ where &self.executor, method, call_data, - extensions, + &mut *extensions, &runtime_code, self.spawn_handle.clone(), call_context, @@ -253,7 +256,7 @@ where &self.executor, method, call_data, - extensions, + &mut *extensions, &runtime_code, self.spawn_handle.clone(), call_context, @@ -300,7 +303,7 @@ where method, call_data, &runtime_code, - self.execution_extensions.extensions(at_hash, at_number), + &mut self.execution_extensions.extensions(at_hash, at_number), ) .map_err(Into::into) } diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 60156b4932dbd..9cb450bcd5cad 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1714,6 +1714,7 @@ where Some(params.storage_transaction_cache), params.recorder, params.call_context, + params.extensions, ) .map_err(Into::into) } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 35d4053a226db..40a22e5c6ee29 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -198,7 +198,7 @@ mod execution { method: &'a str, call_data: &'a [u8], overlay: &'a mut OverlayedChanges, - extensions: Extensions, + extensions: &'a mut Extensions, storage_transaction_cache: Option<&'a mut StorageTransactionCache>, runtime_code: &'a RuntimeCode<'a>, stats: StateMachineStats, @@ -233,7 +233,7 @@ mod execution { exec: &'a Exec, method: &'a str, call_data: &'a [u8], - mut extensions: Extensions, + extensions: &'a mut Extensions, runtime_code: &'a RuntimeCode, spawn_handle: impl SpawnNamed + Send + 'static, context: CallContext, @@ -297,7 +297,7 @@ mod execution { .enter_runtime() .expect("StateMachine is never called from the runtime; qed"); - let mut ext = Ext::new(self.overlay, cache, self.backend, Some(&mut self.extensions)); + let mut ext = Ext::new(self.overlay, cache, self.backend, Some(self.extensions)); let ext_id = ext.id; @@ -356,7 +356,7 @@ mod execution { method, call_data, runtime_code, - Default::default(), + &mut Default::default(), ) } @@ -377,7 +377,7 @@ mod execution { method: &str, call_data: &[u8], runtime_code: &RuntimeCode, - extensions: Extensions, + mut extensions: &mut Extensions, ) -> Result<(Vec, StorageProof), Box> where S: trie_backend_essence::TrieBackendStorage, @@ -460,7 +460,7 @@ mod execution { exec, method, call_data, - Extensions::default(), + &mut Extensions::default(), runtime_code, spawn_handle, CallContext::Offchain, From a3ff3e34248df1086ede855dc35f8eb00e85571d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 4 Apr 2023 22:37:14 +0200 Subject: [PATCH 12/55] Fix merge errors and remove `ExecutionContext` --- client/offchain/src/lib.rs | 6 +-- client/service/src/client/call_executor.rs | 2 +- .../api/proc-macro/src/impl_runtime_apis.rs | 2 +- primitives/consensus/common/src/lib.rs | 10 ----- primitives/core/src/lib.rs | 38 ------------------- 5 files changed, 5 insertions(+), 53 deletions(-) diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index a200d0e1e92be..0ad85a5b53367 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -48,7 +48,7 @@ use sc_network::{NetworkPeers, NetworkStateInfo}; use sc_transaction_pool_api::OffchainSubmitTransaction; use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_core::{offchain, traits::SpawnNamed}; -use sp_keystore::{KeystoreExt, SyncCryptoStorePtr}; +use sp_keystore::{KeystoreExt, KeystorePtr}; use sp_runtime::{ generic::BlockId, traits::{self, Header}, @@ -101,7 +101,7 @@ pub struct OffchainWorkerOptions { /// Provides access to the runtime api. pub runtime_api_provider: Arc, /// Provides access to the keystore. - pub keystore: Option, + pub keystore: Option, /// Provides access to the offchain database. /// /// Use [`NoOffchainStorage`] as type when passing `None` to have some type that works. @@ -124,7 +124,7 @@ pub struct OffchainWorkers { thread_pool: Mutex, shared_http_client: api::SharedClient, enable_http_requests: bool, - keystore: Option, + keystore: Option, offchain_db: Option>, transaction_pool: Option>>, network_provider: Arc, diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 00187737411b1..fc819520efd81 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -22,7 +22,7 @@ use sc_client_api::{ }; use sc_executor::{RuntimeVersion, RuntimeVersionOf}; use sp_api::{ProofRecorder, StorageTransactionCache}; -use sp_core::traits::{CallContext, CodeExecutor, RuntimeCode, SpawnNamed}; +use sp_core::traits::{CallContext, CodeExecutor, RuntimeCode}; use sp_externalities::Extensions; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; use sp_state_machine::{backend::AsTrieBackend, Ext, OverlayedChanges, StateMachine, StorageProof}; diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 2acf00ed29e67..615fd8378fd79 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -457,7 +457,7 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> { input.items.push(parse_quote! { fn __runtime_api_internal_call_api_at( &self, - at: <__SR_API_BLOCK__ as #crate_::BlockT>::Hash, + at: <__SrApiBlock__ as #crate_::BlockT>::Hash, params: std::vec::Vec, fn_name: &dyn Fn(#crate_::RuntimeVersion) -> &'static str, ) -> std::result::Result, #crate_::ApiError> { diff --git a/primitives/consensus/common/src/lib.rs b/primitives/consensus/common/src/lib.rs index 215b4448b4a8e..d47f3bafc4220 100644 --- a/primitives/consensus/common/src/lib.rs +++ b/primitives/consensus/common/src/lib.rs @@ -71,16 +71,6 @@ pub enum BlockOrigin { File, } -impl From for sp_core::ExecutionContext { - fn from(origin: BlockOrigin) -> Self { - if origin == BlockOrigin::NetworkInitialSync { - sp_core::ExecutionContext::Syncing - } else { - sp_core::ExecutionContext::Importing - } - } -} - /// Environment for a Consensus instance. /// /// Creates proposer instance. diff --git a/primitives/core/src/lib.rs b/primitives/core/src/lib.rs index efccd0378e95a..a44e11c8ada88 100644 --- a/primitives/core/src/lib.rs +++ b/primitives/core/src/lib.rs @@ -92,44 +92,6 @@ pub use sp_storage as storage; #[doc(hidden)] pub use sp_std; -/// Context for executing a call into the runtime. -pub enum ExecutionContext { - /// Context used for general block import (including locally authored blocks). - Importing, - /// Context used for importing blocks as part of an initial sync of the blockchain. - /// - /// We distinguish between major sync and import so that validators who are running - /// their initial sync (or catching up after some time offline) can use the faster - /// native runtime (since we can reasonably assume the network as a whole has already - /// come to a broad consensus on the block and it probably hasn't been crafted - /// specifically to attack this node), but when importing blocks at the head of the - /// chain in normal operation they can use the safer Wasm version. - Syncing, - /// Context used for block construction. - BlockConstruction, - /// Context used for offchain calls. - /// - /// This allows passing offchain extension and customizing available capabilities. - OffchainCall(Option<(Box, offchain::Capabilities)>), -} - -impl ExecutionContext { - /// Returns the capabilities of particular context. - pub fn capabilities(&self) -> offchain::Capabilities { - use ExecutionContext::*; - - match self { - Importing | Syncing | BlockConstruction => offchain::Capabilities::empty(), - // Enable keystore, transaction pool and Offchain DB reads by default for offchain - // calls. - OffchainCall(None) => - offchain::Capabilities::KEYSTORE | - offchain::Capabilities::OFFCHAIN_DB_READ | - offchain::Capabilities::TRANSACTION_POOL, - OffchainCall(Some((_, capabilities))) => *capabilities, - } - } -} /// Hex-serialized shim for `Vec`. #[derive(PartialEq, Eq, Clone, RuntimeDebug)] From afbc691a9049adaa083754f81e4ecdb88cbe2d0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 4 Apr 2023 23:29:45 +0200 Subject: [PATCH 13/55] Move registration of `ReadRuntimeVersionExt` to `ExecutionExtension` Instead of registering `ReadRuntimeVersionExt` in `sp-state-machine` it is moved to `ExecutionExtension` which provides the default extensions. --- bin/node/testing/src/bench.rs | 9 +++++++-- client/api/src/execution_extensions.rs | 18 ++++++------------ client/service/src/builder.rs | 1 + client/service/src/client/call_executor.rs | 1 + client/service/src/client/client.rs | 1 + primitives/core/src/traits.rs | 10 ++++++++++ primitives/state-machine/src/lib.rs | 6 ++---- test-utils/client/src/lib.rs | 3 ++- .../benchmarking-cli/src/pallet/command.rs | 3 ++- .../cli/src/commands/execute_block.rs | 2 +- .../cli/src/commands/fast_forward.rs | 4 ++-- .../cli/src/commands/follow_chain.rs | 2 +- .../cli/src/commands/offchain_worker.rs | 2 +- utils/frame/try-runtime/cli/src/lib.rs | 5 +++-- 14 files changed, 40 insertions(+), 27 deletions(-) diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index d6bcf6e252fe0..392b78241d288 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -411,11 +411,16 @@ impl BenchDb { let client = sc_service::new_client( backend.clone(), - executor, + executor.clone(), genesis_block_builder, None, None, - ExecutionExtensions::new(profile.into_execution_strategies(), None, None), + ExecutionExtensions::new( + profile.into_execution_strategies(), + None, + None, + Arc::new(executor), + ), Box::new(task_executor.clone()), None, None, diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index ffa670f7bc628..9344afbd3e6dd 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -27,6 +27,7 @@ use parking_lot::RwLock; use sc_transaction_pool_api::OffchainSubmitTransaction; use sp_core::{ offchain::{self, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt}, + traits::{ReadRuntimeVersion, ReadRuntimeVersionExt}, ExecutionContext, }; use sp_externalities::{Extension, Extensions}; @@ -173,18 +174,7 @@ pub struct ExecutionExtensions { // during initialization. transaction_pool: RwLock>>>, extensions_factory: RwLock>>, -} - -impl Default for ExecutionExtensions { - fn default() -> Self { - Self { - strategies: Default::default(), - keystore: None, - offchain_db: None, - transaction_pool: RwLock::new(None), - extensions_factory: RwLock::new(Box::new(())), - } - } + read_runtime_version: Arc, } impl ExecutionExtensions { @@ -193,6 +183,7 @@ impl ExecutionExtensions { strategies: ExecutionStrategies, keystore: Option, offchain_db: Option>, + read_runtime_version: Arc, ) -> Self { let transaction_pool = RwLock::new(None); let extensions_factory = Box::new(()); @@ -202,6 +193,7 @@ impl ExecutionExtensions { offchain_db, extensions_factory: RwLock::new(extensions_factory), transaction_pool, + read_runtime_version, } } @@ -271,6 +263,8 @@ impl ExecutionExtensions { ))); } + extensions.register(ReadRuntimeVersionExt::new(self.read_runtime_version.clone())); + extensions } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 5d639431f427b..d399b315414f8 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -181,6 +181,7 @@ where config.execution_strategies.clone(), Some(keystore_container.keystore()), sc_offchain::OffchainDb::factory_from_backend(&*backend), + Arc::new(executor.clone()), ); let wasm_runtime_substitutes = config diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 4d019be908b86..ef36768febdbd 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -427,6 +427,7 @@ mod tests { Default::default(), None, None, + Arc::new(executor.clone()), )), }; diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 3adb6d8976969..eee7e6b82363c 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -243,6 +243,7 @@ where Default::default(), keystore, sc_offchain::OffchainDb::factory_from_backend(&*backend), + Arc::new(executor.clone()), ); let call_executor = diff --git a/primitives/core/src/traits.rs b/primitives/core/src/traits.rs index 51327868474a0..40137053ab752 100644 --- a/primitives/core/src/traits.rs +++ b/primitives/core/src/traits.rs @@ -157,6 +157,16 @@ pub trait ReadRuntimeVersion: Send + Sync { ) -> Result, String>; } +impl ReadRuntimeVersion for std::sync::Arc { + fn read_runtime_version( + &self, + wasm_code: &[u8], + ext: &mut dyn Externalities, + ) -> Result, String> { + (**self).read_runtime_version(wasm_code, ext) + } +} + sp_externalities::decl_extension! { /// An extension that provides functionality to read version information from a given wasm blob. pub struct ReadRuntimeVersionExt(Box); diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index c68cf4d004529..0001d0026c394 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -163,7 +163,7 @@ mod execution { use sp_core::{ hexdisplay::HexDisplay, storage::{ChildInfo, ChildType, PrefixedStorageKey}, - traits::{CallContext, CodeExecutor, ReadRuntimeVersionExt, RuntimeCode}, + traits::{CallContext, CodeExecutor, RuntimeCode}, }; use sp_externalities::Extensions; use std::{ @@ -322,12 +322,10 @@ mod execution { exec: &'a Exec, method: &'a str, call_data: &'a [u8], - mut extensions: Extensions, + extensions: Extensions, runtime_code: &'a RuntimeCode, context: CallContext, ) -> Self { - extensions.register(ReadRuntimeVersionExt::new(exec.clone())); - Self { backend, exec, diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 5e3c9f703fea1..c4572061c48af 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -290,12 +290,13 @@ impl }); let executor = LocalCallExecutor::new( self.backend.clone(), - executor, + executor.clone(), Default::default(), ExecutionExtensions::new( self.execution_strategies.clone(), self.keystore.clone(), sc_offchain::OffchainDb::factory_from_backend(&*self.backend), + Arc::new(executor), ), ) .expect("Creates LocalCallExecutor"); diff --git a/utils/frame/benchmarking-cli/src/pallet/command.rs b/utils/frame/benchmarking-cli/src/pallet/command.rs index 5016d65b89beb..4d583950d7b3e 100644 --- a/utils/frame/benchmarking-cli/src/pallet/command.rs +++ b/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -35,7 +35,7 @@ use sp_core::{ testing::{TestOffchainExt, TestTransactionPoolExt}, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt, }, - traits::CallContext, + traits::{CallContext, ReadRuntimeVersionExt}, }; use sp_externalities::Extensions; use sp_keystore::{testing::MemoryKeystore, KeystoreExt}; @@ -225,6 +225,7 @@ impl PalletCmd { extensions.register(OffchainWorkerExt::new(offchain.clone())); extensions.register(OffchainDbExt::new(offchain)); extensions.register(TransactionPoolExt::new(pool)); + extensions.register(ReadRuntimeVersionExt::new(executor.clone())); extensions }; diff --git a/utils/frame/try-runtime/cli/src/commands/execute_block.rs b/utils/frame/try-runtime/cli/src/commands/execute_block.rs index 561bc57c70c5a..48dab6b9bd1b3 100644 --- a/utils/frame/try-runtime/cli/src/commands/execute_block.rs +++ b/utils/frame/try-runtime/cli/src/commands/execute_block.rs @@ -133,7 +133,7 @@ where &executor, "TryRuntime_execute_block", &payload, - full_extensions(), + full_extensions(executor.clone()), shared.export_proof, )?; diff --git a/utils/frame/try-runtime/cli/src/commands/fast_forward.rs b/utils/frame/try-runtime/cli/src/commands/fast_forward.rs index 75c48c3c402f3..0c517c02fbe81 100644 --- a/utils/frame/try-runtime/cli/src/commands/fast_forward.rs +++ b/utils/frame/try-runtime/cli/src/commands/fast_forward.rs @@ -103,7 +103,7 @@ async fn dry_run( executor, method, data, - full_extensions(), + full_extensions(executor.clone()), )?; Ok(::decode(&mut &*result)?) @@ -121,7 +121,7 @@ async fn run( executor, method, data, - full_extensions(), + full_extensions(executor.clone()), )?; let storage_changes = changes.drain_storage_changes( diff --git a/utils/frame/try-runtime/cli/src/commands/follow_chain.rs b/utils/frame/try-runtime/cli/src/commands/follow_chain.rs index 2a67d269c87f1..413c68550b76d 100644 --- a/utils/frame/try-runtime/cli/src/commands/follow_chain.rs +++ b/utils/frame/try-runtime/cli/src/commands/follow_chain.rs @@ -149,7 +149,7 @@ where &executor, "TryRuntime_execute_block", (block, command.state_root_check, command.try_state.clone()).encode().as_ref(), - full_extensions(), + full_extensions(executor.clone()), shared .export_proof .as_ref() diff --git a/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs b/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs index 352d552a3f358..4da6f07836cd1 100644 --- a/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs +++ b/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs @@ -97,7 +97,7 @@ where &executor, "OffchainWorkerApi_offchain_worker", &payload, - full_extensions(), + full_extensions(executor.clone()), )?; Ok(()) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 733eab7f5a262..db690e5086a6c 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -377,7 +377,7 @@ use sp_core::{ OffchainDbExt, OffchainWorkerExt, TransactionPoolExt, }, storage::well_known_keys, - traits::{CallContext, ReadRuntimeVersion}, + traits::{CallContext, ReadRuntimeVersion, ReadRuntimeVersionExt}, twox_128, H256, }; use sp_externalities::Extensions; @@ -810,7 +810,7 @@ where } /// Build all extensions that we typically use. -pub(crate) fn full_extensions() -> Extensions { +pub(crate) fn full_extensions(wasm_executor: WasmExecutor) -> Extensions { let mut extensions = Extensions::default(); let (offchain, _offchain_state) = TestOffchainExt::new(); let (pool, _pool_state) = TestTransactionPoolExt::new(); @@ -819,6 +819,7 @@ pub(crate) fn full_extensions() -> Extensions { extensions.register(OffchainWorkerExt::new(offchain)); extensions.register(KeystoreExt::new(keystore)); extensions.register(TransactionPoolExt::new(pool)); + extensions.register(ReadRuntimeVersionExt::new(wasm_executor)); extensions } From 906fb5a4044787bdf44d0705a1da37e8bcaa8318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 5 Apr 2023 23:27:52 +0200 Subject: [PATCH 14/55] Fix compilation --- client/api/src/execution_extensions.rs | 14 +++++++------- primitives/state-machine/src/lib.rs | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index d3de81393f3c4..260d4516ab0bb 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -23,12 +23,10 @@ //! extensions to support APIs for particular execution context & capabilities. use parking_lot::RwLock; -use sp_core::{ - traits::{ReadRuntimeVersion, ReadRuntimeVersionExt}, -}; +use sp_core::traits::{ReadRuntimeVersion, ReadRuntimeVersionExt}; use sp_externalities::{Extension, Extensions}; use sp_runtime::traits::{Block as BlockT, NumberFor}; -use std::marker::PhantomData; +use std::{marker::PhantomData, sync::Arc}; /// Generate the starting set of [`Extensions`]. /// @@ -102,11 +100,12 @@ pub struct ExecutionExtensions { impl ExecutionExtensions { /// Create new `ExecutionExtensions` given a `keystore` and `ExecutionStrategies`. - pub fn new(extensions_factory: Option + 'static>, + pub fn new( + extensions_factory: Option>>, read_runtime_version: Arc, ) -> Self { Self { - extensions_factory: RwLock::new(extensions_factory.map(Box::new).unwrap_or_else(|| Box::new(()))), + extensions_factory: RwLock::new(extensions_factory.unwrap_or_else(|| Box::new(()))), read_runtime_version, } } @@ -123,7 +122,8 @@ impl ExecutionExtensions { block_hash: Block::Hash, block_number: NumberFor, ) -> Extensions { - let extensions = self.extensions_factory.read().extensions_for(block_hash, block_number); + let mut extensions = + self.extensions_factory.read().extensions_for(block_hash, block_number); extensions.register(ReadRuntimeVersionExt::new(self.read_runtime_version.clone())); diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index f5cd1da8dfa93..f6b422d138e66 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -233,7 +233,7 @@ mod execution { exec: &'a Exec, method: &'a str, call_data: &'a [u8], - extensions: Extensions, + extensions: &'a mut Extensions, runtime_code: &'a RuntimeCode, context: CallContext, ) -> Self { @@ -369,7 +369,7 @@ mod execution { method: &str, call_data: &[u8], runtime_code: &RuntimeCode, - mut extensions: &mut Extensions, + extensions: &mut Extensions, ) -> Result<(Vec, StorageProof), Box> where S: trie_backend_essence::TrieBackendStorage, From 20f3c0293ad6c9f252c5e8721af7b55074c19e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 11 Apr 2023 01:35:10 +0200 Subject: [PATCH 15/55] Register the global extensions inside runtime api instance --- .../rpc-spec-v2/src/chain_head/test_utils.rs | 9 ++++++++ client/service/src/client/call_executor.rs | 4 ---- client/service/src/client/client.rs | 17 +++++++++++---- .../api/proc-macro/src/impl_runtime_apis.rs | 21 +++++++++++++++++++ primitives/api/src/lib.rs | 9 ++++++++ primitives/externalities/src/extensions.rs | 8 +++++++ 6 files changed, 60 insertions(+), 8 deletions(-) diff --git a/client/rpc-spec-v2/src/chain_head/test_utils.rs b/client/rpc-spec-v2/src/chain_head/test_utils.rs index ee563debb4502..536ba7882d356 100644 --- a/client/rpc-spec-v2/src/chain_head/test_utils.rs +++ b/client/rpc-spec-v2/src/chain_head/test_utils.rs @@ -217,8 +217,17 @@ impl> CallApiAt for ChainHeadMock fn state_at(&self, at: Block::Hash) -> Result { self.client.state_at(at) } + + fn initialize_extensions( + &self, + at: ::Hash, + extensions: &mut sp_api::Extensions, + ) -> Result<(), sp_api::ApiError> { + self.client.initialize_extensions(at, extensions) + } } + impl> BlockBackend for ChainHeadMockClient { diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 32872cc5aa40d..3ed2211c1af9e 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -203,12 +203,8 @@ where ) -> Result, sp_blockchain::Error> { let mut storage_transaction_cache = storage_transaction_cache.map(|c| c.borrow_mut()); - let at_number = - self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(at_hash))?; let state = self.backend.state_at(at_hash)?; - // let extensions = self.execution_extensions.extensions(at_hash, at_number); - let changes = &mut *changes.borrow_mut(); // It is important to extract the runtime code here before we create the proof diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 403cd5eb8e3b3..99bfb419668a9 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -234,10 +234,7 @@ where Block: BlockT, B: backend::LocalBackend + 'static, { - let extensions = ExecutionExtensions::new( - None, - Arc::new(executor.clone()), - ); + let extensions = ExecutionExtensions::new(None, Arc::new(executor.clone())); let call_executor = LocalCallExecutor::new(backend.clone(), executor, config.clone(), extensions)?; @@ -1724,6 +1721,18 @@ where fn state_at(&self, at: Block::Hash) -> Result { self.state_at(at).map_err(Into::into) } + + fn initialize_extensions( + &self, + at: Block::Hash, + extensions: &mut sp_externalities::Extensions, + ) -> Result<(), sp_api::ApiError> { + let block_number = self.expect_block_number_from_id(&BlockId::Hash(at))?; + + extensions.merge(self.executor.execution_extensions().extensions(at, block_number)); + + Ok(()) + } } /// NOTE: only use this implementation when you are sure there are NO consensus-level BlockImport diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 615fd8378fd79..23fc033d8e540 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -233,6 +233,7 @@ fn generate_runtime_api_base_structures() -> Result { recorder: std::option::Option<#crate_::ProofRecorder>, call_context: #crate_::CallContext, extensions: std::cell::RefCell<#crate_::Extensions>, + extensions_generated_for: std::cell::RefCell>, } #[cfg(any(feature = "std", test))] @@ -345,6 +346,7 @@ fn generate_runtime_api_base_structures() -> Result { storage_transaction_cache: std::default::Default::default(), call_context: #crate_::CallContext::Offchain, extensions: std::default::Default::default(), + extensions_generated_for: std::default::Default::default(), }.into() } } @@ -473,6 +475,25 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> { at, )?; + match &mut *std::cell::RefCell::borrow_mut(&self.extensions_generated_for) { + Some(generated_for) => { + if *generated_for != at { + return std::result::Result::Err( + #crate_::ApiError::UsingSameInstanceForDifferentBlocks + ) + } + }, + generated_for @ None => { + #crate_::CallApiAt::<__SrApiBlock__>::initialize_extensions( + self.call, + at, + &mut std::cell::RefCell::borrow_mut(&self.extensions), + )?; + + *generated_for = Some(at); + } + } + let params = #crate_::CallApiAtParams { at, function: (*fn_name)(version), diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index 32fed43a08b88..4ca380ab9efff 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -525,6 +525,8 @@ pub enum ApiError { Application(#[from] Box), #[error("Api called for an unknown Block: {0}")] UnknownBlock(String), + #[error("Using the same api instance to call into multiple independent blocks.")] + UsingSameInstanceForDifferentBlocks, } /// Extends the runtime api implementation with some common functionality. @@ -635,6 +637,13 @@ pub trait CallApiAt { /// Get the state `at` the given block. fn state_at(&self, at: Block::Hash) -> Result; + + /// Initialize the `extensions` for the given block `at` by using the global extensions factory. + fn initialize_extensions( + &self, + at: Block::Hash, + extensions: &mut Extensions, + ) -> Result<(), ApiError>; } /// Auxiliary wrapper that holds an api instance and binds it to the given lifetime. diff --git a/primitives/externalities/src/extensions.rs b/primitives/externalities/src/extensions.rs index 84155227a713e..6007cbde189b6 100644 --- a/primitives/externalities/src/extensions.rs +++ b/primitives/externalities/src/extensions.rs @@ -190,6 +190,14 @@ impl Extensions { pub fn iter_mut(&mut self) -> impl Iterator)> { self.extensions.iter_mut() } + + /// Merge `other` into `self`. + /// + /// If both contain the same extension, the extension instance of `other` will be present + /// afterwards in `self`. + pub fn merge(&mut self, other: Self) { + self.extensions.extend(other.extensions); + } } impl Extend for Extensions { From 033a32b3bb13d4e50ec850d64dca92ee2800c0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 27 Apr 2023 00:20:53 +0200 Subject: [PATCH 16/55] Fixes --- bin/node-template/node/src/service.rs | 2 +- bin/node/cli/src/service.rs | 2 +- bin/node/testing/src/bench.rs | 6 ++---- client/consensus/babe/src/lib.rs | 1 - client/service/src/lib.rs | 5 +++-- client/service/test/src/client/mod.rs | 12 ++++++------ primitives/state-machine/src/lib.rs | 6 ++++-- utils/frame/benchmarking-cli/src/pallet/command.rs | 8 ++++---- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 46e5d3ad51cef..8f509ab8d650a 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -184,7 +184,7 @@ pub fn new_full(mut config: Configuration) -> Result sc_offchain::OffchainWorkers::new(sc_offchain::OffchainWorkerOptions { runtime_api_provider: client.clone(), is_validator: config.role.is_authority(), - keystore: Some(keystore_container.sync_keystore()), + keystore: Some(keystore_container.keystore()), offchain_db: backend.offchain_storage(), transaction_pool: Some(transaction_pool.clone()), network_provider: network.clone(), diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 69aeb49192f61..fea422e21da86 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -362,7 +362,7 @@ pub fn new_full_base( "offchain-work", sc_offchain::OffchainWorkers::new(sc_offchain::OffchainWorkerOptions { runtime_api_provider: client.clone(), - keystore: Some(keystore_container.sync_keystore()), + keystore: Some(keystore_container.keystore()), offchain_db: backend.offchain_storage(), transaction_pool: Some(transaction_pool.clone()), network_provider: network.clone(), diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 0b3c534e1189e..d1284d8fbc820 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -40,6 +40,7 @@ use kitchensink_runtime::{ }; use node_primitives::Block; use sc_block_builder::BlockBuilderProvider; +use sc_client_api::execution_extensions::ExecutionExtensions; use sc_client_db::PruningMode; use sc_consensus::{BlockImport, BlockImportParams, ForkChoiceStrategy, ImportResult, ImportedAux}; use sc_executor::{NativeElseWasmExecutor, WasmExecutionMethod, WasmtimeInstantiationStrategy}; @@ -410,10 +411,7 @@ impl BenchDb { genesis_block_builder, None, None, - ExecutionExtensions::new( - None, - Arc::new(executor), - ), + ExecutionExtensions::new(None, Arc::new(executor)), Box::new(task_executor.clone()), None, None, diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index c39ae521ac61a..6de66449ec056 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -116,7 +116,6 @@ use sp_blockchain::{ use sp_consensus::{BlockOrigin, Environment, Error as ConsensusError, Proposer, SelectChain}; use sp_consensus_babe::inherents::BabeInherentData; use sp_consensus_slots::Slot; -use sp_core::crypto::ByteArray; use sp_inherents::{CreateInherentDataProviders, InherentData, InherentDataProvider}; use sp_keystore::KeystorePtr; use sp_runtime::{ diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 607b796c3cf9c..054ba2c49e24a 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -56,8 +56,9 @@ use sp_runtime::{ pub use self::{ builder::{ build_network, new_client, new_db_backend, new_full_client, new_full_parts, - new_full_parts_with_genesis_builder, spawn_tasks, BuildNetworkParams, KeystoreContainer, - NetworkStarter, SpawnTasksParams, TFullBackend, TFullCallExecutor, TFullClient, + new_full_parts_with_genesis_builder, new_native_or_wasm_executor, new_wasm_executor, + spawn_tasks, BuildNetworkParams, KeystoreContainer, NetworkStarter, SpawnTasksParams, + TFullBackend, TFullCallExecutor, TFullClient, }, client::{ClientConfig, LocalCallExecutor}, error::Error, diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 9cf59a277dda7..6dcd6c6cba02e 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -87,7 +87,7 @@ fn construct_block( &new_native_or_wasm_executor(), "Core_initialize_block", &header.encode(), - Default::default(), + &mut Default::default(), &runtime_code, CallContext::Onchain, ) @@ -101,7 +101,7 @@ fn construct_block( &new_native_or_wasm_executor(), "BlockBuilder_apply_extrinsic", &tx.encode(), - Default::default(), + &mut Default::default(), &runtime_code, CallContext::Onchain, ) @@ -115,7 +115,7 @@ fn construct_block( &new_native_or_wasm_executor(), "BlockBuilder_finalize_block", &[], - Default::default(), + &mut Default::default(), &runtime_code, CallContext::Onchain, ) @@ -188,7 +188,7 @@ fn construct_genesis_should_work_with_native() { &new_native_or_wasm_executor(), "Core_execute_block", &b1data, - Default::default(), + &mut Default::default(), &runtime_code, CallContext::Onchain, ) @@ -221,7 +221,7 @@ fn construct_genesis_should_work_with_wasm() { &new_native_or_wasm_executor(), "Core_execute_block", &b1data, - Default::default(), + &mut Default::default(), &runtime_code, CallContext::Onchain, ) @@ -254,7 +254,7 @@ fn construct_genesis_with_bad_transaction_should_panic() { &new_native_or_wasm_executor(), "Core_execute_block", &b1data, - Default::default(), + &mut Default::default(), &runtime_code, CallContext::Onchain, ) diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index f6b422d138e66..f0867e4f92b39 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1174,6 +1174,7 @@ mod tests { let backend = trie_backend::tests::test_trie(state_version, None, None); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); + let mut execution_extensions = &mut Default::default(); let mut state_machine = StateMachine::new( &backend, @@ -1185,7 +1186,7 @@ mod tests { }, "test", &[], - Default::default(), + &mut execution_extensions, &wasm_code, CallContext::Offchain, ); @@ -1202,6 +1203,7 @@ mod tests { let backend = trie_backend::tests::test_trie(state_version, None, None); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); + let mut execution_extensions = &mut Default::default(); let mut state_machine = StateMachine::new( &backend, @@ -1213,7 +1215,7 @@ mod tests { }, "test", &[], - Default::default(), + &mut execution_extensions, &wasm_code, CallContext::Offchain, ); diff --git a/utils/frame/benchmarking-cli/src/pallet/command.rs b/utils/frame/benchmarking-cli/src/pallet/command.rs index 54a70db2308de..cdc09ee065168 100644 --- a/utils/frame/benchmarking-cli/src/pallet/command.rs +++ b/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -239,7 +239,7 @@ impl PalletCmd { &executor, "Benchmark_benchmark_metadata", &(self.extra).encode(), - extensions(), + &mut extensions(), &sp_state_machine::backend::BackendRuntimeCode::new(state).runtime_code()?, CallContext::Offchain, ) @@ -375,7 +375,7 @@ impl PalletCmd { 1, // no need to do internal repeats ) .encode(), - extensions(), + &mut extensions(), &sp_state_machine::backend::BackendRuntimeCode::new(state) .runtime_code()?, CallContext::Offchain, @@ -415,7 +415,7 @@ impl PalletCmd { self.repeat, ) .encode(), - extensions(), + &mut extensions(), &sp_state_machine::backend::BackendRuntimeCode::new(state) .runtime_code()?, CallContext::Offchain, @@ -447,7 +447,7 @@ impl PalletCmd { self.repeat, ) .encode(), - extensions(), + &mut extensions(), &sp_state_machine::backend::BackendRuntimeCode::new(state) .runtime_code()?, CallContext::Offchain, From 97befb64890f59a548e657fbda5fa3a754c175e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 1 May 2023 23:54:45 +0200 Subject: [PATCH 17/55] Fix `generate_initial_session_keys` by passing the keystore extension --- Cargo.lock | 1 + client/service/src/builder.rs | 1 + primitives/keystore/src/lib.rs | 95 ++++++++++++++++++++++++++++++++++ primitives/session/Cargo.toml | 2 + primitives/session/src/lib.rs | 7 ++- 5 files changed, 105 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 250cb4dd12f70..6acd0cb99a646 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10855,6 +10855,7 @@ dependencies = [ "scale-info", "sp-api", "sp-core", + "sp-keystore", "sp-runtime", "sp-staking", "sp-std", diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 273bf3a50681a..ba2279a9dda1f 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -410,6 +410,7 @@ where client.clone(), chain_info.best_hash, config.dev_key_seed.clone().map(|s| vec![s]).unwrap_or_default(), + keystore.clone(), ) .map_err(|e| Error::Application(Box::new(e)))?; diff --git a/primitives/keystore/src/lib.rs b/primitives/keystore/src/lib.rs index 5b41f3b80043d..6b839042d4f06 100644 --- a/primitives/keystore/src/lib.rs +++ b/primitives/keystore/src/lib.rs @@ -214,6 +214,101 @@ pub trait Keystore: Send + Sync { } } +impl Keystore for Arc { + fn sr25519_public_keys(&self, key_type: KeyTypeId) -> Vec { + (**self).sr25519_public_keys(key_type) + } + + fn sr25519_generate_new( + &self, + key_type: KeyTypeId, + seed: Option<&str>, + ) -> Result { + (**self).sr25519_generate_new(key_type, seed) + } + + fn sr25519_sign( + &self, + key_type: KeyTypeId, + public: &sr25519::Public, + msg: &[u8], + ) -> Result, Error> { + (**self).sr25519_sign(key_type, public, msg) + } + + fn sr25519_vrf_sign( + &self, + key_type: KeyTypeId, + public: &sr25519::Public, + transcript: &sr25519::vrf::VrfTranscript, + ) -> Result, Error> { + (**self).sr25519_vrf_sign(key_type, public, transcript) + } + + fn ed25519_public_keys(&self, key_type: KeyTypeId) -> Vec { + (**self).ed25519_public_keys(key_type) + } + + fn ed25519_generate_new( + &self, + key_type: KeyTypeId, + seed: Option<&str>, + ) -> Result { + (**self).ed25519_generate_new(key_type, seed) + } + + fn ed25519_sign( + &self, + key_type: KeyTypeId, + public: &ed25519::Public, + msg: &[u8], + ) -> Result, Error> { + (**self).ed25519_sign(key_type, public, msg) + } + + fn ecdsa_public_keys(&self, key_type: KeyTypeId) -> Vec { + (**self).ecdsa_public_keys(key_type) + } + + fn ecdsa_generate_new( + &self, + key_type: KeyTypeId, + seed: Option<&str>, + ) -> Result { + (**self).ecdsa_generate_new(key_type, seed) + } + + fn ecdsa_sign( + &self, + key_type: KeyTypeId, + public: &ecdsa::Public, + msg: &[u8], + ) -> Result, Error> { + (**self).ecdsa_sign(key_type, public, msg) + } + + fn ecdsa_sign_prehashed( + &self, + key_type: KeyTypeId, + public: &ecdsa::Public, + msg: &[u8; 32], + ) -> Result, Error> { + (**self).ecdsa_sign_prehashed(key_type, public, msg) + } + + fn insert(&self, key_type: KeyTypeId, suri: &str, public: &[u8]) -> Result<(), ()> { + (**self).insert(key_type, suri, public) + } + + fn keys(&self, key_type: KeyTypeId) -> Result>, Error> { + (**self).keys(key_type) + } + + fn has_keys(&self, public_keys: &[(Vec, KeyTypeId)]) -> bool { + (**self).has_keys(public_keys) + } +} + /// A shared pointer to a keystore implementation. pub type KeystorePtr = Arc; diff --git a/primitives/session/Cargo.toml b/primitives/session/Cargo.toml index 6f362974b91b4..dba143454a818 100644 --- a/primitives/session/Cargo.toml +++ b/primitives/session/Cargo.toml @@ -20,6 +20,7 @@ sp-core = { version = "7.0.0", default-features = false, path = "../core" } sp-runtime = { version = "7.0.0", optional = true, path = "../runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } sp-std = { version = "5.0.0", default-features = false, path = "../std" } +sp-keystore = { version = "0.13.0", path = "../keystore", optional = true } [features] default = [ "std" ] @@ -31,4 +32,5 @@ std = [ "sp-runtime/std", "sp-staking/std", "sp-std/std", + "sp-keystore", ] diff --git a/primitives/session/src/lib.rs b/primitives/session/src/lib.rs index 642aa2a21143e..9fa99a5be8f97 100644 --- a/primitives/session/src/lib.rs +++ b/primitives/session/src/lib.rs @@ -112,17 +112,22 @@ pub fn generate_initial_session_keys( client: std::sync::Arc, at: Block::Hash, seeds: Vec, + keystore: sp_keystore::KeystorePtr, ) -> Result<(), sp_api::ApiError> where Block: BlockT, T: ProvideRuntimeApi, T::Api: SessionKeys, { + use sp_api::ApiExt; + if seeds.is_empty() { return Ok(()) } - let runtime_api = client.runtime_api(); + let mut runtime_api = client.runtime_api(); + + runtime_api.register_extension(sp_keystore::KeystoreExt::new(keystore)); for seed in seeds { runtime_api.generate_session_keys(at, Some(seed.as_bytes().to_vec()))?; From c76f8fdced0e3a9be3b45476f465af883ff176ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 2 May 2023 14:36:38 +0200 Subject: [PATCH 18/55] Fix the grandpa tests --- client/consensus/grandpa/src/tests.rs | 2 +- test-utils/runtime/src/lib.rs | 31 ++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/client/consensus/grandpa/src/tests.rs b/client/consensus/grandpa/src/tests.rs index 7a3f862d3602b..80b95635a2c3f 100644 --- a/client/consensus/grandpa/src/tests.rs +++ b/client/consensus/grandpa/src/tests.rs @@ -1988,7 +1988,7 @@ async fn grandpa_environment_doesnt_send_equivocation_reports_for_itself() { // keys it should work equivocation.identity = TryFrom::try_from(&[1; 32][..]).unwrap(); let equivocation_proof = sp_consensus_grandpa::Equivocation::Prevote(equivocation); - assert!(environment.report_equivocation(equivocation_proof).is_ok()); + environment.report_equivocation(equivocation_proof).unwrap(); } #[tokio::test] diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 4909738f57ff4..32e4825779923 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -46,13 +46,11 @@ use frame_system::limits::{BlockLength, BlockWeights}; use sp_api::{decl_runtime_apis, impl_runtime_apis}; pub use sp_core::hash::H256; use sp_inherents::{CheckInherentsResult, InherentData}; -#[cfg(feature = "std")] -use sp_runtime::traits::NumberFor; use sp_runtime::{ create_runtime_str, impl_opaque_keys, traits::{ BlakeTwo256, BlindCheckable, Block as BlockT, Extrinsic as ExtrinsicT, GetNodeBlockType, - GetRuntimeBlockType, IdentityLookup, Verify, + GetRuntimeBlockType, IdentityLookup, NumberFor, Verify, }, transaction_validity::{ InvalidTransaction, TransactionSource, TransactionValidity, TransactionValidityError, @@ -1217,6 +1215,33 @@ cfg_if! { } } + impl sp_consensus_grandpa::GrandpaApi for Runtime { + fn grandpa_authorities() -> sp_consensus_grandpa::AuthorityList { + Vec::new() + } + + fn current_set_id() -> sp_consensus_grandpa::SetId { + 0 + } + + fn submit_report_equivocation_unsigned_extrinsic( + _equivocation_proof: sp_consensus_grandpa::EquivocationProof< + ::Hash, + NumberFor, + >, + _key_owner_proof: sp_consensus_grandpa::OpaqueKeyOwnershipProof, + ) -> Option<()> { + None + } + + fn generate_key_ownership_proof( + _set_id: sp_consensus_grandpa::SetId, + _authority_id: sp_consensus_grandpa::AuthorityId, + ) -> Option { + None + } + } + impl sp_offchain::OffchainWorkerApi for Runtime { fn offchain_worker(header: &::Header) { let ex = Extrinsic::IncludeData(header.number.encode()); From 13347fbb3fbc6b88bf72b65b11e6ef286031315e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 2 May 2023 16:36:52 +0200 Subject: [PATCH 19/55] Fix more tests --- Cargo.lock | 1 + client/rpc/Cargo.toml | 1 + client/rpc/src/author/mod.rs | 11 +++++++---- client/rpc/src/author/tests.rs | 3 +-- client/rpc/src/state/tests.rs | 6 +++--- primitives/application-crypto/test/src/ecdsa.rs | 13 ++++++++----- primitives/application-crypto/test/src/ed25519.rs | 13 ++++++++----- primitives/application-crypto/test/src/sr25519.rs | 13 ++++++++----- primitives/keystore/src/lib.rs | 2 +- test-utils/client/src/lib.rs | 8 -------- 10 files changed, 38 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6acd0cb99a646..868e189bf5588 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9369,6 +9369,7 @@ dependencies = [ "log", "parity-scale-codec", "parking_lot 0.12.1", + "pretty_assertions", "sc-block-builder", "sc-chain-spec", "sc-client-api", diff --git a/client/rpc/Cargo.toml b/client/rpc/Cargo.toml index a22f657878812..cfa536b96191e 100644 --- a/client/rpc/Cargo.toml +++ b/client/rpc/Cargo.toml @@ -49,6 +49,7 @@ sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/comm tokio = "1.22.0" sp-io = { version = "7.0.0", path = "../../primitives/io" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +pretty_assertions = "1.2.1" [features] test-helpers = [] diff --git a/client/rpc/src/author/mod.rs b/client/rpc/src/author/mod.rs index 00a126500e26d..d8e79acb06959 100644 --- a/client/rpc/src/author/mod.rs +++ b/client/rpc/src/author/mod.rs @@ -37,10 +37,10 @@ use sc_transaction_pool_api::{ error::IntoPoolError, BlockHash, InPoolTransaction, TransactionFor, TransactionPool, TransactionSource, TxHash, }; -use sp_api::ProvideRuntimeApi; +use sp_api::{ProvideRuntimeApi, ApiExt}; use sp_blockchain::HeaderBackend; use sp_core::Bytes; -use sp_keystore::KeystorePtr; +use sp_keystore::{KeystorePtr, KeystoreExt}; use sp_runtime::{generic, traits::Block as BlockT}; use sp_session::SessionKeys; @@ -122,8 +122,11 @@ where self.deny_unsafe.check_if_safe()?; let best_block_hash = self.client.info().best_hash; - self.client - .runtime_api() + let mut runtime_api = self.client.runtime_api(); + + runtime_api.register_extension(KeystoreExt::new(self.keystore.clone())); + + runtime_api .generate_session_keys(best_block_hash, None) .map(Into::into) .map_err(|api_err| Error::Client(Box::new(api_err)).into()) diff --git a/client/rpc/src/author/tests.rs b/client/rpc/src/author/tests.rs index fbbd1a92fdaa7..7a78a34450e4a 100644 --- a/client/rpc/src/author/tests.rs +++ b/client/rpc/src/author/tests.rs @@ -65,8 +65,7 @@ struct TestSetup { impl Default for TestSetup { fn default() -> Self { let keystore = Arc::new(MemoryKeystore::new()); - let client_builder = substrate_test_runtime_client::TestClientBuilder::new(); - let client = Arc::new(client_builder.set_keystore(keystore.clone()).build()); + let client = Arc::new(substrate_test_runtime_client::TestClientBuilder::new().build()); let spawner = sp_core::testing::TaskExecutor::new(); let pool = diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index ae193e662b0e7..ddd1d453792f8 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -486,12 +486,12 @@ async fn should_return_runtime_version() { \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\ [\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\ [\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",2],\ - [\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]],\ - \"transactionVersion\":1,\"stateVersion\":1}"; + [\"0xed99c5acb25eedf5\",3],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\ + [\"0xbc9d89904f5b923f\",1]],\"transactionVersion\":1,\"stateVersion\":1}"; let runtime_version = api.runtime_version(None.into()).unwrap(); let serialized = serde_json::to_string(&runtime_version).unwrap(); - assert_eq!(serialized, result); + pretty_assertions::assert_eq!(serialized, result); let deserialized: RuntimeVersion = serde_json::from_str(result).unwrap(); assert_eq!(deserialized, runtime_version); diff --git a/primitives/application-crypto/test/src/ecdsa.rs b/primitives/application-crypto/test/src/ecdsa.rs index 99ca6f4c4adf2..396683a91ac02 100644 --- a/primitives/application-crypto/test/src/ecdsa.rs +++ b/primitives/application-crypto/test/src/ecdsa.rs @@ -16,13 +16,13 @@ // limitations under the License. //! Integration tests for ecdsa -use sp_api::ProvideRuntimeApi; +use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_application_crypto::ecdsa::AppPair; use sp_core::{ crypto::{ByteArray, Pair}, testing::ECDSA, }; -use sp_keystore::{testing::MemoryKeystore, Keystore}; +use sp_keystore::{testing::MemoryKeystore, Keystore, KeystoreExt}; use std::sync::Arc; use substrate_test_runtime_client::{ runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, @@ -31,9 +31,12 @@ use substrate_test_runtime_client::{ #[test] fn ecdsa_works_in_runtime() { let keystore = Arc::new(MemoryKeystore::new()); - let test_client = TestClientBuilder::new().set_keystore(keystore.clone()).build(); - let (signature, public) = test_client - .runtime_api() + let test_client = TestClientBuilder::new().build(); + + let mut runtime_api = test_client.runtime_api(); + runtime_api.register_extension(KeystoreExt::new(keystore.clone())); + + let (signature, public) = runtime_api .test_ecdsa_crypto(test_client.chain_info().genesis_hash) .expect("Tests `ecdsa` crypto."); diff --git a/primitives/application-crypto/test/src/ed25519.rs b/primitives/application-crypto/test/src/ed25519.rs index f4553f95bf1f8..f0ceccdcebfcd 100644 --- a/primitives/application-crypto/test/src/ed25519.rs +++ b/primitives/application-crypto/test/src/ed25519.rs @@ -17,13 +17,13 @@ //! Integration tests for ed25519 -use sp_api::ProvideRuntimeApi; +use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_application_crypto::ed25519::AppPair; use sp_core::{ crypto::{ByteArray, Pair}, testing::ED25519, }; -use sp_keystore::{testing::MemoryKeystore, Keystore}; +use sp_keystore::{testing::MemoryKeystore, Keystore, KeystoreExt}; use std::sync::Arc; use substrate_test_runtime_client::{ runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, @@ -32,9 +32,12 @@ use substrate_test_runtime_client::{ #[test] fn ed25519_works_in_runtime() { let keystore = Arc::new(MemoryKeystore::new()); - let test_client = TestClientBuilder::new().set_keystore(keystore.clone()).build(); - let (signature, public) = test_client - .runtime_api() + let test_client = TestClientBuilder::new().build(); + + let mut runtime_api = test_client.runtime_api(); + runtime_api.register_extension(KeystoreExt::new(keystore.clone())); + + let (signature, public) = runtime_api .test_ed25519_crypto(test_client.chain_info().genesis_hash) .expect("Tests `ed25519` crypto."); diff --git a/primitives/application-crypto/test/src/sr25519.rs b/primitives/application-crypto/test/src/sr25519.rs index 736521d7d9f3a..3c62270395f04 100644 --- a/primitives/application-crypto/test/src/sr25519.rs +++ b/primitives/application-crypto/test/src/sr25519.rs @@ -17,13 +17,13 @@ //! Integration tests for sr25519 -use sp_api::ProvideRuntimeApi; +use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_application_crypto::sr25519::AppPair; use sp_core::{ crypto::{ByteArray, Pair}, testing::SR25519, }; -use sp_keystore::{testing::MemoryKeystore, Keystore}; +use sp_keystore::{testing::MemoryKeystore, Keystore, KeystoreExt}; use std::sync::Arc; use substrate_test_runtime_client::{ runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, @@ -32,9 +32,12 @@ use substrate_test_runtime_client::{ #[test] fn sr25519_works_in_runtime() { let keystore = Arc::new(MemoryKeystore::new()); - let test_client = TestClientBuilder::new().set_keystore(keystore.clone()).build(); - let (signature, public) = test_client - .runtime_api() + let test_client = TestClientBuilder::new().build(); + + let mut runtime_api = test_client.runtime_api(); + runtime_api.register_extension(KeystoreExt::new(keystore.clone())); + + let (signature, public) = runtime_api .test_sr25519_crypto(test_client.chain_info().genesis_hash) .expect("Tests `sr25519` crypto."); diff --git a/primitives/keystore/src/lib.rs b/primitives/keystore/src/lib.rs index 6b839042d4f06..ba494c313d3c3 100644 --- a/primitives/keystore/src/lib.rs +++ b/primitives/keystore/src/lib.rs @@ -214,7 +214,7 @@ pub trait Keystore: Send + Sync { } } -impl Keystore for Arc { +impl Keystore for Arc { fn sr25519_public_keys(&self, key_type: KeyTypeId) -> Vec { (**self).sr25519_public_keys(key_type) } diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 06364a050b812..25307504a09ee 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -65,7 +65,6 @@ pub struct TestClientBuilder, StorageChild>, backend: Arc, _executor: std::marker::PhantomData, - keystore: Option, fork_blocks: ForkBlocks, bad_blocks: BadBlocks, enable_offchain_indexing_api: bool, @@ -113,7 +112,6 @@ impl child_storage_extension: Default::default(), genesis_init: Default::default(), _executor: Default::default(), - keystore: None, fork_blocks: None, bad_blocks: None, enable_offchain_indexing_api: false, @@ -121,12 +119,6 @@ impl } } - /// Set the keystore that should be used by the externalities. - pub fn set_keystore(mut self, keystore: KeystorePtr) -> Self { - self.keystore = Some(keystore); - self - } - /// Alter the genesis storage parameters. pub fn genesis_init_mut(&mut self) -> &mut G { &mut self.genesis_init From 0bd9113afe29a95ce2dafe50eee1c24c251b9966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 2 May 2023 17:34:22 +0200 Subject: [PATCH 20/55] Fix more tests --- Cargo.lock | 1 + client/rpc-spec-v2/Cargo.toml | 1 + client/rpc-spec-v2/src/chain_head/tests.rs | 6 +++--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 868e189bf5588..c30c11e228f3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9441,6 +9441,7 @@ dependencies = [ "log", "parity-scale-codec", "parking_lot 0.12.1", + "pretty_assertions", "sc-block-builder", "sc-chain-spec", "sc-client-api", diff --git a/client/rpc-spec-v2/Cargo.toml b/client/rpc-spec-v2/Cargo.toml index 23b96877f3b17..4aee2a2839a57 100644 --- a/client/rpc-spec-v2/Cargo.toml +++ b/client/rpc-spec-v2/Cargo.toml @@ -45,3 +45,4 @@ sp-maybe-compressed-blob = { version = "4.1.0-dev", path = "../../primitives/may sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" } sc-utils = { version = "4.0.0-dev", path = "../utils" } assert_matches = "1.3.0" +pretty_assertions = "1.2.1" diff --git a/client/rpc-spec-v2/src/chain_head/tests.rs b/client/rpc-spec-v2/src/chain_head/tests.rs index 1d5cb8da26305..1695fa39f7785 100644 --- a/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/client/rpc-spec-v2/src/chain_head/tests.rs @@ -181,8 +181,8 @@ async fn follow_with_runtime() { \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\ [\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\ [\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",2],\ - [\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]],\ - \"transactionVersion\":1,\"stateVersion\":1}"; + [\"0xed99c5acb25eedf5\",3],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\ + [\"0xbc9d89904f5b923f\",1]],\"transactionVersion\":1,\"stateVersion\":1}"; let runtime: RuntimeVersion = serde_json::from_str(runtime_str).unwrap(); let finalized_block_runtime = @@ -193,7 +193,7 @@ async fn follow_with_runtime() { finalized_block_runtime, runtime_updates: false, }); - assert_eq!(event, expected); + pretty_assertions::assert_eq!(event, expected); // Import a new block without runtime changes. // The runtime field must be None in this case. From d457ac9f0278033570968582e559c010397d0685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 3 May 2023 15:50:33 +0200 Subject: [PATCH 21/55] Don't set any heap pages if there isn't an override --- test-utils/runtime/src/genesismap.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test-utils/runtime/src/genesismap.rs b/test-utils/runtime/src/genesismap.rs index 9e00dd29999f8..3c4edb2d29caa 100644 --- a/test-utils/runtime/src/genesismap.rs +++ b/test-utils/runtime/src/genesismap.rs @@ -64,14 +64,14 @@ impl GenesisConfig { }) .map(|(k, v)| (blake2_256(&k[..])[..].to_vec(), v.to_vec())) .chain( - vec![ - (well_known_keys::CODE.into(), wasm_runtime), - ( - well_known_keys::HEAP_PAGES.into(), - vec![].and(&(self.heap_pages_override.unwrap_or(16_u64))), - ), - ] - .into_iter(), + vec![(well_known_keys::CODE.into(), wasm_runtime)] + .into_iter() + .chain( + self.heap_pages_override + .into_iter() + .map(|h| (well_known_keys::HEAP_PAGES.into(), h.encode())), + ) + .into_iter(), ) .collect(); map.insert(twox_128(&b"sys:auth"[..])[..].to_vec(), self.authorities.encode()); From 4e4b5e4af03a7b5c2cb90145d28b25f4e20dc1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 3 May 2023 16:30:00 +0200 Subject: [PATCH 22/55] Fix small fallout --- client/rpc/src/dev/tests.rs | 4 ++-- client/service/test/src/client/mod.rs | 28 ++++++++------------------- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/client/rpc/src/dev/tests.rs b/client/rpc/src/dev/tests.rs index 9beb01182585a..a80a79c860432 100644 --- a/client/rpc/src/dev/tests.rs +++ b/client/rpc/src/dev/tests.rs @@ -43,8 +43,8 @@ async fn block_stats_work() { .await .unwrap(), Some(BlockStats { - witness_len: 630, - witness_compact_len: 534, + witness_len: 612, + witness_compact_len: 516, block_len: 99, num_extrinsics: 0, }), diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 6dcd6c6cba02e..859a7c0f4c0a4 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -1703,21 +1703,20 @@ fn storage_keys_prefix_and_start_key_works() { let block_hash = client.info().best_hash; - let child_root = b":child_storage:default:child".to_vec(); + let child_root = array_bytes::bytes2hex("", b":child_storage:default:child"); let prefix = StorageKey(array_bytes::hex2bytes_unchecked("3a")); let child_prefix = StorageKey(b"sec".to_vec()); let res: Vec<_> = client .storage_keys(block_hash, Some(&prefix), None) .unwrap() - .map(|x| x.0) + .map(|x| array_bytes::bytes2hex("", &x.0)) .collect(); assert_eq!( res, [ - child_root.clone(), - array_bytes::hex2bytes_unchecked("3a636f6465"), - array_bytes::hex2bytes_unchecked("3a686561707061676573"), + &child_root, + "3a636f6465", ] ); @@ -1728,20 +1727,9 @@ fn storage_keys_prefix_and_start_key_works() { Some(&StorageKey(array_bytes::hex2bytes_unchecked("3a636f6465"))), ) .unwrap() - .map(|x| x.0) - .collect(); - assert_eq!(res, [array_bytes::hex2bytes_unchecked("3a686561707061676573")]); - - let res: Vec<_> = client - .storage_keys( - block_hash, - Some(&prefix), - Some(&StorageKey(array_bytes::hex2bytes_unchecked("3a686561707061676573"))), - ) - .unwrap() - .map(|x| x.0) + .map(|x| array_bytes::bytes2hex("", &x.0)) .collect(); - assert_eq!(res, Vec::>::new()); + assert_eq!(res, Vec::::new()); let res: Vec<_> = client .child_storage_keys(block_hash, child_info.clone(), Some(&child_prefix), None) @@ -1824,13 +1812,13 @@ fn storage_keys_works() { res, [ "3a636f6465", - "3a686561707061676573", "52008686cc27f6e5ed83a216929942f8bcd32a396f09664a5698f81371934b56", "5348d72ac6cc66e5d8cbecc27b0e0677503b845fe2382d819f83001781788fd5", "5c2d5fda66373dabf970e4fb13d277ce91c5233473321129d32b5a8085fa8133", "6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081", "66484000ed3f75c95fc7b03f39c20ca1e1011e5999278247d3b2f5e3c3273808", "7d5007603a7f5dd729d51d93cf695d6465789443bb967c0d1fe270e388c96eaa", + "811ecfaadcf5f2ee1d67393247e2f71a1662d433e8ce7ff89fb0d4aa9561820b", ] ); @@ -1848,13 +1836,13 @@ fn storage_keys_works() { assert_eq!( res, [ - "3a686561707061676573", "52008686cc27f6e5ed83a216929942f8bcd32a396f09664a5698f81371934b56", "5348d72ac6cc66e5d8cbecc27b0e0677503b845fe2382d819f83001781788fd5", "5c2d5fda66373dabf970e4fb13d277ce91c5233473321129d32b5a8085fa8133", "6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081", "66484000ed3f75c95fc7b03f39c20ca1e1011e5999278247d3b2f5e3c3273808", "7d5007603a7f5dd729d51d93cf695d6465789443bb967c0d1fe270e388c96eaa", + "811ecfaadcf5f2ee1d67393247e2f71a1662d433e8ce7ff89fb0d4aa9561820b", ] ); From 7d48c14cc1d2f661d8d8920168b074d3a9d07f10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 3 May 2023 17:30:13 +0200 Subject: [PATCH 23/55] FMT --- bin/node/bench/src/construct.rs | 4 +--- bin/node/bench/src/import.rs | 6 +----- client/api/src/call_executor.rs | 2 +- client/api/src/execution_extensions.rs | 2 -- client/consensus/grandpa/src/lib.rs | 4 ++-- client/offchain/src/api.rs | 4 ++-- client/rpc-spec-v2/src/chain_head/chain_head.rs | 7 +------ client/rpc-spec-v2/src/chain_head/test_utils.rs | 1 - client/rpc/src/author/mod.rs | 4 ++-- client/rpc/src/state/state_full.rs | 7 +------ client/service/src/builder.rs | 2 +- client/service/test/src/client/mod.rs | 8 +------- primitives/api/proc-macro/src/decl_runtime_apis.rs | 3 +-- primitives/api/src/lib.rs | 4 ++-- primitives/api/test/tests/runtime_calls.rs | 10 +++------- primitives/core/src/lib.rs | 1 - primitives/runtime/src/runtime_logger.rs | 3 +-- test-utils/client/src/lib.rs | 5 +---- 18 files changed, 21 insertions(+), 56 deletions(-) diff --git a/bin/node/bench/src/construct.rs b/bin/node/bench/src/construct.rs index 67c37bc323a77..4f3ca07f86b9d 100644 --- a/bin/node/bench/src/construct.rs +++ b/bin/node/bench/src/construct.rs @@ -100,9 +100,7 @@ impl core::BenchmarkDescription for ConstructionBenchmarkDescription { fn name(&self) -> Cow<'static, str> { format!( "Block construction ({:?}/{}, {:?} backend)", - self.block_type, - self.size, - self.database_type, + self.block_type, self.size, self.database_type, ) .into() } diff --git a/bin/node/bench/src/import.rs b/bin/node/bench/src/import.rs index e1882f1ed6eb5..78b280076e0bd 100644 --- a/bin/node/bench/src/import.rs +++ b/bin/node/bench/src/import.rs @@ -83,11 +83,7 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription { fn setup(self: Box) -> Box { let mut bench_db = BenchDb::with_key_types(self.database_type, 50_000, self.key_types); let block = bench_db.generate_block(self.block_type.to_content(self.size.transactions())); - Box::new(ImportBenchmark { - database: bench_db, - block_type: self.block_type, - block, - }) + Box::new(ImportBenchmark { database: bench_db, block_type: self.block_type, block }) } fn name(&self) -> Cow<'static, str> { diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index 8152f6e7b68fc..cecc1053d1b9d 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -20,10 +20,10 @@ use sc_executor::{RuntimeVersion, RuntimeVersionOf}; use sp_core::traits::CallContext; +use sp_externalities::Extensions; use sp_runtime::traits::Block as BlockT; use sp_state_machine::{OverlayedChanges, StorageProof}; use std::cell::RefCell; -use sp_externalities::Extensions; use crate::execution_extensions::ExecutionExtensions; use sp_api::{ProofRecorder, StorageTransactionCache}; diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 6fb69d0b10a84..eeaef40da1edf 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -129,8 +129,6 @@ impl ExecutionExtensions { extensions.register(ReadRuntimeVersionExt::new(self.read_runtime_version.clone())); - extensions.register(ReadRuntimeVersionExt::new(self.read_runtime_version.clone())); - extensions } } diff --git a/client/consensus/grandpa/src/lib.rs b/client/consensus/grandpa/src/lib.rs index 8322b49f2b44b..c200f1f6cb1c6 100644 --- a/client/consensus/grandpa/src/lib.rs +++ b/client/consensus/grandpa/src/lib.rs @@ -64,8 +64,8 @@ use prometheus_endpoint::{PrometheusError, Registry}; use sc_client_api::{ backend::{AuxStore, Backend}, utils::is_descendent_of, - BlockchainEvents, CallExecutor, ExecutorProvider, Finalizer, LockImportRun, - StorageProvider, TransactionFor, + BlockchainEvents, CallExecutor, ExecutorProvider, Finalizer, LockImportRun, StorageProvider, + TransactionFor, }; use sc_consensus::BlockImport; use sc_network::types::ProtocolName; diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index df3f4c986ccb9..4f437a700dc69 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -25,8 +25,8 @@ pub use http::SharedClient; use libp2p::{Multiaddr, PeerId}; use sp_core::{ offchain::{ - self, HttpError, HttpRequestId, HttpRequestStatus, OffchainStorage, - OpaqueMultiaddr, OpaqueNetworkState, StorageKind, Timestamp, + self, HttpError, HttpRequestId, HttpRequestStatus, OffchainStorage, OpaqueMultiaddr, + OpaqueNetworkState, StorageKind, Timestamp, }, OpaquePeerId, }; diff --git a/client/rpc-spec-v2/src/chain_head/chain_head.rs b/client/rpc-spec-v2/src/chain_head/chain_head.rs index 59eb90529b91b..85b11178d0a17 100644 --- a/client/rpc-spec-v2/src/chain_head/chain_head.rs +++ b/client/rpc-spec-v2/src/chain_head/chain_head.rs @@ -390,12 +390,7 @@ where let res = client .executor() - .call( - hash, - &function, - &call_parameters, - CallContext::Offchain, - ) + .call(hash, &function, &call_parameters, CallContext::Offchain) .map(|result| { let result = format!("0x{:?}", HexDisplay::from(&result)); ChainHeadEvent::Done(ChainHeadResult { result }) diff --git a/client/rpc-spec-v2/src/chain_head/test_utils.rs b/client/rpc-spec-v2/src/chain_head/test_utils.rs index 536ba7882d356..54c585932a744 100644 --- a/client/rpc-spec-v2/src/chain_head/test_utils.rs +++ b/client/rpc-spec-v2/src/chain_head/test_utils.rs @@ -227,7 +227,6 @@ impl> CallApiAt for ChainHeadMock } } - impl> BlockBackend for ChainHeadMockClient { diff --git a/client/rpc/src/author/mod.rs b/client/rpc/src/author/mod.rs index d8e79acb06959..16f8c91b70360 100644 --- a/client/rpc/src/author/mod.rs +++ b/client/rpc/src/author/mod.rs @@ -37,10 +37,10 @@ use sc_transaction_pool_api::{ error::IntoPoolError, BlockHash, InPoolTransaction, TransactionFor, TransactionPool, TransactionSource, TxHash, }; -use sp_api::{ProvideRuntimeApi, ApiExt}; +use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_blockchain::HeaderBackend; use sp_core::Bytes; -use sp_keystore::{KeystorePtr, KeystoreExt}; +use sp_keystore::{KeystoreExt, KeystorePtr}; use sp_runtime::{generic, traits::Block as BlockT}; use sp_session::SessionKeys; diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index 9911c5652842c..9604d9165f987 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -198,12 +198,7 @@ where .and_then(|block| { self.client .executor() - .call( - block, - &method, - &call_data, - CallContext::Offchain, - ) + .call(block, &method, &call_data, CallContext::Offchain) .map(Into::into) }) .map_err(client_err) diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index ba2279a9dda1f..9effa0e3a24ed 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -180,7 +180,7 @@ where let client = { let extensions = sc_client_api::execution_extensions::ExecutionExtensions::new( - None, + None, Arc::new(executor.clone()), ); diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 859a7c0f4c0a4..065294a6c4fac 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -1712,13 +1712,7 @@ fn storage_keys_prefix_and_start_key_works() { .unwrap() .map(|x| array_bytes::bytes2hex("", &x.0)) .collect(); - assert_eq!( - res, - [ - &child_root, - "3a636f6465", - ] - ); + assert_eq!(res, [&child_root, "3a636f6465",]); let res: Vec<_> = client .storage_keys( diff --git a/primitives/api/proc-macro/src/decl_runtime_apis.rs b/primitives/api/proc-macro/src/decl_runtime_apis.rs index a0b3082c4a67d..172748797e9d5 100644 --- a/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -357,8 +357,7 @@ impl<'a> ToClientSideDecl<'a> { items.into_iter().for_each(|i| match i { TraitItem::Fn(method) => { - let fn_decl = - self.create_method_decl(method, trait_generics_num); + let fn_decl = self.create_method_decl(method, trait_generics_num); result.push(fn_decl.into()); }, r => result.push(r), diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index cec9fe2c9f97b..9439931f2ee79 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -76,10 +76,10 @@ pub use codec::{self, Decode, DecodeLimit, Encode}; #[cfg(feature = "std")] pub use hash_db::Hasher; #[doc(hidden)] -pub use sp_core::offchain; -#[doc(hidden)] pub use scale_info; #[doc(hidden)] +pub use sp_core::offchain; +#[doc(hidden)] #[cfg(not(feature = "std"))] pub use sp_core::to_substrate_wasm_fn_return_value; #[doc(hidden)] diff --git a/primitives/api/test/tests/runtime_calls.rs b/primitives/api/test/tests/runtime_calls.rs index 11b3b24d25b61..df6516a349259 100644 --- a/primitives/api/test/tests/runtime_calls.rs +++ b/primitives/api/test/tests/runtime_calls.rs @@ -17,9 +17,7 @@ use sp_api::{Core, ProvideRuntimeApi}; use sp_runtime::traits::{HashFor, Header as HeaderT}; -use sp_state_machine::{ - create_proof_check_backend, execution_proof_check_on_trie_backend, -}; +use sp_state_machine::{create_proof_check_backend, execution_proof_check_on_trie_backend}; use substrate_test_runtime_client::{ prelude::*, runtime::{Block, Header, TestAPI, Transfer}, @@ -53,8 +51,7 @@ fn calling_runtime_signature_changed_old_function() { #[test] fn use_trie_function() { - let client = TestClientBuilder::new() - .build(); + let client = TestClientBuilder::new().build(); let runtime_api = client.runtime_api(); let best_hash = client.chain_info().best_hash; assert_eq!(runtime_api.use_trie(best_hash).unwrap(), 2); @@ -82,8 +79,7 @@ fn initialize_block_works() { #[test] fn record_proof_works() { - let (client, longest_chain) = TestClientBuilder::new() - .build_with_longest_chain(); + let (client, longest_chain) = TestClientBuilder::new().build_with_longest_chain(); let storage_root = *futures::executor::block_on(longest_chain.best_chain()).unwrap().state_root(); diff --git a/primitives/core/src/lib.rs b/primitives/core/src/lib.rs index e5ff2a6d805b2..abe6e746cd2f6 100644 --- a/primitives/core/src/lib.rs +++ b/primitives/core/src/lib.rs @@ -93,7 +93,6 @@ pub use sp_storage as storage; #[doc(hidden)] pub use sp_std; - /// Hex-serialized shim for `Vec`. #[derive(PartialEq, Eq, Clone, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord))] diff --git a/primitives/runtime/src/runtime_logger.rs b/primitives/runtime/src/runtime_logger.rs index 851e5e3c37d53..b7374b8b6f6c8 100644 --- a/primitives/runtime/src/runtime_logger.rs +++ b/primitives/runtime/src/runtime_logger.rs @@ -68,8 +68,7 @@ mod tests { use sp_api::ProvideRuntimeApi; use std::{env, str::FromStr}; use substrate_test_runtime_client::{ - runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, - TestClientBuilderExt, + runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, }; #[test] diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 25307504a09ee..996676e250d41 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -266,10 +266,7 @@ impl self.backend.clone(), executor.clone(), Default::default(), - ExecutionExtensions::new( - None, - Arc::new(executor), - ), + ExecutionExtensions::new(None, Arc::new(executor)), ) .expect("Creates LocalCallExecutor"); From 05223f1e19e8c71c59f4b974b116a921911b18e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 7 May 2023 22:26:37 +0200 Subject: [PATCH 24/55] Fix tests --- .../src/chain_head/subscription/inner.rs | 1 - client/rpc-spec-v2/src/chain_head/tests.rs | 8 ++++---- client/rpc/src/state/tests.rs | 6 +++--- primitives/api/test/tests/runtime_calls.rs | 10 ---------- test-utils/runtime/src/lib.rs | 3 +-- test-utils/runtime/src/substrate_test_pallet.rs | 14 ++++++++++---- 6 files changed, 18 insertions(+), 24 deletions(-) diff --git a/client/rpc-spec-v2/src/chain_head/subscription/inner.rs b/client/rpc-spec-v2/src/chain_head/subscription/inner.rs index 8865daa83cba2..beff4af9f7072 100644 --- a/client/rpc-spec-v2/src/chain_head/subscription/inner.rs +++ b/client/rpc-spec-v2/src/chain_head/subscription/inner.rs @@ -539,7 +539,6 @@ mod tests { genesis_block_builder, None, None, - None, Box::new(TaskExecutor::new()), client_config, ) diff --git a/client/rpc-spec-v2/src/chain_head/tests.rs b/client/rpc-spec-v2/src/chain_head/tests.rs index 62a9d6e60af79..4ff10f366707b 100644 --- a/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/client/rpc-spec-v2/src/chain_head/tests.rs @@ -188,9 +188,10 @@ async fn follow_with_runtime() { let runtime_str = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\ [\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\ - [\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",2],\ - [\"0xed99c5acb25eedf5\",3],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\ - [\"0xbc9d89904f5b923f\",1]],\"transactionVersion\":1,\"stateVersion\":1}"; + [\"0xbc9d89904f5b923f\",1],[\"0xc6e9a76309f39b09\",2],[\"0xdd718d5cc53262d4\",1],\ + [\"0xcbca25e39f142387\",2],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\ + [\"0xed99c5acb25eedf5\",3]],\"transactionVersion\":1,\"stateVersion\":1}"; + let runtime: RuntimeVersion = serde_json::from_str(runtime_str).unwrap(); let finalized_block_runtime = @@ -1365,7 +1366,6 @@ async fn pin_block_references() { genesis_block_builder, None, None, - None, Box::new(TaskExecutor::new()), client_config, ) diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 2c31ca57156c3..7c73c1e1cd8da 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -516,9 +516,9 @@ async fn should_return_runtime_version() { let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\ [\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\ - [\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",2],\ - [\"0xed99c5acb25eedf5\",3],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\ - [\"0xbc9d89904f5b923f\",1]],\"transactionVersion\":1,\"stateVersion\":1}"; + [\"0xbc9d89904f5b923f\",1],[\"0xc6e9a76309f39b09\",2],[\"0xdd718d5cc53262d4\",1],\ + [\"0xcbca25e39f142387\",2],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\ + [\"0xed99c5acb25eedf5\",3]],\"transactionVersion\":1,\"stateVersion\":1}"; let runtime_version = api.runtime_version(None.into()).unwrap(); let serialized = serde_json::to_string(&runtime_version).unwrap(); diff --git a/primitives/api/test/tests/runtime_calls.rs b/primitives/api/test/tests/runtime_calls.rs index 2f4520f102e27..a6d4c8f8581d8 100644 --- a/primitives/api/test/tests/runtime_calls.rs +++ b/primitives/api/test/tests/runtime_calls.rs @@ -38,16 +38,6 @@ fn calling_runtime_function() { assert_eq!(runtime_api.benchmark_add_one(best_hash, &1).unwrap(), 2); } -#[test] -fn calling_native_runtime_function() { - calling_function_with_strat(ExecutionStrategy::NativeWhenPossible); -} - -#[test] -fn calling_wasm_runtime_function() { - calling_function_with_strat(ExecutionStrategy::AlwaysWasm); -} - #[test] fn calling_native_runtime_signature_changed_function() { let client = TestClientBuilder::new().build(); diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index e037bebdaf88e..0ec095af4edca 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -999,10 +999,9 @@ mod tests { use sc_block_builder::BlockBuilderProvider; use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_consensus::BlockOrigin; - use sp_core::{storage::well_known_keys::HEAP_PAGES, ExecutionContext}; + use sp_core::{storage::well_known_keys::HEAP_PAGES, traits::CallContext}; use sp_keyring::AccountKeyring; use sp_runtime::{traits::SignedExtension, transaction_validity::InvalidTransaction}; - use sp_state_machine::ExecutionStrategy; use substrate_test_runtime_client::{ prelude::*, runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, }; diff --git a/test-utils/runtime/src/substrate_test_pallet.rs b/test-utils/runtime/src/substrate_test_pallet.rs index 98ebd550b9eef..82604d0263559 100644 --- a/test-utils/runtime/src/substrate_test_pallet.rs +++ b/test-utils/runtime/src/substrate_test_pallet.rs @@ -23,8 +23,11 @@ use crate::AuthorityId; use frame_support::{pallet_prelude::*, storage}; -use sp_runtime::transaction_validity::{ - InvalidTransaction, TransactionSource, TransactionValidity, ValidTransaction, +use sp_runtime::{ + traits::Hash, + transaction_validity::{ + InvalidTransaction, TransactionSource, TransactionValidity, ValidTransaction, + }, }; use sp_std::prelude::*; @@ -38,7 +41,7 @@ pub mod pallet { use crate::TransferData; use frame_system::pallet_prelude::*; use sp_core::storage::well_known_keys; - use sp_runtime::{transaction_validity::TransactionPriority, Perbill}; + use sp_runtime::{traits::BlakeTwo256, transaction_validity::TransactionPriority, Perbill}; #[pallet::pallet] #[pallet::without_storage_info] @@ -225,7 +228,10 @@ pub mod pallet { Call::deposit_log_digest_item { .. } | Call::storage_change { .. } | Call::read { .. } | - Call::read_and_panic { .. } => Ok(Default::default()), + Call::read_and_panic { .. } => Ok(ValidTransaction { + provides: vec![BlakeTwo256::hash_of(call).encode()], + ..Default::default() + }), _ => Err(TransactionValidityError::Invalid(InvalidTransaction::Call)), } } From 6ca673774cf6a2f715e82e19a85557fdb860ad12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 8 May 2023 16:19:59 +0200 Subject: [PATCH 25/55] More tests --- client/service/test/src/client/mod.rs | 14 ++--- test-utils/runtime/src/lib.rs | 76 +++++++++++++++++++++------ 2 files changed, 67 insertions(+), 23 deletions(-) diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 5bc07f7125474..01ef525afe787 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -1728,7 +1728,7 @@ fn storage_keys_works() { sp_tracing::try_init_simple(); let expected_keys = - substrate_test_runtime::storage_key_generator::get_expected_storage_hashed_keys(); + substrate_test_runtime::storage_key_generator::get_expected_storage_hashed_keys(false); let client = substrate_test_runtime_client::new(); let block_hash = client.info().best_hash; @@ -1767,10 +1767,10 @@ fn storage_keys_works() { res, expected_keys .iter() - .filter(|&i| i > &"3a636f64".to_string()) + .filter(|&i| *i > "3a636f64") .take(8) .cloned() - .collect::>() + .collect::>() ); // Starting at a complete key the first key is skipped. @@ -1788,10 +1788,10 @@ fn storage_keys_works() { res, expected_keys .iter() - .filter(|&i| i > &"3a636f6465".to_string()) + .filter(|&i| *i > "3a636f6465") .take(8) .cloned() - .collect::>() + .collect::>() ); const SOME_BALANCE_KEY : &str = "26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9e2c1dc507e2035edbbd8776c440d870460c57f0008067cc01c5ff9eb2e2f9b3a94299a915a91198bd1021a6c55596f57"; @@ -1809,10 +1809,10 @@ fn storage_keys_works() { res, expected_keys .iter() - .filter(|&i| i > &SOME_BALANCE_KEY.to_string()) + .filter(|&i| *i > SOME_BALANCE_KEY) .take(8) .cloned() - .collect::>() + .collect::>() ); } diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 0ec095af4edca..e0c2decb39214 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -841,8 +841,12 @@ pub mod storage_key_generator { /// Generate the hashed storage keys from the raw literals. These keys are expected to be be in /// storage with given substrate-test runtime. - pub fn generate_expected_storage_hashed_keys() -> Vec { - let literals: Vec<&[u8]> = vec![b":code", b":extrinsic_index", b":heappages"]; + pub fn generate_expected_storage_hashed_keys(custom_heap_pages: bool) -> Vec { + let mut literals: Vec<&[u8]> = vec![b":code", b":extrinsic_index"]; + + if custom_heap_pages { + literals.push(b":heappages"); + } let keys: Vec> = vec![ vec![b"Babe", b"Authorities"], @@ -902,8 +906,11 @@ pub mod storage_key_generator { /// that would be generated by `generate_expected_storage_hashed_keys`. This list is provided /// for the debugging convenience only. Value of each hex-string is documented with the literal /// origin. - pub fn get_expected_storage_hashed_keys() -> Vec { - [ + /// + /// `custom_heap_pages`: Should be set to `true` when the state contains the `:heap_pages` key + /// aka when overriding the heap pages to be used by the executor. + pub fn get_expected_storage_hashed_keys(custom_heap_pages: bool) -> Vec<&'static str> { + let mut res = vec![ //System|:__STORAGE_VERSION__: "00771836bebdd29870ff246d305c578c4e7b9012096b41c4eb3aaf947f6ea429", //SubstrateTest|Authorities @@ -973,20 +980,25 @@ pub mod storage_key_generator { "3a636f6465", // :extrinsic_index "3a65787472696e7369635f696e646578", - // :heappages - "3a686561707061676573", // Balances|:__STORAGE_VERSION__: "c2261276cc9d1f8598ea4b6a74b15c2f4e7b9012096b41c4eb3aaf947f6ea429", // Balances|TotalIssuance "c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80", - ].into_iter().map(String::from).collect::>() + ]; + + if custom_heap_pages { + // :heappages + res.push("3a686561707061676573"); + } + + res } #[test] fn expected_keys_vec_are_matching() { assert_eq!( - storage_key_generator::get_expected_storage_hashed_keys(), - storage_key_generator::generate_expected_storage_hashed_keys(), + storage_key_generator::get_expected_storage_hashed_keys(false), + storage_key_generator::generate_expected_storage_hashed_keys(false), ); } } @@ -995,13 +1007,16 @@ pub mod storage_key_generator { mod tests { use super::*; use codec::Encode; - use frame_support::dispatch::DispatchInfo; + use frame_support::{dispatch::DispatchInfo, pallet_prelude::ValidTransaction}; use sc_block_builder::BlockBuilderProvider; use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_consensus::BlockOrigin; use sp_core::{storage::well_known_keys::HEAP_PAGES, traits::CallContext}; use sp_keyring::AccountKeyring; - use sp_runtime::{traits::SignedExtension, transaction_validity::InvalidTransaction}; + use sp_runtime::{ + traits::{Hash, SignedExtension}, + transaction_validity::InvalidTransaction, + }; use substrate_test_runtime_client::{ prelude::*, runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, }; @@ -1093,7 +1108,7 @@ mod tests { .cloned() .map(storage_key_generator::hex) .collect::>(), - storage_key_generator::get_expected_storage_hashed_keys() + storage_key_generator::get_expected_storage_hashed_keys(false) ); } @@ -1132,7 +1147,15 @@ mod tests { log: DigestItem::Other(vec![]) }, ), - Ok(Default::default()), + Ok(ValidTransaction { + provides: vec![BlakeTwo256::hash_of( + &substrate_test_pallet::Call::::deposit_log_digest_item { + log: DigestItem::Other(vec![]) + } + ) + .encode()], + ..Default::default() + }), ); assert_eq!( @@ -1140,7 +1163,16 @@ mod tests { TransactionSource::External, &substrate_test_pallet::Call::storage_change { key: vec![], value: None }, ), - Ok(Default::default()), + Ok(ValidTransaction { + provides: vec![BlakeTwo256::hash_of( + &substrate_test_pallet::Call::::storage_change { + key: vec![], + value: None + } + ) + .encode()], + ..Default::default() + }), ); assert_eq!( @@ -1148,7 +1180,13 @@ mod tests { TransactionSource::External, &substrate_test_pallet::Call::read { count: 0 }, ), - Ok(Default::default()), + Ok(ValidTransaction { + provides: vec![BlakeTwo256::hash_of( + &substrate_test_pallet::Call::::read { count: 0 } + ) + .encode()], + ..Default::default() + }), ); assert_eq!( @@ -1156,7 +1194,13 @@ mod tests { TransactionSource::External, &substrate_test_pallet::Call::read_and_panic { count: 0 }, ), - Ok(Default::default()), + Ok(ValidTransaction { + provides: vec![BlakeTwo256::hash_of( + &substrate_test_pallet::Call::::read_and_panic { count: 0 } + ) + .encode()], + ..Default::default() + }), ); }); } From 5942c967c53a33a78a06059a41686e4e2d3f9352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 9 May 2023 20:23:08 +0200 Subject: [PATCH 26/55] Offchain worker custom extensions --- Cargo.lock | 1 + client/offchain/Cargo.toml | 1 + client/offchain/src/lib.rs | 28 +++++++++++++++++++--- primitives/core/src/offchain/mod.rs | 26 +++++++++----------- primitives/externalities/src/extensions.rs | 6 +++++ 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a2ac776da69a..a80d099d5e400 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9519,6 +9519,7 @@ dependencies = [ "sp-api", "sp-consensus", "sp-core", + "sp-externalities", "sp-keystore", "sp-offchain", "sp-runtime", diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index f8146d2c30762..a72339746e0ac 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -39,6 +39,7 @@ sp-core = { version = "7.0.0", path = "../../primitives/core" } sp-offchain = { version = "4.0.0-dev", path = "../../primitives/offchain" } sp-runtime = { version = "7.0.0", path = "../../primitives/runtime" } sp-keystore = { version = "0.13.0", path = "../../primitives/keystore" } +sp-externalities = { version = "0.13.0", path = "../../primitives/externalities" } log = "0.4.17" [dev-dependencies] diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 21c14e71e2790..02a83330553fa 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -54,6 +54,7 @@ use sp_runtime::{ traits::{self, Header}, }; use threadpool::ThreadPool; +use sp_externalities::Extension; mod api; @@ -97,7 +98,7 @@ impl offchain::OffchainStorage for NoOffchainStorage { } /// Options for [`OffchainWorkers`] -pub struct OffchainWorkerOptions { +pub struct OffchainWorkerOptions { /// Provides access to the runtime api. pub runtime_api_provider: Arc, /// Provides access to the keystore. @@ -116,6 +117,20 @@ pub struct OffchainWorkerOptions { /// /// If not enabled, any http request will panic. pub enable_http_requests: bool, + /// Callback to create custom [`Extension`]s that should be registered for the + /// `offchain_worker` runtime call. + /// + /// These [`Extension`]s are registered along-side the default extensions and are accessible in + /// the host functions. + /// + /// # Example: + /// + /// ```nocompile + /// custom_extensions: |block_hash| { + /// vec![MyCustomExtension::new()] + /// } + /// ``` + pub custom_extensions: CE, } /// An offchain workers manager. @@ -129,11 +144,12 @@ pub struct OffchainWorkers { transaction_pool: Option>>, network_provider: Arc, is_validator: bool, + custom_extensions: Box Vec>>, } impl OffchainWorkers { /// Creates new [`OffchainWorkers`]. - pub fn new( + pub fn new Vec> + 'static>( OffchainWorkerOptions { runtime_api_provider, keystore, @@ -142,7 +158,8 @@ impl OffchainWorkers { network_provider, is_validator, enable_http_requests, - }: OffchainWorkerOptions, + custom_extensions, + }: OffchainWorkerOptions, ) -> Self { Self { runtime_api_provider, @@ -157,6 +174,7 @@ impl OffchainWorkers { transaction_pool, is_validator, network_provider, + custom_extensions: Box::new(custom_extensions), } } } @@ -247,6 +265,7 @@ where let keystore = self.keystore.clone(); let db = self.offchain_db.clone(); let tx_pool = self.transaction_pool.clone(); + let custom_extensions = (*self.custom_extensions)(hash); self.spawn_worker(move || { let mut runtime = client.runtime_api(); @@ -273,6 +292,8 @@ where offchain::LimitedExternalities::new(capabilities, api), )); + custom_extensions.into_iter().for_each(|ext| runtime.register_extension(ext)); + let run = if version == 2 { runtime.offchain_worker(hash, &header) } else { @@ -483,6 +504,7 @@ mod tests { network_provider: network, is_validator: false, enable_http_requests: false, + custom_extensions: |_| Vec::new(), }); futures::executor::block_on(offchain.on_block_imported(&header)); diff --git a/primitives/core/src/offchain/mod.rs b/primitives/core/src/offchain/mod.rs index a9e6639807023..d395941848589 100644 --- a/primitives/core/src/offchain/mod.rs +++ b/primitives/core/src/offchain/mod.rs @@ -260,26 +260,22 @@ impl Timestamp { bitflags::bitflags! { /// Execution context extra capabilities. pub struct Capabilities: u32 { - /// Access to transaction pool. - const TRANSACTION_POOL = 0b0000_0000_0001; /// External http calls. - const HTTP = 0b0000_0000_0010; + const HTTP = 0b0000_0000_0001; /// Keystore access. - const KEYSTORE = 0b0000_0000_0100; + const KEYSTORE = 0b0000_0000_0010; /// Randomness source. - const RANDOMNESS = 0b0000_0000_1000; + const RANDOMNESS = 0b0000_0000_0100; /// Access to opaque network state. - const NETWORK_STATE = 0b0000_0001_0000; + const NETWORK_STATE = 0b0000_0000_1000; /// Access to offchain worker DB (read only). - const OFFCHAIN_DB_READ = 0b0000_0010_0000; + const OFFCHAIN_DB_READ = 0b0000_0001_0000; /// Access to offchain worker DB (writes). - const OFFCHAIN_DB_WRITE = 0b0000_0100_0000; + const OFFCHAIN_DB_WRITE = 0b0000_0010_0000; /// Manage the authorized nodes - const NODE_AUTHORIZATION = 0b0000_1000_0000; + const NODE_AUTHORIZATION = 0b0000_0100_0000; /// Access time related functionality - const TIME = 0b0001_0000_0000; - /// Access the statement store. - const STATEMENT_STORE = 0b0010_0000_0000; + const TIME = 0b0000_1000_0000; } } @@ -785,8 +781,8 @@ mod tests { assert!(!none.contains(Capabilities::KEYSTORE)); assert!(all.contains(Capabilities::KEYSTORE)); assert!(some.contains(Capabilities::KEYSTORE)); - assert!(!none.contains(Capabilities::TRANSACTION_POOL)); - assert!(all.contains(Capabilities::TRANSACTION_POOL)); - assert!(!some.contains(Capabilities::TRANSACTION_POOL)); + assert!(!none.contains(Capabilities::RANDOMNESS)); + assert!(all.contains(Capabilities::RANDOMNESS)); + assert!(!some.contains(Capabilities::TIME)); } } diff --git a/primitives/externalities/src/extensions.rs b/primitives/externalities/src/extensions.rs index 6007cbde189b6..2d204c52ec8a8 100644 --- a/primitives/externalities/src/extensions.rs +++ b/primitives/externalities/src/extensions.rs @@ -42,6 +42,12 @@ pub trait Extension: Send + Any { fn as_mut_any(&mut self) -> &mut dyn Any; } +impl Extension for Box { + fn as_mut_any(&mut self) -> &mut dyn Any { + (**self).as_mut_any() + } +} + /// Macro for declaring an extension that usable with [`Extensions`]. /// /// The extension will be an unit wrapper struct that implements [`Extension`], `Deref` and From 5b253f8757de20cd95f274eb35531a5b0a683987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 9 May 2023 22:45:01 +0200 Subject: [PATCH 27/55] More fixes --- bin/node-template/node/src/service.rs | 1 + bin/node/cli/benches/block_production.rs | 11 ------ bin/node/cli/benches/transaction_pool.rs | 9 ----- bin/node/cli/src/service.rs | 48 +++++++++++++----------- client/offchain/src/lib.rs | 6 +-- client/statement-store/src/lib.rs | 9 ++++- 6 files changed, 38 insertions(+), 46 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 8f509ab8d650a..55206e72bd024 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -189,6 +189,7 @@ pub fn new_full(mut config: Configuration) -> Result transaction_pool: Some(transaction_pool.clone()), network_provider: network.clone(), enable_http_requests: true, + custom_extensions: |_| vec![], }) .run(client.clone(), task_manager.spawn_handle()) .boxed(), diff --git a/bin/node/cli/benches/block_production.rs b/bin/node/cli/benches/block_production.rs index 527b145c62c46..5ee538d18d6a8 100644 --- a/bin/node/cli/benches/block_production.rs +++ b/bin/node/cli/benches/block_production.rs @@ -21,7 +21,6 @@ use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughpu use kitchensink_runtime::{constants::currency::*, BalancesCall}; use node_cli::service::{create_extrinsic, FullClient}; use sc_block_builder::{BlockBuilderProvider, BuiltBlock, RecordProof}; -use sc_client_api::execution_extensions::ExecutionStrategies; use sc_consensus::{ block_import::{BlockImportParams, ForkChoiceStrategy}, BlockImport, StateAction, @@ -56,9 +55,6 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase { let spec = Box::new(node_cli::chain_spec::development_config()); - // NOTE: We enforce the use of the WASM runtime to benchmark block production using WASM. - let execution_strategy = sc_client_api::ExecutionStrategy::AlwaysWasm; - let config = Configuration { impl_name: "BenchmarkImpl".into(), impl_version: "1.0".into(), @@ -77,13 +73,6 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase { wasm_method: WasmExecutionMethod::Compiled { instantiation_strategy: WasmtimeInstantiationStrategy::PoolingCopyOnWrite, }, - execution_strategies: ExecutionStrategies { - syncing: execution_strategy, - importing: execution_strategy, - block_construction: execution_strategy, - offchain_worker: execution_strategy, - other: execution_strategy, - }, rpc_addr: None, rpc_max_connections: Default::default(), rpc_cors: None, diff --git a/bin/node/cli/benches/transaction_pool.rs b/bin/node/cli/benches/transaction_pool.rs index 44ebe1e7d4fe6..d3e8c02a958f7 100644 --- a/bin/node/cli/benches/transaction_pool.rs +++ b/bin/node/cli/benches/transaction_pool.rs @@ -23,7 +23,6 @@ use futures::{future, StreamExt}; use kitchensink_runtime::{constants::currency::*, BalancesCall, SudoCall}; use node_cli::service::{create_extrinsic, fetch_nonce, FullClient, TransactionPool}; use node_primitives::AccountId; -use sc_client_api::execution_extensions::ExecutionStrategies; use sc_service::{ config::{ BlocksPruning, DatabaseSource, KeystoreConfig, NetworkConfiguration, OffchainWorkerConfig, @@ -70,14 +69,6 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase { blocks_pruning: BlocksPruning::KeepAll, chain_spec: spec, wasm_method: Default::default(), - // NOTE: we enforce the use of the native runtime to make the errors more debuggable - execution_strategies: ExecutionStrategies { - syncing: sc_client_api::ExecutionStrategy::NativeWhenPossible, - importing: sc_client_api::ExecutionStrategy::NativeWhenPossible, - block_construction: sc_client_api::ExecutionStrategy::NativeWhenPossible, - offchain_worker: sc_client_api::ExecutionStrategy::NativeWhenPossible, - other: sc_client_api::ExecutionStrategy::NativeWhenPossible, - }, rpc_addr: None, rpc_max_connections: Default::default(), rpc_cors: None, diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index b4acce47a23bd..1abe3485ac96a 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -379,24 +379,6 @@ pub fn new_full_base( warp_sync_params: Some(WarpSyncParams::WithProvider(warp_sync)), })?; - if config.offchain_worker.enabled { - task_manager.spawn_handle().spawn( - "offchain-workers-runner", - "offchain-work", - sc_offchain::OffchainWorkers::new(sc_offchain::OffchainWorkerOptions { - runtime_api_provider: client.clone(), - keystore: Some(keystore_container.keystore()), - offchain_db: backend.offchain_storage(), - transaction_pool: Some(transaction_pool.clone()), - network_provider: network.clone(), - is_validator: config.role.is_authority(), - enable_http_requests: true, - }) - .run(client.clone(), task_manager.spawn_handle()) - .boxed(), - ); - } - let role = config.role.clone(); let force_authoring = config.force_authoring; let backoff_authoring_blocks = @@ -404,10 +386,11 @@ pub fn new_full_base( let name = config.network.node_name.clone(); let enable_grandpa = !config.disable_grandpa; let prometheus_registry = config.prometheus_registry().cloned(); + let enable_offchain_worker = config.offchain_worker.enabled; let rpc_handlers = sc_service::spawn_tasks(sc_service::SpawnTasksParams { config, - backend, + backend: backend.clone(), client: client.clone(), keystore: keystore_container.keystore(), network: network.clone(), @@ -532,14 +515,14 @@ pub fn new_full_base( // need a keystore, regardless of which protocol we use below. let keystore = if role.is_authority() { Some(keystore_container.keystore()) } else { None }; - let config = grandpa::Config { + let grandpa_config = grandpa::Config { // FIXME #1578 make this available through chainspec gossip_duration: std::time::Duration::from_millis(333), justification_period: 512, name: Some(name), observer_enabled: false, keystore, - local_role: role, + local_role: role.clone(), telemetry: telemetry.as_ref().map(|x| x.handle()), protocol_name: grandpa_protocol_name, }; @@ -552,7 +535,7 @@ pub fn new_full_base( // been tested extensively yet and having most nodes in a network run it // could lead to finality stalls. let grandpa_config = grandpa::GrandpaParams { - config, + config: grandpa_config, link: grandpa_link, network: network.clone(), sync: Arc::new(sync_service.clone()), @@ -591,6 +574,27 @@ pub fn new_full_base( statement_handler.run(), ); + if enable_offchain_worker { + task_manager.spawn_handle().spawn( + "offchain-workers-runner", + "offchain-work", + sc_offchain::OffchainWorkers::new(sc_offchain::OffchainWorkerOptions { + runtime_api_provider: client.clone(), + keystore: Some(keystore_container.keystore()), + offchain_db: backend.offchain_storage(), + transaction_pool: Some(transaction_pool.clone()), + network_provider: network.clone(), + is_validator: role.is_authority(), + enable_http_requests: true, + custom_extensions: move |_| { + vec![Box::new(statement_store.clone().as_statement_store_ext()) as Box<_>] + }, + }) + .run(client.clone(), task_manager.spawn_handle()) + .boxed(), + ); + } + network_starter.start_network(); Ok(NewFullBase { task_manager, diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 02a83330553fa..5d8a7c842847e 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -48,13 +48,13 @@ use sc_network::{NetworkPeers, NetworkStateInfo}; use sc_transaction_pool_api::OffchainSubmitTransaction; use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_core::{offchain, traits::SpawnNamed}; +use sp_externalities::Extension; use sp_keystore::{KeystoreExt, KeystorePtr}; use sp_runtime::{ generic::BlockId, traits::{self, Header}, }; use threadpool::ThreadPool; -use sp_externalities::Extension; mod api; @@ -144,12 +144,12 @@ pub struct OffchainWorkers { transaction_pool: Option>>, network_provider: Arc, is_validator: bool, - custom_extensions: Box Vec>>, + custom_extensions: Box Vec> + Send>, } impl OffchainWorkers { /// Creates new [`OffchainWorkers`]. - pub fn new Vec> + 'static>( + pub fn new Vec> + Send + 'static>( OffchainWorkerOptions { runtime_api_provider, keystore, diff --git a/client/statement-store/src/lib.rs b/client/statement-store/src/lib.rs index 178c40f552994..8a0a9cf1bc235 100644 --- a/client/statement-store/src/lib.rs +++ b/client/statement-store/src/lib.rs @@ -59,7 +59,9 @@ use sp_blockchain::HeaderBackend; use sp_core::{hexdisplay::HexDisplay, traits::SpawnNamed, Decode, Encode}; use sp_runtime::traits::Block as BlockT; use sp_statement_store::{ - runtime_api::{InvalidStatement, StatementSource, ValidStatement, ValidateStatement}, + runtime_api::{ + InvalidStatement, StatementSource, StatementStoreExt, ValidStatement, ValidateStatement, + }, AccountId, BlockHash, Channel, DecryptionKey, Hash, NetworkPriority, Proof, Result, Statement, SubmitResult, Topic, }; @@ -695,6 +697,11 @@ impl Store { fn set_time(&mut self, time: u64) { self.time_override = Some(time); } + + /// Returns `self` as [`StatementStoreExt`]. + pub fn as_statement_store_ext(self: Arc) -> StatementStoreExt { + StatementStoreExt::new(self) + } } impl StatementStore for Store { From 16714b6338e69718f0f7be1e2552458758dbeb98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 23 May 2023 20:12:40 +0200 Subject: [PATCH 28/55] Make offchain tx pool creation reusable Introduces an `OffchainTransactionPoolFactory` for creating offchain transactions pools that can be registered in the runtime externalities context. This factory will be required for a later pr to make the creation of offchain transaction pools easier. --- Cargo.lock | 2 + client/api/src/execution_extensions.rs | 55 ++++--------------- client/offchain/src/lib.rs | 35 +++--------- client/transaction-pool/api/Cargo.toml | 2 + client/transaction-pool/api/src/lib.rs | 76 ++++++++++++++++++++++---- client/transaction-pool/src/lib.rs | 18 ++++-- 6 files changed, 100 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae6557c1b8631..a043bb0221508 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10124,9 +10124,11 @@ dependencies = [ "async-trait", "futures", "log", + "parity-scale-codec", "serde", "serde_json", "sp-blockchain", + "sp-core", "sp-runtime", "thiserror", ] diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 9ff4b6db418ad..20dc34d19b1de 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -22,20 +22,16 @@ //! strategy for the runtime calls and provide the right `Externalities` //! extensions to support APIs for particular execution context & capabilities. -use codec::Decode; use parking_lot::RwLock; -use sc_transaction_pool_api::OffchainSubmitTransaction; +use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sp_core::{ - offchain::{self, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt}, + offchain::{self, OffchainDbExt, OffchainWorkerExt}, traits::{ReadRuntimeVersion, ReadRuntimeVersionExt}, ExecutionContext, }; use sp_externalities::{Extension, Extensions}; use sp_keystore::{KeystoreExt, KeystorePtr}; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, NumberFor}, -}; +use sp_runtime::traits::{Block as BlockT, NumberFor}; pub use sp_state_machine::ExecutionStrategy; use sp_state_machine::{DefaultHandler, ExecutionManager}; use std::{ @@ -168,11 +164,7 @@ pub struct ExecutionExtensions { offchain_db: Option>, // FIXME: these three are only RwLock because of https://github.com/paritytech/substrate/issues/4587 // remove when fixed. - // To break retain cycle between `Client` and `TransactionPool` we require this - // extension to be a `Weak` reference. - // That's also the reason why it's being registered lazily instead of - // during initialization. - transaction_pool: RwLock>>>, + transaction_pool_factory: RwLock>>, extensions_factory: RwLock>>, statement_store: RwLock>>, read_runtime_version: Arc, @@ -194,7 +186,7 @@ impl ExecutionExtensions { keystore, offchain_db, extensions_factory: RwLock::new(extensions_factory), - transaction_pool, + transaction_pool_factory: transaction_pool, statement_store, read_runtime_version, } @@ -211,11 +203,11 @@ impl ExecutionExtensions { } /// Register transaction pool extension. - pub fn register_transaction_pool(&self, pool: &Arc) - where - T: OffchainSubmitTransaction + 'static, - { - *self.transaction_pool.write() = Some(Arc::downgrade(pool) as _); + pub fn register_transaction_pool_factory( + &self, + factory: OffchainTransactionPoolFactory, + ) { + *self.transaction_pool_factory.write() = Some(factory); } /// Register statement store extension. @@ -245,11 +237,8 @@ impl ExecutionExtensions { } if capabilities.contains(offchain::Capabilities::TRANSACTION_POOL) { - if let Some(pool) = self.transaction_pool.read().as_ref().and_then(|x| x.upgrade()) { - extensions.register(TransactionPoolExt(Box::new(TransactionPoolAdapter { - at: BlockId::Hash(block_hash), - pool, - }) as _)); + if let Some(pool) = self.transaction_pool_factory.read().as_ref() { + extensions.register(pool.offchain_transaction_pool(block_hash)); } } @@ -303,23 +292,3 @@ impl ExecutionExtensions { (manager, self.extensions(block_hash, block_number, context)) } } - -/// A wrapper type to pass `BlockId` to the actual transaction pool. -struct TransactionPoolAdapter { - at: BlockId, - pool: Arc>, -} - -impl offchain::TransactionPool for TransactionPoolAdapter { - fn submit_transaction(&mut self, data: Vec) -> Result<(), ()> { - let xt = match Block::Extrinsic::decode(&mut &*data) { - Ok(xt) => xt, - Err(e) => { - log::warn!("Unable to decode extrinsic: {:?}: {}", data, e); - return Err(()) - }, - }; - - self.pool.submit_at(&self.at, xt) - } -} diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 013bb27980e29..af8d183105a55 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -248,16 +248,15 @@ mod tests { use sc_client_api::Backend as _; use sc_network::{config::MultiaddrWithPeerId, types::ProtocolName}; use sc_peerset::ReputationChange; - use sc_transaction_pool::{BasicPool, FullChainApi}; + use sc_transaction_pool::BasicPool; use sc_transaction_pool_api::{InPoolTransaction, TransactionPool}; use sp_consensus::BlockOrigin; - use sp_runtime::generic::BlockId; use std::{collections::HashSet, sync::Arc}; use substrate_test_runtime_client::{ runtime::{ - substrate_test_pallet::pallet::Call as PalletCall, Block, ExtrinsicBuilder, RuntimeCall, + substrate_test_pallet::pallet::Call as PalletCall, ExtrinsicBuilder, RuntimeCall, }, - ClientBlockImportExt, DefaultTestClientBuilderExt, TestClient, TestClientBuilderExt, + ClientBlockImportExt, DefaultTestClientBuilderExt, TestClientBuilderExt, }; struct TestNetwork(); @@ -338,34 +337,14 @@ mod tests { } } - struct TestPool(Arc, Block>>); - - impl sc_transaction_pool_api::OffchainSubmitTransaction for TestPool { - fn submit_at( - &self, - at: &BlockId, - extrinsic: ::Extrinsic, - ) -> Result<(), ()> { - let source = sc_transaction_pool_api::TransactionSource::Local; - futures::executor::block_on(self.0.submit_one(&at, source, extrinsic)) - .map(|_| ()) - .map_err(|_| ()) - } - } - #[test] fn should_call_into_runtime_and_produce_extrinsic() { sp_tracing::try_init_simple(); let client = Arc::new(substrate_test_runtime_client::new()); let spawner = sp_core::testing::TaskExecutor::new(); - let pool = TestPool(BasicPool::new_full( - Default::default(), - true.into(), - None, - spawner, - client.clone(), - )); + let pool = + BasicPool::new_full(Default::default(), true.into(), None, spawner, client.clone()); let network = Arc::new(TestNetwork()); let header = client.header(client.chain_info().genesis_hash).unwrap().unwrap(); @@ -374,9 +353,9 @@ mod tests { futures::executor::block_on(offchain.on_block_imported(&header, network, false)); // then - assert_eq!(pool.0.status().ready, 1); + assert_eq!(pool.status().ready, 1); assert!(matches!( - pool.0.ready().next().unwrap().data().function, + pool.ready().next().unwrap().data().function, RuntimeCall::SubstrateTest(PalletCall::storage_change { .. }) )); } diff --git a/client/transaction-pool/api/Cargo.toml b/client/transaction-pool/api/Cargo.toml index e14a3ff4f3839..a930761467b50 100644 --- a/client/transaction-pool/api/Cargo.toml +++ b/client/transaction-pool/api/Cargo.toml @@ -10,11 +10,13 @@ description = "Transaction pool client facing API." [dependencies] async-trait = "0.1.57" +codec = { package = "parity-scale-codec", version = "3.2.2" } futures = "0.3.21" log = "0.4.17" serde = { version = "1.0.136", features = ["derive"] } thiserror = "1.0.30" sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } +sp-core = { version = "7.0.0", default-features = false, path = "../../../primitives/core" } sp-runtime = { version = "7.0.0", default-features = false, path = "../../../primitives/runtime" } [dev-dependencies] diff --git a/client/transaction-pool/api/src/lib.rs b/client/transaction-pool/api/src/lib.rs index 428d0aed62efe..9e97acaf4338f 100644 --- a/client/transaction-pool/api/src/lib.rs +++ b/client/transaction-pool/api/src/lib.rs @@ -24,11 +24,17 @@ pub mod error; use async_trait::async_trait; use futures::{Future, Stream}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use sp_core::offchain::TransactionPoolExt; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Member, NumberFor}, }; -use std::{collections::HashMap, hash::Hash, pin::Pin, sync::Arc}; +use std::{ + collections::HashMap, + hash::Hash, + pin::Pin, + sync::{Arc, Weak}, +}; const LOG_TARGET: &str = "txpool::api"; @@ -329,29 +335,27 @@ pub trait LocalTransactionPool: Send + Sync { /// `TransactionSource::Local`. fn submit_local( &self, - at: &BlockId, + at: ::Hash, xt: LocalTransactionFor, ) -> Result; } -/// An abstraction for transaction pool. +/// An abstraction for [`LocalTransactionPool`] /// -/// This trait is used by offchain calls to be able to submit transactions. -/// The main use case is for offchain workers, to feed back the results of computations, -/// but since the transaction pool access is a separate `ExternalitiesExtension` it can -/// be also used in context of other offchain calls. For one may generate and submit -/// a transaction for some misbehavior reports (say equivocation). -pub trait OffchainSubmitTransaction: Send + Sync { +/// We want to use a transaction pool in [`OffchainTransactionPoolFactory`] in a `Arc` without +/// bleeding the associated types besides the `Block`. Thus, this abstraction here exists to achieve +/// the wrapping in a `Arc`. +trait OffchainSubmitTransaction: Send + Sync { /// Submit transaction. /// /// The transaction will end up in the pool and be propagated to others. - fn submit_at(&self, at: &BlockId, extrinsic: Block::Extrinsic) -> Result<(), ()>; + fn submit_at(&self, at: Block::Hash, extrinsic: Block::Extrinsic) -> Result<(), ()>; } impl OffchainSubmitTransaction for TPool { fn submit_at( &self, - at: &BlockId, + at: ::Hash, extrinsic: ::Extrinsic, ) -> Result<(), ()> { log::debug!( @@ -372,6 +376,56 @@ impl OffchainSubmitTransaction for TP } } +/// Factory for creating [`TransactionPoolExt`]s. +/// +/// This provides an easy way for creating [`TransactionPoolExt`] extensions for registering them in +/// the wasm execution environment to send transactions from an offchain call to the runtime. +#[derive(Clone)] +pub struct OffchainTransactionPoolFactory { + // To break retain cycle between `Client` and `TransactionPool` we require this + // extension to be a `Weak` reference. + pool: Weak>, +} + +impl OffchainTransactionPoolFactory { + /// Creates a new instance using the given `tx_pool`. + pub fn new + 'static>(tx_pool: &Arc) -> Self { + Self { pool: Arc::downgrade(tx_pool) as Weak<_> } + } + + /// Returns an instance of [`TransactionPoolExt`] bound to the given `block_hash`. + /// + /// Transactions that are being submitted by this instance will be submitted with `block_hash` + /// as context for validation. + pub fn offchain_transaction_pool(&self, block_hash: Block::Hash) -> TransactionPoolExt { + TransactionPoolExt::new(OffchainTransactionPool { pool: self.pool.clone(), block_hash }) + } +} + +/// Wraps a `pool` and `block_hash` to implemented [`sp_core::offchain::TransactionPool`]. +struct OffchainTransactionPool { + block_hash: Block::Hash, + pool: Weak>, +} + +impl sp_core::offchain::TransactionPool for OffchainTransactionPool { + fn submit_transaction(&mut self, extrinsic: Vec) -> Result<(), ()> { + let extrinsic = match codec::Decode::decode(&mut &extrinsic[..]) { + Ok(t) => t, + Err(e) => { + log::error!( + target: LOG_TARGET, + "Failed to decode extrinsic in `OffchainTransactionPool::submit_transaction`: {e:?}" + ); + + return Err(()) + }, + }; + + self.pool.upgrade().ok_or(())?.submit_at(self.block_hash, extrinsic) + } +} + /// Wrapper functions to keep the API backwards compatible over the wire for the old RPC spec. mod v1_compatible { use serde::{Deserialize, Deserializer, Serialize, Serializer}; diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index c3a85a373ba31..1b438bd7e4f2f 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -52,8 +52,8 @@ use std::{ use graph::{ExtrinsicHash, IsValidator}; use sc_transaction_pool_api::{ error::Error as TxPoolError, ChainEvent, ImportNotificationStream, MaintainedTransactionPool, - PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, TransactionPool, TransactionSource, - TransactionStatusStreamFor, TxHash, + OffchainTransactionPoolFactory, PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, + TransactionPool, TransactionSource, TransactionStatusStreamFor, TxHash, }; use sp_core::traits::SpawnEssentialNamed; use sp_runtime::{ @@ -397,7 +397,9 @@ where )); // make transaction pool available for off-chain runtime calls. - client.execution_extensions().register_transaction_pool(&pool); + client + .execution_extensions() + .register_transaction_pool_factory(OffchainTransactionPoolFactory::new(&pool)); pool } @@ -421,7 +423,7 @@ where fn submit_local( &self, - at: &BlockId, + at: Block::Hash, xt: sc_transaction_pool_api::LocalTransactionFor, ) -> Result { use sp_runtime::{ @@ -430,7 +432,11 @@ where let validity = self .api - .validate_transaction_blocking(at, TransactionSource::Local, xt.clone())? + .validate_transaction_blocking( + &BlockId::hash(at), + TransactionSource::Local, + xt.clone(), + )? .map_err(|e| { Self::Error::Pool(match e { TransactionValidityError::Invalid(i) => TxPoolError::InvalidTransaction(i), @@ -441,7 +447,7 @@ where let (hash, bytes) = self.pool.validated_pool().api().hash_and_length(&xt); let block_number = self .api - .block_id_to_number(at)? + .block_id_to_number(&BlockId::hash(at))? .ok_or_else(|| error::Error::BlockIdConversion(format!("{:?}", at)))?; let validated = ValidatedTransaction::valid_at( From 902f1a4fbdb64af5341423a478b5a578d9ad10f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 25 May 2023 23:49:00 +0200 Subject: [PATCH 29/55] Fixes --- Cargo.lock | 1 + client/offchain/src/lib.rs | 38 +++---------------- client/transaction-pool/api/src/lib.rs | 33 ++++++++++------ client/transaction-pool/src/lib.rs | 4 +- test-utils/runtime/src/lib.rs | 2 +- .../runtime/src/substrate_test_pallet.rs | 2 +- 6 files changed, 33 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1abdcca5c731c..6304b1ba3426d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5453,6 +5453,7 @@ dependencies = [ "sc-executor", "sc-keystore", "sc-network", + "sc-offchain", "sc-rpc", "sc-rpc-api", "sc-service", diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 853d4d3996db5..ed3583117a738 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -37,7 +37,6 @@ use std::{fmt, sync::Arc}; -use codec::Decode; use futures::{ future::{ready, Future}, prelude::*, @@ -45,15 +44,12 @@ use futures::{ use parking_lot::Mutex; use sc_client_api::BlockchainEvents; use sc_network::{NetworkPeers, NetworkStateInfo}; -use sc_transaction_pool_api::OffchainSubmitTransaction; +use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_core::{offchain, traits::SpawnNamed}; use sp_externalities::Extension; use sp_keystore::{KeystoreExt, KeystorePtr}; -use sp_runtime::{ - generic::BlockId, - traits::{self, Header}, -}; +use sp_runtime::traits::{self, Header}; use threadpool::ThreadPool; mod api; @@ -108,7 +104,7 @@ pub struct OffchainWorkerOptions { /// Use [`NoOffchainStorage`] as type when passing `None` to have some type that works. pub offchain_db: Option, /// Provides access to the transaction pool. - pub transaction_pool: Option>>, + pub transaction_pool: Option>, /// Provides access to network information. pub network_provider: Arc, /// Is the node running as validator? @@ -141,7 +137,7 @@ pub struct OffchainWorkers { enable_http_requests: bool, keystore: Option, offchain_db: Option>, - transaction_pool: Option>>, + transaction_pool: Option>, network_provider: Arc, is_validator: bool, custom_extensions: Box Vec> + Send>, @@ -277,9 +273,7 @@ where } if let Some(pool) = tx_pool { - runtime.register_extension(offchain::TransactionPoolExt(Box::new( - TransactionPoolAdapter { hash, pool: pool.clone() }, - ) as _)); + runtime.register_extension(pool.offchain_transaction_pool(hash)); } if let Some(offchain_db) = db { @@ -332,26 +326,6 @@ where } } -/// A wrapper type to pass `BlockId` to the actual transaction pool. -struct TransactionPoolAdapter { - hash: Block::Hash, - pool: Arc>, -} - -impl offchain::TransactionPool for TransactionPoolAdapter { - fn submit_transaction(&mut self, data: Vec) -> Result<(), ()> { - let xt = match Block::Extrinsic::decode(&mut &*data) { - Ok(xt) => xt, - Err(e) => { - log::warn!("Unable to decode extrinsic: {:?}: {}", data, e); - return Err(()) - }, - }; - - self.pool.submit_at(&BlockId::Hash(self.hash), xt) - } -} - #[cfg(test)] mod tests { use super::*; @@ -466,7 +440,7 @@ mod tests { runtime_api_provider: client, keystore: None, offchain_db: None::, - transaction_pool: Some(Arc::new(pool.clone())), + transaction_pool: Some(OffchainTransactionPoolFactory::new(pool.clone())), network_provider: network, is_validator: false, enable_http_requests: false, diff --git a/client/transaction-pool/api/src/lib.rs b/client/transaction-pool/api/src/lib.rs index 9e97acaf4338f..9535950b3a5cf 100644 --- a/client/transaction-pool/api/src/lib.rs +++ b/client/transaction-pool/api/src/lib.rs @@ -29,12 +29,7 @@ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Member, NumberFor}, }; -use std::{ - collections::HashMap, - hash::Hash, - pin::Pin, - sync::{Arc, Weak}, -}; +use std::{collections::HashMap, hash::Hash, pin::Pin, sync::Arc}; const LOG_TARGET: &str = "txpool::api"; @@ -340,6 +335,22 @@ pub trait LocalTransactionPool: Send + Sync { ) -> Result; } +implLocalTransactionPool for Arc { + type Block = T::Block; + + type Hash = T::Hash; + + type Error = T::Error; + + fn submit_local( + &self, + at: ::Hash, + xt: LocalTransactionFor, + ) -> Result { + (**self).submit_local(at, xt) + } +} + /// An abstraction for [`LocalTransactionPool`] /// /// We want to use a transaction pool in [`OffchainTransactionPoolFactory`] in a `Arc` without @@ -384,13 +395,13 @@ impl OffchainSubmitTransaction for TP pub struct OffchainTransactionPoolFactory { // To break retain cycle between `Client` and `TransactionPool` we require this // extension to be a `Weak` reference. - pool: Weak>, + pool: Arc>, } impl OffchainTransactionPoolFactory { /// Creates a new instance using the given `tx_pool`. - pub fn new + 'static>(tx_pool: &Arc) -> Self { - Self { pool: Arc::downgrade(tx_pool) as Weak<_> } + pub fn new + 'static>(tx_pool: T) -> Self { + Self { pool: Arc::new(tx_pool) as Arc<_> } } /// Returns an instance of [`TransactionPoolExt`] bound to the given `block_hash`. @@ -405,7 +416,7 @@ impl OffchainTransactionPoolFactory { /// Wraps a `pool` and `block_hash` to implemented [`sp_core::offchain::TransactionPool`]. struct OffchainTransactionPool { block_hash: Block::Hash, - pool: Weak>, + pool: Arc>, } impl sp_core::offchain::TransactionPool for OffchainTransactionPool { @@ -422,7 +433,7 @@ impl sp_core::offchain::TransactionPool for OffchainTransactionPo }, }; - self.pool.upgrade().ok_or(())?.submit_at(self.block_hash, extrinsic) + self.pool.submit_at(self.block_hash, extrinsic) } } diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index 1dae40fa74e47..80e5925194c68 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -52,8 +52,8 @@ use std::{ use graph::{ExtrinsicHash, IsValidator}; use sc_transaction_pool_api::{ error::Error as TxPoolError, ChainEvent, ImportNotificationStream, MaintainedTransactionPool, - OffchainTransactionPoolFactory, PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, - TransactionPool, TransactionSource, TransactionStatusStreamFor, TxHash, + PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, TransactionPool, TransactionSource, + TransactionStatusStreamFor, TxHash, }; use sp_core::traits::SpawnEssentialNamed; use sp_runtime::{ diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 13c0650b2c2a7..0f1c7c72521e8 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1001,7 +1001,7 @@ pub mod storage_key_generator { mod tests { use super::*; use codec::Encode; - use frame_support::{dispatch::DispatchInfo, pallet_prelude::ValidTransaction}; + use frame_support::dispatch::DispatchInfo; use sc_block_builder::BlockBuilderProvider; use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_consensus::BlockOrigin; diff --git a/test-utils/runtime/src/substrate_test_pallet.rs b/test-utils/runtime/src/substrate_test_pallet.rs index 97728383da690..b4382d070207d 100644 --- a/test-utils/runtime/src/substrate_test_pallet.rs +++ b/test-utils/runtime/src/substrate_test_pallet.rs @@ -24,7 +24,7 @@ use frame_support::{pallet_prelude::*, storage}; use sp_core::sr25519::Public; use sp_runtime::{ - traits::{BlakeTwo256, Hash}, + traits::Hash, transaction_validity::{ InvalidTransaction, TransactionSource, TransactionValidity, ValidTransaction, }, From 671f9b95def660038238731485f7735c39e3f533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 25 May 2023 23:49:00 +0200 Subject: [PATCH 30/55] Fixes --- bin/node-template/node/src/service.rs | 5 ++++- bin/node/cli/src/service.rs | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index c2b4a4d4ec1f7..8178aafb6fdf7 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -8,6 +8,7 @@ use sc_consensus_grandpa::SharedVoterState; pub use sc_executor::NativeElseWasmExecutor; use sc_service::{error::Error as ServiceError, Configuration, TaskManager, WarpSyncParams}; use sc_telemetry::{Telemetry, TelemetryWorker}; +use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sp_consensus_aura::sr25519::AuthorityPair as AuraPair; use std::{sync::Arc, time::Duration}; @@ -187,7 +188,9 @@ pub fn new_full(config: Configuration) -> Result { is_validator: config.role.is_authority(), keystore: Some(keystore_container.keystore()), offchain_db: backend.offchain_storage(), - transaction_pool: Some(transaction_pool.clone()), + transaction_pool: Some(OffchainTransactionPoolFactory::new( + transaction_pool.clone(), + )), network_provider: network.clone(), enable_http_requests: true, custom_extensions: |_| vec![], diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 7f34e934121e6..d372b7ca7da4f 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -37,6 +37,7 @@ use sc_network_sync::SyncingService; use sc_service::{config::Configuration, error::Error as ServiceError, RpcHandlers, TaskManager}; use sc_statement_store::Store as StatementStore; use sc_telemetry::{Telemetry, TelemetryWorker}; +use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sp_api::ProvideRuntimeApi; use sp_core::crypto::Pair; use sp_runtime::{generic, traits::Block as BlockT, SaturatedConversion}; @@ -584,7 +585,9 @@ pub fn new_full_base( runtime_api_provider: client.clone(), keystore: Some(keystore_container.keystore()), offchain_db: backend.offchain_storage(), - transaction_pool: Some(transaction_pool.clone()), + transaction_pool: Some(OffchainTransactionPoolFactory::new( + transaction_pool.clone(), + )), network_provider: network.clone(), is_validator: role.is_authority(), enable_http_requests: true, From 59a6fa0d5c2b5d004c0e2bd4a91a07a008f3e955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 29 May 2023 17:27:32 +0100 Subject: [PATCH 31/55] Set offchain transaction pool in BABE before using it in the runtime --- Cargo.lock | 1 + client/consensus/babe/Cargo.toml | 1 + client/consensus/babe/src/lib.rs | 70 ++++++++++++++++++++++++------ client/consensus/babe/src/tests.rs | 20 +++++++++ 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6304b1ba3426d..adcbbb1e90b4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9022,6 +9022,7 @@ dependencies = [ "sc-network", "sc-network-test", "sc-telemetry", + "sc-transaction-pool-api", "scale-info", "sp-api", "sp-application-crypto", diff --git a/client/consensus/babe/Cargo.toml b/client/consensus/babe/Cargo.toml index 2382d064d0219..ab399246c21e5 100644 --- a/client/consensus/babe/Cargo.toml +++ b/client/consensus/babe/Cargo.toml @@ -32,6 +32,7 @@ sc-consensus-epochs = { version = "0.10.0-dev", path = "../epochs" } sc-consensus-slots = { version = "0.10.0-dev", path = "../slots" } sc-keystore = { version = "4.0.0-dev", path = "../../keystore" } sc-telemetry = { version = "4.0.0-dev", path = "../../telemetry" } +sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../transaction-pool/api" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-application-crypto = { version = "7.0.0", path = "../../../primitives/application-crypto" } sp-block-builder = { version = "4.0.0-dev", path = "../../../primitives/block-builder" } diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index 6de66449ec056..77e557aa0e153 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -106,6 +106,7 @@ use sc_consensus_slots::{ SlotInfo, StorageChanges, }; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_TRACE}; +use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_application_crypto::AppCrypto; use sp_block_builder::BlockBuilder as BlockBuilderApi; @@ -116,6 +117,7 @@ use sp_blockchain::{ use sp_consensus::{BlockOrigin, Environment, Error as ConsensusError, Proposer, SelectChain}; use sp_consensus_babe::inherents::BabeInherentData; use sp_consensus_slots::Slot; +use sp_core::traits::SpawnEssentialNamed; use sp_inherents::{CreateInherentDataProviders, InherentData, InherentDataProvider}; use sp_keystore::KeystorePtr; use sp_runtime::{ @@ -991,6 +993,7 @@ pub struct BabeVerifier { config: BabeConfiguration, epoch_changes: SharedEpochChanges, telemetry: Option, + offchain_tx_pool: OffchainTransactionPoolFactory, } impl BabeVerifier @@ -1096,8 +1099,12 @@ where }; // submit equivocation report at best block. - self.client - .runtime_api() + let mut runtime_api = self.client.runtime_api(); + + // Register the offchain tx pool to be able to use it from the runtime. + runtime_api.register_extension(self.offchain_tx_pool.offchain_transaction_pool(best_hash)); + + runtime_api .submit_report_equivocation_unsigned_extrinsic( best_hash, equivocation_proof, @@ -1765,6 +1772,38 @@ where Ok((import, link)) } +/// Parameters passed to [`import_queue`]. +pub struct ImportQueueParams<'a, Block: BlockT, BI, Client, CIDP, SelectChain, Spawn> { + /// The BABE link that is created by [`block_import`]. + pub babe_link: BabeLink, + /// The block import that should be wrapped. + pub block_import: BI, + /// Optional justification import. + pub justification_import: Option>, + /// The client to interact with the internals of the node. + pub client: Arc, + /// A [`SelectChain`] implementation. + /// + /// Used to determine the best block that should be used as basis when sending an equivocation + /// report. + pub select_chain: SelectChain, + /// Used to crate the inherent data providers. + /// + /// These inherent data providers are then used to create the inherent data that is + /// passed to the `check_inherents` runtime call. + pub create_inherent_data_providers: CIDP, + /// Spawner for spawning futures. + pub spawner: &'a Spawn, + /// Registry for prometheus metrics. + pub registry: Option<&'a Registry>, + /// Optional telemetry handle to report telemetry events. + pub telemetry: Option, + /// The offchain transaction pool factory. + /// + /// Will be used when sending equivocation reports. + pub offchain_tx_pool: OffchainTransactionPoolFactory, +} + /// Start an import queue for the BABE consensus algorithm. /// /// This method returns the import queue, some data that needs to be passed to the block authoring @@ -1774,19 +1813,22 @@ where /// /// The block import object provided must be the `BabeBlockImport` or a wrapper /// of it, otherwise crucial import logic will be omitted. -pub fn import_queue( - babe_link: BabeLink, - block_import: Inner, - justification_import: Option>, - client: Arc, - select_chain: SelectChain, - create_inherent_data_providers: CIDP, - spawner: &impl sp_core::traits::SpawnEssentialNamed, - registry: Option<&Registry>, - telemetry: Option, +pub fn import_queue( + ImportQueueParams { + babe_link, + block_import, + justification_import, + client, + select_chain, + create_inherent_data_providers, + spawner, + registry, + telemetry, + offchain_tx_pool, + }: ImportQueueParams<'_, Block, BI, Client, CIDP, SelectChain, Spawn>, ) -> ClientResult<(DefaultImportQueue, BabeWorkerHandle)> where - Inner: BlockImport< + BI: BlockImport< Block, Error = ConsensusError, Transaction = sp_api::TransactionFor, @@ -1804,6 +1846,7 @@ where SelectChain: sp_consensus::SelectChain + 'static, CIDP: CreateInherentDataProviders + Send + Sync + 'static, CIDP::InherentDataProviders: InherentDataProviderExt + Send + Sync, + Spawn: SpawnEssentialNamed, { const HANDLE_BUFFER_SIZE: usize = 1024; @@ -1814,6 +1857,7 @@ where epoch_changes: babe_link.epoch_changes.clone(), telemetry, client: client.clone(), + offchain_tx_pool, }; let (worker_tx, worker_rx) = channel(HANDLE_BUFFER_SIZE); diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index 59b4076e2fd01..987a8ed65102a 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -26,6 +26,7 @@ use sc_consensus::{BoxBlockImport, BoxJustificationImport}; use sc_consensus_epochs::{EpochIdentifier, EpochIdentifierPosition}; use sc_consensus_slots::BackoffAuthoringOnFinalizedHeadLagging; use sc_network_test::{Block as TestBlock, *}; +use sc_transaction_pool_api::LocalTransactionPool; use sp_application_crypto::key_types::BABE; use sp_consensus::{DisableProofRecording, NoNetwork as DummyOracle, Proposal}; use sp_consensus_babe::{ @@ -214,6 +215,24 @@ impl Verifier for TestVerifier { } } +struct FakeTxPool; + +impl LocalTransactionPool for FakeTxPool { + type Block = TestBlock; + + type Hash = ::Hash; + + type Error = sc_transaction_pool_api::error::Error; + + fn submit_local( + &self, + _: ::Hash, + _: ::Extrinsic, + ) -> Result { + unimplemented!() + } +} + pub struct PeerData { link: BabeLink, block_import: Mutex< @@ -283,6 +302,7 @@ impl TestNetFactory for BabeTestNet { config: data.link.config.clone(), epoch_changes: data.link.epoch_changes.clone(), telemetry: None, + offchain_tx_pool: OffchainTransactionPoolFactory::new(FakeTxPool), }, mutator: MUTATOR.with(|m| m.borrow().clone()), } From 91459764e12bfeab3daa81b59924ea49ffcba9bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 31 May 2023 00:29:44 +0100 Subject: [PATCH 32/55] Add the `offchain_tx_pool` to Grandpa as well --- Cargo.lock | 1 + client/consensus/babe/src/tests.rs | 22 ++--------------- client/consensus/grandpa/Cargo.toml | 1 + client/consensus/grandpa/src/environment.rs | 11 +++++++-- client/consensus/grandpa/src/lib.rs | 11 +++++++++ client/consensus/grandpa/src/tests.rs | 8 ++++++ client/transaction-pool/api/src/lib.rs | 27 +++++++++++++++++++-- 7 files changed, 57 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index adcbbb1e90b4b..af87346385f4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9174,6 +9174,7 @@ dependencies = [ "sc-network-gossip", "sc-network-test", "sc-telemetry", + "sc-transaction-pool-api", "sc-utils", "serde", "serde_json", diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index 987a8ed65102a..e2052a6bb5d9d 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -26,7 +26,7 @@ use sc_consensus::{BoxBlockImport, BoxJustificationImport}; use sc_consensus_epochs::{EpochIdentifier, EpochIdentifierPosition}; use sc_consensus_slots::BackoffAuthoringOnFinalizedHeadLagging; use sc_network_test::{Block as TestBlock, *}; -use sc_transaction_pool_api::LocalTransactionPool; +use sc_transaction_pool_api::{LocalTransactionPool, RejectAllTxPool}; use sp_application_crypto::key_types::BABE; use sp_consensus::{DisableProofRecording, NoNetwork as DummyOracle, Proposal}; use sp_consensus_babe::{ @@ -215,24 +215,6 @@ impl Verifier for TestVerifier { } } -struct FakeTxPool; - -impl LocalTransactionPool for FakeTxPool { - type Block = TestBlock; - - type Hash = ::Hash; - - type Error = sc_transaction_pool_api::error::Error; - - fn submit_local( - &self, - _: ::Hash, - _: ::Extrinsic, - ) -> Result { - unimplemented!() - } -} - pub struct PeerData { link: BabeLink, block_import: Mutex< @@ -302,7 +284,7 @@ impl TestNetFactory for BabeTestNet { config: data.link.config.clone(), epoch_changes: data.link.epoch_changes.clone(), telemetry: None, - offchain_tx_pool: OffchainTransactionPoolFactory::new(FakeTxPool), + offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }, mutator: MUTATOR.with(|m| m.borrow().clone()), } diff --git a/client/consensus/grandpa/Cargo.toml b/client/consensus/grandpa/Cargo.toml index 67c58d37a2cf5..aab3f46d6ebbf 100644 --- a/client/consensus/grandpa/Cargo.toml +++ b/client/consensus/grandpa/Cargo.toml @@ -32,6 +32,7 @@ prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0. sc-block-builder = { version = "0.10.0-dev", path = "../../block-builder" } sc-chain-spec = { version = "4.0.0-dev", path = "../../../client/chain-spec" } sc-client-api = { version = "4.0.0-dev", path = "../../api" } +sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../transaction-pool/api" } sc-consensus = { version = "0.10.0-dev", path = "../common" } sc-network = { version = "0.10.0-dev", path = "../../network" } sc-network-gossip = { version = "0.10.0-dev", path = "../../network-gossip" } diff --git a/client/consensus/grandpa/src/environment.rs b/client/consensus/grandpa/src/environment.rs index 67820a59cc943..19dcc32d62723 100644 --- a/client/consensus/grandpa/src/environment.rs +++ b/client/consensus/grandpa/src/environment.rs @@ -40,6 +40,8 @@ use sc_client_api::{ utils::is_descendent_of, }; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; +use sc_transaction_pool_api::OffchainTransactionPoolFactory; +use sp_api::ApiExt; use sp_blockchain::HeaderMetadata; use sp_consensus::SelectChain as SelectChainT; use sp_consensus_grandpa::{ @@ -444,6 +446,7 @@ pub(crate) struct Environment< pub(crate) metrics: Option, pub(crate) justification_sender: Option>, pub(crate) telemetry: Option, + pub(crate) offchain_tx_pool: OffchainTransactionPoolFactory, pub(crate) _phantom: PhantomData, } @@ -570,8 +573,12 @@ where // submit equivocation report at **best** block let equivocation_proof = EquivocationProof::new(authority_set.set_id, equivocation); - self.client - .runtime_api() + let mut runtime_api = self.client.runtime_api(); + + runtime_api + .register_extension(self.offchain_tx_pool.offchain_transaction_pool(best_block_hash)); + + runtime_api .submit_report_equivocation_unsigned_extrinsic( best_block_hash, equivocation_proof, diff --git a/client/consensus/grandpa/src/lib.rs b/client/consensus/grandpa/src/lib.rs index 967b64204bda3..4f8cde54399b8 100644 --- a/client/consensus/grandpa/src/lib.rs +++ b/client/consensus/grandpa/src/lib.rs @@ -70,6 +70,7 @@ use sc_client_api::{ use sc_consensus::BlockImport; use sc_network::types::ProtocolName; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; +use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; use sp_api::ProvideRuntimeApi; use sp_application_crypto::AppCrypto; @@ -687,6 +688,11 @@ pub struct GrandpaParams { pub shared_voter_state: SharedVoterState, /// TelemetryHandle instance. pub telemetry: Option, + /// Offchain transaction pool factory. + /// + /// This will be used to create an offchain transaction pool instance for sending an + /// equivocation report from the runtime. + pub offchain_tx_pool: OffchainTransactionPoolFactory, } /// Returns the configuration value to put in @@ -736,6 +742,7 @@ where prometheus_registry, shared_voter_state, telemetry, + offchain_tx_pool, } = grandpa_params; // NOTE: we have recently removed `run_grandpa_observer` from the public @@ -810,6 +817,7 @@ where shared_voter_state, justification_sender, telemetry, + offchain_tx_pool, ); let voter_work = voter_work.map(|res| match res { @@ -879,6 +887,7 @@ where shared_voter_state: SharedVoterState, justification_sender: GrandpaJustificationSender, telemetry: Option, + offchain_tx_pool: OffchainTransactionPoolFactory, ) -> Self { let metrics = match prometheus_registry.as_ref().map(Metrics::register) { Some(Ok(metrics)) => Some(metrics), @@ -903,6 +912,7 @@ where metrics: metrics.as_ref().map(|m| m.environment.clone()), justification_sender: Some(justification_sender), telemetry: telemetry.clone(), + offchain_tx_pool, _phantom: PhantomData, }); @@ -1054,6 +1064,7 @@ where metrics: self.env.metrics.clone(), justification_sender: self.env.justification_sender.clone(), telemetry: self.telemetry.clone(), + offchain_tx_pool: self.env.offchain_tx_pool.clone(), _phantom: PhantomData, }); diff --git a/client/consensus/grandpa/src/tests.rs b/client/consensus/grandpa/src/tests.rs index 274859a5f4456..69612608c0ab5 100644 --- a/client/consensus/grandpa/src/tests.rs +++ b/client/consensus/grandpa/src/tests.rs @@ -33,6 +33,7 @@ use sc_network_test::{ Block, BlockImportAdapter, FullPeerConfig, Hash, PassThroughVerifier, Peer, PeersClient, PeersFullClient, TestClient, TestNetFactory, }; +use sc_transaction_pool_api::RejectAllTxPool; use sp_api::{ApiRef, ProvideRuntimeApi}; use sp_consensus::{BlockOrigin, Error as ConsensusError, SelectChain}; use sp_consensus_grandpa::{ @@ -331,6 +332,7 @@ fn initialize_grandpa( voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), + offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), telemetry: None, }; let voter = @@ -481,6 +483,7 @@ async fn finalize_3_voters_1_full_observer() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, + offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }; run_grandpa_voter(grandpa_params).expect("all in order with client and network") @@ -573,6 +576,7 @@ async fn transition_3_voters_twice_1_full_observer() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, + offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }; voters @@ -1040,6 +1044,7 @@ async fn voter_persists_its_votes() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, + offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }; run_grandpa_voter(grandpa_params).expect("all in order with client and network") @@ -1083,6 +1088,7 @@ async fn voter_persists_its_votes() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, + offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }; run_grandpa_voter(grandpa_params) @@ -1293,6 +1299,7 @@ async fn voter_catches_up_to_latest_round_when_behind() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, + offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }; Box::pin(run_grandpa_voter(grandpa_params).expect("all in order with client and network")) @@ -1422,6 +1429,7 @@ where justification_sender: None, telemetry: None, _phantom: PhantomData, + offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), } } diff --git a/client/transaction-pool/api/src/lib.rs b/client/transaction-pool/api/src/lib.rs index 9535950b3a5cf..347232166f643 100644 --- a/client/transaction-pool/api/src/lib.rs +++ b/client/transaction-pool/api/src/lib.rs @@ -29,7 +29,7 @@ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Member, NumberFor}, }; -use std::{collections::HashMap, hash::Hash, pin::Pin, sync::Arc}; +use std::{collections::HashMap, hash::Hash, marker::PhantomData, pin::Pin, sync::Arc}; const LOG_TARGET: &str = "txpool::api"; @@ -335,7 +335,7 @@ pub trait LocalTransactionPool: Send + Sync { ) -> Result; } -implLocalTransactionPool for Arc { +impl LocalTransactionPool for Arc { type Block = T::Block; type Hash = T::Hash; @@ -460,6 +460,29 @@ mod v1_compatible { } } +/// Transaction pool that rejects all submitted transactions. +/// +/// Could be used for example in tests. +pub struct RejectAllTxPool(PhantomData); + +impl Default for RejectAllTxPool { + fn default() -> Self { + Self(PhantomData) + } +} + +impl LocalTransactionPool for RejectAllTxPool { + type Block = Block; + + type Hash = Block::Hash; + + type Error = error::Error; + + fn submit_local(&self, _: Block::Hash, _: Block::Extrinsic) -> Result { + Err(error::Error::ImmediatelyDropped) + } +} + #[cfg(test)] mod tests { use super::*; From c29dcb3feb3cb55714ef0572ffd0c1916bbc63b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 31 May 2023 00:42:52 +0100 Subject: [PATCH 33/55] Fix the nodes --- bin/node-template/node/src/service.rs | 3 ++- bin/node/cli/src/service.rs | 33 +++++++++++++++------------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 8178aafb6fdf7..b6f974bde7dc6 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -237,7 +237,7 @@ pub fn new_full(config: Configuration) -> Result { let proposer_factory = sc_basic_authorship::ProposerFactory::new( task_manager.spawn_handle(), client.clone(), - transaction_pool, + transaction_pool.clone(), prometheus_registry.as_ref(), telemetry.as_ref().map(|x| x.handle()), ); @@ -313,6 +313,7 @@ pub fn new_full(config: Configuration) -> Result { prometheus_registry, shared_voter_state: SharedVoterState::empty(), telemetry: telemetry.as_ref().map(|x| x.handle()), + offchain_tx_pool: OffchainTransactionPoolFactory::new(transaction_pool), }; // the GRANDPA voter task is considered infallible, i.e. diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index d372b7ca7da4f..6f1b5052d90fa 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -206,27 +206,29 @@ pub fn new_partial( )?; let slot_duration = babe_link.config().slot_duration(); - let (import_queue, babe_worker_handle) = sc_consensus_babe::import_queue( - babe_link.clone(), - block_import.clone(), - Some(Box::new(justification_import)), - client.clone(), - select_chain.clone(), - move |_, ()| async move { - let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); + let (import_queue, babe_worker_handle) = + sc_consensus_babe::import_queue(sc_consensus_babe::ImportQueueParams { + babe_link: babe_link.clone(), + block_import: block_import.clone(), + justification_import: Some(Box::new(justification_import)), + client: client.clone(), + select_chain: select_chain.clone(), + create_inherent_data_providers: move |_, ()| async move { + let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); - let slot = + let slot = sp_consensus_babe::inherents::InherentDataProvider::from_timestamp_and_slot_duration( *timestamp, slot_duration, ); - Ok((slot, timestamp)) - }, - &task_manager.spawn_essential_handle(), - config.prometheus_registry(), - telemetry.as_ref().map(|x| x.handle()), - )?; + Ok((slot, timestamp)) + }, + spawner: &task_manager.spawn_essential_handle(), + registry: config.prometheus_registry(), + telemetry: telemetry.as_ref().map(|x| x.handle()), + offchain_tx_pool: OffchainTransactionPoolFactory::new(transaction_pool.clone()), + })?; let import_setup = (block_import, grandpa_link, babe_link); @@ -546,6 +548,7 @@ pub fn new_full_base( voting_rule: grandpa::VotingRulesBuilder::default().build(), prometheus_registry: prometheus_registry.clone(), shared_voter_state, + offchain_tx_pool: OffchainTransactionPoolFactory::new(transaction_pool.clone()), }; // the GRANDPA voter task is considered infallible, i.e. From 23578b1fadd384897e9d08f13cd8482d0bd99ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 2 Jun 2023 14:04:01 +0100 Subject: [PATCH 34/55] Print some error when using the old warnings --- client/cli/src/params/import_params.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs index f70265f560e16..5fa3771378582 100644 --- a/client/cli/src/params/import_params.rs +++ b/client/cli/src/params/import_params.rs @@ -100,6 +100,8 @@ impl ImportParams { /// Get the WASM execution method from the parameters pub fn wasm_method(&self) -> sc_service::config::WasmExecutionMethod { + self.execution_strategies.check_usage_and_print_deprecation_warning(); + crate::execution_method_from_cli(self.wasm_method, self.wasmtime_instantiation_strategy) } @@ -152,3 +154,23 @@ pub struct ExecutionStrategiesParams { )] pub execution: Option, } + +impl ExecutionStrategiesParams { + /// Check if one of the parameters is still passed and print a warning if so. + fn check_usage_and_print_deprecation_warning(&self) { + for (param, name) in [ + (&self.execution_syncing, "execution-syncing"), + (&self.execution_import_block, "execution-import-block"), + (&self.execution_block_construction, "execution-block-construction"), + (&self.execution_offchain_worker, "execution-offchain-worker"), + (&self.execution_other, "execution-other"), + (&self.execution, "execution"), + ] { + if param.is_some() { + eprintln!( + "CLI parameter `--{name}` is deprecated and will be removed in the future!" + ); + } + } + } +} From 0e0325cbc6fd093ee68a3cad477c9b3576524035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 2 Jun 2023 14:38:41 +0100 Subject: [PATCH 35/55] Fix merge issues --- client/offchain/Cargo.toml | 6 +- primitives/api/Cargo.toml | 1 + .../api/proc-macro/src/impl_runtime_apis.rs | 84 +++++++------------ primitives/session/Cargo.toml | 2 +- 4 files changed, 37 insertions(+), 56 deletions(-) diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index 317511e78f2de..831ca5d6d6fc9 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -36,9 +36,9 @@ sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } sp-core = { version = "21.0.0", path = "../../primitives/core" } sp-offchain = { version = "4.0.0-dev", path = "../../primitives/offchain" } -sp-runtime = { version = "7.0.0", path = "../../primitives/runtime" } -sp-keystore = { version = "0.13.0", path = "../../primitives/keystore" } -sp-externalities = { version = "0.13.0", path = "../../primitives/externalities" } +sp-runtime = { version = "24.0.0", path = "../../primitives/runtime" } +sp-keystore = { version = "0.27.0", path = "../../primitives/keystore" } +sp-externalities = { version = "0.19.0", path = "../../primitives/externalities" } log = "0.4.17" [dev-dependencies] diff --git a/primitives/api/Cargo.toml b/primitives/api/Cargo.toml index 9f1ab2a6db0d5..c346f536dc36c 100644 --- a/primitives/api/Cargo.toml +++ b/primitives/api/Cargo.toml @@ -18,6 +18,7 @@ sp-api-proc-macro = { version = "4.0.0-dev", path = "proc-macro" } sp-core = { version = "21.0.0", default-features = false, path = "../core" } sp-std = { version = "8.0.0", default-features = false, path = "../std" } sp-runtime = { version = "24.0.0", default-features = false, path = "../runtime" } +sp-externalities = { version = "0.19.0", default-features = false, optional = true, path = "../externalities" } sp-version = { version = "22.0.0", default-features = false, path = "../version" } sp-state-machine = { version = "0.28.0", default-features = false, optional = true, path = "../state-machine" } sp-trie = { version = "22.0.0", default-features = false, optional = true, path = "../trie" } diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 19efd02cc0208..7a4a6c70f675b 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -226,38 +226,18 @@ fn generate_runtime_api_base_structures() -> Result { Ok(quote!( pub struct RuntimeApi {} #crate_::std_enabled! { - /// Implements all runtime apis for the client side. - pub struct RuntimeApiImpl + 'static> { - call: &'static C, - commit_on_success: std::cell::RefCell, - changes: std::cell::RefCell<#crate_::OverlayedChanges>, - storage_transaction_cache: std::cell::RefCell< - #crate_::StorageTransactionCache - >, - recorder: std::option::Option<#crate_::ProofRecorder>, - call_context: #crate_::CallContext, - extensions: std::cell::RefCell<#crate_::Extensions>, - extensions_generated_for: std::cell::RefCell>, - } - - impl> #crate_::ApiExt for - RuntimeApiImpl - { - type StateBackend = C::StateBackend; - - fn execute_in_transaction #crate_::TransactionOutcome, R>( - &self, - call: F, - ) -> R where Self: Sized { - self.start_transaction(); - - *std::cell::RefCell::borrow_mut(&self.commit_on_success) = false; - let res = call(self); - *std::cell::RefCell::borrow_mut(&self.commit_on_success) = true; - - self.commit_or_rollback(std::matches!(res, #crate_::TransactionOutcome::Commit(_))); - - res.into_inner() + /// Implements all runtime apis for the client side. + pub struct RuntimeApiImpl + 'static> { + call: &'static C, + commit_on_success: std::cell::RefCell, + changes: std::cell::RefCell<#crate_::OverlayedChanges>, + storage_transaction_cache: std::cell::RefCell< + #crate_::StorageTransactionCache + >, + recorder: std::option::Option<#crate_::ProofRecorder>, + call_context: #crate_::CallContext, + extensions: std::cell::RefCell<#crate_::Extensions>, + extensions_generated_for: std::cell::RefCell>, } impl> #crate_::ApiExt for @@ -341,16 +321,15 @@ fn generate_runtime_api_base_structures() -> Result { state_version, ) } - } - fn set_call_context(&mut self, call_context: #crate_::CallContext) { - self.call_context = call_context; - } + fn set_call_context(&mut self, call_context: #crate_::CallContext) { + self.call_context = call_context; + } - fn register_extension(&mut self, extension: E) { - std::cell::RefCell::borrow_mut(&self.extensions).register(extension); + fn register_extension(&mut self, extension: E) { + std::cell::RefCell::borrow_mut(&self.extensions).register(extension); + } } - } impl #crate_::ConstructRuntimeApi for RuntimeApi @@ -359,19 +338,20 @@ fn generate_runtime_api_base_structures() -> Result { { type RuntimeApi = RuntimeApiImpl; - fn construct_runtime_api<'a>( - call: &'a C, - ) -> #crate_::ApiRef<'a, Self::RuntimeApi> { - RuntimeApiImpl { - call: unsafe { std::mem::transmute(call) }, - commit_on_success: true.into(), - changes: std::default::Default::default(), - recorder: std::default::Default::default(), - storage_transaction_cache: std::default::Default::default(), - call_context: #crate_::CallContext::Offchain, - extensions: std::default::Default::default(), - extensions_generated_for: std::default::Default::default(), - }.into() + fn construct_runtime_api<'a>( + call: &'a C, + ) -> #crate_::ApiRef<'a, Self::RuntimeApi> { + RuntimeApiImpl { + call: unsafe { std::mem::transmute(call) }, + commit_on_success: true.into(), + changes: std::default::Default::default(), + recorder: std::default::Default::default(), + storage_transaction_cache: std::default::Default::default(), + call_context: #crate_::CallContext::Offchain, + extensions: std::default::Default::default(), + extensions_generated_for: std::default::Default::default(), + }.into() + } } impl> RuntimeApiImpl { diff --git a/primitives/session/Cargo.toml b/primitives/session/Cargo.toml index 4aa2631859220..f0c421eb609d3 100644 --- a/primitives/session/Cargo.toml +++ b/primitives/session/Cargo.toml @@ -20,7 +20,7 @@ sp-core = { version = "21.0.0", default-features = false, path = "../core" } sp-runtime = { version = "24.0.0", optional = true, path = "../runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } sp-std = { version = "8.0.0", default-features = false, path = "../std" } -sp-keystore = { version = "0.13.0", path = "../keystore", optional = true } +sp-keystore = { version = "0.27.0", path = "../keystore", optional = true } [features] default = [ "std" ] From 65248e91a37c665f4268017284471817bbfd66ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 5 Jun 2023 14:03:03 +0200 Subject: [PATCH 36/55] Fix compilation --- Cargo.lock | 1 + client/consensus/babe/rpc/Cargo.toml | 1 + client/consensus/babe/rpc/src/lib.rs | 25 ++++++++++++++----------- client/consensus/babe/src/tests.rs | 2 +- client/transaction-pool/src/lib.rs | 4 ++-- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 42a3521269244..7dd871d0af6d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9133,6 +9133,7 @@ dependencies = [ "sc-consensus-epochs", "sc-keystore", "sc-rpc-api", + "sc-transaction-pool-api", "serde", "serde_json", "sp-api", diff --git a/client/consensus/babe/rpc/Cargo.toml b/client/consensus/babe/rpc/Cargo.toml index c0b20eb11c95a..5a5ce80fc4368 100644 --- a/client/consensus/babe/rpc/Cargo.toml +++ b/client/consensus/babe/rpc/Cargo.toml @@ -20,6 +20,7 @@ thiserror = "1.0" sc-consensus-babe = { version = "0.10.0-dev", path = "../" } sc-consensus-epochs = { version = "0.10.0-dev", path = "../../epochs" } sc-rpc-api = { version = "0.10.0-dev", path = "../../../rpc-api" } +sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../transaction-pool/api" } sp-api = { version = "4.0.0-dev", path = "../../../../primitives/api" } sp-application-crypto = { version = "23.0.0", path = "../../../../primitives/application-crypto" } sp-blockchain = { version = "4.0.0-dev", path = "../../../../primitives/blockchain" } diff --git a/client/consensus/babe/rpc/src/lib.rs b/client/consensus/babe/rpc/src/lib.rs index 1ae15cc5453d7..617b5d4a5d232 100644 --- a/client/consensus/babe/rpc/src/lib.rs +++ b/client/consensus/babe/rpc/src/lib.rs @@ -186,6 +186,8 @@ impl From for JsonRpseeError { #[cfg(test)] mod tests { use super::*; + use sc_consensus_babe::ImportQueueParams; + use sc_transaction_pool_api::{OffchainTransactionPoolFactory, RejectAllTxPool}; use sp_consensus_babe::inherents::InherentDataProvider; use sp_core::{crypto::key_types::BABE, testing::TaskExecutor}; use sp_keyring::Sr25519Keyring; @@ -219,22 +221,23 @@ mod tests { sc_consensus_babe::block_import(config.clone(), client.clone(), client.clone()) .expect("can initialize block-import"); - let (_, babe_worker_handle) = sc_consensus_babe::import_queue( - link.clone(), - block_import.clone(), - None, - client.clone(), - longest_chain.clone(), - move |_, _| async move { + let (_, babe_worker_handle) = sc_consensus_babe::import_queue(ImportQueueParams { + babe_link: link.clone(), + block_import: block_import.clone(), + justification_import: None, + client: client.clone(), + select_chain: longest_chain.clone(), + create_inherent_data_providers: move |_, _| async move { Ok((InherentDataProvider::from_timestamp_and_slot_duration( 0.into(), slot_duration, ),)) }, - &task_executor, - None, - None, - ) + spawner: &task_executor, + registry: None, + telemetry: None, + offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + }) .unwrap(); Babe::new(client.clone(), babe_worker_handle, keystore, longest_chain, deny_unsafe) diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index e2052a6bb5d9d..6aa1c0840c674 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -26,7 +26,7 @@ use sc_consensus::{BoxBlockImport, BoxJustificationImport}; use sc_consensus_epochs::{EpochIdentifier, EpochIdentifierPosition}; use sc_consensus_slots::BackoffAuthoringOnFinalizedHeadLagging; use sc_network_test::{Block as TestBlock, *}; -use sc_transaction_pool_api::{LocalTransactionPool, RejectAllTxPool}; +use sc_transaction_pool_api::RejectAllTxPool; use sp_application_crypto::key_types::BABE; use sp_consensus::{DisableProofRecording, NoNetwork as DummyOracle, Proposal}; use sp_consensus_babe::{ diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index 1dae40fa74e47..80e5925194c68 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -52,8 +52,8 @@ use std::{ use graph::{ExtrinsicHash, IsValidator}; use sc_transaction_pool_api::{ error::Error as TxPoolError, ChainEvent, ImportNotificationStream, MaintainedTransactionPool, - OffchainTransactionPoolFactory, PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, - TransactionPool, TransactionSource, TransactionStatusStreamFor, TxHash, + PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, TransactionPool, TransactionSource, + TransactionStatusStreamFor, TxHash, }; use sp_core::traits::SpawnEssentialNamed; use sp_runtime::{ From 67b17307af7e49f75c4e2751be7d0a98f56efcb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 5 Jun 2023 14:06:05 +0200 Subject: [PATCH 37/55] Rename `babe_link` --- bin/node/cli/src/service.rs | 2 +- client/consensus/babe/rpc/src/lib.rs | 2 +- client/consensus/babe/src/lib.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 6f1b5052d90fa..c096420b0dc90 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -208,7 +208,7 @@ pub fn new_partial( let slot_duration = babe_link.config().slot_duration(); let (import_queue, babe_worker_handle) = sc_consensus_babe::import_queue(sc_consensus_babe::ImportQueueParams { - babe_link: babe_link.clone(), + link: babe_link.clone(), block_import: block_import.clone(), justification_import: Some(Box::new(justification_import)), client: client.clone(), diff --git a/client/consensus/babe/rpc/src/lib.rs b/client/consensus/babe/rpc/src/lib.rs index 617b5d4a5d232..b4582646daf35 100644 --- a/client/consensus/babe/rpc/src/lib.rs +++ b/client/consensus/babe/rpc/src/lib.rs @@ -222,7 +222,7 @@ mod tests { .expect("can initialize block-import"); let (_, babe_worker_handle) = sc_consensus_babe::import_queue(ImportQueueParams { - babe_link: link.clone(), + link: link.clone(), block_import: block_import.clone(), justification_import: None, client: client.clone(), diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index 77e557aa0e153..e4defd53df1a3 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -1775,7 +1775,7 @@ where /// Parameters passed to [`import_queue`]. pub struct ImportQueueParams<'a, Block: BlockT, BI, Client, CIDP, SelectChain, Spawn> { /// The BABE link that is created by [`block_import`]. - pub babe_link: BabeLink, + pub link: BabeLink, /// The block import that should be wrapped. pub block_import: BI, /// Optional justification import. @@ -1815,7 +1815,7 @@ pub struct ImportQueueParams<'a, Block: BlockT, BI, Client, CIDP, SelectChain, S /// of it, otherwise crucial import logic will be omitted. pub fn import_queue( ImportQueueParams { - babe_link, + link: babe_link, block_import, justification_import, client, From b1bb12d9879b5974b51db6764ce6d4e96e74f509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 5 Jun 2023 16:41:13 +0200 Subject: [PATCH 38/55] Rename to `offchain_tx_pool_factory` --- bin/node-template/node/src/service.rs | 2 +- bin/node/cli/src/service.rs | 4 ++-- client/consensus/babe/rpc/src/lib.rs | 2 +- client/consensus/babe/src/lib.rs | 10 +++++----- client/consensus/babe/src/tests.rs | 2 +- client/consensus/grandpa/src/environment.rs | 4 ++-- client/consensus/grandpa/src/lib.rs | 12 ++++++------ client/consensus/grandpa/src/tests.rs | 14 +++++++------- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index b6f974bde7dc6..4cc3450d06f8e 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -313,7 +313,7 @@ pub fn new_full(config: Configuration) -> Result { prometheus_registry, shared_voter_state: SharedVoterState::empty(), telemetry: telemetry.as_ref().map(|x| x.handle()), - offchain_tx_pool: OffchainTransactionPoolFactory::new(transaction_pool), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(transaction_pool), }; // the GRANDPA voter task is considered infallible, i.e. diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index c096420b0dc90..c57274c4af745 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -227,7 +227,7 @@ pub fn new_partial( spawner: &task_manager.spawn_essential_handle(), registry: config.prometheus_registry(), telemetry: telemetry.as_ref().map(|x| x.handle()), - offchain_tx_pool: OffchainTransactionPoolFactory::new(transaction_pool.clone()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(transaction_pool.clone()), })?; let import_setup = (block_import, grandpa_link, babe_link); @@ -548,7 +548,7 @@ pub fn new_full_base( voting_rule: grandpa::VotingRulesBuilder::default().build(), prometheus_registry: prometheus_registry.clone(), shared_voter_state, - offchain_tx_pool: OffchainTransactionPoolFactory::new(transaction_pool.clone()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(transaction_pool.clone()), }; // the GRANDPA voter task is considered infallible, i.e. diff --git a/client/consensus/babe/rpc/src/lib.rs b/client/consensus/babe/rpc/src/lib.rs index b4582646daf35..ce9932640c1a4 100644 --- a/client/consensus/babe/rpc/src/lib.rs +++ b/client/consensus/babe/rpc/src/lib.rs @@ -236,7 +236,7 @@ mod tests { spawner: &task_executor, registry: None, telemetry: None, - offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }) .unwrap(); diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index e4defd53df1a3..7c36a6b45a380 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -993,7 +993,7 @@ pub struct BabeVerifier { config: BabeConfiguration, epoch_changes: SharedEpochChanges, telemetry: Option, - offchain_tx_pool: OffchainTransactionPoolFactory, + offchain_tx_pool_factory: OffchainTransactionPoolFactory, } impl BabeVerifier @@ -1102,7 +1102,7 @@ where let mut runtime_api = self.client.runtime_api(); // Register the offchain tx pool to be able to use it from the runtime. - runtime_api.register_extension(self.offchain_tx_pool.offchain_transaction_pool(best_hash)); + runtime_api.register_extension(self.offchain_tx_pool_factory.offchain_transaction_pool(best_hash)); runtime_api .submit_report_equivocation_unsigned_extrinsic( @@ -1801,7 +1801,7 @@ pub struct ImportQueueParams<'a, Block: BlockT, BI, Client, CIDP, SelectChain, S /// The offchain transaction pool factory. /// /// Will be used when sending equivocation reports. - pub offchain_tx_pool: OffchainTransactionPoolFactory, + pub offchain_tx_pool_factory: OffchainTransactionPoolFactory, } /// Start an import queue for the BABE consensus algorithm. @@ -1824,7 +1824,7 @@ pub fn import_queue( spawner, registry, telemetry, - offchain_tx_pool, + offchain_tx_pool_factory, }: ImportQueueParams<'_, Block, BI, Client, CIDP, SelectChain, Spawn>, ) -> ClientResult<(DefaultImportQueue, BabeWorkerHandle)> where @@ -1857,7 +1857,7 @@ where epoch_changes: babe_link.epoch_changes.clone(), telemetry, client: client.clone(), - offchain_tx_pool, + offchain_tx_pool_factory, }; let (worker_tx, worker_rx) = channel(HANDLE_BUFFER_SIZE); diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index 6aa1c0840c674..3d3ba9c535a7f 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -284,7 +284,7 @@ impl TestNetFactory for BabeTestNet { config: data.link.config.clone(), epoch_changes: data.link.epoch_changes.clone(), telemetry: None, - offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }, mutator: MUTATOR.with(|m| m.borrow().clone()), } diff --git a/client/consensus/grandpa/src/environment.rs b/client/consensus/grandpa/src/environment.rs index 19dcc32d62723..47e056462207c 100644 --- a/client/consensus/grandpa/src/environment.rs +++ b/client/consensus/grandpa/src/environment.rs @@ -446,7 +446,7 @@ pub(crate) struct Environment< pub(crate) metrics: Option, pub(crate) justification_sender: Option>, pub(crate) telemetry: Option, - pub(crate) offchain_tx_pool: OffchainTransactionPoolFactory, + pub(crate) offchain_tx_pool_factory: OffchainTransactionPoolFactory, pub(crate) _phantom: PhantomData, } @@ -576,7 +576,7 @@ where let mut runtime_api = self.client.runtime_api(); runtime_api - .register_extension(self.offchain_tx_pool.offchain_transaction_pool(best_block_hash)); + .register_extension(self.offchain_tx_pool_factory.offchain_transaction_pool(best_block_hash)); runtime_api .submit_report_equivocation_unsigned_extrinsic( diff --git a/client/consensus/grandpa/src/lib.rs b/client/consensus/grandpa/src/lib.rs index 4f8cde54399b8..ffcd899a53f4f 100644 --- a/client/consensus/grandpa/src/lib.rs +++ b/client/consensus/grandpa/src/lib.rs @@ -692,7 +692,7 @@ pub struct GrandpaParams { /// /// This will be used to create an offchain transaction pool instance for sending an /// equivocation report from the runtime. - pub offchain_tx_pool: OffchainTransactionPoolFactory, + pub offchain_tx_pool_factory: OffchainTransactionPoolFactory, } /// Returns the configuration value to put in @@ -742,7 +742,7 @@ where prometheus_registry, shared_voter_state, telemetry, - offchain_tx_pool, + offchain_tx_pool_factory, } = grandpa_params; // NOTE: we have recently removed `run_grandpa_observer` from the public @@ -817,7 +817,7 @@ where shared_voter_state, justification_sender, telemetry, - offchain_tx_pool, + offchain_tx_pool_factory, ); let voter_work = voter_work.map(|res| match res { @@ -887,7 +887,7 @@ where shared_voter_state: SharedVoterState, justification_sender: GrandpaJustificationSender, telemetry: Option, - offchain_tx_pool: OffchainTransactionPoolFactory, + offchain_tx_pool_factory: OffchainTransactionPoolFactory, ) -> Self { let metrics = match prometheus_registry.as_ref().map(Metrics::register) { Some(Ok(metrics)) => Some(metrics), @@ -912,7 +912,7 @@ where metrics: metrics.as_ref().map(|m| m.environment.clone()), justification_sender: Some(justification_sender), telemetry: telemetry.clone(), - offchain_tx_pool, + offchain_tx_pool_factory, _phantom: PhantomData, }); @@ -1064,7 +1064,7 @@ where metrics: self.env.metrics.clone(), justification_sender: self.env.justification_sender.clone(), telemetry: self.telemetry.clone(), - offchain_tx_pool: self.env.offchain_tx_pool.clone(), + offchain_tx_pool_factory: self.env.offchain_tx_pool_factory.clone(), _phantom: PhantomData, }); diff --git a/client/consensus/grandpa/src/tests.rs b/client/consensus/grandpa/src/tests.rs index 69612608c0ab5..65311b8934ed4 100644 --- a/client/consensus/grandpa/src/tests.rs +++ b/client/consensus/grandpa/src/tests.rs @@ -332,7 +332,7 @@ fn initialize_grandpa( voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), - offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), telemetry: None, }; let voter = @@ -483,7 +483,7 @@ async fn finalize_3_voters_1_full_observer() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, - offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }; run_grandpa_voter(grandpa_params).expect("all in order with client and network") @@ -576,7 +576,7 @@ async fn transition_3_voters_twice_1_full_observer() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, - offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }; voters @@ -1044,7 +1044,7 @@ async fn voter_persists_its_votes() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, - offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }; run_grandpa_voter(grandpa_params).expect("all in order with client and network") @@ -1088,7 +1088,7 @@ async fn voter_persists_its_votes() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, - offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }; run_grandpa_voter(grandpa_params) @@ -1299,7 +1299,7 @@ async fn voter_catches_up_to_latest_round_when_behind() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, - offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), }; Box::pin(run_grandpa_voter(grandpa_params).expect("all in order with client and network")) @@ -1429,7 +1429,7 @@ where justification_sender: None, telemetry: None, _phantom: PhantomData, - offchain_tx_pool: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), } } From 85ccb899bebd3255192b846e23c771088345fc68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 14 Jun 2023 16:56:44 +0200 Subject: [PATCH 39/55] Cleanup --- Cargo.lock | 1 - client/service/Cargo.toml | 1 - client/service/src/builder.rs | 10 ++-------- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e79070885c16d..42b7911c6c770 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10040,7 +10040,6 @@ dependencies = [ "sc-network-light", "sc-network-sync", "sc-network-transactions", - "sc-offchain", "sc-rpc", "sc-rpc-server", "sc-rpc-spec-v2", diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index a064b71d0fb12..c3b484129593f 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -69,7 +69,6 @@ sc-rpc-spec-v2 = { version = "0.10.0-dev", path = "../rpc-spec-v2" } sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" } sc-informant = { version = "0.10.0-dev", path = "../informant" } sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" } -sc-offchain = { version = "4.0.0-dev", path = "../offchain" } prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.10.0-dev" } sc-tracing = { version = "4.0.0-dev", path = "../tracing" } sc-sysinfo = { version = "6.0.0-dev", path = "../sysinfo" } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 3cf6dabbd969e..8c8eaebb834d0 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -320,19 +320,14 @@ where /// Shared network instance implementing a set of mandatory traits. pub trait SpawnTaskNetwork: - sc_offchain::NetworkProvider + NetworkStateInfo + NetworkStatusProvider + Send + Sync + 'static + NetworkStateInfo + NetworkStatusProvider + Send + Sync + 'static { } impl SpawnTaskNetwork for T where Block: BlockT, - T: sc_offchain::NetworkProvider - + NetworkStateInfo - + NetworkStatusProvider - + Send - + Sync - + 'static, + T: NetworkStateInfo + NetworkStatusProvider + Send + Sync + 'static, { } @@ -386,7 +381,6 @@ where + Send + 'static, >::Api: sp_api::Metadata - + sc_offchain::OffchainWorkerApi + sp_transaction_pool::runtime_api::TaggedTransactionQueue + sp_session::SessionKeys + sp_api::ApiExt, From 5042fb01763147c3d97636dd883e2083c76aaf91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 14 Jun 2023 17:04:59 +0200 Subject: [PATCH 40/55] FMT --- client/consensus/babe/rpc/src/lib.rs | 4 +++- client/consensus/babe/src/lib.rs | 3 ++- client/consensus/babe/src/tests.rs | 4 +++- client/consensus/grandpa/src/environment.rs | 5 ++-- client/consensus/grandpa/src/tests.rs | 24 ++++++++++++++----- .../unlock_and_unreserve_all_funds.rs | 13 ++++++++-- .../unlock_and_unreserve_all_funds.rs | 15 +++++++++--- 7 files changed, 52 insertions(+), 16 deletions(-) diff --git a/client/consensus/babe/rpc/src/lib.rs b/client/consensus/babe/rpc/src/lib.rs index ce9932640c1a4..bffe026ea6ef6 100644 --- a/client/consensus/babe/rpc/src/lib.rs +++ b/client/consensus/babe/rpc/src/lib.rs @@ -236,7 +236,9 @@ mod tests { spawner: &task_executor, registry: None, telemetry: None, - offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new( + RejectAllTxPool::default(), + ), }) .unwrap(); diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index 7c36a6b45a380..ede26e0e19444 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -1102,7 +1102,8 @@ where let mut runtime_api = self.client.runtime_api(); // Register the offchain tx pool to be able to use it from the runtime. - runtime_api.register_extension(self.offchain_tx_pool_factory.offchain_transaction_pool(best_hash)); + runtime_api + .register_extension(self.offchain_tx_pool_factory.offchain_transaction_pool(best_hash)); runtime_api .submit_report_equivocation_unsigned_extrinsic( diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index 3d3ba9c535a7f..384e45228b599 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -284,7 +284,9 @@ impl TestNetFactory for BabeTestNet { config: data.link.config.clone(), epoch_changes: data.link.epoch_changes.clone(), telemetry: None, - offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new( + RejectAllTxPool::default(), + ), }, mutator: MUTATOR.with(|m| m.borrow().clone()), } diff --git a/client/consensus/grandpa/src/environment.rs b/client/consensus/grandpa/src/environment.rs index 47e056462207c..eeb9ff4468291 100644 --- a/client/consensus/grandpa/src/environment.rs +++ b/client/consensus/grandpa/src/environment.rs @@ -575,8 +575,9 @@ where let mut runtime_api = self.client.runtime_api(); - runtime_api - .register_extension(self.offchain_tx_pool_factory.offchain_transaction_pool(best_block_hash)); + runtime_api.register_extension( + self.offchain_tx_pool_factory.offchain_transaction_pool(best_block_hash), + ); runtime_api .submit_report_equivocation_unsigned_extrinsic( diff --git a/client/consensus/grandpa/src/tests.rs b/client/consensus/grandpa/src/tests.rs index 65311b8934ed4..4fbeed71a1bf5 100644 --- a/client/consensus/grandpa/src/tests.rs +++ b/client/consensus/grandpa/src/tests.rs @@ -332,7 +332,9 @@ fn initialize_grandpa( voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), - offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new( + RejectAllTxPool::default(), + ), telemetry: None, }; let voter = @@ -483,7 +485,9 @@ async fn finalize_3_voters_1_full_observer() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, - offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new( + RejectAllTxPool::default(), + ), }; run_grandpa_voter(grandpa_params).expect("all in order with client and network") @@ -576,7 +580,9 @@ async fn transition_3_voters_twice_1_full_observer() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, - offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new( + RejectAllTxPool::default(), + ), }; voters @@ -1044,7 +1050,9 @@ async fn voter_persists_its_votes() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, - offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new( + RejectAllTxPool::default(), + ), }; run_grandpa_voter(grandpa_params).expect("all in order with client and network") @@ -1088,7 +1096,9 @@ async fn voter_persists_its_votes() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, - offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new( + RejectAllTxPool::default(), + ), }; run_grandpa_voter(grandpa_params) @@ -1299,7 +1309,9 @@ async fn voter_catches_up_to_latest_round_when_behind() { prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), telemetry: None, - offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(RejectAllTxPool::default()), + offchain_tx_pool_factory: OffchainTransactionPoolFactory::new( + RejectAllTxPool::default(), + ), }; Box::pin(run_grandpa_voter(grandpa_params).expect("all in order with client and network")) diff --git a/frame/democracy/src/migrations/unlock_and_unreserve_all_funds.rs b/frame/democracy/src/migrations/unlock_and_unreserve_all_funds.rs index e23a7c4f5f74c..1f46070d1d7fc 100644 --- a/frame/democracy/src/migrations/unlock_and_unreserve_all_funds.rs +++ b/frame/democracy/src/migrations/unlock_and_unreserve_all_funds.rs @@ -150,8 +150,17 @@ where log::info!(target: LOG_TARGET, "Total accounts: {:?}", all_accounts.len()); log::info!(target: LOG_TARGET, "Total stake to unlock: {:?}", total_stake_to_unlock); - log::info!(target: LOG_TARGET, "Total deposit to unreserve: {:?}", total_deposits_to_unreserve); - log::info!(target: LOG_TARGET, "Bugged deposits: {}/{}", bugged_deposits, account_deposits.len()); + log::info!( + target: LOG_TARGET, + "Total deposit to unreserve: {:?}", + total_deposits_to_unreserve + ); + log::info!( + target: LOG_TARGET, + "Bugged deposits: {}/{}", + bugged_deposits, + account_deposits.len() + ); Ok(account_reserved_before.encode()) } diff --git a/frame/elections-phragmen/src/migrations/unlock_and_unreserve_all_funds.rs b/frame/elections-phragmen/src/migrations/unlock_and_unreserve_all_funds.rs index 0115e4148d7bb..f566f84e51a2b 100644 --- a/frame/elections-phragmen/src/migrations/unlock_and_unreserve_all_funds.rs +++ b/frame/elections-phragmen/src/migrations/unlock_and_unreserve_all_funds.rs @@ -165,11 +165,20 @@ where let total_stake_to_unlock = account_staked_sums.clone().into_values().sum::>(); let total_deposits_to_unreserve = account_deposited_sums.clone().into_values().sum::>(); - log::info!(target: LOG_TARGET, "Total accounts: {:?}", all_accounts.len()); + log::info!(target: LOG_TARGET, "Total accounts: {:?}", all_accounts.len()); log::info!(target: LOG_TARGET, "Total stake to unlock: {:?}", total_stake_to_unlock); - log::info!(target: LOG_TARGET, "Total deposit to unreserve: {:?}", total_deposits_to_unreserve); + log::info!( + target: LOG_TARGET, + "Total deposit to unreserve: {:?}", + total_deposits_to_unreserve + ); if bugged_deposits > 0 { - log::warn!(target: LOG_TARGET, "Bugged deposits: {}/{}", bugged_deposits, all_accounts.len()); + log::warn!( + target: LOG_TARGET, + "Bugged deposits: {}/{}", + bugged_deposits, + all_accounts.len() + ); } Ok(account_reserved_before.encode()) From 99a5ca8a92f5ef16839020444a1ed74fdedc7b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 15 Jun 2023 14:48:44 +0200 Subject: [PATCH 41/55] Fix benchmark name --- scripts/ci/gitlab/pipeline/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 533acd59e094d..65cc7041342f6 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -115,8 +115,8 @@ cargo-check-benches: rusty-cachier cache upload ;; 2) - cargo run --locked --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json - | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json + cargo run --locked --release -p node-bench -- ::node::import::sr25519::transfer_keep_alive::paritydb::small --json + | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::sr25519::transfer_keep_alive::paritydb::small.json ;; esac From 32e69a2ec9cca28d626e1e849614799cbf0587ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 15 Jun 2023 15:28:26 +0200 Subject: [PATCH 42/55] Fix `try-runtime` --- utils/frame/try-runtime/cli/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index b61395a47b9e1..fbc55ad1dce6f 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -858,7 +858,7 @@ pub(crate) fn state_machine_call( executor: &WasmExecutor, method: &'static str, data: &[u8], - extensions: Extensions, + mut extensions: Extensions, ) -> sc_cli::Result<(OverlayedChanges, Vec)> { let mut changes = Default::default(); let encoded_results = StateMachine::new( @@ -867,7 +867,7 @@ pub(crate) fn state_machine_call( executor, method, data, - extensions, + &mut extensions, &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend).runtime_code()?, CallContext::Offchain, ) @@ -887,7 +887,7 @@ pub(crate) fn state_machine_call_with_proof, method: &'static str, data: &[u8], - extensions: Extensions, + mut extensions: Extensions, maybe_export_proof: Option, ) -> sc_cli::Result<(OverlayedChanges, Vec)> { use parity_scale_codec::Encode; @@ -906,7 +906,7 @@ pub(crate) fn state_machine_call_with_proof Date: Thu, 15 Jun 2023 23:28:37 +0200 Subject: [PATCH 43/55] Remove `--execution` CLI args --- bin/node-template/pallets/template/src/weights.rs | 1 - bin/node/cli/tests/benchmark_block_works.rs | 2 +- frame/benchmarking/README.md | 1 - frame/nfts/src/weights.rs | 1 - scripts/ci/gitlab/pipeline/test.yml | 2 +- scripts/run_all_benchmarks.sh | 2 -- utils/frame/benchmarking-cli/src/block/cmd.rs | 4 ++-- utils/frame/benchmarking-cli/src/overhead/README.md | 5 ++--- 8 files changed, 6 insertions(+), 12 deletions(-) diff --git a/bin/node-template/pallets/template/src/weights.rs b/bin/node-template/pallets/template/src/weights.rs index e8fbc09bad8e9..7c42936e09f29 100644 --- a/bin/node-template/pallets/template/src/weights.rs +++ b/bin/node-template/pallets/template/src/weights.rs @@ -19,7 +19,6 @@ // * // --steps=50 // --repeat=20 -// --execution=wasm // --wasm-execution=compiled // --output // pallets/template/src/weights.rs diff --git a/bin/node/cli/tests/benchmark_block_works.rs b/bin/node/cli/tests/benchmark_block_works.rs index 50103a66a4d40..09c2f262e2c29 100644 --- a/bin/node/cli/tests/benchmark_block_works.rs +++ b/bin/node/cli/tests/benchmark_block_works.rs @@ -39,7 +39,7 @@ async fn benchmark_block_works() { .arg(base_dir.path()) .args(["--from", "1", "--to", "1"]) .args(["--repeat", "1"]) - .args(["--execution", "wasm", "--wasm-execution", "compiled"]) + .args(["--wasm-execution", "compiled"]) .status() .unwrap(); diff --git a/frame/benchmarking/README.md b/frame/benchmarking/README.md index 76673c5f69b33..dc6a184435df6 100644 --- a/frame/benchmarking/README.md +++ b/frame/benchmarking/README.md @@ -175,7 +175,6 @@ Then you can run a benchmark like so: ```bash ./target/production/substrate benchmark pallet \ --chain dev \ # Configurable Chain Spec - --execution=wasm \ # Always test with Wasm --wasm-execution=compiled \ # Always used `wasm-time` --pallet pallet_balances \ # Select the pallet --extrinsic transfer \ # Select the extrinsic diff --git a/frame/nfts/src/weights.rs b/frame/nfts/src/weights.rs index 19a61974a61a7..7d52c74974de2 100644 --- a/frame/nfts/src/weights.rs +++ b/frame/nfts/src/weights.rs @@ -32,7 +32,6 @@ // --repeat=20 // --pallet=pallet_nfts // --extrinsic=* -// --execution=wasm // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/nfts/src/weights.rs diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 65cc7041342f6..86575693a23e0 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -304,7 +304,7 @@ quick-benchmarks: WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings" script: - rusty-cachier snapshot create - - time cargo run --locked --release --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + - time cargo run --locked --release --features runtime-benchmarks -- benchmark pallet --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 - rusty-cachier cache upload test-frame-examples-compile-to-wasm: diff --git a/scripts/run_all_benchmarks.sh b/scripts/run_all_benchmarks.sh index 727b49e26afe5..83848100a7e51 100755 --- a/scripts/run_all_benchmarks.sh +++ b/scripts/run_all_benchmarks.sh @@ -119,7 +119,6 @@ for PALLET in "${PALLETS[@]}"; do --repeat=20 \ --pallet="$PALLET" \ --extrinsic="*" \ - --execution=wasm \ --wasm-execution=compiled \ --heap-pages=4096 \ --output="$WEIGHT_FILE" \ @@ -137,7 +136,6 @@ echo "[+] Benchmarking block and extrinsic overheads..." OUTPUT=$( $SUBSTRATE benchmark overhead \ --chain=dev \ - --execution=wasm \ --wasm-execution=compiled \ --weight-path="./frame/support/src/weights/" \ --header="./HEADER-APACHE2" \ diff --git a/utils/frame/benchmarking-cli/src/block/cmd.rs b/utils/frame/benchmarking-cli/src/block/cmd.rs index 0192372fa33a7..90b71cd78c2a4 100644 --- a/utils/frame/benchmarking-cli/src/block/cmd.rs +++ b/utils/frame/benchmarking-cli/src/block/cmd.rs @@ -39,12 +39,12 @@ use super::bench::{Benchmark, BenchmarkParams}; /// did not use more weight than declared which would otherwise be an issue. /// To test this with a dev node, first create one with a temp directory: /// -/// $ substrate --dev -d /tmp/my-dev --execution wasm --wasm-execution compiled +/// $ substrate --dev -d /tmp/my-dev --wasm-execution compiled /// /// And wait some time to let it produce 3 blocks. Then benchmark them with: /// /// $ substrate benchmark-block --from 1 --to 3 --dev -d /tmp/my-dev -/// --execution wasm --wasm-execution compiled --pruning archive +/// --wasm-execution compiled --pruning archive /// /// The output will be similar to this: /// diff --git a/utils/frame/benchmarking-cli/src/overhead/README.md b/utils/frame/benchmarking-cli/src/overhead/README.md index 85bcc7fa36f2d..390bc09e41701 100644 --- a/utils/frame/benchmarking-cli/src/overhead/README.md +++ b/utils/frame/benchmarking-cli/src/overhead/README.md @@ -103,12 +103,12 @@ Writing weights to "extrinsic_weights.rs" The complete command for Polkadot looks like this: ```sh -cargo run --profile=production -- benchmark overhead --chain=polkadot-dev --execution=wasm --wasm-execution=compiled --weight-path=runtime/polkadot/constants/src/weights/ +cargo run --profile=production -- benchmark overhead --chain=polkadot-dev --wasm-execution=compiled --weight-path=runtime/polkadot/constants/src/weights/ ``` This will overwrite the the [block_weights.rs](https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/block_weights.rs) and [extrinsic_weights.rs](https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/extrinsic_weights.rs) files in the Polkadot runtime directory. You can try the same for *Rococo* and to see that the results slightly differ. -👉 It is paramount to use `--profile=production`, `--execution=wasm` and `--wasm-execution=compiled` as the results are otherwise useless. +👉 It is paramount to use `--profile=production` and `--wasm-execution=compiled` as the results are otherwise useless. ## Output Interpretation @@ -122,7 +122,6 @@ Minimizing this is important to have a large transaction throughput. - `--weight-path` Set the output directory or file to write the weights to. - `--repeat` Set the repetitions of both benchmarks. - `--warmup` Set the rounds of warmup before measuring. -- `--execution` Should be set to `wasm` for correct results. - `--wasm-execution` Should be set to `compiled` for correct results. - [`--mul`](../shared/README.md#arguments) - [`--add`](../shared/README.md#arguments) From 8960448fea7f32b26e9dafea9bb58ced5db0c839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 16 Jun 2023 11:47:13 +0200 Subject: [PATCH 44/55] Make clippy happy --- client/service/src/client/call_executor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 65f1a8497350a..facde72321db4 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -233,7 +233,7 @@ where &self.executor, method, call_data, - &mut *extensions, + &mut extensions, &runtime_code, call_context, ) @@ -248,7 +248,7 @@ where &self.executor, method, call_data, - &mut *extensions, + &mut extensions, &runtime_code, call_context, ) From 42301cd7c4d6ea626f99c39633364891f868901b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 16 Jun 2023 12:03:03 +0200 Subject: [PATCH 45/55] Forward bls functions --- primitives/keystore/src/lib.rs | 60 ++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/primitives/keystore/src/lib.rs b/primitives/keystore/src/lib.rs index 72f37d9f21e1a..4a0ec74c6f830 100644 --- a/primitives/keystore/src/lib.rs +++ b/primitives/keystore/src/lib.rs @@ -174,37 +174,36 @@ pub trait Keystore: Send + Sync { msg: &[u8; 32], ) -> Result, Error>; - #[cfg(feature = "bls-experimental")] /// Returns all bls12-381 public keys for the given key type. + #[cfg(feature = "bls-experimental")] fn bls381_public_keys(&self, id: KeyTypeId) -> Vec; - #[cfg(feature = "bls-experimental")] /// Returns all bls12-377 public keys for the given key type. + #[cfg(feature = "bls-experimental")] fn bls377_public_keys(&self, id: KeyTypeId) -> Vec; - #[cfg(feature = "bls-experimental")] /// Generate a new bls381 key pair for the given key type and an optional seed. /// /// Returns an `bls381::Public` key of the generated key pair or an `Err` if /// something failed during key generation. + #[cfg(feature = "bls-experimental")] fn bls381_generate_new( &self, key_type: KeyTypeId, seed: Option<&str>, ) -> Result; - #[cfg(feature = "bls-experimental")] /// Generate a new bls377 key pair for the given key type and an optional seed. /// /// Returns an `bls377::Public` key of the generated key pair or an `Err` if /// something failed during key generation. + #[cfg(feature = "bls-experimental")] fn bls377_generate_new( &self, key_type: KeyTypeId, seed: Option<&str>, ) -> Result; - #[cfg(feature = "bls-experimental")] /// Generate a bls381 signature for a given message. /// /// Receives [`KeyTypeId`] and a [`bls381::Public`] key to be able to map @@ -213,6 +212,7 @@ pub trait Keystore: Send + Sync { /// Returns an [`bls381::Signature`] or `None` in case the given `key_type` /// and `public` combination doesn't exist in the keystore. /// An `Err` will be returned if generating the signature itself failed. + #[cfg(feature = "bls-experimental")] fn bls381_sign( &self, key_type: KeyTypeId, @@ -220,7 +220,6 @@ pub trait Keystore: Send + Sync { msg: &[u8], ) -> Result, Error>; - #[cfg(feature = "bls-experimental")] /// Generate a bls377 signature for a given message. /// /// Receives [`KeyTypeId`] and a [`bls377::Public`] key to be able to map @@ -229,6 +228,7 @@ pub trait Keystore: Send + Sync { /// Returns an [`bls377::Signature`] or `None` in case the given `key_type` /// and `public` combination doesn't exist in the keystore. /// An `Err` will be returned if generating the signature itself failed. + #[cfg(feature = "bls-experimental")] fn bls377_sign( &self, key_type: KeyTypeId, @@ -411,6 +411,54 @@ impl Keystore for Arc { fn has_keys(&self, public_keys: &[(Vec, KeyTypeId)]) -> bool { (**self).has_keys(public_keys) } + + #[cfg(feature = "bls-experimental")] + fn bls381_public_keys(&self, id: KeyTypeId) -> Vec { + (**self).bls381_public_keys(id) + } + + #[cfg(feature = "bls-experimental")] + fn bls377_public_keys(&self, id: KeyTypeId) -> Vec { + (**self).bls377_public_keys(id) + } + + #[cfg(feature = "bls-experimental")] + fn bls381_generate_new( + &self, + key_type: KeyTypeId, + seed: Option<&str>, + ) -> Result { + (**self).bls381_generate_new(key_type, seed) + } + + #[cfg(feature = "bls-experimental")] + fn bls377_generate_new( + &self, + key_type: KeyTypeId, + seed: Option<&str>, + ) -> Result { + (**self).bls377_generate_new(key_type, seed) + } + + #[cfg(feature = "bls-experimental")] + fn bls381_sign( + &self, + key_type: KeyTypeId, + public: &bls381::Public, + msg: &[u8], + ) -> Result, Error> { + (**self).bls381_sign(key_type, public, msg) + } + + #[cfg(feature = "bls-experimental")] + fn bls377_sign( + &self, + key_type: KeyTypeId, + public: &bls377::Public, + msg: &[u8], + ) -> Result, Error> { + (**self).bls377_sign(key_type, public, msg) + } } /// A shared pointer to a keystore implementation. From e4bfccca23788842bc842dc81b917aff704835ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 16 Jun 2023 12:53:54 +0200 Subject: [PATCH 46/55] Fix docs --- client/offchain/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 999fcfd114135..960aa067d08d8 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -65,7 +65,7 @@ pub trait NetworkProvider: NetworkStateInfo + NetworkPeers {} impl NetworkProvider for T where T: NetworkStateInfo + NetworkPeers {} -/// Special type that implements [`OffchainStorage`]. +/// Special type that implements [`OffchainStorage`](offchain::OffchainStorage). /// /// This type can not be constructed and should only be used when passing `None` as `offchain_db` to /// [`OffchainWorkerOptions`] to make the compiler happy. From 22c56032a0bc7d1a292f8f5053d04691c75670d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Tue, 20 Jun 2023 23:00:22 +0200 Subject: [PATCH 47/55] Update UI tests --- .../tests/ui/mock_only_self_reference.stderr | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/primitives/api/test/tests/ui/mock_only_self_reference.stderr b/primitives/api/test/tests/ui/mock_only_self_reference.stderr index 430f63eee1660..f088e8f2de59d 100644 --- a/primitives/api/test/tests/ui/mock_only_self_reference.stderr +++ b/primitives/api/test/tests/ui/mock_only_self_reference.stderr @@ -48,42 +48,3 @@ error[E0050]: method `test2` has 2 parameters but the declaration in trait `Api: | |_^ expected 3 parameters, found 2 | = note: this error originates in the macro `sp_api::mock_impl_runtime_apis` (in Nightly builds, run with -Z macro-backtrace for more info) - -error[E0050]: method `test_with_context` has 3 parameters but the declaration in trait `Api::test_with_context` has 4 - --> tests/ui/mock_only_self_reference.rs:12:1 - | -3 | / sp_api::decl_runtime_apis! { -4 | | pub trait Api { -5 | | fn test(data: u64); - | |_________________________- trait requires 4 parameters -... -12 | / sp_api::mock_impl_runtime_apis! { -13 | | impl Api for MockApi { -14 | | fn test(self, data: u64) {} -15 | | -16 | | fn test2(&mut self, data: u64) {} -17 | | } -18 | | } - | |_^ expected 4 parameters, found 3 - | - = note: this error originates in the macro `sp_api::mock_impl_runtime_apis` (in Nightly builds, run with -Z macro-backtrace for more info) - -error[E0050]: method `test2_with_context` has 3 parameters but the declaration in trait `Api::test2_with_context` has 4 - --> tests/ui/mock_only_self_reference.rs:12:1 - | -3 | / sp_api::decl_runtime_apis! { -4 | | pub trait Api { -5 | | fn test(data: u64); -6 | | fn test2(data: u64); - | |__________________________- trait requires 4 parameters -... -12 | / sp_api::mock_impl_runtime_apis! { -13 | | impl Api for MockApi { -14 | | fn test(self, data: u64) {} -15 | | -16 | | fn test2(&mut self, data: u64) {} -17 | | } -18 | | } - | |_^ expected 4 parameters, found 3 - | - = note: this error originates in the macro `sp_api::mock_impl_runtime_apis` (in Nightly builds, run with -Z macro-backtrace for more info) From 74ac48e6bbab3bc1d6dd1fc026d6cecbef6a2ef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 24 Jun 2023 23:17:51 +0200 Subject: [PATCH 48/55] Update client/api/src/execution_extensions.rs Co-authored-by: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> --- client/api/src/execution_extensions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index eeaef40da1edf..a6bbc19e82a05 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -99,7 +99,7 @@ pub struct ExecutionExtensions { } impl ExecutionExtensions { - /// Create new `ExecutionExtensions` given a `keystore` and `ExecutionStrategies`. + /// Create new `ExecutionExtensions` given an `extensions_factory`. pub fn new( extensions_factory: Option>>, read_runtime_version: Arc, From b07fde8c24821056cdd06b8fd298cbdf51e9272d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sat, 24 Jun 2023 23:25:03 +0200 Subject: [PATCH 49/55] Apply suggestions from code review Co-authored-by: Koute --- client/offchain/src/lib.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 960aa067d08d8..ef9bb2fe72d2c 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -70,10 +70,7 @@ impl NetworkProvider for T where T: NetworkStateInfo + NetworkPeers {} /// This type can not be constructed and should only be used when passing `None` as `offchain_db` to /// [`OffchainWorkerOptions`] to make the compiler happy. #[derive(Clone)] -pub struct NoOffchainStorage { - // Ensure no one can construct this type - _priv: (), -} +pub enum NoOffchainStorage {} impl offchain::OffchainStorage for NoOffchainStorage { fn set(&mut self, _: &[u8], _: &[u8], _: &[u8]) { From ec9e8720540484e4159a5ff524c507675445623f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 25 Jun 2023 00:58:42 +0200 Subject: [PATCH 50/55] Update client/cli/src/params/import_params.rs Co-authored-by: Koute --- client/cli/src/params/import_params.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs index 5fa3771378582..bfa54a35058f6 100644 --- a/client/cli/src/params/import_params.rs +++ b/client/cli/src/params/import_params.rs @@ -168,7 +168,7 @@ impl ExecutionStrategiesParams { ] { if param.is_some() { eprintln!( - "CLI parameter `--{name}` is deprecated and will be removed in the future!" + "CLI parameter `--{name}` has no effect anymore and will be removed in the future!" ); } } From e4781c459a13da456f03ae0bee8baa0457fbfb75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Sun, 25 Jun 2023 00:58:52 +0200 Subject: [PATCH 51/55] Update client/api/src/execution_extensions.rs Co-authored-by: Koute --- client/api/src/execution_extensions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index a6bbc19e82a05..4dbe1fc1d8ebd 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -106,7 +106,7 @@ impl ExecutionExtensions { ) -> Self { Self { extensions_factory: extensions_factory - .map(|f| RwLock::new(f)) + .map(RwLock::new) .unwrap_or_else(|| RwLock::new(Box::new(()))), read_runtime_version, } From 3c71aa160b8ee594aecf924c66dcadbb8b8c8cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Fri, 30 Jun 2023 18:01:13 +0200 Subject: [PATCH 52/55] Pass the offchain storage to the MMR RPC --- Cargo.lock | 1 + bin/node/cli/src/service.rs | 3 +- bin/node/rpc/src/lib.rs | 35 ++++--- client/merkle-mountain-range/rpc/src/lib.rs | 29 ++++-- client/offchain/src/api.rs | 109 +------------------- client/offchain/src/lib.rs | 6 +- primitives/core/Cargo.toml | 2 + primitives/core/src/offchain/storage.rs | 96 ++++++++++++++++- 8 files changed, 148 insertions(+), 133 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76bcd637dca6b..67c00c391440b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11035,6 +11035,7 @@ dependencies = [ "substrate-bip39", "thiserror", "tiny-bip39", + "tracing", "w3f-bls", "zeroize", ] diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index d4b2d14064bc8..487c6c48f0061 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -281,9 +281,10 @@ pub fn new_partial( finality_provider: finality_proof_provider.clone(), }, statement_store: rpc_statement_store.clone(), + backend: rpc_backend.clone(), }; - node_rpc::create_full(deps, rpc_backend.clone()).map_err(Into::into) + node_rpc::create_full(deps).map_err(Into::into) }; (rpc_extensions_builder, shared_voter_state2) diff --git a/bin/node/rpc/src/lib.rs b/bin/node/rpc/src/lib.rs index 5ab96bf1c7064..40c4741dbc1c8 100644 --- a/bin/node/rpc/src/lib.rs +++ b/bin/node/rpc/src/lib.rs @@ -90,12 +90,23 @@ pub struct FullDeps { pub grandpa: GrandpaDeps, /// Shared statement store reference. pub statement_store: Arc, + /// The backend used by the node. + pub backend: Arc, } /// Instantiate all Full RPC extensions. pub fn create_full( - deps: FullDeps, - backend: Arc, + FullDeps { + client, + pool, + select_chain, + chain_spec, + deny_unsafe, + babe, + grandpa, + statement_store, + backend, + }: FullDeps, ) -> Result, Box> where C: ProvideRuntimeApi @@ -130,16 +141,6 @@ where use substrate_state_trie_migration_rpc::{StateMigration, StateMigrationApiServer}; let mut io = RpcModule::new(()); - let FullDeps { - client, - pool, - select_chain, - chain_spec, - deny_unsafe, - babe, - grandpa, - statement_store, - } = deps; let BabeDeps { keystore, babe_worker_handle } = babe; let GrandpaDeps { @@ -159,7 +160,15 @@ where // Making synchronous calls in light client freezes the browser currently, // more context: https://github.com/paritytech/substrate/pull/3480 // These RPCs should use an asynchronous caller instead. - io.merge(Mmr::new(client.clone()).into_rpc())?; + io.merge( + Mmr::new( + client.clone(), + backend + .offchain_storage() + .ok_or_else(|| "Backend doesn't provide an offchain storage")?, + ) + .into_rpc(), + )?; io.merge(TransactionPayment::new(client.clone()).into_rpc())?; io.merge( Babe::new(client.clone(), babe_worker_handle.clone(), keystore, select_chain, deny_unsafe) diff --git a/client/merkle-mountain-range/rpc/src/lib.rs b/client/merkle-mountain-range/rpc/src/lib.rs index 024023f57c7a5..5be82b600d914 100644 --- a/client/merkle-mountain-range/rpc/src/lib.rs +++ b/client/merkle-mountain-range/rpc/src/lib.rs @@ -30,9 +30,12 @@ use jsonrpsee::{ }; use serde::{Deserialize, Serialize}; -use sp_api::{NumberFor, ProvideRuntimeApi}; +use sp_api::{ApiExt, NumberFor, ProvideRuntimeApi}; use sp_blockchain::HeaderBackend; -use sp_core::Bytes; +use sp_core::{ + offchain::{storage::OffchainDb, OffchainDbExt, OffchainStorage}, + Bytes, +}; use sp_mmr_primitives::{Error as MmrError, Proof}; use sp_runtime::traits::Block as BlockT; @@ -127,26 +130,28 @@ pub trait MmrApi { } /// MMR RPC methods. -pub struct Mmr { +pub struct Mmr { client: Arc, + offchain_db: OffchainDb, _marker: PhantomData, } -impl Mmr { +impl Mmr { /// Create new `Mmr` with the given reference to the client. - pub fn new(client: Arc) -> Self { - Self { client, _marker: Default::default() } + pub fn new(client: Arc, offchain_storage: S) -> Self { + Self { client, _marker: Default::default(), offchain_db: OffchainDb::new(offchain_storage) } } } #[async_trait] -impl MmrApiServer<::Hash, NumberFor, MmrHash> - for Mmr +impl MmrApiServer<::Hash, NumberFor, MmrHash> + for Mmr where Block: BlockT, Client: Send + Sync + 'static + ProvideRuntimeApi + HeaderBackend, Client::Api: MmrRuntimeApi>, MmrHash: Codec + Send + Sync + 'static, + S: OffchainStorage + 'static, { fn mmr_root(&self, at: Option<::Hash>) -> RpcResult { let block_hash = at.unwrap_or_else(|| @@ -166,11 +171,13 @@ where best_known_block_number: Option>, at: Option<::Hash>, ) -> RpcResult::Hash>> { - let api = self.client.runtime_api(); + let mut api = self.client.runtime_api(); let block_hash = at.unwrap_or_else(|| // If the block hash is not supplied assume the best block. self.client.info().best_hash); + api.register_extension(OffchainDbExt::new(self.offchain_db.clone())); + let (leaves, proof) = api .generate_proof(block_hash, block_numbers, best_known_block_number) .map_err(runtime_error_into_rpc_error)? @@ -180,7 +187,7 @@ where } fn verify_proof(&self, proof: LeavesProof<::Hash>) -> RpcResult { - let api = self.client.runtime_api(); + let mut api = self.client.runtime_api(); let leaves = Decode::decode(&mut &proof.leaves.0[..]) .map_err(|e| CallError::InvalidParams(anyhow::Error::new(e)))?; @@ -188,6 +195,8 @@ where let decoded_proof = Decode::decode(&mut &proof.proof.0[..]) .map_err(|e| CallError::InvalidParams(anyhow::Error::new(e)))?; + api.register_extension(OffchainDbExt::new(self.offchain_db.clone())); + api.verify_proof(proof.block_hash, leaves, decoded_proof) .map_err(runtime_error_into_rpc_error)? .map_err(mmr_error_into_rpc_error)?; diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index 499d83c745a72..aaf5dc11a4095 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -25,8 +25,8 @@ pub use http::SharedClient; use libp2p::{Multiaddr, PeerId}; use sp_core::{ offchain::{ - self, HttpError, HttpRequestId, HttpRequestStatus, OffchainStorage, OpaqueMultiaddr, - OpaqueNetworkState, StorageKind, Timestamp, + self, HttpError, HttpRequestId, HttpRequestStatus, OpaqueMultiaddr, OpaqueNetworkState, + Timestamp, }, OpaquePeerId, }; @@ -36,107 +36,6 @@ mod http; mod timestamp; -fn unavailable_yet(name: &str) -> R { - tracing::error!( - target: super::LOG_TARGET, - "The {:?} API is not available for offchain workers yet. Follow \ - https://github.com/paritytech/substrate/issues/1458 for details", - name - ); - Default::default() -} - -const LOCAL_DB: &str = "LOCAL (fork-aware) DB"; - -/// Offchain DB reference. -#[derive(Debug, Clone)] -pub struct Db { - /// Persistent storage database. - persistent: Storage, -} - -impl Db { - /// Create new instance of Offchain DB. - pub fn new(persistent: Storage) -> Self { - Self { persistent } - } - - /// Create new instance of Offchain DB, backed by the given backend. - pub fn from_backend(backend: &Backend) -> Option> - where - Backend: sc_client_api::Backend, - Block: sp_runtime::traits::Block, - { - sc_client_api::Backend::offchain_storage(backend).map(|db| Db::new(db)) - } -} - -impl offchain::DbExternalities for Db { - fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { - tracing::debug!( - target: "offchain-worker::storage", - ?kind, - key = ?array_bytes::bytes2hex("", key), - value = ?array_bytes::bytes2hex("", value), - "Write", - ); - match kind { - StorageKind::PERSISTENT => self.persistent.set(STORAGE_PREFIX, key, value), - StorageKind::LOCAL => unavailable_yet(LOCAL_DB), - } - } - - fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { - tracing::debug!( - target: "offchain-worker::storage", - ?kind, - key = ?array_bytes::bytes2hex("", key), - "Clear", - ); - match kind { - StorageKind::PERSISTENT => self.persistent.remove(STORAGE_PREFIX, key), - StorageKind::LOCAL => unavailable_yet(LOCAL_DB), - } - } - - fn local_storage_compare_and_set( - &mut self, - kind: StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool { - tracing::debug!( - target: "offchain-worker::storage", - ?kind, - key = ?array_bytes::bytes2hex("", key), - new_value = ?array_bytes::bytes2hex("", new_value), - old_value = ?old_value.as_ref().map(|s| array_bytes::bytes2hex("", s)), - "CAS", - ); - match kind { - StorageKind::PERSISTENT => - self.persistent.compare_and_set(STORAGE_PREFIX, key, old_value, new_value), - StorageKind::LOCAL => unavailable_yet(LOCAL_DB), - } - } - - fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { - let result = match kind { - StorageKind::PERSISTENT => self.persistent.get(STORAGE_PREFIX, key), - StorageKind::LOCAL => unavailable_yet(LOCAL_DB), - }; - tracing::debug!( - target: "offchain-worker::storage", - ?kind, - key = ?array_bytes::bytes2hex("", key), - result = ?result.as_ref().map(|s| array_bytes::bytes2hex("", s)), - "Read", - ); - result - } -} - /// Asynchronous offchain API. /// /// NOTE this is done to prevent recursive calls into the runtime @@ -326,7 +225,7 @@ mod tests { config::MultiaddrWithPeerId, types::ProtocolName, NetworkPeers, NetworkStateInfo, ReputationChange, }; - use sp_core::offchain::{DbExternalities, Externalities}; + use sp_core::offchain::{storage::OffchainDb, DbExternalities, Externalities, StorageKind}; use std::time::SystemTime; pub(super) struct TestNetwork(); @@ -416,7 +315,7 @@ mod tests { } fn offchain_db() -> Db { - Db::new(LocalStorage::new_test()) + OffchainDb::new(LocalStorage::new_test()) } #[test] diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index ef9bb2fe72d2c..4c11a5cb7294d 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -54,7 +54,7 @@ use threadpool::ThreadPool; mod api; -pub use api::Db as OffchainDb; +pub use sp_core::offchain::storage::OffchainDb; pub use sp_offchain::{OffchainWorkerApi, STORAGE_PREFIX}; const LOG_TARGET: &str = "offchain-worker"; @@ -133,7 +133,7 @@ pub struct OffchainWorkers { shared_http_client: api::SharedClient, enable_http_requests: bool, keystore: Option, - offchain_db: Option>, + offchain_db: Option>, transaction_pool: Option>, network_provider: Arc, is_validator: bool, @@ -163,7 +163,7 @@ impl OffchainWorkers { shared_http_client: api::SharedClient::new(), enable_http_requests, keystore, - offchain_db: offchain_db.map(|d| api::Db::new(d)), + offchain_db: offchain_db.map(OffchainDb::new), transaction_pool, is_validator, network_provider, diff --git a/primitives/core/Cargo.toml b/primitives/core/Cargo.toml index 92933fdbd9aff..0f0ee83eb4cce 100644 --- a/primitives/core/Cargo.toml +++ b/primitives/core/Cargo.toml @@ -39,6 +39,7 @@ sp-externalities = { version = "0.19.0", optional = true, path = "../externaliti futures = { version = "0.3.21", optional = true } dyn-clonable = { version = "0.9.0", optional = true } thiserror = { version = "1.0.30", optional = true } +tracing = { version = "0.1.29", optional = true } bitflags = "1.3" paste = "1.0.7" @@ -113,6 +114,7 @@ std = [ "futures/thread-pool", "libsecp256k1/std", "dyn-clonable", + "tracing", ] # Serde support without relying on std features. diff --git a/primitives/core/src/offchain/storage.rs b/primitives/core/src/offchain/storage.rs index 3a114de5bfa3c..64cba1670bf8a 100644 --- a/primitives/core/src/offchain/storage.rs +++ b/primitives/core/src/offchain/storage.rs @@ -17,12 +17,14 @@ //! In-memory implementation of offchain workers database. -use crate::offchain::OffchainStorage; +use crate::offchain::{OffchainStorage, StorageKind, STORAGE_PREFIX, DbExternalities}; use std::{ collections::hash_map::{Entry, HashMap}, iter::Iterator, }; +const LOG_TARGET: &str = "offchain-worker::storage"; + /// In-memory storage for offchain workers. #[derive(Debug, Clone, Default)] pub struct InMemOffchainStorage { @@ -88,3 +90,95 @@ impl OffchainStorage for InMemOffchainStorage { } } } + +fn unavailable_yet(name: &str) -> R { + tracing::error!( + target: LOG_TARGET, + "The {:?} API is not available for offchain workers yet. Follow \ + https://github.com/paritytech/substrate/issues/1458 for details", + name + ); + Default::default() +} + +const LOCAL_DB: &str = "LOCAL (fork-aware) DB"; + +/// Offchain DB that implements [`DbExternalities`] for [`OffchainStorage`]. +#[derive(Debug, Clone)] +pub struct OffchainDb { + /// Persistent storage database. + persistent: Storage, +} + +impl OffchainDb { + /// Create new instance of Offchain DB. + pub fn new(persistent: Storage) -> Self { + Self { persistent } + } +} + +impl DbExternalities for OffchainDb { + fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { + tracing::debug!( + target: LOG_TARGET, + ?kind, + key = ?array_bytes::bytes2hex("", key), + value = ?array_bytes::bytes2hex("", value), + "Write", + ); + match kind { + StorageKind::PERSISTENT => self.persistent.set(STORAGE_PREFIX, key, value), + StorageKind::LOCAL => unavailable_yet(LOCAL_DB), + } + } + + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { + tracing::debug!( + target: LOG_TARGET, + ?kind, + key = ?array_bytes::bytes2hex("", key), + "Clear", + ); + match kind { + StorageKind::PERSISTENT => self.persistent.remove(STORAGE_PREFIX, key), + StorageKind::LOCAL => unavailable_yet(LOCAL_DB), + } + } + + fn local_storage_compare_and_set( + &mut self, + kind: StorageKind, + key: &[u8], + old_value: Option<&[u8]>, + new_value: &[u8], + ) -> bool { + tracing::debug!( + target: LOG_TARGET, + ?kind, + key = ?array_bytes::bytes2hex("", key), + new_value = ?array_bytes::bytes2hex("", new_value), + old_value = ?old_value.as_ref().map(|s| array_bytes::bytes2hex("", s)), + "CAS", + ); + match kind { + StorageKind::PERSISTENT => + self.persistent.compare_and_set(STORAGE_PREFIX, key, old_value, new_value), + StorageKind::LOCAL => unavailable_yet(LOCAL_DB), + } + } + + fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { + let result = match kind { + StorageKind::PERSISTENT => self.persistent.get(STORAGE_PREFIX, key), + StorageKind::LOCAL => unavailable_yet(LOCAL_DB), + }; + tracing::debug!( + target: LOG_TARGET, + ?kind, + key = ?array_bytes::bytes2hex("", key), + result = ?result.as_ref().map(|s| array_bytes::bytes2hex("", s)), + "Read", + ); + result + } +} From 7079d0171dfacbadc32ba204dca6ee55f1aeff29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 10 Jul 2023 22:29:58 +0200 Subject: [PATCH 53/55] Update client/api/src/execution_extensions.rs Co-authored-by: Sebastian Kunert --- client/api/src/execution_extensions.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 4dbe1fc1d8ebd..6f927105df0bf 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -36,7 +36,6 @@ pub trait ExtensionsFactory: Send + Sync { /// /// - `block_hash`: The hash of the block in the context that extensions will be used. /// - `block_number`: The number of the block in the context that extensions will be used. - /// - `capabilities`: The capabilities fn extensions_for(&self, block_hash: Block::Hash, block_number: NumberFor) -> Extensions; } From ad87780aa8bd28b505d583109e3869974bd1d048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 10 Jul 2023 22:46:04 +0200 Subject: [PATCH 54/55] Review comments --- client/consensus/babe/rpc/Cargo.toml | 2 +- client/transaction-pool/api/src/lib.rs | 2 -- primitives/core/src/offchain/mod.rs | 16 ++++++++-------- primitives/externalities/src/extensions.rs | 4 ++-- primitives/keystore/src/lib.rs | 4 ++-- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/client/consensus/babe/rpc/Cargo.toml b/client/consensus/babe/rpc/Cargo.toml index 5a5ce80fc4368..7b16ea84c4366 100644 --- a/client/consensus/babe/rpc/Cargo.toml +++ b/client/consensus/babe/rpc/Cargo.toml @@ -20,7 +20,6 @@ thiserror = "1.0" sc-consensus-babe = { version = "0.10.0-dev", path = "../" } sc-consensus-epochs = { version = "0.10.0-dev", path = "../../epochs" } sc-rpc-api = { version = "0.10.0-dev", path = "../../../rpc-api" } -sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../transaction-pool/api" } sp-api = { version = "4.0.0-dev", path = "../../../../primitives/api" } sp-application-crypto = { version = "23.0.0", path = "../../../../primitives/application-crypto" } sp-blockchain = { version = "4.0.0-dev", path = "../../../../primitives/blockchain" } @@ -35,5 +34,6 @@ serde_json = "1.0.85" tokio = "1.22.0" sc-consensus = { version = "0.10.0-dev", path = "../../../consensus/common" } sc-keystore = { version = "4.0.0-dev", path = "../../../keystore" } +sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../transaction-pool/api" } sp-keyring = { version = "24.0.0", path = "../../../../primitives/keyring" } substrate-test-runtime-client = { version = "2.0.0", path = "../../../../test-utils/runtime/client" } diff --git a/client/transaction-pool/api/src/lib.rs b/client/transaction-pool/api/src/lib.rs index bbc6141d7f688..32fe30f4584f0 100644 --- a/client/transaction-pool/api/src/lib.rs +++ b/client/transaction-pool/api/src/lib.rs @@ -407,8 +407,6 @@ impl OffchainSubmitTransaction for TP /// the wasm execution environment to send transactions from an offchain call to the runtime. #[derive(Clone)] pub struct OffchainTransactionPoolFactory { - // To break retain cycle between `Client` and `TransactionPool` we require this - // extension to be a `Weak` reference. pool: Arc>, } diff --git a/primitives/core/src/offchain/mod.rs b/primitives/core/src/offchain/mod.rs index 3ea0bfce70f1f..cef495dfaacdc 100644 --- a/primitives/core/src/offchain/mod.rs +++ b/primitives/core/src/offchain/mod.rs @@ -261,21 +261,21 @@ bitflags::bitflags! { /// Execution context extra capabilities. pub struct Capabilities: u32 { /// External http calls. - const HTTP = 0b0000_0000_0001; + const HTTP = 1 << 0; /// Keystore access. - const KEYSTORE = 0b0000_0000_0010; + const KEYSTORE = 1 << 2; /// Randomness source. - const RANDOMNESS = 0b0000_0000_0100; + const RANDOMNESS = 1 << 3; /// Access to opaque network state. - const NETWORK_STATE = 0b0000_0000_1000; + const NETWORK_STATE = 1 << 4; /// Access to offchain worker DB (read only). - const OFFCHAIN_DB_READ = 0b0000_0001_0000; + const OFFCHAIN_DB_READ = 1 << 5; /// Access to offchain worker DB (writes). - const OFFCHAIN_DB_WRITE = 0b0000_0010_0000; + const OFFCHAIN_DB_WRITE = 1 << 6; /// Manage the authorized nodes - const NODE_AUTHORIZATION = 0b0000_0100_0000; + const NODE_AUTHORIZATION = 1 << 7; /// Access time related functionality - const TIME = 0b0000_1000_0000; + const TIME = 1 << 8; } } diff --git a/primitives/externalities/src/extensions.rs b/primitives/externalities/src/extensions.rs index 2d204c52ec8a8..8b0bbd2c5921b 100644 --- a/primitives/externalities/src/extensions.rs +++ b/primitives/externalities/src/extensions.rs @@ -199,8 +199,8 @@ impl Extensions { /// Merge `other` into `self`. /// - /// If both contain the same extension, the extension instance of `other` will be present - /// afterwards in `self`. + /// If both contain the same extension, the extension instance of `other` will overwrite the + /// instance found in `self`. pub fn merge(&mut self, other: Self) { self.extensions.extend(other.extensions); } diff --git a/primitives/keystore/src/lib.rs b/primitives/keystore/src/lib.rs index 4a0ec74c6f830..338e1588c56b5 100644 --- a/primitives/keystore/src/lib.rs +++ b/primitives/keystore/src/lib.rs @@ -471,7 +471,7 @@ sp_externalities::decl_extension! { impl KeystoreExt { /// Create a new instance of `KeystoreExt` - pub fn new(keystore: T) -> Self { - Self(Arc::new(keystore)) + pub fn new(keystore: impl Into>) -> Self { + Self(keystore.into()) } } From 792fbccf65d09dbf7ed7c66775b83920feca0f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Mon, 10 Jul 2023 23:16:12 +0200 Subject: [PATCH 55/55] Fixes --- client/offchain/src/api.rs | 2 +- client/rpc/src/author/mod.rs | 2 +- primitives/core/src/offchain/storage.rs | 2 +- primitives/keystore/src/lib.rs | 11 +++++++++-- primitives/session/src/lib.rs | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index aaf5dc11a4095..e6b0e30f20378 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -314,7 +314,7 @@ mod tests { AsyncApi::new(mock, false, shared_client) } - fn offchain_db() -> Db { + fn offchain_db() -> OffchainDb { OffchainDb::new(LocalStorage::new_test()) } diff --git a/client/rpc/src/author/mod.rs b/client/rpc/src/author/mod.rs index 16f8c91b70360..feee22641ef34 100644 --- a/client/rpc/src/author/mod.rs +++ b/client/rpc/src/author/mod.rs @@ -124,7 +124,7 @@ where let best_block_hash = self.client.info().best_hash; let mut runtime_api = self.client.runtime_api(); - runtime_api.register_extension(KeystoreExt::new(self.keystore.clone())); + runtime_api.register_extension(KeystoreExt::from(self.keystore.clone())); runtime_api .generate_session_keys(best_block_hash, None) diff --git a/primitives/core/src/offchain/storage.rs b/primitives/core/src/offchain/storage.rs index 64cba1670bf8a..4db839f1a451a 100644 --- a/primitives/core/src/offchain/storage.rs +++ b/primitives/core/src/offchain/storage.rs @@ -17,7 +17,7 @@ //! In-memory implementation of offchain workers database. -use crate::offchain::{OffchainStorage, StorageKind, STORAGE_PREFIX, DbExternalities}; +use crate::offchain::{DbExternalities, OffchainStorage, StorageKind, STORAGE_PREFIX}; use std::{ collections::hash_map::{Entry, HashMap}, iter::Iterator, diff --git a/primitives/keystore/src/lib.rs b/primitives/keystore/src/lib.rs index 338e1588c56b5..07583d11d527a 100644 --- a/primitives/keystore/src/lib.rs +++ b/primitives/keystore/src/lib.rs @@ -471,7 +471,14 @@ sp_externalities::decl_extension! { impl KeystoreExt { /// Create a new instance of `KeystoreExt` - pub fn new(keystore: impl Into>) -> Self { - Self(keystore.into()) + /// + /// This is more performant as we don't need to wrap keystore in another [`Arc`]. + pub fn from(keystore: KeystorePtr) -> Self { + Self(keystore) + } + + /// Create a new instance of `KeystoreExt` using the given `keystore`. + pub fn new(keystore: T) -> Self { + Self(Arc::new(keystore)) } } diff --git a/primitives/session/src/lib.rs b/primitives/session/src/lib.rs index 9fa99a5be8f97..45395e9766f55 100644 --- a/primitives/session/src/lib.rs +++ b/primitives/session/src/lib.rs @@ -127,7 +127,7 @@ where let mut runtime_api = client.runtime_api(); - runtime_api.register_extension(sp_keystore::KeystoreExt::new(keystore)); + runtime_api.register_extension(sp_keystore::KeystoreExt::from(keystore)); for seed in seeds { runtime_api.generate_session_keys(at, Some(seed.as_bytes().to_vec()))?;