From a5a85b18486638d16637031a2747eeab53f02761 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 31 May 2021 11:20:59 +0200 Subject: [PATCH 1/5] Inject for remote-ext --- frame/system/src/offchain.rs | 2 +- utils/frame/remote-externalities/src/lib.rs | 105 ++++---------------- 2 files changed, 20 insertions(+), 87 deletions(-) diff --git a/frame/system/src/offchain.rs b/frame/system/src/offchain.rs index fe601f995ce51..6769923bc04b8 100644 --- a/frame/system/src/offchain.rs +++ b/frame/system/src/offchain.rs @@ -447,7 +447,7 @@ pub trait AppCrypto { // TODO [#5663] Could this be just `T::Signature as traits::Verify>::Signer`? // Seems that this may cause issues with bounds resolution. pub trait SigningTypes: crate::Config { - /// A public key that is capable of identifing `AccountId`s. + /// A public key that is capable of identifying `AccountId`s. /// /// Usually that's either a raw crypto public key (e.g. `sr25519::Public`) or /// an aggregate type for multiple crypto public keys, like `MulitSigner`. diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 46aa583b9b2a5..8e0ecbfc6324d 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -19,87 +19,6 @@ //! //! An equivalent of `sp_io::TestExternalities` that can load its state from a remote substrate //! based chain, or a local state snapshot file. -//! -//! #### Runtime to Test Against -//! -//! While not absolutely necessary, you most likely need a `Runtime` equivalent in your test setup -//! through which you can infer storage types. There are two options here: -//! -//! 1. Build a mock runtime, similar how to you would build one in a pallet test (see example -//! below). The very important point here is that this mock needs to hold real values for types -//! that matter for you, based on the chain of interest. Some typical ones are: -//! -//! - `sp_runtime::AccountId32` as `AccountId`. -//! - `u32` as `BlockNumber`. -//! - `u128` as Balance. -//! -//! Once you have your `Runtime`, you can use it for storage type resolution and do things like -//! `>::storage_getter()` or `>::get()`. -//! -//! 2. Or, you can use a real runtime. -//! -//! ### Example -//! -//! With a test runtime -//! -//! ```ignore -//! use remote_externalities::Builder; -//! -//! #[derive(Clone, Eq, PartialEq, Debug, Default)] -//! pub struct TestRuntime; -//! -//! use frame_system as system; -//! impl_outer_origin! { -//! pub enum Origin for TestRuntime {} -//! } -//! -//! impl frame_system::Config for TestRuntime { -//! .. -//! // we only care about these two for now. The rest can be mock. The block number type of -//! // kusama is u32. -//! type BlockNumber = u32; -//! type Header = Header; -//! .. -//! } -//! -//! #[test] -//! fn test_runtime_works() { -//! let hash: Hash = -//! hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into(); -//! let parent: Hash = -//! hex!["540922e96a8fcaf945ed23c6f09c3e189bd88504ec945cc2171deaebeaf2f37e"].into(); -//! Builder::new() -//! .at(hash) -//! .module("System") -//! .build() -//! .execute_with(|| { -//! assert_eq!( -//! // note: the hash corresponds to 3098546. We can check only the parent. -//! // https://polkascan.io/kusama/block/3098546 -//! >::block_hash(3098545u32), -//! parent, -//! ) -//! }); -//! } -//! ``` -//! -//! Or with the real kusama runtime. -//! -//! ```ignore -//! use remote_externalities::Builder; -//! use kusama_runtime::Runtime; -//! -//! #[test] -//! fn test_runtime_works() { -//! let hash: Hash = -//! hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into(); -//! Builder::new() -//! .at(hash) -//! .module("Staking") -//! .build() -//! .execute_with(|| assert_eq!(>::validator_count(), 400)); -//! } -//! ``` use std::{ fs, @@ -235,8 +154,10 @@ impl Default for SnapshotConfig { /// Builder for remote-externalities. pub struct Builder { - /// Pallets to inject their prefix into the externalities. + /// certain custom key-pairs to be injected into the externalities. inject: Vec, + /// Certain prefixes the be injected as well. The raw prefix must be given. + raw_prefixes: Vec>, /// connectivity mode, online or offline. mode: Mode, } @@ -245,7 +166,7 @@ pub struct Builder { // that. impl Default for Builder { fn default() -> Self { - Self { inject: Default::default(), mode: Default::default() } + Self { inject: Default::default(), mode: Default::default(), raw_prefixes: Default::default() } } } @@ -394,7 +315,7 @@ impl Builder { /// initialize `Self` from state snapshot. Panics if the file does not exist. fn load_state_snapshot(&self, path: &Path) -> Result, &'static str> { - info!(target: LOG_TARGET, "scraping keypairs from state snapshot {:?}", path,); + info!(target: LOG_TARGET, "scraping key-pairs from state snapshot {:?}", path,); let bytes = fs::read(path).map_err(|_| "fs::read failed.")?; Decode::decode(&mut &*bytes).map_err(|_| "decode failed") } @@ -407,9 +328,9 @@ impl Builder { .at .expect("online config must be initialized by this point; qed.") .clone(); - info!(target: LOG_TARGET, "scraping keypairs from remote @ {:?}", at); + info!(target: LOG_TARGET, "scraping key-pairs from remote @ {:?}", at); - let keys_and_values = if config.modules.len() > 0 { + let mut keys_and_values = if config.modules.len() > 0 { let mut filtered_kv = vec![]; for f in config.modules.iter() { let hashed_prefix = StorageKey(twox_128(f.as_bytes()).to_vec()); @@ -429,6 +350,12 @@ impl Builder { self.rpc_get_pairs_paged(StorageKey(vec![]), at).await? }; + for prefix in &self.raw_prefixes { + info!(target: LOG_TARGET, "adding data for raw prefix: {:?}", HexDisplay::from(prefix)); + let additional_key_values = self.rpc_get_pairs_paged(StorageKey(prefix.to_vec()), at).await?; + keys_and_values.extend(additional_key_values); + } + Ok(keys_and_values) } @@ -491,6 +418,12 @@ impl Builder { self } + /// Inject a manual raw prefixes + pub fn raw_prefix(mut self, raw: &[u8]) -> Self { + self.raw_prefixes.push(raw.to_vec()); + self + } + /// Configure a state snapshot to be used. pub fn mode(mut self, mode: Mode) -> Self { self.mode = mode; From e3b089b33d8ee42e476a91b0fa48f868f4312246 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Tue, 1 Jun 2021 08:16:02 +0200 Subject: [PATCH 2/5] Update utils/frame/remote-externalities/src/lib.rs Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com> --- utils/frame/remote-externalities/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 8e0ecbfc6324d..aa04f356b0986 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -156,7 +156,7 @@ impl Default for SnapshotConfig { pub struct Builder { /// certain custom key-pairs to be injected into the externalities. inject: Vec, - /// Certain prefixes the be injected as well. The raw prefix must be given. + /// Storage entry key prefixes to be injected into the externalities. The raw prefix must be given. raw_prefixes: Vec>, /// connectivity mode, online or offline. mode: Mode, From a2ac0ff45e9ebea5d0f00754ea043b4cc0eaa4b6 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Tue, 1 Jun 2021 08:16:20 +0200 Subject: [PATCH 3/5] Update utils/frame/remote-externalities/src/lib.rs Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com> --- utils/frame/remote-externalities/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index aa04f356b0986..a2b2ec11c46cb 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -154,7 +154,7 @@ impl Default for SnapshotConfig { /// Builder for remote-externalities. pub struct Builder { - /// certain custom key-pairs to be injected into the externalities. + /// Custom key-pairs to be injected into the externalities. inject: Vec, /// Storage entry key prefixes to be injected into the externalities. The raw prefix must be given. raw_prefixes: Vec>, From 21f4d90051c81d8a9b9d37717bcc1e9e13ca6a4d Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Tue, 1 Jun 2021 12:33:54 +0200 Subject: [PATCH 4/5] Apply suggestions from code review --- utils/frame/remote-externalities/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index a2b2ec11c46cb..6bff384044072 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -157,7 +157,7 @@ pub struct Builder { /// Custom key-pairs to be injected into the externalities. inject: Vec, /// Storage entry key prefixes to be injected into the externalities. The raw prefix must be given. - raw_prefixes: Vec>, + hashed_prefixes: Vec>, /// connectivity mode, online or offline. mode: Mode, } @@ -166,7 +166,7 @@ pub struct Builder { // that. impl Default for Builder { fn default() -> Self { - Self { inject: Default::default(), mode: Default::default(), raw_prefixes: Default::default() } + Self { inject: Default::default(), mode: Default::default(), hashed_prefixes: Default::default() } } } @@ -350,7 +350,7 @@ impl Builder { self.rpc_get_pairs_paged(StorageKey(vec![]), at).await? }; - for prefix in &self.raw_prefixes { + for prefix in &self.hashed_prefixes { info!(target: LOG_TARGET, "adding data for raw prefix: {:?}", HexDisplay::from(prefix)); let additional_key_values = self.rpc_get_pairs_paged(StorageKey(prefix.to_vec()), at).await?; keys_and_values.extend(additional_key_values); @@ -418,9 +418,9 @@ impl Builder { self } - /// Inject a manual raw prefixes - pub fn raw_prefix(mut self, raw: &[u8]) -> Self { - self.raw_prefixes.push(raw.to_vec()); + /// Inject a hashed prefix. This is treated as-is, and should be pre-hashed. + pub fn inject_hashed_prefix(mut self, hashed: &[u8]) -> Self { + self.hashed_prefixes.push(hashed.to_vec()); self } From d5012099930fd3777f9063a47ba6422168c4f789 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Tue, 1 Jun 2021 12:34:45 +0200 Subject: [PATCH 5/5] Apply suggestions from code review --- utils/frame/remote-externalities/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 6bff384044072..3ea97fc9d365b 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -156,7 +156,7 @@ impl Default for SnapshotConfig { pub struct Builder { /// Custom key-pairs to be injected into the externalities. inject: Vec, - /// Storage entry key prefixes to be injected into the externalities. The raw prefix must be given. + /// Storage entry key prefixes to be injected into the externalities. The *hashed* prefix must be given. hashed_prefixes: Vec>, /// connectivity mode, online or offline. mode: Mode, @@ -351,7 +351,7 @@ impl Builder { }; for prefix in &self.hashed_prefixes { - info!(target: LOG_TARGET, "adding data for raw prefix: {:?}", HexDisplay::from(prefix)); + info!(target: LOG_TARGET, "adding data for hashed prefix: {:?}", HexDisplay::from(prefix)); let additional_key_values = self.rpc_get_pairs_paged(StorageKey(prefix.to_vec()), at).await?; keys_and_values.extend(additional_key_values); }