diff --git a/Cargo.lock b/Cargo.lock index 903b76bff8023..e2321d7fa99d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4987,6 +4987,7 @@ dependencies = [ "sp-keystore", "sp-runtime", "substrate-frame-rpc-system", + "substrate-state-trie-migration-rpc", ] [[package]] @@ -6472,6 +6473,7 @@ dependencies = [ "sp-runtime", "sp-std", "sp-tracing", + "substrate-state-trie-migration-rpc", "thousands", "tokio", "zstd", @@ -10300,7 +10302,6 @@ dependencies = [ "sp-trie", "thiserror", "tracing", - "trie-db", "trie-root", ] @@ -10628,6 +10629,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "substrate-state-trie-migration-rpc" +version = "4.0.0-dev" +dependencies = [ + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "log 0.4.14", + "parity-scale-codec", + "sc-client-api", + "sc-rpc-api", + "scale-info", + "serde", + "serde_json", + "sp-core", + "sp-io", + "sp-runtime", + "sp-state-machine", + "sp-std", + "sp-trie", + "trie-db", +] + [[package]] name = "substrate-test-client" version = "2.0.1" diff --git a/Cargo.toml b/Cargo.toml index bce23456b27e5..13657dd1234a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,7 @@ members = [ "frame/staking", "frame/staking/reward-curve", "frame/staking/reward-fn", + "frame/state-trie-migration", "frame/sudo", "frame/support", "frame/support/procedural", @@ -210,6 +211,7 @@ members = [ "utils/frame/remote-externalities", "utils/frame/frame-utilities-cli", "utils/frame/try-runtime/cli", + "utils/frame/rpc/state-trie-migration-rpc", "utils/frame/rpc/support", "utils/frame/rpc/system", "utils/frame/generate-bags", diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 10d39f278f5f3..038c14029cf31 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -252,6 +252,7 @@ pub fn new_partial( let keystore = keystore_container.sync_keystore(); let chain_spec = config.chain_spec.cloned_box(); + let rpc_backend = backend.clone(); let rpc_extensions_builder = move |deny_unsafe, subscription_executor| { let deps = node_rpc::FullDeps { client: client.clone(), @@ -273,7 +274,7 @@ pub fn new_partial( }, }; - node_rpc::create_full(deps).map_err(Into::into) + node_rpc::create_full(deps, rpc_backend.clone()).map_err(Into::into) }; (rpc_extensions_builder, rpc_setup) diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml index 95ea3f8174fad..6c18e70f0d634 100644 --- a/bin/node/rpc/Cargo.toml +++ b/bin/node/rpc/Cargo.toml @@ -35,3 +35,4 @@ sp-consensus-babe = { version = "0.10.0-dev", path = "../../../primitives/consen sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../client/transaction-pool/api" } substrate-frame-rpc-system = { version = "4.0.0-dev", path = "../../../utils/frame/rpc/system" } +substrate-state-trie-migration-rpc = { version = "4.0.0-dev", path = "../../../utils/frame/rpc/state-trie-migration-rpc/" } diff --git a/bin/node/rpc/src/lib.rs b/bin/node/rpc/src/lib.rs index 31f2f41086885..09f350ed3dcf1 100644 --- a/bin/node/rpc/src/lib.rs +++ b/bin/node/rpc/src/lib.rs @@ -99,6 +99,7 @@ pub type IoHandler = jsonrpc_core::IoHandler; /// Instantiate all Full RPC extensions. pub fn create_full( deps: FullDeps, + backend: Arc, ) -> Result, Box> where C: ProvideRuntimeApi @@ -159,6 +160,10 @@ where finality_provider, ))); + io.extend_with(substrate_state_trie_migration_rpc::StateMigrationApi::to_delegate( + substrate_state_trie_migration_rpc::MigrationRpc::new(client.clone(), backend, deny_unsafe), + )); + io.extend_with(sc_sync_state_rpc::SyncStateRpcApi::to_delegate( sc_sync_state_rpc::SyncStateRpcHandler::new( chain_spec, diff --git a/frame/state-trie-migration/Cargo.toml b/frame/state-trie-migration/Cargo.toml index fb8bccb52d1f2..762fd85f13b62 100644 --- a/frame/state-trie-migration/Cargo.toml +++ b/frame/state-trie-migration/Cargo.toml @@ -21,6 +21,7 @@ sp-std = { default-features = false, path = "../../primitives/std" } sp-io = { default-features = false, path = "../../primitives/io" } sp-core = { default-features = false, path = "../../primitives/core" } sp-runtime = { default-features = false, path = "../../primitives/runtime" } +substrate-state-trie-migration-rpc = { optional = true, path = "../../utils/frame/rpc/state-trie-migration-rpc" } frame-support = { default-features = false, path = "../support" } frame-system = { default-features = false, path = "../system" } @@ -49,9 +50,9 @@ std = [ "sp-core/std", "sp-io/std", "sp-runtime/std", - "sp-std/std" + "sp-std/std", ] runtime-benchmarks = ["frame-benchmarking"] try-runtime = ["frame-support/try-runtime"] -remote-test = [ "std", "zstd", "serde", "thousands", "remote-externalities" ] +remote-test = [ "std", "zstd", "serde", "thousands", "remote-externalities", "substrate-state-trie-migration-rpc" ] diff --git a/frame/state-trie-migration/src/lib.rs b/frame/state-trie-migration/src/lib.rs index 4de130e9ac06b..a5a077c54e579 100644 --- a/frame/state-trie-migration/src/lib.rs +++ b/frame/state-trie-migration/src/lib.rs @@ -123,6 +123,13 @@ pub mod pallet { } } + #[derive(Clone, Encode, Decode, scale_info::TypeInfo, PartialEq, Eq)] + pub(crate) enum Progress { + ToStart, + LastKey(Vec), + Complete, + } + /// A migration task stored in state. /// /// It tracks the last top and child keys read. @@ -130,20 +137,13 @@ pub mod pallet { #[codec(mel_bound(T: Config))] #[scale_info(skip_type_params(T))] pub struct MigrationTask { - /// The last top key that we migrated. + /// The current top trie migration progress. + pub(crate) progress_top: Progress, + /// The current child trie migration progress. /// - /// If it does not exist, it means that the migration is done and no further keys exist. - pub(crate) last_top: Option>, - /// The last child key that we have processed. - /// - /// This is a child key under the current `self.last_top`. - /// - /// If this is set, no further top keys are processed until the child key migration is - /// complete. - pub(crate) last_child: Option>, - - /// A marker to indicate if the previous tick was a child tree migration or not. - pub(crate) prev_tick_child: bool, + /// If `ToStart`, no further top keys are processed until the child key migration is + /// `Complete`. + pub(crate) progress_child: Progress, /// Dynamic counter for the number of items that we have processed in this execution from /// the top trie. @@ -182,18 +182,22 @@ pub mod pallet { pub(crate) _ph: sp_std::marker::PhantomData, } + impl sp_std::fmt::Debug for Progress { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + match self { + Progress::ToStart => f.write_str("To start"), + Progress::LastKey(key) => + write!(f, "Last: {:?}", sp_core::hexdisplay::HexDisplay::from(key)), + Progress::Complete => f.write_str("Complete"), + } + } + } + impl sp_std::fmt::Debug for MigrationTask { fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { f.debug_struct("MigrationTask") - .field( - "top", - &self.last_top.as_ref().map(|d| sp_core::hexdisplay::HexDisplay::from(d)), - ) - .field( - "child", - &self.last_child.as_ref().map(|d| sp_core::hexdisplay::HexDisplay::from(d)), - ) - .field("prev_tick_child", &self.prev_tick_child) + .field("top", &self.progress_top) + .field("child", &self.progress_child) .field("dyn_top_items", &self.dyn_top_items) .field("dyn_child_items", &self.dyn_child_items) .field("dyn_size", &self.dyn_size) @@ -207,12 +211,11 @@ pub mod pallet { impl Default for MigrationTask { fn default() -> Self { Self { - last_top: Some(Default::default()), - last_child: Default::default(), + progress_top: Progress::ToStart, + progress_child: Progress::ToStart, dyn_child_items: Default::default(), dyn_top_items: Default::default(), dyn_size: Default::default(), - prev_tick_child: Default::default(), _ph: Default::default(), size: Default::default(), top_items: Default::default(), @@ -224,7 +227,7 @@ pub mod pallet { impl MigrationTask { /// Return true if the task is finished. pub(crate) fn finished(&self) -> bool { - self.last_top.is_none() && self.last_child.is_none() + matches!(self.progress_top, Progress::Complete) } /// Check if there's any work left, or if we have exhausted the limits already. @@ -269,51 +272,36 @@ pub mod pallet { /// /// This function is *the* core of this entire pallet. fn migrate_tick(&mut self) { - match (self.last_top.as_ref(), self.last_child.as_ref()) { - (Some(_), Some(_)) => { + match (&self.progress_top, &self.progress_child) { + (Progress::ToStart, _) => { + self.migrate_top(); + }, + (Progress::LastKey(_), Progress::LastKey(_)) => { // we're in the middle of doing work on a child tree. self.migrate_child(); }, - (Some(ref top_key), None) => { - // we have a top key and no child key. 3 possibilities exist: - // 1. we continue the top key migrations. - // 2. this is the root of a child key, and we start processing child keys (and - // should call `migrate_child`). + (Progress::LastKey(top_key), Progress::ToStart) => { // 3. this is the root of a child key, and we are finishing all child-keys (and // should call `migrate_top`). // NOTE: this block is written intentionally to verbosely for easy of // verification. - match ( - top_key.starts_with(DEFAULT_CHILD_STORAGE_KEY_PREFIX), - self.prev_tick_child, - ) { - (false, false) => { - // continue the top key migration - self.migrate_top(); - }, - (true, false) => { - self.last_child = Some(Default::default()); - self.migrate_child(); - self.prev_tick_child = true; - }, - (true, true) => { - // we're done with migrating a child-root. - self.prev_tick_child = false; - self.migrate_top(); - }, - (false, true) => { - // should never happen. - log!(error, "LOGIC ERROR: unreachable code [0]."); - Pallet::::halt(); - }, - }; + if !top_key.starts_with(DEFAULT_CHILD_STORAGE_KEY_PREFIX) { + // we continue the top key migrations. + // continue the top key migration + self.migrate_top(); + } else { + // this is the root of a child key, and we start processing child keys (and + // should call `migrate_child`). + self.migrate_child(); + } }, - (None, Some(_)) => { - log!(error, "LOGIC ERROR: unreachable code [1]."); - Pallet::::halt() + (Progress::LastKey(_), Progress::Complete) => { + // we're done with migrating a child-root. + self.migrate_top(); + self.progress_child = Progress::ToStart; }, - (None, None) => { + (Progress::Complete, _) => { // nada }, } @@ -324,19 +312,26 @@ pub mod pallet { /// It updates the dynamic counters. fn migrate_child(&mut self) { use sp_io::default_child_storage as child_io; - let (last_child, last_top) = match (&self.last_child, &self.last_top) { - (Some(last_child), Some(last_top)) => (last_child, last_top), + let (maybe_current_child, child_root) = match (&self.progress_child, &self.progress_top) + { + (Progress::LastKey(last_child), Progress::LastKey(last_top)) => { + let child_root = Pallet::::transform_child_key_or_halt(&last_top); + let maybe_current_child = child_io::next_key(child_root, &last_child); + (maybe_current_child, child_root) + }, + (Progress::ToStart, Progress::LastKey(last_top)) => { + let child_root = Pallet::::transform_child_key_or_halt(&last_top); + // Start with the empty key as first key. + (Some(Vec::new()), child_root) + }, _ => { - // defensive: this function is only called when both of these values exist. - // much that we can do otherwise.. + // defensive: there must be an ongoing top migration. frame_support::defensive!("cannot migrate child key."); return }, }; - let child_root = Pallet::::transform_child_key_or_halt(&last_top); - let maybe_current_child = child_io::next_key(child_root, &last_child); - if let Some(ref current_child) = maybe_current_child { + if let Some(current_child) = maybe_current_child.as_ref() { let added_size = if let Some(data) = child_io::get(child_root, ¤t_child) { child_io::set(child_root, current_child, &data); data.len() as u32 @@ -348,25 +343,28 @@ pub mod pallet { } log!(trace, "migrated a child key, next_child_key: {:?}", maybe_current_child); - self.last_child = maybe_current_child; + self.progress_child = match maybe_current_child { + Some(last_child) => Progress::LastKey(last_child), + None => Progress::Complete, + } } /// Migrate the current top key, setting it to its new value, if one exists. /// /// It updates the dynamic counters. fn migrate_top(&mut self) { - let last_top = match &self.last_top { - Some(last_top) => last_top, - None => { - // defensive: this function is only called when this value exist. - // much that we can do otherwise.. + let maybe_current_top = match &self.progress_top { + Progress::LastKey(last_top) => sp_io::storage::next_key(last_top), + // Start with the empty key as first key. + Progress::ToStart => Some(Vec::new()), + Progress::Complete => { + // defensive: there must be an ongoing top migration. frame_support::defensive!("cannot migrate top key."); return }, }; - let maybe_current_top = sp_io::storage::next_key(last_top); - if let Some(ref current_top) = maybe_current_top { + if let Some(current_top) = maybe_current_top.as_ref() { let added_size = if let Some(data) = sp_io::storage::get(¤t_top) { sp_io::storage::set(¤t_top, &data); data.len() as u32 @@ -378,7 +376,10 @@ pub mod pallet { } log!(trace, "migrated a top key, next_top_key = {:?}", maybe_current_top); - self.last_top = maybe_current_top; + self.progress_top = match maybe_current_top { + Some(last_top) => Progress::LastKey(last_top), + None => Progress::Complete, + } } } @@ -811,7 +812,7 @@ mod benchmarks { continue_migrate_wrong_witness { let null = MigrationLimits::default(); let caller = frame_benchmarking::whitelisted_caller(); - let bad_witness = MigrationTask { last_top: Some(vec![1u8]), ..Default::default() }; + let bad_witness = MigrationTask { progress_top: Progress::LastKey(vec![1u8]), ..Default::default() }; }: { assert!( StateTrieMigration::::continue_migrate( @@ -1141,15 +1142,12 @@ mod test { } #[test] - #[ignore] fn detects_value_in_empty_top_key() { let limit = MigrationLimits { item: 1, size: 1000 }; let initial_keys = Some(vec![(vec![], vec![66u8; 77])]); let mut ext = new_test_ext(StateVersion::V0, false, initial_keys.clone(), None); let root_upgraded = ext.execute_with(|| { - sp_io::storage::set(&[], &vec![66u8; 77]); - AutoLimits::::put(Some(limit)); let root = run_to_block(30).0; @@ -1168,9 +1166,7 @@ mod test { } #[test] - #[ignore] fn detects_value_in_first_child_key() { - use frame_support::storage::child; let limit = MigrationLimits { item: 1, size: 1000 }; let initial_child = Some(vec![(b"chk1".to_vec(), vec![], vec![66u8; 77])]); let mut ext = new_test_ext(StateVersion::V0, false, None, initial_child.clone()); @@ -1186,7 +1182,6 @@ mod test { let mut ext2 = new_test_ext(StateVersion::V1, false, None, initial_child); let root = ext2.execute_with(|| { - child::put(&child::ChildInfo::new_default(b"chk1"), &[], &vec![66u8; 77]); AutoLimits::::put(Some(limit)); run_to_block(30).0 }); @@ -1214,7 +1209,7 @@ mod test { // eventually everything is over. assert!(matches!( StateTrieMigration::migration_process(), - MigrationTask { last_child: None, last_top: None, .. } + MigrationTask { progress_top: Progress::Complete, .. } )); root }); @@ -1276,7 +1271,10 @@ mod test { Origin::signed(1), MigrationLimits { item: 5, size: 100 }, 100, - MigrationTask { last_top: Some(vec![1u8]), ..Default::default() } + MigrationTask { + progress_top: Progress::LastKey(vec![1u8]), + ..Default::default() + } ), Error::::BadWitness ); @@ -1451,7 +1449,8 @@ pub(crate) mod remote_tests { // set the version to 1, as if the upgrade happened. ext.state_version = sp_core::storage::StateVersion::V1; - let (top_left, child_left) = ext.as_backend().essence().check_migration_state().unwrap(); + let (top_left, child_left) = + substrate_state_trie_migration_rpc::migration_status(&ext.as_backend()).unwrap(); assert!( top_left > 0, "no node needs migrating, this probably means that state was initialized with `StateVersion::V1`", @@ -1509,7 +1508,8 @@ pub(crate) mod remote_tests { ) }); - let (top_left, child_left) = ext.as_backend().essence().check_migration_state().unwrap(); + let (top_left, child_left) = + substrate_state_trie_migration_rpc::migration_status(&ext.as_backend()).unwrap(); assert_eq!(top_left, 0); assert_eq!(child_left, 0); } diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 2a6ab80133ee6..80651130575ea 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -18,7 +18,6 @@ log = { version = "0.4.11", optional = true } thiserror = { version = "1.0.30", optional = true } parking_lot = { version = "0.12.0", optional = true } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.23.1", default-features = false } trie-root = { version = "0.17.0", default-features = false } sp-trie = { version = "6.0.0", path = "../trie", default-features = false } sp-core = { version = "6.0.0", path = "../core", default-features = false } @@ -47,7 +46,6 @@ std = [ "sp-externalities/std", "sp-std/std", "sp-trie/std", - "trie-db/std", "trie-root/std", "log", "thiserror", diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index b0eb543824379..8531e4907d6a7 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -23,7 +23,7 @@ use codec::Encode; use hash_db::{self, AsHashDB, HashDB, HashDBRef, Hasher, Prefix}; #[cfg(feature = "std")] use parking_lot::RwLock; -use sp_core::storage::{ChildInfo, ChildType, PrefixedStorageKey, StateVersion}; +use sp_core::storage::{ChildInfo, ChildType, StateVersion}; use sp_std::{boxed::Box, vec::Vec}; use sp_trie::{ child_delta_trie_root, delta_trie_root, empty_child_trie_root, read_child_trie_value, @@ -36,10 +36,6 @@ use sp_trie::{ use std::collections::HashMap; #[cfg(feature = "std")] use std::sync::Arc; -use trie_db::{ - node::{NodePlan, ValuePlan}, - TrieDBNodeIterator, -}; #[cfg(not(feature = "std"))] macro_rules! format { @@ -433,72 +429,6 @@ where ); } - /// Check remaining state item to migrate. Note this function should be remove when all state - /// migration did finished as it is only an utility. - // original author: @cheme - pub fn check_migration_state(&self) -> Result<(u64, u64)> { - let threshold: u32 = sp_core::storage::TRIE_VALUE_NODE_THRESHOLD; - let mut nb_to_migrate = 0; - let mut nb_to_migrate_child = 0; - - let trie = sp_trie::trie_types::TrieDB::new(self, &self.root) - .map_err(|e| format!("TrieDB creation error: {}", e))?; - let iter_node = TrieDBNodeIterator::new(&trie) - .map_err(|e| format!("TrieDB node iterator error: {}", e))?; - for node in iter_node { - let node = node.map_err(|e| format!("TrieDB node iterator error: {}", e))?; - match node.2.node_plan() { - NodePlan::Leaf { value, .. } | - NodePlan::NibbledBranch { value: Some(value), .. } => - if let ValuePlan::Inline(range) = value { - if (range.end - range.start) as u32 >= threshold { - nb_to_migrate += 1; - } - }, - _ => (), - } - } - - let mut child_roots: Vec<(ChildInfo, Vec)> = Vec::new(); - // get all child trie roots - for key_value in trie.iter().map_err(|e| format!("TrieDB node iterator error: {}", e))? { - let (key, value) = - key_value.map_err(|e| format!("TrieDB node iterator error: {}", e))?; - if key[..] - .starts_with(sp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX) - { - let prefixed_key = PrefixedStorageKey::new(key); - let (_type, unprefixed) = ChildType::from_prefixed_key(&prefixed_key).unwrap(); - child_roots.push((ChildInfo::new_default(unprefixed), value)); - } - } - for (child_info, root) in child_roots { - let mut child_root = H::Out::default(); - let storage = KeySpacedDB::new(self, child_info.keyspace()); - - child_root.as_mut()[..].copy_from_slice(&root[..]); - let trie = sp_trie::trie_types::TrieDB::new(&storage, &child_root) - .map_err(|e| format!("New child TrieDB error: {}", e))?; - let iter_node = TrieDBNodeIterator::new(&trie) - .map_err(|e| format!("TrieDB node iterator error: {}", e))?; - for node in iter_node { - let node = node.map_err(|e| format!("Child TrieDB node iterator error: {}", e))?; - match node.2.node_plan() { - NodePlan::Leaf { value, .. } | - NodePlan::NibbledBranch { value: Some(value), .. } => - if let ValuePlan::Inline(range) = value { - if (range.end - range.start) as u32 >= threshold { - nb_to_migrate_child += 1; - } - }, - _ => (), - } - } - } - - Ok((nb_to_migrate, nb_to_migrate_child)) - } - /// Returns all `(key, value)` pairs in the trie. pub fn pairs(&self) -> Vec<(StorageKey, StorageValue)> { let collect_all = || -> sp_std::result::Result<_, Box>> { diff --git a/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml b/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml new file mode 100644 index 0000000000000..deb641a89a466 --- /dev/null +++ b/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "substrate-state-trie-migration-rpc" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "Node-specific RPC methods for interaction with state trie migration." +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +serde = { version = "1", features = ["derive"] } +log = { version = "0.4.14", default-features = false } + +sp-std = { path = "../../../../primitives/std" } +sp-io = { path = "../../../../primitives/io" } +sp-core = { path = "../../../../primitives/core" } +sp-state-machine = { path = "../../../../primitives/state-machine" } +sp-trie = { path = "../../../../primitives/trie" } +trie-db = { version = "0.23.1" } + +jsonrpc-core = "18.0.0" +jsonrpc-core-client = "18.0.0" +jsonrpc-derive = "18.0.0" + +# Substrate Dependencies +sc-client-api = { version = "4.0.0-dev", path = "../../../../client/api" } +sc-rpc-api = { version = "0.10.0-dev", path = "../../../../client/rpc-api" } +sp-runtime = { version = "6.0.0", path = "../../../../primitives/runtime" } + +[dev-dependencies] +serde_json = "1" diff --git a/utils/frame/rpc/state-trie-migration-rpc/README.md b/utils/frame/rpc/state-trie-migration-rpc/README.md new file mode 100644 index 0000000000000..03bbfdf1b5939 --- /dev/null +++ b/utils/frame/rpc/state-trie-migration-rpc/README.md @@ -0,0 +1,3 @@ +Node-specific RPC methods for interaction with trie migration. + +License: Apache-2.0 diff --git a/utils/frame/rpc/state-trie-migration-rpc/src/lib.rs b/utils/frame/rpc/state-trie-migration-rpc/src/lib.rs new file mode 100644 index 0000000000000..98a3cf964843c --- /dev/null +++ b/utils/frame/rpc/state-trie-migration-rpc/src/lib.rs @@ -0,0 +1,164 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rpc for state migration. + +use jsonrpc_core::{Error, ErrorCode, Result}; +use jsonrpc_derive::rpc; +use sc_rpc_api::DenyUnsafe; +use serde::{Deserialize, Serialize}; +use sp_runtime::{generic::BlockId, traits::Block as BlockT}; +use std::sync::Arc; + +use sp_core::{ + storage::{ChildInfo, ChildType, PrefixedStorageKey}, + Hasher, +}; +use sp_state_machine::Backend; +use sp_trie::{trie_types::TrieDB, KeySpacedDB, Trie}; +use trie_db::{ + node::{NodePlan, ValuePlan}, + TrieDBNodeIterator, +}; + +fn count_migrate<'a, H: Hasher>( + storage: &'a dyn trie_db::HashDBRef>, + root: &'a H::Out, +) -> std::result::Result<(u64, TrieDB<'a, H>), String> { + let mut nb = 0u64; + let trie = TrieDB::new(storage, root).map_err(|e| format!("TrieDB creation error: {}", e))?; + let iter_node = + TrieDBNodeIterator::new(&trie).map_err(|e| format!("TrieDB node iterator error: {}", e))?; + for node in iter_node { + let node = node.map_err(|e| format!("TrieDB node iterator error: {}", e))?; + match node.2.node_plan() { + NodePlan::Leaf { value, .. } | NodePlan::NibbledBranch { value: Some(value), .. } => + if let ValuePlan::Inline(range) = value { + if (range.end - range.start) as u32 >= + sp_core::storage::TRIE_VALUE_NODE_THRESHOLD + { + nb += 1; + } + }, + _ => (), + } + } + Ok((nb, trie)) +} + +/// Check trie migration status. +pub fn migration_status(backend: &B) -> std::result::Result<(u64, u64), String> +where + H: Hasher, + H::Out: codec::Codec, + B: Backend, +{ + let trie_backend = if let Some(backend) = backend.as_trie_backend() { + backend + } else { + return Err("No access to trie from backend.".to_string()) + }; + let essence = trie_backend.essence(); + let (nb_to_migrate, trie) = count_migrate(essence, &essence.root())?; + + let mut nb_to_migrate_child = 0; + let mut child_roots: Vec<(ChildInfo, Vec)> = Vec::new(); + // get all child trie roots + for key_value in trie.iter().map_err(|e| format!("TrieDB node iterator error: {}", e))? { + let (key, value) = key_value.map_err(|e| format!("TrieDB node iterator error: {}", e))?; + if key[..].starts_with(sp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX) + { + let prefixed_key = PrefixedStorageKey::new(key); + let (_type, unprefixed) = ChildType::from_prefixed_key(&prefixed_key).unwrap(); + child_roots.push((ChildInfo::new_default(unprefixed), value)); + } + } + for (child_info, root) in child_roots { + let mut child_root = H::Out::default(); + let storage = KeySpacedDB::new(essence, child_info.keyspace()); + + child_root.as_mut()[..].copy_from_slice(&root[..]); + nb_to_migrate_child += count_migrate(&storage, &child_root)?.0; + } + + Ok((nb_to_migrate, nb_to_migrate_child)) +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct MigrationStatusResult { + top_remaining_to_migrate: u64, + child_remaining_to_migrate: u64, +} + +/// Migration RPC methods. +#[rpc] +pub trait StateMigrationApi { + /// Check current migration state. + /// + /// This call is performed locally without submitting any transactions. Thus executing this + /// won't change any state. Nonetheless it is a VERY costy call that should be + /// only exposed to trusted peers. + #[rpc(name = "state_trieMigrationStatus")] + fn call(&self, at: Option) -> Result; +} + +/// An implementation of state migration specific RPC methods. +pub struct MigrationRpc { + client: Arc, + backend: Arc, + deny_unsafe: DenyUnsafe, + _marker: std::marker::PhantomData<(B, BA)>, +} + +impl MigrationRpc { + /// Create new state migration rpc for the given reference to the client. + pub fn new(client: Arc, backend: Arc, deny_unsafe: DenyUnsafe) -> Self { + MigrationRpc { client, backend, deny_unsafe, _marker: Default::default() } + } +} + +impl StateMigrationApi<::Hash> for MigrationRpc +where + B: BlockT, + C: Send + Sync + 'static + sc_client_api::HeaderBackend, + BA: 'static + sc_client_api::backend::Backend, +{ + fn call(&self, at: Option<::Hash>) -> Result { + if let Err(err) = self.deny_unsafe.check_if_safe() { + return Err(err.into()) + } + + let block_id = BlockId::hash(at.unwrap_or_else(|| self.client.info().best_hash)); + let state = self.backend.state_at(block_id).map_err(error_into_rpc_err)?; + let (top, child) = migration_status(&state).map_err(error_into_rpc_err)?; + + Ok(MigrationStatusResult { + top_remaining_to_migrate: top, + child_remaining_to_migrate: child, + }) + } +} + +fn error_into_rpc_err(err: impl std::fmt::Display) -> Error { + Error { + code: ErrorCode::InternalError, + message: "Error while checking migration state".into(), + data: Some(err.to_string().into()), + } +}