From 611af08dd9bfb1af1327a33be4263b4993ce43de Mon Sep 17 00:00:00 2001 From: mm-near Date: Tue, 7 Feb 2023 18:48:21 +0100 Subject: [PATCH 1/8] first version of the CLI tool for flat storage --- Cargo.lock | 83 ++++++++ chain/chain/src/flat_storage_creator.rs | 7 +- core/store/src/flat_state.rs | 64 +++++- neard/Cargo.toml | 1 + neard/src/cli.rs | 7 + tools/flat-storage/Cargo.toml | 31 +++ tools/flat-storage/README.md | 1 + tools/flat-storage/src/lib.rs | 269 ++++++++++++++++++++++++ 8 files changed, 449 insertions(+), 14 deletions(-) create mode 100644 tools/flat-storage/Cargo.toml create mode 100644 tools/flat-storage/README.md create mode 100644 tools/flat-storage/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 86cfe3b5047..12a3a9d106f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1398,6 +1398,31 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crossterm" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" +dependencies = [ + "bitflags", + "crossterm_winapi", + "libc", + "mio", + "parking_lot 0.12.1", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -3261,6 +3286,33 @@ dependencies = [ "tracing", ] +[[package]] +name = "near-flat-storage" +version = "0.0.0" +dependencies = [ + "anyhow", + "borsh", + "clap 3.1.18", + "near-chain", + "near-chain-configs", + "near-crypto", + "near-epoch-manager", + "near-network", + "near-primitives", + "near-primitives-core", + "near-store", + "near-test-contracts", + "nearcore", + "node-runtime", + "num-rational", + "rayon", + "serde", + "serde_json", + "tempfile", + "tqdm", + "tracing", +] + [[package]] name = "near-indexer" version = "0.0.0" @@ -3955,6 +4007,7 @@ dependencies = [ "near-chain-configs", "near-client", "near-dyn-configs", + "near-flat-storage", "near-jsonrpc-primitives", "near-mirror", "near-network", @@ -5642,6 +5695,27 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +[[package]] +name = "signal-hook" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -6298,6 +6372,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" +[[package]] +name = "tqdm" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f056dcbe6bffd4595b23fba18affc5bd8c36ebd3199a112b1046f14f57b7323" +dependencies = [ + "crossterm", +] + [[package]] name = "tracing" version = "0.1.36" diff --git a/chain/chain/src/flat_storage_creator.rs b/chain/chain/src/flat_storage_creator.rs index 837aaa58151..1290a9b160b 100644 --- a/chain/chain/src/flat_storage_creator.rs +++ b/chain/chain/src/flat_storage_creator.rs @@ -188,7 +188,7 @@ impl FlatStorageShardCreator { /// Creates flat storage when all intermediate steps are finished. /// Returns boolean indicating if flat storage was created. #[cfg(feature = "protocol_feature_flat_state")] - pub(crate) fn update_status( + pub fn update_status( &mut self, chain_store: &ChainStore, thread_pool: &rayon::ThreadPool, @@ -202,7 +202,7 @@ impl FlatStorageShardCreator { let final_head = chain_store.final_head()?; let final_height = final_head.height; - if final_height > self.start_height { + if final_height >= self.start_height { // If it holds, deltas for all blocks after final head are saved to disk, because they have bigger // heights than one on which we launched a node. Check that it is true: for height in final_height + 1..=chain_store.head()?.height { @@ -372,7 +372,8 @@ impl FlatStorageShardCreator { merged_delta.merge(delta.as_ref()); } - if old_flat_head != &flat_head { + if (old_flat_head != &flat_head) || (flat_head == chain_final_head.last_block_hash) + { // If flat head changes, save all changes to store. let old_height = chain_store.get_block_height(&old_flat_head).unwrap(); let height = chain_store.get_block_height(&flat_head).unwrap(); diff --git a/core/store/src/flat_state.rs b/core/store/src/flat_state.rs index 50f6d9f8869..1aa97606895 100644 --- a/core/store/src/flat_state.rs +++ b/core/store/src/flat_state.rs @@ -425,11 +425,7 @@ impl FlatStateDelta { use near_o11y::metrics::IntGauge; use near_primitives::errors::StorageError; -#[cfg(feature = "protocol_feature_flat_state")] -use near_primitives::shard_layout::account_id_to_shard_id; use near_primitives::shard_layout::ShardLayout; -#[cfg(feature = "protocol_feature_flat_state")] -use near_primitives::trie_key::trie_key_parsers::parse_account_id_from_raw_key; use std::sync::{Arc, RwLock}; #[cfg(feature = "protocol_feature_flat_state")] use tracing::info; @@ -573,8 +569,11 @@ pub mod store_helper { use crate::{FlatStateDelta, Store, StoreUpdate}; use borsh::{BorshDeserialize, BorshSerialize}; use byteorder::ReadBytesExt; + use near_primitives::errors::StorageError; use near_primitives::hash::CryptoHash; + use near_primitives::shard_layout::{account_id_to_shard_id, ShardLayout}; use near_primitives::state::ValueRef; + use near_primitives::trie_key::trie_key_parsers::parse_account_id_from_raw_key; use near_primitives::types::ShardId; use std::sync::Arc; @@ -735,6 +734,54 @@ pub mod store_helper { pub fn remove_flat_storage_creation_status(store_update: &mut StoreUpdate, shard_id: ShardId) { store_update.delete(crate::DBCol::FlatStateMisc, &creation_status_key(shard_id)); } + + /// Iterate over flat storage entries for a given shard. + /// It reads data only from the 'main' column - which represents the state as of final head.k + /// + /// WARNING: flat storage keeps changing, so the results might be inconsistent, unless you're running + /// this method on the shapshot of the data. + pub fn iter_flat_state_entries<'a>( + shard_layout: ShardLayout, + shard_id: u64, + store: &'a Store, + from: Option>, + to: Option>, + ) -> impl Iterator, Box<[u8]>)> + 'a { + store.iter(crate::DBCol::FlatState).filter_map(move |result| { + if let Ok((key, value)) = result { + // Currently all the data in flat storage is 'together' - so we have to parse the key, + // to see if this element belongs to this shard. + if let Ok(key_in_shard) = key_belongs_to_shard(&key, &shard_layout, shard_id) { + if key_in_shard { + // Right now this function is very slow, as we iterate over whole flat storage DB (and ignore most of the keys). + // We should add support to our Database object to handle range iterators. + if from.as_ref().map_or(true, |x| x.as_slice() <= key.as_ref()) + && to.as_ref().map_or(true, |x| x.as_slice() >= key.as_ref()) + { + return Some((key, value)); + } + } + } + } + return None; + }) + } + + /// Currently all the data in flat storage is 'together' - so we have to parse the key, + /// to see if this element belongs to this shard. + pub fn key_belongs_to_shard( + key: &Box<[u8]>, + shard_layout: &ShardLayout, + shard_id: u64, + ) -> Result { + let account_id = parse_account_id_from_raw_key(key) + .map_err(|e| StorageError::StorageInconsistentState(e.to_string()))? + .ok_or(StorageError::FlatStorageError(format!( + "Failed to find account id in flat storage key {:?}", + key + )))?; + Ok(account_id_to_shard_id(&account_id, &shard_layout) == shard_id) + } } #[cfg(not(feature = "protocol_feature_flat_state"))] @@ -1092,13 +1139,8 @@ impl FlatStorageState { for item in guard.store.iter(crate::DBCol::FlatState) { let (key, _) = item.map_err(|e| StorageError::StorageInconsistentState(e.to_string()))?; - let account_id = parse_account_id_from_raw_key(&key) - .map_err(|e| StorageError::StorageInconsistentState(e.to_string()))? - .ok_or(StorageError::FlatStorageError(format!( - "Failed to find account id in flat storage key {:?}", - key - )))?; - if account_id_to_shard_id(&account_id, &shard_layout) == shard_id { + + if store_helper::key_belongs_to_shard(&key, &shard_layout, shard_id)? { removed_items += 1; store_update.delete(crate::DBCol::FlatState, &key); } diff --git a/neard/Cargo.toml b/neard/Cargo.toml index 9af29ad4772..c4215f248a5 100644 --- a/neard/Cargo.toml +++ b/neard/Cargo.toml @@ -35,6 +35,7 @@ near-chain-configs = { path = "../core/chain-configs" } near-client = { path = "../chain/client" } near-cold-store-tool = { path = "../tools/cold-store", package = "cold-store-tool" } near-dyn-configs = { path = "../core/dyn-configs" } +near-flat-storage = { path = "../tools/flat-storage" } near-jsonrpc-primitives = { path = "../chain/jsonrpc-primitives" } near-mirror = { path = "../tools/mirror" } near-network = { path = "../chain/network" } diff --git a/neard/src/cli.rs b/neard/src/cli.rs index 62104d2a57f..a46310d4593 100644 --- a/neard/src/cli.rs +++ b/neard/src/cli.rs @@ -6,6 +6,7 @@ use near_chain_configs::GenesisValidationMode; use near_client::ConfigUpdater; use near_cold_store_tool::ColdStoreCommand; use near_dyn_configs::{UpdateableConfigLoader, UpdateableConfigLoaderError, UpdateableConfigs}; +use near_flat_storage::FlatStorageCommand; use near_jsonrpc_primitives::types::light_client::RpcLightClientExecutionProofResponse; use near_mirror::MirrorCommand; use near_network::tcp; @@ -117,6 +118,9 @@ impl NeardCmd { NeardSubCommand::StateParts(cmd) => { cmd.run()?; } + NeardSubCommand::FlatStorage(cmd) => { + cmd.run(&home_dir)?; + } }; Ok(()) } @@ -226,6 +230,9 @@ pub(super) enum NeardSubCommand { /// Connects to a NEAR node and sends state parts requests after the handshake is completed. StateParts(StatePartsCommand), + + /// Flat storage related tooling. + FlatStorage(FlatStorageCommand), } #[derive(Parser)] diff --git a/tools/flat-storage/Cargo.toml b/tools/flat-storage/Cargo.toml new file mode 100644 index 00000000000..bc601bc27c3 --- /dev/null +++ b/tools/flat-storage/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "near-flat-storage" +version = "0.0.0" +authors.workspace = true +publish = false +edition.workspace = true + +[dependencies] +anyhow.workspace = true +borsh.workspace = true +clap.workspace = true +num-rational.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile.workspace = true +tracing.workspace = true +rayon.workspace = true + +tqdm = "0.4.4" + +near-chain = { path = "../../chain/chain" } +near-chain-configs = { path = "../../core/chain-configs" } +near-crypto = { path = "../../core/crypto" } +near-epoch-manager = { path = "../../chain/epoch-manager" } +near-network = { path = "../../chain/network" } +near-primitives = { path = "../../core/primitives" } +near-primitives-core = { path = "../../core/primitives-core" } +near-store = { path = "../../core/store" } +near-test-contracts = { path = "../../runtime/near-test-contracts" } +nearcore = { path = "../../nearcore" } +node-runtime = { path = "../../runtime/runtime" } \ No newline at end of file diff --git a/tools/flat-storage/README.md b/tools/flat-storage/README.md new file mode 100644 index 00000000000..3d484711e2c --- /dev/null +++ b/tools/flat-storage/README.md @@ -0,0 +1 @@ +Set of tools for debugging and experimenting with flat storage. \ No newline at end of file diff --git a/tools/flat-storage/src/lib.rs b/tools/flat-storage/src/lib.rs new file mode 100644 index 00000000000..87bb9d12ef2 --- /dev/null +++ b/tools/flat-storage/src/lib.rs @@ -0,0 +1,269 @@ +/// Tools for modifying flat storage - should be used only for experimentation & debugging. +use clap::Parser; +use near_chain::{ + flat_storage_creator::FlatStorageShardCreator, types::RuntimeAdapter, ChainStore, + ChainStoreAccess, +}; +use near_epoch_manager::EpochManagerAdapter; +use near_primitives::{state::ValueRef, trie_key::trie_key_parsers::parse_account_id_from_raw_key}; +use near_store::{ + flat_state::store_helper::{self, get_flat_head, get_flat_storage_creation_status}, + Mode, NodeStorage, Store, StoreOpener, +}; +use nearcore::{load_config, NearConfig, NightshadeRuntime}; +use std::{path::PathBuf, sync::Arc, time::Duration}; +use tqdm::tqdm; + +#[derive(Parser)] +pub struct FlatStorageCommand { + #[clap(subcommand)] + subcmd: SubCommand, +} + +#[derive(Parser)] +#[clap(subcommand_required = true, arg_required_else_help = true)] +enum SubCommand { + /// View the current state of flat storage + View, + + /// Reset the flat storage state (remove all the contents) + Reset(ResetCmd), + + /// Init the flat storage state, by copying from trie + Init(InitCmd), + + /// Verify flat storage state + Verify(VerifyCmd), + + /// Temporary command to set the store version (useful as long flat + /// storage is enabled only during nightly with separate DB version). + SetStoreVersion(SetStoreVersionCmd), +} + +#[derive(Parser)] +pub struct SetStoreVersionCmd { + version: u32, +} + +#[derive(Parser)] +pub struct ResetCmd { + shard_id: u64, +} + +#[derive(Parser)] + +pub struct InitCmd { + shard_id: u64, + + #[clap(default_value = "3")] + num_threads: usize, +} + +#[derive(Parser)] +pub struct VerifyCmd { + shard_id: u64, +} + +impl FlatStorageCommand { + fn get_db( + opener: &StoreOpener, + home_dir: &PathBuf, + near_config: &NearConfig, + mode: Mode, + ) -> (NodeStorage, Arc, ChainStore, Store) { + let node_storage = opener.open_in_mode(mode).unwrap(); + let hot_runtime = Arc::new(NightshadeRuntime::from_config( + home_dir, + node_storage.get_hot_store(), + &near_config, + )); + let chain_store = ChainStore::new(node_storage.get_hot_store(), 0, false); + let hot_store = node_storage.get_hot_store(); + (node_storage, hot_runtime, chain_store, hot_store) + } + + pub fn run(&self, home_dir: &PathBuf) -> anyhow::Result<()> { + let near_config = + load_config(home_dir, near_chain_configs::GenesisValidationMode::Full).unwrap(); + let opener = NodeStorage::opener(home_dir, &near_config.config.store, None); + + match &self.subcmd { + SubCommand::View => { + let (_, hot_runtime, chain_store, hot_store) = + Self::get_db(&opener, home_dir, &near_config, near_store::Mode::ReadOnly); + let tip = chain_store.final_head().unwrap(); + let shards = hot_runtime.num_shards(&tip.epoch_id).unwrap(); + println!("DB version: {:?}", hot_store.get_db_version()); + println!("Current final tip @{:?} - shards: {:?}", tip.height, shards); + + for shard in 0..shards { + let head_hash = get_flat_head(&hot_store, shard); + if let Some(head_hash) = head_hash { + let head_header = chain_store.get_block_header(&head_hash); + let creation_status = get_flat_storage_creation_status(&hot_store, shard); + println!( + "Shard: {:?} - flat storage @{:?} Details: {:?}", + shard, + head_header.map(|header| header.height()), + creation_status + ); + } else { + println!("Shard: {:?} - no flat storage.", shard) + } + } + } + SubCommand::SetStoreVersion(set_version) => { + let rw_storage = opener.open_in_mode(near_store::Mode::ReadWriteExisting).unwrap(); + let rw_store = rw_storage.get_hot_store(); + println!("Setting storage DB version to: {:?}", set_version.version); + rw_store.set_db_version(set_version.version)?; + } + SubCommand::Reset(reset_cmd) => { + let (_, rw_hot_runtime, rw_chain_store, _) = Self::get_db( + &opener, + home_dir, + &near_config, + near_store::Mode::ReadWriteExisting, + ); + let tip = rw_chain_store.final_head().unwrap(); + + // TODO: there should be a method that 'loads' the current flat storage state based on Storage. + rw_hot_runtime.create_flat_storage_state_for_shard( + reset_cmd.shard_id, + tip.height, + &rw_chain_store, + ); + + rw_hot_runtime + .remove_flat_storage_state_for_shard(reset_cmd.shard_id, &tip.epoch_id)?; + } + SubCommand::Init(init_cmd) => { + let (_, rw_hot_runtime, rw_chain_store, rw_hot_store) = Self::get_db( + &opener, + home_dir, + &near_config, + near_store::Mode::ReadWriteExisting, + ); + + let tip = rw_chain_store.final_head().unwrap(); + + let mut creator = + FlatStorageShardCreator::new(init_cmd.shard_id, tip.height, rw_hot_runtime); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(init_cmd.num_threads) + .build() + .unwrap(); + + loop { + let status = creator.update_status(&rw_chain_store, &pool)?; + if status { + break; + } + let current_status = + get_flat_storage_creation_status(&rw_hot_store, init_cmd.shard_id); + println!("Status: {:?}", current_status); + + std::thread::sleep(Duration::from_secs(1)); + } + + println!("Flat storage initialization finished."); + } + SubCommand::Verify(verify_cmd) => { + let (_, hot_runtime, chain_store, hot_store) = + Self::get_db(&opener, home_dir, &near_config, near_store::Mode::ReadOnly); + + let head_hash = get_flat_head(&hot_store, verify_cmd.shard_id).expect(&format!( + "Flat storage head missing for shard {:?}.", + verify_cmd.shard_id + )); + let block_header = chain_store.get_block_header(&head_hash).unwrap(); + let block = chain_store.get_block(&head_hash).unwrap(); + println!( + "Verifying flat storage for shard {:?} - flat head @{:?} ({:?})", + verify_cmd.shard_id, + block_header.height(), + block_header.hash() + ); + let chunks_collection = block.chunks(); + let shard_chunk_header = + chunks_collection.get(verify_cmd.shard_id as usize).unwrap(); + // TODO: this might be wrong.. + let state_root = shard_chunk_header.prev_state_root(); + + println!("Verifying using the {:?} as state_root", state_root); + let tip = chain_store.final_head().unwrap(); + + hot_runtime.create_flat_storage_state_for_shard( + verify_cmd.shard_id, + tip.height, + &chain_store, + ); + + let trie = hot_runtime + .get_view_trie_for_shard(verify_cmd.shard_id, &head_hash, state_root) + .unwrap(); + + let shard_layout = hot_runtime.get_shard_layout(block_header.epoch_id()).unwrap(); + + let all_entries = store_helper::iter_flat_state_entries( + shard_layout, + verify_cmd.shard_id, + &hot_store, + None, + None, + ); + + let trie_iter = trie.iter().unwrap().filter(|entry| { + let result_copy = &entry.clone().unwrap().0; + let result = &result_copy[..]; + parse_account_id_from_raw_key(result).unwrap().is_some() + }); + + let mut verified = 0; + let mut success = true; + for (item_trie, item_flat) in tqdm(std::iter::zip(trie_iter, all_entries)) { + let value_ref = ValueRef::decode((*item_flat.1).try_into().unwrap()); + verified += 1; + + let item_trie = item_trie.unwrap(); + if item_trie.0 != *item_flat.0 { + println!( + "Different keys {:?} in trie, {:?} in flat. ", + item_trie.0, item_flat.0 + ); + success = false; + break; + } + if item_trie.1.len() != value_ref.length as usize { + println!( + "Wrong values for key: {:?} length trie: {:?} vs {:?}", + item_trie.0, + item_trie.1.len(), + value_ref.length + ); + success = false; + break; + } + + if near_primitives::hash::hash(&item_trie.1) != value_ref.hash { + println!( + "Wrong hash for key: {:?} length trie: {:?} vs {:?}", + item_trie.0, + near_primitives::hash::hash(&item_trie.1), + value_ref.hash + ); + success = false; + break; + } + } + if success { + println!("Success - verified {:?} nodes", verified); + } else { + println!("FAILED - on node {:?}", verified); + } + } + } + + Ok(()) + } +} From 7a5027f80cfe5d1250a5b4cce9a29d6e5677776a Mon Sep 17 00:00:00 2001 From: mm-near Date: Tue, 7 Feb 2023 19:08:57 +0100 Subject: [PATCH 2/8] put under the compilation flag --- Cargo.lock | 10 -- neard/Cargo.toml | 3 +- neard/src/cli.rs | 5 +- tools/flat-storage/Cargo.toml | 16 +- tools/flat-storage/src/flat.rs | 269 ++++++++++++++++++++++++++++++++ tools/flat-storage/src/lib.rs | 271 +-------------------------------- 6 files changed, 283 insertions(+), 291 deletions(-) create mode 100644 tools/flat-storage/src/flat.rs diff --git a/Cargo.lock b/Cargo.lock index 12a3a9d106f..ab8a030cd39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3295,22 +3295,12 @@ dependencies = [ "clap 3.1.18", "near-chain", "near-chain-configs", - "near-crypto", "near-epoch-manager", - "near-network", "near-primitives", - "near-primitives-core", "near-store", - "near-test-contracts", "nearcore", - "node-runtime", - "num-rational", "rayon", - "serde", - "serde_json", - "tempfile", "tqdm", - "tracing", ] [[package]] diff --git a/neard/Cargo.toml b/neard/Cargo.toml index c4215f248a5..62e5165399f 100644 --- a/neard/Cargo.toml +++ b/neard/Cargo.toml @@ -63,7 +63,7 @@ delay_detector = ["nearcore/delay_detector"] rosetta_rpc = ["nearcore/rosetta_rpc"] json_rpc = ["nearcore/json_rpc"] protocol_feature_fix_staking_threshold = ["nearcore/protocol_feature_fix_staking_threshold"] -protocol_feature_flat_state = ["nearcore/protocol_feature_flat_state"] +protocol_feature_flat_state = ["nearcore/protocol_feature_flat_state", "near-flat-storage/protocol_feature_flat_state"] protocol_feature_nep366_delegate_action = [ "nearcore/protocol_feature_nep366_delegate_action", "near-state-viewer/protocol_feature_nep366_delegate_action", @@ -73,6 +73,7 @@ nightly = [ "nightly_protocol", "nearcore/nightly", "near-state-viewer/nightly", + "protocol_feature_flat_state", ] nightly_protocol = ["nearcore/nightly_protocol"] diff --git a/neard/src/cli.rs b/neard/src/cli.rs index a46310d4593..92b335897b6 100644 --- a/neard/src/cli.rs +++ b/neard/src/cli.rs @@ -6,7 +6,8 @@ use near_chain_configs::GenesisValidationMode; use near_client::ConfigUpdater; use near_cold_store_tool::ColdStoreCommand; use near_dyn_configs::{UpdateableConfigLoader, UpdateableConfigLoaderError, UpdateableConfigs}; -use near_flat_storage::FlatStorageCommand; +#[cfg(feature = "protocol_feature_flat_state")] +use near_flat_storage::flat::FlatStorageCommand; use near_jsonrpc_primitives::types::light_client::RpcLightClientExecutionProofResponse; use near_mirror::MirrorCommand; use near_network::tcp; @@ -118,6 +119,7 @@ impl NeardCmd { NeardSubCommand::StateParts(cmd) => { cmd.run()?; } + #[cfg(feature = "protocol_feature_flat_state")] NeardSubCommand::FlatStorage(cmd) => { cmd.run(&home_dir)?; } @@ -231,6 +233,7 @@ pub(super) enum NeardSubCommand { /// Connects to a NEAR node and sends state parts requests after the handshake is completed. StateParts(StatePartsCommand), + #[cfg(feature = "protocol_feature_flat_state")] /// Flat storage related tooling. FlatStorage(FlatStorageCommand), } diff --git a/tools/flat-storage/Cargo.toml b/tools/flat-storage/Cargo.toml index bc601bc27c3..fe7c2a92680 100644 --- a/tools/flat-storage/Cargo.toml +++ b/tools/flat-storage/Cargo.toml @@ -9,23 +9,19 @@ edition.workspace = true anyhow.workspace = true borsh.workspace = true clap.workspace = true -num-rational.workspace = true -serde.workspace = true -serde_json.workspace = true -tempfile.workspace = true -tracing.workspace = true rayon.workspace = true tqdm = "0.4.4" near-chain = { path = "../../chain/chain" } near-chain-configs = { path = "../../core/chain-configs" } -near-crypto = { path = "../../core/crypto" } + near-epoch-manager = { path = "../../chain/epoch-manager" } -near-network = { path = "../../chain/network" } near-primitives = { path = "../../core/primitives" } -near-primitives-core = { path = "../../core/primitives-core" } near-store = { path = "../../core/store" } -near-test-contracts = { path = "../../runtime/near-test-contracts" } nearcore = { path = "../../nearcore" } -node-runtime = { path = "../../runtime/runtime" } \ No newline at end of file + +[features] +protocol_feature_flat_state = ["nearcore/protocol_feature_flat_state", +"near-chain/protocol_feature_flat_state", +"near-store/protocol_feature_flat_state",] diff --git a/tools/flat-storage/src/flat.rs b/tools/flat-storage/src/flat.rs new file mode 100644 index 00000000000..87bb9d12ef2 --- /dev/null +++ b/tools/flat-storage/src/flat.rs @@ -0,0 +1,269 @@ +/// Tools for modifying flat storage - should be used only for experimentation & debugging. +use clap::Parser; +use near_chain::{ + flat_storage_creator::FlatStorageShardCreator, types::RuntimeAdapter, ChainStore, + ChainStoreAccess, +}; +use near_epoch_manager::EpochManagerAdapter; +use near_primitives::{state::ValueRef, trie_key::trie_key_parsers::parse_account_id_from_raw_key}; +use near_store::{ + flat_state::store_helper::{self, get_flat_head, get_flat_storage_creation_status}, + Mode, NodeStorage, Store, StoreOpener, +}; +use nearcore::{load_config, NearConfig, NightshadeRuntime}; +use std::{path::PathBuf, sync::Arc, time::Duration}; +use tqdm::tqdm; + +#[derive(Parser)] +pub struct FlatStorageCommand { + #[clap(subcommand)] + subcmd: SubCommand, +} + +#[derive(Parser)] +#[clap(subcommand_required = true, arg_required_else_help = true)] +enum SubCommand { + /// View the current state of flat storage + View, + + /// Reset the flat storage state (remove all the contents) + Reset(ResetCmd), + + /// Init the flat storage state, by copying from trie + Init(InitCmd), + + /// Verify flat storage state + Verify(VerifyCmd), + + /// Temporary command to set the store version (useful as long flat + /// storage is enabled only during nightly with separate DB version). + SetStoreVersion(SetStoreVersionCmd), +} + +#[derive(Parser)] +pub struct SetStoreVersionCmd { + version: u32, +} + +#[derive(Parser)] +pub struct ResetCmd { + shard_id: u64, +} + +#[derive(Parser)] + +pub struct InitCmd { + shard_id: u64, + + #[clap(default_value = "3")] + num_threads: usize, +} + +#[derive(Parser)] +pub struct VerifyCmd { + shard_id: u64, +} + +impl FlatStorageCommand { + fn get_db( + opener: &StoreOpener, + home_dir: &PathBuf, + near_config: &NearConfig, + mode: Mode, + ) -> (NodeStorage, Arc, ChainStore, Store) { + let node_storage = opener.open_in_mode(mode).unwrap(); + let hot_runtime = Arc::new(NightshadeRuntime::from_config( + home_dir, + node_storage.get_hot_store(), + &near_config, + )); + let chain_store = ChainStore::new(node_storage.get_hot_store(), 0, false); + let hot_store = node_storage.get_hot_store(); + (node_storage, hot_runtime, chain_store, hot_store) + } + + pub fn run(&self, home_dir: &PathBuf) -> anyhow::Result<()> { + let near_config = + load_config(home_dir, near_chain_configs::GenesisValidationMode::Full).unwrap(); + let opener = NodeStorage::opener(home_dir, &near_config.config.store, None); + + match &self.subcmd { + SubCommand::View => { + let (_, hot_runtime, chain_store, hot_store) = + Self::get_db(&opener, home_dir, &near_config, near_store::Mode::ReadOnly); + let tip = chain_store.final_head().unwrap(); + let shards = hot_runtime.num_shards(&tip.epoch_id).unwrap(); + println!("DB version: {:?}", hot_store.get_db_version()); + println!("Current final tip @{:?} - shards: {:?}", tip.height, shards); + + for shard in 0..shards { + let head_hash = get_flat_head(&hot_store, shard); + if let Some(head_hash) = head_hash { + let head_header = chain_store.get_block_header(&head_hash); + let creation_status = get_flat_storage_creation_status(&hot_store, shard); + println!( + "Shard: {:?} - flat storage @{:?} Details: {:?}", + shard, + head_header.map(|header| header.height()), + creation_status + ); + } else { + println!("Shard: {:?} - no flat storage.", shard) + } + } + } + SubCommand::SetStoreVersion(set_version) => { + let rw_storage = opener.open_in_mode(near_store::Mode::ReadWriteExisting).unwrap(); + let rw_store = rw_storage.get_hot_store(); + println!("Setting storage DB version to: {:?}", set_version.version); + rw_store.set_db_version(set_version.version)?; + } + SubCommand::Reset(reset_cmd) => { + let (_, rw_hot_runtime, rw_chain_store, _) = Self::get_db( + &opener, + home_dir, + &near_config, + near_store::Mode::ReadWriteExisting, + ); + let tip = rw_chain_store.final_head().unwrap(); + + // TODO: there should be a method that 'loads' the current flat storage state based on Storage. + rw_hot_runtime.create_flat_storage_state_for_shard( + reset_cmd.shard_id, + tip.height, + &rw_chain_store, + ); + + rw_hot_runtime + .remove_flat_storage_state_for_shard(reset_cmd.shard_id, &tip.epoch_id)?; + } + SubCommand::Init(init_cmd) => { + let (_, rw_hot_runtime, rw_chain_store, rw_hot_store) = Self::get_db( + &opener, + home_dir, + &near_config, + near_store::Mode::ReadWriteExisting, + ); + + let tip = rw_chain_store.final_head().unwrap(); + + let mut creator = + FlatStorageShardCreator::new(init_cmd.shard_id, tip.height, rw_hot_runtime); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(init_cmd.num_threads) + .build() + .unwrap(); + + loop { + let status = creator.update_status(&rw_chain_store, &pool)?; + if status { + break; + } + let current_status = + get_flat_storage_creation_status(&rw_hot_store, init_cmd.shard_id); + println!("Status: {:?}", current_status); + + std::thread::sleep(Duration::from_secs(1)); + } + + println!("Flat storage initialization finished."); + } + SubCommand::Verify(verify_cmd) => { + let (_, hot_runtime, chain_store, hot_store) = + Self::get_db(&opener, home_dir, &near_config, near_store::Mode::ReadOnly); + + let head_hash = get_flat_head(&hot_store, verify_cmd.shard_id).expect(&format!( + "Flat storage head missing for shard {:?}.", + verify_cmd.shard_id + )); + let block_header = chain_store.get_block_header(&head_hash).unwrap(); + let block = chain_store.get_block(&head_hash).unwrap(); + println!( + "Verifying flat storage for shard {:?} - flat head @{:?} ({:?})", + verify_cmd.shard_id, + block_header.height(), + block_header.hash() + ); + let chunks_collection = block.chunks(); + let shard_chunk_header = + chunks_collection.get(verify_cmd.shard_id as usize).unwrap(); + // TODO: this might be wrong.. + let state_root = shard_chunk_header.prev_state_root(); + + println!("Verifying using the {:?} as state_root", state_root); + let tip = chain_store.final_head().unwrap(); + + hot_runtime.create_flat_storage_state_for_shard( + verify_cmd.shard_id, + tip.height, + &chain_store, + ); + + let trie = hot_runtime + .get_view_trie_for_shard(verify_cmd.shard_id, &head_hash, state_root) + .unwrap(); + + let shard_layout = hot_runtime.get_shard_layout(block_header.epoch_id()).unwrap(); + + let all_entries = store_helper::iter_flat_state_entries( + shard_layout, + verify_cmd.shard_id, + &hot_store, + None, + None, + ); + + let trie_iter = trie.iter().unwrap().filter(|entry| { + let result_copy = &entry.clone().unwrap().0; + let result = &result_copy[..]; + parse_account_id_from_raw_key(result).unwrap().is_some() + }); + + let mut verified = 0; + let mut success = true; + for (item_trie, item_flat) in tqdm(std::iter::zip(trie_iter, all_entries)) { + let value_ref = ValueRef::decode((*item_flat.1).try_into().unwrap()); + verified += 1; + + let item_trie = item_trie.unwrap(); + if item_trie.0 != *item_flat.0 { + println!( + "Different keys {:?} in trie, {:?} in flat. ", + item_trie.0, item_flat.0 + ); + success = false; + break; + } + if item_trie.1.len() != value_ref.length as usize { + println!( + "Wrong values for key: {:?} length trie: {:?} vs {:?}", + item_trie.0, + item_trie.1.len(), + value_ref.length + ); + success = false; + break; + } + + if near_primitives::hash::hash(&item_trie.1) != value_ref.hash { + println!( + "Wrong hash for key: {:?} length trie: {:?} vs {:?}", + item_trie.0, + near_primitives::hash::hash(&item_trie.1), + value_ref.hash + ); + success = false; + break; + } + } + if success { + println!("Success - verified {:?} nodes", verified); + } else { + println!("FAILED - on node {:?}", verified); + } + } + } + + Ok(()) + } +} diff --git a/tools/flat-storage/src/lib.rs b/tools/flat-storage/src/lib.rs index 87bb9d12ef2..72e62bdc6c5 100644 --- a/tools/flat-storage/src/lib.rs +++ b/tools/flat-storage/src/lib.rs @@ -1,269 +1,2 @@ -/// Tools for modifying flat storage - should be used only for experimentation & debugging. -use clap::Parser; -use near_chain::{ - flat_storage_creator::FlatStorageShardCreator, types::RuntimeAdapter, ChainStore, - ChainStoreAccess, -}; -use near_epoch_manager::EpochManagerAdapter; -use near_primitives::{state::ValueRef, trie_key::trie_key_parsers::parse_account_id_from_raw_key}; -use near_store::{ - flat_state::store_helper::{self, get_flat_head, get_flat_storage_creation_status}, - Mode, NodeStorage, Store, StoreOpener, -}; -use nearcore::{load_config, NearConfig, NightshadeRuntime}; -use std::{path::PathBuf, sync::Arc, time::Duration}; -use tqdm::tqdm; - -#[derive(Parser)] -pub struct FlatStorageCommand { - #[clap(subcommand)] - subcmd: SubCommand, -} - -#[derive(Parser)] -#[clap(subcommand_required = true, arg_required_else_help = true)] -enum SubCommand { - /// View the current state of flat storage - View, - - /// Reset the flat storage state (remove all the contents) - Reset(ResetCmd), - - /// Init the flat storage state, by copying from trie - Init(InitCmd), - - /// Verify flat storage state - Verify(VerifyCmd), - - /// Temporary command to set the store version (useful as long flat - /// storage is enabled only during nightly with separate DB version). - SetStoreVersion(SetStoreVersionCmd), -} - -#[derive(Parser)] -pub struct SetStoreVersionCmd { - version: u32, -} - -#[derive(Parser)] -pub struct ResetCmd { - shard_id: u64, -} - -#[derive(Parser)] - -pub struct InitCmd { - shard_id: u64, - - #[clap(default_value = "3")] - num_threads: usize, -} - -#[derive(Parser)] -pub struct VerifyCmd { - shard_id: u64, -} - -impl FlatStorageCommand { - fn get_db( - opener: &StoreOpener, - home_dir: &PathBuf, - near_config: &NearConfig, - mode: Mode, - ) -> (NodeStorage, Arc, ChainStore, Store) { - let node_storage = opener.open_in_mode(mode).unwrap(); - let hot_runtime = Arc::new(NightshadeRuntime::from_config( - home_dir, - node_storage.get_hot_store(), - &near_config, - )); - let chain_store = ChainStore::new(node_storage.get_hot_store(), 0, false); - let hot_store = node_storage.get_hot_store(); - (node_storage, hot_runtime, chain_store, hot_store) - } - - pub fn run(&self, home_dir: &PathBuf) -> anyhow::Result<()> { - let near_config = - load_config(home_dir, near_chain_configs::GenesisValidationMode::Full).unwrap(); - let opener = NodeStorage::opener(home_dir, &near_config.config.store, None); - - match &self.subcmd { - SubCommand::View => { - let (_, hot_runtime, chain_store, hot_store) = - Self::get_db(&opener, home_dir, &near_config, near_store::Mode::ReadOnly); - let tip = chain_store.final_head().unwrap(); - let shards = hot_runtime.num_shards(&tip.epoch_id).unwrap(); - println!("DB version: {:?}", hot_store.get_db_version()); - println!("Current final tip @{:?} - shards: {:?}", tip.height, shards); - - for shard in 0..shards { - let head_hash = get_flat_head(&hot_store, shard); - if let Some(head_hash) = head_hash { - let head_header = chain_store.get_block_header(&head_hash); - let creation_status = get_flat_storage_creation_status(&hot_store, shard); - println!( - "Shard: {:?} - flat storage @{:?} Details: {:?}", - shard, - head_header.map(|header| header.height()), - creation_status - ); - } else { - println!("Shard: {:?} - no flat storage.", shard) - } - } - } - SubCommand::SetStoreVersion(set_version) => { - let rw_storage = opener.open_in_mode(near_store::Mode::ReadWriteExisting).unwrap(); - let rw_store = rw_storage.get_hot_store(); - println!("Setting storage DB version to: {:?}", set_version.version); - rw_store.set_db_version(set_version.version)?; - } - SubCommand::Reset(reset_cmd) => { - let (_, rw_hot_runtime, rw_chain_store, _) = Self::get_db( - &opener, - home_dir, - &near_config, - near_store::Mode::ReadWriteExisting, - ); - let tip = rw_chain_store.final_head().unwrap(); - - // TODO: there should be a method that 'loads' the current flat storage state based on Storage. - rw_hot_runtime.create_flat_storage_state_for_shard( - reset_cmd.shard_id, - tip.height, - &rw_chain_store, - ); - - rw_hot_runtime - .remove_flat_storage_state_for_shard(reset_cmd.shard_id, &tip.epoch_id)?; - } - SubCommand::Init(init_cmd) => { - let (_, rw_hot_runtime, rw_chain_store, rw_hot_store) = Self::get_db( - &opener, - home_dir, - &near_config, - near_store::Mode::ReadWriteExisting, - ); - - let tip = rw_chain_store.final_head().unwrap(); - - let mut creator = - FlatStorageShardCreator::new(init_cmd.shard_id, tip.height, rw_hot_runtime); - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(init_cmd.num_threads) - .build() - .unwrap(); - - loop { - let status = creator.update_status(&rw_chain_store, &pool)?; - if status { - break; - } - let current_status = - get_flat_storage_creation_status(&rw_hot_store, init_cmd.shard_id); - println!("Status: {:?}", current_status); - - std::thread::sleep(Duration::from_secs(1)); - } - - println!("Flat storage initialization finished."); - } - SubCommand::Verify(verify_cmd) => { - let (_, hot_runtime, chain_store, hot_store) = - Self::get_db(&opener, home_dir, &near_config, near_store::Mode::ReadOnly); - - let head_hash = get_flat_head(&hot_store, verify_cmd.shard_id).expect(&format!( - "Flat storage head missing for shard {:?}.", - verify_cmd.shard_id - )); - let block_header = chain_store.get_block_header(&head_hash).unwrap(); - let block = chain_store.get_block(&head_hash).unwrap(); - println!( - "Verifying flat storage for shard {:?} - flat head @{:?} ({:?})", - verify_cmd.shard_id, - block_header.height(), - block_header.hash() - ); - let chunks_collection = block.chunks(); - let shard_chunk_header = - chunks_collection.get(verify_cmd.shard_id as usize).unwrap(); - // TODO: this might be wrong.. - let state_root = shard_chunk_header.prev_state_root(); - - println!("Verifying using the {:?} as state_root", state_root); - let tip = chain_store.final_head().unwrap(); - - hot_runtime.create_flat_storage_state_for_shard( - verify_cmd.shard_id, - tip.height, - &chain_store, - ); - - let trie = hot_runtime - .get_view_trie_for_shard(verify_cmd.shard_id, &head_hash, state_root) - .unwrap(); - - let shard_layout = hot_runtime.get_shard_layout(block_header.epoch_id()).unwrap(); - - let all_entries = store_helper::iter_flat_state_entries( - shard_layout, - verify_cmd.shard_id, - &hot_store, - None, - None, - ); - - let trie_iter = trie.iter().unwrap().filter(|entry| { - let result_copy = &entry.clone().unwrap().0; - let result = &result_copy[..]; - parse_account_id_from_raw_key(result).unwrap().is_some() - }); - - let mut verified = 0; - let mut success = true; - for (item_trie, item_flat) in tqdm(std::iter::zip(trie_iter, all_entries)) { - let value_ref = ValueRef::decode((*item_flat.1).try_into().unwrap()); - verified += 1; - - let item_trie = item_trie.unwrap(); - if item_trie.0 != *item_flat.0 { - println!( - "Different keys {:?} in trie, {:?} in flat. ", - item_trie.0, item_flat.0 - ); - success = false; - break; - } - if item_trie.1.len() != value_ref.length as usize { - println!( - "Wrong values for key: {:?} length trie: {:?} vs {:?}", - item_trie.0, - item_trie.1.len(), - value_ref.length - ); - success = false; - break; - } - - if near_primitives::hash::hash(&item_trie.1) != value_ref.hash { - println!( - "Wrong hash for key: {:?} length trie: {:?} vs {:?}", - item_trie.0, - near_primitives::hash::hash(&item_trie.1), - value_ref.hash - ); - success = false; - break; - } - } - if success { - println!("Success - verified {:?} nodes", verified); - } else { - println!("FAILED - on node {:?}", verified); - } - } - } - - Ok(()) - } -} +#[cfg(feature = "protocol_feature_flat_state")] +pub mod flat; From de0afc2cb48d50c225251f1e125df4c38d95ccde Mon Sep 17 00:00:00 2001 From: mm-near Date: Wed, 8 Feb 2023 12:25:59 +0100 Subject: [PATCH 3/8] small fixes --- chain/chain/src/flat_storage_creator.rs | 2 +- .../src/tests/client/flat_storage.rs | 58 +++++++++++++++++-- tools/flat-storage/src/flat.rs | 2 +- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/chain/chain/src/flat_storage_creator.rs b/chain/chain/src/flat_storage_creator.rs index 1290a9b160b..b931f34e811 100644 --- a/chain/chain/src/flat_storage_creator.rs +++ b/chain/chain/src/flat_storage_creator.rs @@ -202,7 +202,7 @@ impl FlatStorageShardCreator { let final_head = chain_store.final_head()?; let final_height = final_head.height; - if final_height >= self.start_height { + if final_height > self.start_height { // If it holds, deltas for all blocks after final head are saved to disk, because they have bigger // heights than one on which we launched a node. Check that it is true: for height in final_height + 1..=chain_store.head()?.height { diff --git a/integration-tests/src/tests/client/flat_storage.rs b/integration-tests/src/tests/client/flat_storage.rs index 9e9b83aabd0..1429979b421 100644 --- a/integration-tests/src/tests/client/flat_storage.rs +++ b/integration-tests/src/tests/client/flat_storage.rs @@ -39,12 +39,18 @@ fn setup_env(genesis: &Genesis, store: Store) -> TestEnv { /// We have a pause after processing each block because state data is being fetched in rayon threads, /// but we expect it to finish in <30s because state is small and there is only one state part. /// Returns next block height available to produce. -fn wait_for_flat_storage_creation(env: &mut TestEnv, start_height: BlockHeight) -> BlockHeight { +fn wait_for_flat_storage_creation( + env: &mut TestEnv, + start_height: BlockHeight, + produce_blocks: bool, +) -> BlockHeight { let store = env.clients[0].runtime_adapter.store().clone(); let mut next_height = start_height; let mut prev_status = store_helper::get_flat_storage_creation_status(&store, 0); while next_height < start_height + CREATION_TIMEOUT { - env.produce_block(0, next_height); + if produce_blocks { + env.produce_block(0, next_height); + } env.clients[0].run_flat_storage_creation_step().unwrap(); let status = store_helper::get_flat_storage_creation_status(&store, 0); @@ -68,6 +74,7 @@ fn wait_for_flat_storage_creation(env: &mut TestEnv, start_height: BlockHeight) panic!("Invalid status {prev_status:?} observed during flat storage creation for height {next_height}"); } } + tracing::info!("Flat Creation status: {:?}", status); prev_status = status; next_height += 1; @@ -187,7 +194,7 @@ fn test_flat_storage_creation() { }) ); - wait_for_flat_storage_creation(&mut env, START_HEIGHT + 3); + wait_for_flat_storage_creation(&mut env, START_HEIGHT + 3, true); } /// Check that client can create flat storage on some shard while it already exists on another shard. @@ -249,7 +256,7 @@ fn test_flat_storage_creation_two_shards() { FlatStorageCreationStatus::Ready ); - wait_for_flat_storage_creation(&mut env, START_HEIGHT); + wait_for_flat_storage_creation(&mut env, START_HEIGHT, true); } /// Check that flat storage creation can be started from intermediate state where one @@ -342,7 +349,7 @@ fn test_flat_storage_creation_start_from_state_part() { assert!(env.clients[0].runtime_adapter.get_flat_storage_state_for_shard(0).is_none()); // Run chain for a couple of blocks and check that flat storage for shard 0 is eventually created. - let next_height = wait_for_flat_storage_creation(&mut env, START_HEIGHT); + let next_height = wait_for_flat_storage_creation(&mut env, START_HEIGHT, true); // Check that all the keys are present in flat storage. let block_hash = env.clients[0].chain.get_block_hash_by_height(next_height - 1).unwrap(); @@ -364,3 +371,44 @@ fn test_flat_storage_creation_start_from_state_part() { } } } + +/// Tests the scenario where we start flat storage migration, and get just a few new blocks. +/// (in this test we still generate 3 blocks in order to generate deltas). +#[cfg(feature = "protocol_feature_flat_state")] +#[test] +fn test_cachup_succeeds_even_if_no_new_blocks() { + init_test_logger(); + let genesis = Genesis::test(vec!["test0".parse().unwrap()], 1); + let store = create_test_store(); + + // Process some blocks with flat storage. Then remove flat storage data from disk. + { + let mut env = setup_env(&genesis, store.clone()); + for height in 1..START_HEIGHT { + env.produce_block(0, height); + } + // Remove flat storage. + let block_hash = env.clients[0].chain.get_block_hash_by_height(START_HEIGHT - 1).unwrap(); + let epoch_id = env.clients[0].chain.runtime_adapter.get_epoch_id(&block_hash).unwrap(); + env.clients[0] + .chain + .runtime_adapter + .remove_flat_storage_state_for_shard(0, &epoch_id) + .unwrap(); + } + let mut env = setup_env(&genesis, store.clone()); + assert!(env.clients[0].runtime_adapter.get_flat_storage_state_for_shard(0).is_none()); + assert_eq!( + store_helper::get_flat_storage_creation_status(&store, 0), + FlatStorageCreationStatus::SavingDeltas + ); + // Create 3 more blocks (so that the deltas are generated) - and assume that no new blocks are received. + // In the future, we should also support the scenario where no new blocks are created. + + for block_height in START_HEIGHT + 1..=START_HEIGHT + 3 { + env.produce_block(0, block_height); + } + + assert!(!env.clients[0].run_flat_storage_creation_step().unwrap()); + wait_for_flat_storage_creation(&mut env, START_HEIGHT + 3, false); +} diff --git a/tools/flat-storage/src/flat.rs b/tools/flat-storage/src/flat.rs index 87bb9d12ef2..6840c323a68 100644 --- a/tools/flat-storage/src/flat.rs +++ b/tools/flat-storage/src/flat.rs @@ -148,7 +148,7 @@ impl FlatStorageCommand { let tip = rw_chain_store.final_head().unwrap(); let mut creator = - FlatStorageShardCreator::new(init_cmd.shard_id, tip.height, rw_hot_runtime); + FlatStorageShardCreator::new(init_cmd.shard_id, tip.height - 1, rw_hot_runtime); let pool = rayon::ThreadPoolBuilder::new() .num_threads(init_cmd.num_threads) .build() From 63745909de96addf4b602ea9c083372c68358054 Mon Sep 17 00:00:00 2001 From: mm-near Date: Wed, 8 Feb 2023 13:59:29 +0100 Subject: [PATCH 4/8] add proper state root --- tools/flat-storage/src/flat.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tools/flat-storage/src/flat.rs b/tools/flat-storage/src/flat.rs index 6840c323a68..ab16efd857f 100644 --- a/tools/flat-storage/src/flat.rs +++ b/tools/flat-storage/src/flat.rs @@ -8,7 +8,7 @@ use near_epoch_manager::EpochManagerAdapter; use near_primitives::{state::ValueRef, trie_key::trie_key_parsers::parse_account_id_from_raw_key}; use near_store::{ flat_state::store_helper::{self, get_flat_head, get_flat_storage_creation_status}, - Mode, NodeStorage, Store, StoreOpener, + Mode, NodeStorage, ShardUId, Store, StoreOpener, }; use nearcore::{load_config, NearConfig, NightshadeRuntime}; use std::{path::PathBuf, sync::Arc, time::Duration}; @@ -177,18 +177,23 @@ impl FlatStorageCommand { verify_cmd.shard_id )); let block_header = chain_store.get_block_header(&head_hash).unwrap(); - let block = chain_store.get_block(&head_hash).unwrap(); + let shard_layout = hot_runtime.get_shard_layout(block_header.epoch_id()).unwrap(); + println!( "Verifying flat storage for shard {:?} - flat head @{:?} ({:?})", verify_cmd.shard_id, block_header.height(), block_header.hash() ); - let chunks_collection = block.chunks(); - let shard_chunk_header = - chunks_collection.get(verify_cmd.shard_id as usize).unwrap(); - // TODO: this might be wrong.. - let state_root = shard_chunk_header.prev_state_root(); + let chunk_extra = chain_store + .get_chunk_extra( + &head_hash, + &ShardUId::from_shard_id_and_layout(verify_cmd.shard_id, &shard_layout), + ) + .unwrap(); + + // The state root must be from AFTER applying the final block (that's why we're taking it from the chunk extra). + let state_root = chunk_extra.state_root(); println!("Verifying using the {:?} as state_root", state_root); let tip = chain_store.final_head().unwrap(); @@ -200,11 +205,9 @@ impl FlatStorageCommand { ); let trie = hot_runtime - .get_view_trie_for_shard(verify_cmd.shard_id, &head_hash, state_root) + .get_view_trie_for_shard(verify_cmd.shard_id, &head_hash, state_root.clone()) .unwrap(); - let shard_layout = hot_runtime.get_shard_layout(block_header.epoch_id()).unwrap(); - let all_entries = store_helper::iter_flat_state_entries( shard_layout, verify_cmd.shard_id, From 41f099b4788d33bcb0e7a54375a416174a8080c8 Mon Sep 17 00:00:00 2001 From: mm-near Date: Wed, 22 Feb 2023 12:31:27 +0100 Subject: [PATCH 5/8] review --- tools/flat-storage/Cargo.toml | 8 +++++--- tools/flat-storage/src/flat.rs | 15 +++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tools/flat-storage/Cargo.toml b/tools/flat-storage/Cargo.toml index fe7c2a92680..29c1d66e07b 100644 --- a/tools/flat-storage/Cargo.toml +++ b/tools/flat-storage/Cargo.toml @@ -22,6 +22,8 @@ near-store = { path = "../../core/store" } nearcore = { path = "../../nearcore" } [features] -protocol_feature_flat_state = ["nearcore/protocol_feature_flat_state", -"near-chain/protocol_feature_flat_state", -"near-store/protocol_feature_flat_state",] +protocol_feature_flat_state = [ + "nearcore/protocol_feature_flat_state", + "near-chain/protocol_feature_flat_state", + "near-store/protocol_feature_flat_state", +] diff --git a/tools/flat-storage/src/flat.rs b/tools/flat-storage/src/flat.rs index ab16efd857f..4e0cee79457 100644 --- a/tools/flat-storage/src/flat.rs +++ b/tools/flat-storage/src/flat.rs @@ -32,7 +32,7 @@ enum SubCommand { /// Init the flat storage state, by copying from trie Init(InitCmd), - /// Verify flat storage state + /// Verify flat storage state (it can take up to couple hours if flat storage is very large) Verify(VerifyCmd), /// Temporary command to set the store version (useful as long flat @@ -208,7 +208,7 @@ impl FlatStorageCommand { .get_view_trie_for_shard(verify_cmd.shard_id, &head_hash, state_root.clone()) .unwrap(); - let all_entries = store_helper::iter_flat_state_entries( + let flat_state_entries_iter = store_helper::iter_flat_state_entries( shard_layout, verify_cmd.shard_id, &hot_store, @@ -216,6 +216,7 @@ impl FlatStorageCommand { None, ); + // Trie iterator which skips all the 'delayed' keys - that don't contain the account_id as string. let trie_iter = trie.iter().unwrap().filter(|entry| { let result_copy = &entry.clone().unwrap().0; let result = &result_copy[..]; @@ -224,14 +225,16 @@ impl FlatStorageCommand { let mut verified = 0; let mut success = true; - for (item_trie, item_flat) in tqdm(std::iter::zip(trie_iter, all_entries)) { + for (item_trie, item_flat) in + tqdm(std::iter::zip(trie_iter, flat_state_entries_iter)) + { let value_ref = ValueRef::decode((*item_flat.1).try_into().unwrap()); verified += 1; let item_trie = item_trie.unwrap(); if item_trie.0 != *item_flat.0 { println!( - "Different keys {:?} in trie, {:?} in flat. ", + "Different keys {:?} in trie, {:?} in flat storage. ", item_trie.0, item_flat.0 ); success = false; @@ -239,7 +242,7 @@ impl FlatStorageCommand { } if item_trie.1.len() != value_ref.length as usize { println!( - "Wrong values for key: {:?} length trie: {:?} vs {:?}", + "Different ValueRef::length for key: {:?} in trie: {:?} vs flat storage: {:?}", item_trie.0, item_trie.1.len(), value_ref.length @@ -250,7 +253,7 @@ impl FlatStorageCommand { if near_primitives::hash::hash(&item_trie.1) != value_ref.hash { println!( - "Wrong hash for key: {:?} length trie: {:?} vs {:?}", + "Different ValueRef::hashfor key: {:?} in trie: {:?} vs flat storage: {:?}", item_trie.0, near_primitives::hash::hash(&item_trie.1), value_ref.hash From e2ad32adf227a60cba3fc464f288ad061c9ee9da Mon Sep 17 00:00:00 2001 From: mm-near Date: Wed, 22 Feb 2023 12:34:23 +0100 Subject: [PATCH 6/8] renamed --- neard/src/cli.rs | 2 +- tools/flat-storage/src/{flat.rs => commands.rs} | 0 tools/flat-storage/src/lib.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename tools/flat-storage/src/{flat.rs => commands.rs} (100%) diff --git a/neard/src/cli.rs b/neard/src/cli.rs index 92b335897b6..4c5cc23cea3 100644 --- a/neard/src/cli.rs +++ b/neard/src/cli.rs @@ -7,7 +7,7 @@ use near_client::ConfigUpdater; use near_cold_store_tool::ColdStoreCommand; use near_dyn_configs::{UpdateableConfigLoader, UpdateableConfigLoaderError, UpdateableConfigs}; #[cfg(feature = "protocol_feature_flat_state")] -use near_flat_storage::flat::FlatStorageCommand; +use near_flat_storage::commands::FlatStorageCommand; use near_jsonrpc_primitives::types::light_client::RpcLightClientExecutionProofResponse; use near_mirror::MirrorCommand; use near_network::tcp; diff --git a/tools/flat-storage/src/flat.rs b/tools/flat-storage/src/commands.rs similarity index 100% rename from tools/flat-storage/src/flat.rs rename to tools/flat-storage/src/commands.rs diff --git a/tools/flat-storage/src/lib.rs b/tools/flat-storage/src/lib.rs index 72e62bdc6c5..6e36f4e27bb 100644 --- a/tools/flat-storage/src/lib.rs +++ b/tools/flat-storage/src/lib.rs @@ -1,2 +1,2 @@ #[cfg(feature = "protocol_feature_flat_state")] -pub mod flat; +pub mod commands; From a721f63fcf9f7b1cf50d0e6141a2821a6152f37d Mon Sep 17 00:00:00 2001 From: mm-near Date: Wed, 22 Feb 2023 12:37:47 +0100 Subject: [PATCH 7/8] fix --- Cargo.lock | 2 +- neard/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 021259a0eac..94e8ebfd45c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3350,7 +3350,7 @@ name = "near-flat-storage" version = "0.0.0" dependencies = [ "anyhow", - "borsh", + "borsh 0.10.2", "clap 3.1.18", "near-chain", "near-chain-configs", diff --git a/neard/Cargo.toml b/neard/Cargo.toml index e325c5d0997..bd1ff803c12 100644 --- a/neard/Cargo.toml +++ b/neard/Cargo.toml @@ -65,7 +65,7 @@ delay_detector = ["nearcore/delay_detector"] rosetta_rpc = ["nearcore/rosetta_rpc"] json_rpc = ["nearcore/json_rpc"] protocol_feature_fix_staking_threshold = ["nearcore/protocol_feature_fix_staking_threshold"] -protocol_feature_flat_state = ["nearcore/protocol_feature_flat_state", "near-flat-storage/protocol_feature_flat_state"]] +protocol_feature_flat_state = ["nearcore/protocol_feature_flat_state", "near-flat-storage/protocol_feature_flat_state"] nightly = [ "nightly_protocol", From 7e619d38ef5909ecbe983bf1ed4cf09bcc0faee2 Mon Sep 17 00:00:00 2001 From: mm-near Date: Wed, 22 Feb 2023 13:37:18 +0100 Subject: [PATCH 8/8] fixed compile issue --- tools/flat-storage/src/commands.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/flat-storage/src/commands.rs b/tools/flat-storage/src/commands.rs index 4e0cee79457..4bd1d6cb2fa 100644 --- a/tools/flat-storage/src/commands.rs +++ b/tools/flat-storage/src/commands.rs @@ -85,7 +85,7 @@ impl FlatStorageCommand { pub fn run(&self, home_dir: &PathBuf) -> anyhow::Result<()> { let near_config = load_config(home_dir, near_chain_configs::GenesisValidationMode::Full).unwrap(); - let opener = NodeStorage::opener(home_dir, &near_config.config.store, None); + let opener = NodeStorage::opener(home_dir, false, &near_config.config.store, None); match &self.subcmd { SubCommand::View => {