From 491d092eca96fd1be8a409bda711e9b122bc1b8d Mon Sep 17 00:00:00 2001 From: Trisfald Date: Wed, 10 Jul 2024 16:56:04 +0200 Subject: [PATCH 1/6] fix block apply outside flat storage by re-enabling DBTrie that does not charge costs --- chain/chain/src/runtime/mod.rs | 19 ++++++++++ chain/chain/src/types.rs | 4 +++ core/store/src/trie/mod.rs | 5 +++ tools/state-viewer/src/apply_chunk.rs | 38 ++++++++++++++++---- tools/state-viewer/src/cli.rs | 42 +++++++++++++++++++--- tools/state-viewer/src/commands.rs | 50 ++++++++++++++++++++++----- 6 files changed, 139 insertions(+), 19 deletions(-) diff --git a/chain/chain/src/runtime/mod.rs b/chain/chain/src/runtime/mod.rs index 49eb8794c05..5d1a92c065f 100644 --- a/chain/chain/src/runtime/mod.rs +++ b/chain/chain/src/runtime/mod.rs @@ -745,6 +745,13 @@ impl RuntimeAdapter for NightshadeRuntime { StorageDataSource::Db => { self.tries.get_trie_for_shard(shard_uid, storage_config.state_root) } + StorageDataSource::DbTrieOnly => { + // If there is no flat storage on disk, use trie but simulate costs with enabled + // flat storage by not charging gas for trie nodes. + let mut trie = self.tries.get_trie_for_shard(shard_uid, storage_config.state_root); + trie.dont_charge_gas_for_trie_node_access(); + trie + } StorageDataSource::Recorded(storage) => Trie::from_recorded_storage( storage, storage_config.state_root, @@ -958,6 +965,18 @@ impl RuntimeAdapter for NightshadeRuntime { storage_config.state_root, storage_config.use_flat_storage, )?, + StorageDataSource::DbTrieOnly => { + // If there is no flat storage on disk, use trie but simulate costs with enabled + // flat storage by not charging gas for trie nodes. + let mut trie = self.get_trie_for_shard( + shard_id, + &block.prev_block_hash, + storage_config.state_root, + false, + )?; + trie.dont_charge_gas_for_trie_node_access(); + trie + } StorageDataSource::Recorded(storage) => Trie::from_recorded_storage( storage, storage_config.state_root, diff --git a/chain/chain/src/types.rs b/chain/chain/src/types.rs index 42789ded60f..17dbbe4ed02 100644 --- a/chain/chain/src/types.rs +++ b/chain/chain/src/types.rs @@ -261,6 +261,10 @@ impl ChainGenesis { pub enum StorageDataSource { /// Full state data is present in DB. Db, + /// Trie is present in DB and flat storage is not. + /// Used to reply past blocks and simulate gas costs as if flat storage + /// was present. + DbTrieOnly, /// State data is supplied from state witness, there is no state data /// stored on disk. Recorded(PartialStorage), diff --git a/core/store/src/trie/mod.rs b/core/store/src/trie/mod.rs index 34a5f64e2d5..fc533a51225 100644 --- a/core/store/src/trie/mod.rs +++ b/core/store/src/trie/mod.rs @@ -660,6 +660,11 @@ impl Trie { } } + /// Helper to simulate gas costs as if flat storage was present. + pub fn dont_charge_gas_for_trie_node_access(&mut self) { + self.charge_gas_for_trie_node_access = false; + } + /// Makes a new trie that has everything the same except that access /// through that trie accumulates a state proof for all nodes accessed. pub fn recording_reads(&self) -> Self { diff --git a/tools/state-viewer/src/apply_chunk.rs b/tools/state-viewer/src/apply_chunk.rs index c0ffb7468ee..894702194e6 100644 --- a/tools/state-viewer/src/apply_chunk.rs +++ b/tools/state-viewer/src/apply_chunk.rs @@ -4,7 +4,7 @@ use near_chain::chain::collect_receipts_from_response; use near_chain::migrations::check_if_block_is_first_with_chunk_of_version; use near_chain::types::{ ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, RuntimeAdapter, - RuntimeStorageConfig, + RuntimeStorageConfig, StorageDataSource, }; use near_chain::{ChainStore, ChainStoreAccess}; use near_epoch_manager::{EpochManagerAdapter, EpochManagerHandle}; @@ -87,6 +87,7 @@ pub(crate) fn apply_chunk( target_height: Option, rng: Option, use_flat_storage: bool, + use_trie_for_free: bool, ) -> anyhow::Result<(ApplyChunkResult, Gas)> { let chunk = chain_store.get_chunk(&chunk_hash)?; let chunk_header = chunk.cloned_header(); @@ -131,9 +132,14 @@ pub(crate) fn apply_chunk( shard_id, )?; + let mut storage = RuntimeStorageConfig::new(prev_state_root, use_flat_storage); + if use_trie_for_free { + storage.source = StorageDataSource::DbTrieOnly; + } + Ok(( runtime.apply_chunk( - RuntimeStorageConfig::new(prev_state_root, use_flat_storage), + storage, ApplyChunkReason::UpdateTrackedShard, ApplyChunkShardContext { shard_id, @@ -204,6 +210,7 @@ fn apply_tx_in_block( tx_hash: &CryptoHash, block_hash: CryptoHash, use_flat_storage: bool, + use_trie_for_free: bool, ) -> anyhow::Result { match find_tx_or_receipt(tx_hash, &block_hash, epoch_manager, chain_store)? { Some((hash_type, shard_id)) => { @@ -211,7 +218,7 @@ fn apply_tx_in_block( HashType::Tx => { println!("Found tx in block {} shard {}. equivalent command:\nview_state apply --height {} --shard-id {}\n", &block_hash, shard_id, chain_store.get_block_header(&block_hash)?.height(), shard_id); - let (block, apply_result) = crate::commands::apply_block(block_hash, shard_id, epoch_manager, runtime, chain_store, use_flat_storage); + let (block, apply_result) = crate::commands::apply_block(block_hash, shard_id, epoch_manager, runtime, chain_store, use_flat_storage, use_trie_for_free); crate::commands::check_apply_block_result(&block, &apply_result, epoch_manager, chain_store, shard_id)?; Ok(apply_result) }, @@ -233,6 +240,7 @@ fn apply_tx_in_chunk( chain_store: &mut ChainStore, tx_hash: &CryptoHash, use_flat_storage: bool, + use_trie_for_free: bool, ) -> anyhow::Result> { if chain_store.get_transaction(tx_hash)?.is_none() { return Err(anyhow!("tx with hash {} not known", tx_hash)); @@ -241,6 +249,7 @@ fn apply_tx_in_chunk( println!("Transaction is known but doesn't seem to have been applied. Searching in chunks that haven't been applied..."); let head = chain_store.head()?.height; + let protocol_version = chain_store.head_header()?.latest_protocol_version(); let mut chunk_hashes = vec![]; for item in store.iter(DBCol::ChunkHashesByHeight) { @@ -284,10 +293,11 @@ fn apply_tx_in_chunk( None, None, use_flat_storage, + use_trie_for_free, )?; println!( "resulting chunk extra:\n{:?}", - crate::commands::resulting_chunk_extra(&apply_result, gas_limit) + crate::commands::resulting_chunk_extra(&apply_result, gas_limit, protocol_version) ); results.push(apply_result); } @@ -301,6 +311,7 @@ pub(crate) fn apply_tx( store: Store, tx_hash: CryptoHash, use_flat_storage: bool, + use_trie_for_free: bool, ) -> anyhow::Result> { let mut chain_store = ChainStore::new(store.clone(), genesis_height, false); let outcomes = chain_store.get_outcomes_by_id(&tx_hash)?; @@ -313,6 +324,7 @@ pub(crate) fn apply_tx( &tx_hash, outcome.block_hash, use_flat_storage, + use_trie_for_free, )?]) } else { apply_tx_in_chunk( @@ -322,6 +334,7 @@ pub(crate) fn apply_tx( &mut chain_store, &tx_hash, use_flat_storage, + use_trie_for_free, ) } } @@ -333,6 +346,7 @@ fn apply_receipt_in_block( id: &CryptoHash, block_hash: CryptoHash, use_flat_storage: bool, + use_trie_for_free: bool, ) -> anyhow::Result { match find_tx_or_receipt(id, &block_hash, epoch_manager, chain_store)? { Some((hash_type, shard_id)) => { @@ -343,7 +357,7 @@ fn apply_receipt_in_block( HashType::Receipt => { println!("Found receipt in block {}. Receiver is in shard {}. equivalent command:\nview_state apply --height {} --shard-id {}\n", &block_hash, shard_id, chain_store.get_block_header(&block_hash)?.height(), shard_id); - let (block, apply_result) = crate::commands::apply_block(block_hash, shard_id, epoch_manager, runtime, chain_store, use_flat_storage); + let (block, apply_result) = crate::commands::apply_block(block_hash, shard_id, epoch_manager, runtime, chain_store, use_flat_storage, use_trie_for_free); crate::commands::check_apply_block_result(&block, &apply_result, epoch_manager, chain_store, shard_id)?; Ok(apply_result) }, @@ -363,6 +377,7 @@ fn apply_receipt_in_chunk( chain_store: &mut ChainStore, id: &CryptoHash, use_flat_storage: bool, + use_trie_for_free: bool, ) -> anyhow::Result> { if chain_store.get_receipt(id)?.is_none() { // TODO: handle local/delayed receipts @@ -374,6 +389,7 @@ fn apply_receipt_in_chunk( ); let head = chain_store.head()?.height; + let protocol_version = chain_store.head_header()?.latest_protocol_version(); let mut to_apply = HashSet::new(); let mut non_applied_chunks = HashMap::new(); @@ -441,8 +457,10 @@ fn apply_receipt_in_chunk( None, None, use_flat_storage, + use_trie_for_free, )?; - let chunk_extra = crate::commands::resulting_chunk_extra(&apply_result, gas_limit); + let chunk_extra = + crate::commands::resulting_chunk_extra(&apply_result, gas_limit, protocol_version); println!("resulting chunk extra:\n{:?}", chunk_extra); results.push(apply_result); } @@ -456,6 +474,7 @@ pub(crate) fn apply_receipt( store: Store, id: CryptoHash, use_flat_storage: bool, + use_trie_for_free: bool, ) -> anyhow::Result> { let mut chain_store = ChainStore::new(store.clone(), genesis_height, false); let outcomes = chain_store.get_outcomes_by_id(&id)?; @@ -467,6 +486,7 @@ pub(crate) fn apply_receipt( &id, outcome.block_hash, use_flat_storage, + use_trie_for_free, )?]) } else { apply_receipt_in_chunk( @@ -476,6 +496,7 @@ pub(crate) fn apply_receipt( &mut chain_store, &id, use_flat_storage, + use_trie_for_free, ) } } @@ -593,6 +614,7 @@ mod test { None, Some(rng), false, + false, ) .unwrap(); assert_eq!(apply_result.new_root, new_root); @@ -676,6 +698,7 @@ mod test { store.clone(), tx.get_hash(), false, + false, ) .unwrap(); assert_eq!(results.len(), 1); @@ -695,6 +718,7 @@ mod test { store.clone(), receipt.get_hash(), false, + false, ) .unwrap(); assert_eq!(results.len(), 1); @@ -725,6 +749,7 @@ mod test { store.clone(), tx.get_hash(), false, + false, ) .unwrap(); for result in results { @@ -746,6 +771,7 @@ mod test { store.clone(), receipt.get_hash(), false, + false, ) .unwrap(); for result in results { diff --git a/tools/state-viewer/src/cli.rs b/tools/state-viewer/src/cli.rs index 9a6e71924b7..bc52b0815f9 100644 --- a/tools/state-viewer/src/cli.rs +++ b/tools/state-viewer/src/cli.rs @@ -188,6 +188,10 @@ pub struct ApplyCmd { shard_id: ShardId, #[clap(long)] use_flat_storage: bool, + /// Use the data stored in trie, but without paying extra gas costs. + /// This could be used to simulate flat storage when the latter is not present. + #[clap(long)] + use_trie_for_free: bool, } impl ApplyCmd { @@ -196,6 +200,7 @@ impl ApplyCmd { self.height, self.shard_id, self.use_flat_storage, + self.use_trie_for_free, home_dir, near_config, store, @@ -212,13 +217,25 @@ pub struct ApplyChunkCmd { target_height: Option, #[clap(long)] use_flat_storage: bool, + /// Use the data stored in trie, but without paying extra gas costs. + /// This could be used to simulate flat storage when the latter is not present. + #[clap(long)] + use_trie_for_free: bool, } impl ApplyChunkCmd { pub fn run(self, home_dir: &Path, near_config: NearConfig, store: Store) { let hash = ChunkHash::from(CryptoHash::from_str(&self.chunk_hash).unwrap()); - apply_chunk(home_dir, near_config, store, hash, self.target_height, self.use_flat_storage) - .unwrap() + apply_chunk( + home_dir, + near_config, + store, + hash, + self.target_height, + self.use_flat_storage, + self.use_trie_for_free, + ) + .unwrap() } } @@ -285,12 +302,24 @@ pub struct ApplyReceiptCmd { hash: String, #[clap(long)] use_flat_storage: bool, + /// Use the data stored in trie, but without paying extra gas costs. + /// This could be used to simulate flat storage when the latter is not present. + #[clap(long)] + use_trie_for_free: bool, } impl ApplyReceiptCmd { pub fn run(self, home_dir: &Path, near_config: NearConfig, store: Store) { let hash = CryptoHash::from_str(&self.hash).unwrap(); - apply_receipt(home_dir, near_config, store, hash, self.use_flat_storage).unwrap(); + apply_receipt( + home_dir, + near_config, + store, + hash, + self.use_flat_storage, + self.use_trie_for_free, + ) + .unwrap(); } } @@ -300,12 +329,17 @@ pub struct ApplyTxCmd { hash: String, #[clap(long)] use_flat_storage: bool, + /// Use the data stored in trie, but without paying extra gas costs. + /// This could be used to simulate flat storage when the latter is not present. + #[clap(long)] + use_trie_for_free: bool, } impl ApplyTxCmd { pub fn run(self, home_dir: &Path, near_config: NearConfig, store: Store) { let hash = CryptoHash::from_str(&self.hash).unwrap(); - apply_tx(home_dir, near_config, store, hash, self.use_flat_storage).unwrap(); + apply_tx(home_dir, near_config, store, hash, self.use_flat_storage, self.use_trie_for_free) + .unwrap(); } } diff --git a/tools/state-viewer/src/commands.rs b/tools/state-viewer/src/commands.rs index f81b40c9f86..671edc040a9 100644 --- a/tools/state-viewer/src/commands.rs +++ b/tools/state-viewer/src/commands.rs @@ -17,7 +17,7 @@ use near_chain::chain::collect_receipts_from_response; use near_chain::migrations::check_if_block_is_first_with_chunk_of_version; use near_chain::types::{ ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, RuntimeAdapter, - RuntimeStorageConfig, + RuntimeStorageConfig, StorageDataSource, }; use near_chain::{ChainStore, ChainStoreAccess, ChainStoreUpdate, Error}; use near_chain_configs::GenesisChangeConfig; @@ -37,6 +37,7 @@ use near_primitives::state_record::state_record_to_account_id; use near_primitives::state_record::StateRecord; use near_primitives::trie_key::col::COLUMNS_WITH_ACCOUNT_ID_IN_KEY; use near_primitives::trie_key::TrieKey; +use near_primitives::types::ProtocolVersion; use near_primitives::types::{chunk_extra::ChunkExtra, BlockHeight, EpochId, ShardId}; use near_primitives::version::PROTOCOL_VERSION; use near_primitives_core::types::{Balance, EpochHeight, Gas}; @@ -64,6 +65,7 @@ pub(crate) fn apply_block( runtime: &dyn RuntimeAdapter, chain_store: &mut ChainStore, use_flat_storage: bool, + use_trie_for_free: bool, ) -> (Block, ApplyChunkResult) { let block = chain_store.get_block(&block_hash).unwrap(); let height = block.header().height(); @@ -94,9 +96,15 @@ pub(crate) fn apply_block( ) .unwrap(); + let mut storage = + RuntimeStorageConfig::new(*chunk_inner.prev_state_root(), use_flat_storage); + if use_trie_for_free { + storage.source = StorageDataSource::DbTrieOnly; + } + runtime .apply_chunk( - RuntimeStorageConfig::new(*chunk_inner.prev_state_root(), use_flat_storage), + storage, ApplyChunkReason::UpdateTrackedShard, ApplyChunkShardContext { shard_id, @@ -119,9 +127,14 @@ pub(crate) fn apply_block( chain_store.get_chunk_extra(block.header().prev_hash(), &shard_uid).unwrap(); let prev_block = chain_store.get_block(block.header().prev_hash()).unwrap(); + let mut storage = RuntimeStorageConfig::new(*chunk_extra.state_root(), use_flat_storage); + if use_trie_for_free { + storage.source = StorageDataSource::DbTrieOnly; + } + runtime .apply_chunk( - RuntimeStorageConfig::new(*chunk_extra.state_root(), use_flat_storage), + storage, ApplyChunkReason::UpdateTrackedShard, ApplyChunkShardContext { shard_id, @@ -147,6 +160,7 @@ pub(crate) fn apply_block_at_height( height: BlockHeight, shard_id: ShardId, use_flat_storage: bool, + use_trie_for_free: bool, home_dir: &Path, near_config: NearConfig, store: Store, @@ -168,6 +182,7 @@ pub(crate) fn apply_block_at_height( runtime.as_ref(), &mut chain_store, use_flat_storage, + use_trie_for_free, ); check_apply_block_result( &block, @@ -185,6 +200,7 @@ pub(crate) fn apply_chunk( chunk_hash: ChunkHash, target_height: Option, use_flat_storage: bool, + use_trie_for_free: bool, ) -> anyhow::Result<()> { let epoch_manager = EpochManager::new_arc_handle(store.clone(), &near_config.genesis.config); let runtime = NightshadeRuntime::from_config( @@ -207,8 +223,14 @@ pub(crate) fn apply_chunk( target_height, None, use_flat_storage, + use_trie_for_free, )?; - println!("resulting chunk extra:\n{:?}", resulting_chunk_extra(&apply_result, gas_limit)); + // Most probably `PROTOCOL_VERSION` won't work if the target_height points to a time + // before congestion control has been introduced. + println!( + "resulting chunk extra:\n{:?}", + resulting_chunk_extra(&apply_result, gas_limit, PROTOCOL_VERSION) + ); Ok(()) } @@ -257,6 +279,7 @@ pub(crate) fn apply_receipt( store: Store, hash: CryptoHash, use_flat_storage: bool, + use_trie_for_free: bool, ) -> anyhow::Result<()> { let epoch_manager = EpochManager::new_arc_handle(store.clone(), &near_config.genesis.config); let runtime = NightshadeRuntime::from_config( @@ -273,6 +296,7 @@ pub(crate) fn apply_receipt( store, hash, use_flat_storage, + use_trie_for_free, ) .map(|_| ()) } @@ -283,6 +307,7 @@ pub(crate) fn apply_tx( store: Store, hash: CryptoHash, use_flat_storage: bool, + use_trie_for_free: bool, ) -> anyhow::Result<()> { let epoch_manager = EpochManager::new_arc_handle(store.clone(), &near_config.genesis.config); let runtime = NightshadeRuntime::from_config( @@ -299,6 +324,7 @@ pub(crate) fn apply_tx( store, hash, use_flat_storage, + use_trie_for_free, ) .map(|_| ()) } @@ -557,8 +583,12 @@ pub(crate) fn check_apply_block_result( ) -> anyhow::Result<()> { let height = block.header().height(); let block_hash = block.header().hash(); - let new_chunk_extra = - resulting_chunk_extra(apply_result, block.chunks()[shard_id as usize].gas_limit()); + let protocol_version = block.header().latest_protocol_version(); + let new_chunk_extra = resulting_chunk_extra( + apply_result, + block.chunks()[shard_id as usize].gas_limit(), + protocol_version, + ); println!( "apply chunk for shard {} at height {}, resulting chunk extra {:?}", shard_id, height, &new_chunk_extra, @@ -739,9 +769,11 @@ pub(crate) fn replay_chain( } } -pub(crate) fn resulting_chunk_extra(result: &ApplyChunkResult, gas_limit: Gas) -> ChunkExtra { - // TODO(congestion_control): is it okay to use latest protocol version here for the chunk extra? - let protocol_version = PROTOCOL_VERSION; +pub(crate) fn resulting_chunk_extra( + result: &ApplyChunkResult, + gas_limit: Gas, + protocol_version: ProtocolVersion, +) -> ChunkExtra { let (outcome_root, _) = ApplyChunkResult::compute_outcomes_proof(&result.outcomes); ChunkExtra::new( protocol_version, From 5c2336ec0bf262c10d8595c0557419e1be9dfc8b Mon Sep 17 00:00:00 2001 From: Trisfald Date: Thu, 11 Jul 2024 10:52:34 +0200 Subject: [PATCH 2/6] fix apply range of block in the past --- tools/state-viewer/src/apply_chain_range.rs | 27 ++++++++++++++++++--- tools/state-viewer/src/cli.rs | 9 ++++--- tools/state-viewer/src/commands.rs | 2 ++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/tools/state-viewer/src/apply_chain_range.rs b/tools/state-viewer/src/apply_chain_range.rs index 5639251919a..0c7c900f0a6 100644 --- a/tools/state-viewer/src/apply_chain_range.rs +++ b/tools/state-viewer/src/apply_chain_range.rs @@ -3,7 +3,7 @@ use near_chain::chain::collect_receipts_from_response; use near_chain::migrations::check_if_block_is_first_with_chunk_of_version; use near_chain::types::{ ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, RuntimeAdapter, - RuntimeStorageConfig, + RuntimeStorageConfig, StorageDataSource, }; use near_chain::{ChainStore, ChainStoreAccess, ChainStoreUpdate}; use near_chain_configs::Genesis; @@ -126,6 +126,7 @@ fn apply_block_from_range( csv_file_mutex: &Mutex>, only_contracts: bool, use_flat_storage: bool, + use_trie_for_free: bool, ) { // normally save_trie_changes depends on whether the node is // archival, but here we don't care, and can just set it to false @@ -232,9 +233,16 @@ fn apply_block_from_range( return; } } + + let mut storage = + RuntimeStorageConfig::new(*chunk_inner.prev_state_root(), use_flat_storage); + if use_trie_for_free { + storage.source = StorageDataSource::DbTrieOnly; + } + runtime_adapter .apply_chunk( - RuntimeStorageConfig::new(*chunk_inner.prev_state_root(), use_flat_storage), + storage, ApplyChunkReason::UpdateTrackedShard, ApplyChunkShardContext { shard_id, @@ -258,9 +266,14 @@ fn apply_block_from_range( chain_store.get_chunk_extra(block.header().prev_hash(), &shard_uid).unwrap(); prev_chunk_extra = Some(chunk_extra.clone()); + let mut storage = RuntimeStorageConfig::new(*chunk_extra.state_root(), use_flat_storage); + if use_trie_for_free { + storage.source = StorageDataSource::DbTrieOnly; + } + runtime_adapter .apply_chunk( - RuntimeStorageConfig::new(*chunk_extra.state_root(), use_flat_storage), + storage, ApplyChunkReason::UpdateTrackedShard, ApplyChunkShardContext { shard_id, @@ -377,6 +390,7 @@ pub fn apply_chain_range( csv_file: Option<&mut File>, only_contracts: bool, use_flat_storage: bool, + use_trie_for_free: bool, ) { let parent_span = tracing::debug_span!( target: "state_viewer", @@ -386,7 +400,8 @@ pub fn apply_chain_range( ?end_height, %shard_id, only_contracts, - use_flat_storage) + use_flat_storage, + use_trie_for_free) .entered(); let chain_store = ChainStore::new(store.clone(), genesis.config.genesis_height, false); let (start_height, end_height) = match mode { @@ -394,6 +409,7 @@ pub fn apply_chain_range( // Benchmarking mode requires flat storage and retrieves start and // end heights from flat storage and chain. assert!(use_flat_storage); + assert!(!use_trie_for_free); assert!(start_height.is_none()); assert!(end_height.is_none()); @@ -458,6 +474,7 @@ pub fn apply_chain_range( &csv_file_mutex, only_contracts, use_flat_storage, + use_trie_for_free, ); }; @@ -642,6 +659,7 @@ mod test { None, false, false, + false, ); } @@ -686,6 +704,7 @@ mod test { Some(file.as_file_mut()), false, false, + false, ); let mut csv = String::new(); file.as_file_mut().seek(SeekFrom::Start(0)).unwrap(); diff --git a/tools/state-viewer/src/cli.rs b/tools/state-viewer/src/cli.rs index bc52b0815f9..00a73a885ba 100644 --- a/tools/state-viewer/src/cli.rs +++ b/tools/state-viewer/src/cli.rs @@ -242,14 +242,10 @@ impl ApplyChunkCmd { #[derive(clap::Parser, Copy, Clone, Debug, Eq, PartialEq)] pub enum ApplyRangeMode { /// Applies chunks one after another in order of increasing heights. - /// TODO(#8741): doesn't work. Remove dependency on flat storage - /// by simulating correct costs. Consider reintroducing DbTrieOnly - /// read mode removed at #10490. Sequential, /// Applies chunks in parallel. /// Useful for quick correctness check of applying chunks by comparing /// results with `ChunkExtra`s. - /// TODO(#8741): doesn't work, same as above. Parallel, /// Sequentially applies chunks from flat storage head until chain /// final head, moving flat head forward. Use in combination with @@ -274,6 +270,10 @@ pub struct ApplyRangeCmd { only_contracts: bool, #[clap(long)] use_flat_storage: bool, + /// Use the data stored in trie, but without paying extra gas costs. + /// This could be used to simulate flat storage when the latter is not present. + #[clap(long)] + use_trie_for_free: bool, #[clap(subcommand)] mode: ApplyRangeMode, } @@ -292,6 +292,7 @@ impl ApplyRangeCmd { store, self.only_contracts, self.use_flat_storage, + self.use_trie_for_free, ); } } diff --git a/tools/state-viewer/src/commands.rs b/tools/state-viewer/src/commands.rs index 671edc040a9..798d57a1ead 100644 --- a/tools/state-viewer/src/commands.rs +++ b/tools/state-viewer/src/commands.rs @@ -246,6 +246,7 @@ pub(crate) fn apply_range( store: Store, only_contracts: bool, use_flat_storage: bool, + use_trie_for_free: bool, ) { let mut csv_file = csv_file.map(|filename| std::fs::File::create(filename).unwrap()); @@ -270,6 +271,7 @@ pub(crate) fn apply_range( csv_file.as_mut(), only_contracts, use_flat_storage, + use_trie_for_free, ); } From 85c27c101d1f359d6b75be7b50f502c79ebf6d9e Mon Sep 17 00:00:00 2001 From: Trisfald Date: Thu, 11 Jul 2024 12:14:28 +0200 Subject: [PATCH 3/6] add warnings about avoid using DbTrieOnly in production --- chain/chain/src/runtime/mod.rs | 2 ++ chain/chain/src/types.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/chain/chain/src/runtime/mod.rs b/chain/chain/src/runtime/mod.rs index 5d1a92c065f..2e4077f9046 100644 --- a/chain/chain/src/runtime/mod.rs +++ b/chain/chain/src/runtime/mod.rs @@ -748,6 +748,7 @@ impl RuntimeAdapter for NightshadeRuntime { StorageDataSource::DbTrieOnly => { // If there is no flat storage on disk, use trie but simulate costs with enabled // flat storage by not charging gas for trie nodes. + // WARNING: should never be used in production! Consider this option only for debugging or replaying blocks. let mut trie = self.tries.get_trie_for_shard(shard_uid, storage_config.state_root); trie.dont_charge_gas_for_trie_node_access(); trie @@ -968,6 +969,7 @@ impl RuntimeAdapter for NightshadeRuntime { StorageDataSource::DbTrieOnly => { // If there is no flat storage on disk, use trie but simulate costs with enabled // flat storage by not charging gas for trie nodes. + // WARNING: should never be used in production! Consider this option only for debugging or replaying blocks. let mut trie = self.get_trie_for_shard( shard_id, &block.prev_block_hash, diff --git a/chain/chain/src/types.rs b/chain/chain/src/types.rs index 17dbbe4ed02..95720fb5f24 100644 --- a/chain/chain/src/types.rs +++ b/chain/chain/src/types.rs @@ -264,6 +264,7 @@ pub enum StorageDataSource { /// Trie is present in DB and flat storage is not. /// Used to reply past blocks and simulate gas costs as if flat storage /// was present. + /// WARNING: do not use this variant in production! DbTrieOnly, /// State data is supplied from state witness, there is no state data /// stored on disk. From 7d11e2d49ab5be805da7c7a34e6f29bf6f51892c Mon Sep 17 00:00:00 2001 From: Trisfald Date: Thu, 11 Jul 2024 12:38:39 +0200 Subject: [PATCH 4/6] retrieve protocol version at correct block height --- tools/state-viewer/src/commands.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/state-viewer/src/commands.rs b/tools/state-viewer/src/commands.rs index 798d57a1ead..d1cdc887556 100644 --- a/tools/state-viewer/src/commands.rs +++ b/tools/state-viewer/src/commands.rs @@ -225,11 +225,19 @@ pub(crate) fn apply_chunk( use_flat_storage, use_trie_for_free, )?; + let protocol_version = if let Some(height) = target_height { + // Retrieve the protocol version at the given height. + let block_hash = chain_store.get_block_hash_by_height(height)?; + chain_store.get_block(&block_hash)?.header().latest_protocol_version() + } else { + // No block height specified, fallback to current protocol version. + PROTOCOL_VERSION + }; // Most probably `PROTOCOL_VERSION` won't work if the target_height points to a time // before congestion control has been introduced. println!( "resulting chunk extra:\n{:?}", - resulting_chunk_extra(&apply_result, gas_limit, PROTOCOL_VERSION) + resulting_chunk_extra(&apply_result, gas_limit, protocol_version) ); Ok(()) } From 8551ac56b8245211e82e40a7863d88c55d4f1904 Mon Sep 17 00:00:00 2001 From: Trisfald Date: Thu, 11 Jul 2024 16:16:00 +0200 Subject: [PATCH 5/6] simplify logic by replacing the flags use_flat_storage and use_trie_for_free with an enum --- chain/chain/src/types.rs | 13 +++ tools/state-viewer/src/apply_chain_range.rs | 41 +++------ tools/state-viewer/src/apply_chunk.rs | 95 ++++++--------------- tools/state-viewer/src/cli.rs | 93 ++++++++------------ tools/state-viewer/src/commands.rs | 53 ++++-------- 5 files changed, 103 insertions(+), 192 deletions(-) diff --git a/chain/chain/src/types.rs b/chain/chain/src/types.rs index 95720fb5f24..fe45a2928bd 100644 --- a/chain/chain/src/types.rs +++ b/chain/chain/src/types.rs @@ -287,6 +287,19 @@ impl RuntimeStorageConfig { state_patch: Default::default(), } } + + /// Creates a [RuntimeStorageConfig] with [StorageDataSource::DbTrieOnly]. + /// Flat storage is disabled because it is implied to be missing. + /// + /// This's meant to be used only to replay blocks. + pub fn with_db_trie_only(state_root: StateRoot) -> Self { + Self { + state_root, + use_flat_storage: false, + source: StorageDataSource::DbTrieOnly, + state_patch: Default::default(), + } + } } #[derive(Clone)] diff --git a/tools/state-viewer/src/apply_chain_range.rs b/tools/state-viewer/src/apply_chain_range.rs index 0c7c900f0a6..20e72aaf055 100644 --- a/tools/state-viewer/src/apply_chain_range.rs +++ b/tools/state-viewer/src/apply_chain_range.rs @@ -1,9 +1,8 @@ -use crate::cli::ApplyRangeMode; +use crate::cli::{ApplyRangeMode, StorageSource}; use near_chain::chain::collect_receipts_from_response; use near_chain::migrations::check_if_block_is_first_with_chunk_of_version; use near_chain::types::{ ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, RuntimeAdapter, - RuntimeStorageConfig, StorageDataSource, }; use near_chain::{ChainStore, ChainStoreAccess, ChainStoreUpdate}; use near_chain_configs::Genesis; @@ -125,8 +124,7 @@ fn apply_block_from_range( verbose_output: bool, csv_file_mutex: &Mutex>, only_contracts: bool, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) { // normally save_trie_changes depends on whether the node is // archival, but here we don't care, and can just set it to false @@ -234,15 +232,9 @@ fn apply_block_from_range( } } - let mut storage = - RuntimeStorageConfig::new(*chunk_inner.prev_state_root(), use_flat_storage); - if use_trie_for_free { - storage.source = StorageDataSource::DbTrieOnly; - } - runtime_adapter .apply_chunk( - storage, + storage.create_runtime_storage(*chunk_inner.prev_state_root()), ApplyChunkReason::UpdateTrackedShard, ApplyChunkShardContext { shard_id, @@ -266,14 +258,9 @@ fn apply_block_from_range( chain_store.get_chunk_extra(block.header().prev_hash(), &shard_uid).unwrap(); prev_chunk_extra = Some(chunk_extra.clone()); - let mut storage = RuntimeStorageConfig::new(*chunk_extra.state_root(), use_flat_storage); - if use_trie_for_free { - storage.source = StorageDataSource::DbTrieOnly; - } - runtime_adapter .apply_chunk( - storage, + storage.create_runtime_storage(*chunk_extra.state_root()), ApplyChunkReason::UpdateTrackedShard, ApplyChunkShardContext { shard_id, @@ -389,8 +376,7 @@ pub fn apply_chain_range( verbose_output: bool, csv_file: Option<&mut File>, only_contracts: bool, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) { let parent_span = tracing::debug_span!( target: "state_viewer", @@ -400,16 +386,14 @@ pub fn apply_chain_range( ?end_height, %shard_id, only_contracts, - use_flat_storage, - use_trie_for_free) + ?storage) .entered(); let chain_store = ChainStore::new(store.clone(), genesis.config.genesis_height, false); let (start_height, end_height) = match mode { ApplyRangeMode::Benchmarking => { // Benchmarking mode requires flat storage and retrieves start and // end heights from flat storage and chain. - assert!(use_flat_storage); - assert!(!use_trie_for_free); + assert!(matches!(storage, StorageSource::FlatStorage)); assert!(start_height.is_none()); assert!(end_height.is_none()); @@ -473,8 +457,7 @@ pub fn apply_chain_range( verbose_output, &csv_file_mutex, only_contracts, - use_flat_storage, - use_trie_for_free, + storage, ); }; @@ -559,7 +542,7 @@ mod test { use nearcore::NightshadeRuntime; use crate::apply_chain_range::apply_chain_range; - use crate::cli::ApplyRangeMode; + use crate::cli::{ApplyRangeMode, StorageSource}; fn setup(epoch_length: NumBlocks) -> (Store, Genesis, TestEnv) { let mut genesis = @@ -658,8 +641,7 @@ mod test { true, None, false, - false, - false, + StorageSource::Trie, ); } @@ -703,8 +685,7 @@ mod test { true, Some(file.as_file_mut()), false, - false, - false, + StorageSource::Trie, ); let mut csv = String::new(); file.as_file_mut().seek(SeekFrom::Start(0)).unwrap(); diff --git a/tools/state-viewer/src/apply_chunk.rs b/tools/state-viewer/src/apply_chunk.rs index 894702194e6..d86c1a33c1d 100644 --- a/tools/state-viewer/src/apply_chunk.rs +++ b/tools/state-viewer/src/apply_chunk.rs @@ -4,7 +4,6 @@ use near_chain::chain::collect_receipts_from_response; use near_chain::migrations::check_if_block_is_first_with_chunk_of_version; use near_chain::types::{ ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, RuntimeAdapter, - RuntimeStorageConfig, StorageDataSource, }; use near_chain::{ChainStore, ChainStoreAccess}; use near_epoch_manager::{EpochManagerAdapter, EpochManagerHandle}; @@ -25,6 +24,8 @@ use rand::seq::SliceRandom; use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use crate::cli::StorageSource; + // like ChainStoreUpdate::get_incoming_receipts_for_shard(), but for the case when we don't // know of a block containing the target chunk fn get_incoming_receipts( @@ -86,8 +87,7 @@ pub(crate) fn apply_chunk( chunk_hash: ChunkHash, target_height: Option, rng: Option, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> anyhow::Result<(ApplyChunkResult, Gas)> { let chunk = chain_store.get_chunk(&chunk_hash)?; let chunk_header = chunk.cloned_header(); @@ -119,7 +119,7 @@ pub(crate) fn apply_chunk( ) .context("Failed collecting incoming receipts")?; - if use_flat_storage { + if matches!(storage, StorageSource::FlatStorage) { let shard_uid = epoch_manager.shard_id_to_uid(shard_id as u64, prev_block.header().epoch_id()).unwrap(); runtime.get_flat_storage_manager().create_flat_storage_for_shard(shard_uid).unwrap(); @@ -132,14 +132,9 @@ pub(crate) fn apply_chunk( shard_id, )?; - let mut storage = RuntimeStorageConfig::new(prev_state_root, use_flat_storage); - if use_trie_for_free { - storage.source = StorageDataSource::DbTrieOnly; - } - Ok(( runtime.apply_chunk( - storage, + storage.create_runtime_storage(prev_state_root), ApplyChunkReason::UpdateTrackedShard, ApplyChunkShardContext { shard_id, @@ -209,8 +204,7 @@ fn apply_tx_in_block( chain_store: &mut ChainStore, tx_hash: &CryptoHash, block_hash: CryptoHash, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> anyhow::Result { match find_tx_or_receipt(tx_hash, &block_hash, epoch_manager, chain_store)? { Some((hash_type, shard_id)) => { @@ -218,7 +212,7 @@ fn apply_tx_in_block( HashType::Tx => { println!("Found tx in block {} shard {}. equivalent command:\nview_state apply --height {} --shard-id {}\n", &block_hash, shard_id, chain_store.get_block_header(&block_hash)?.height(), shard_id); - let (block, apply_result) = crate::commands::apply_block(block_hash, shard_id, epoch_manager, runtime, chain_store, use_flat_storage, use_trie_for_free); + let (block, apply_result) = crate::commands::apply_block(block_hash, shard_id, epoch_manager, runtime, chain_store, storage); crate::commands::check_apply_block_result(&block, &apply_result, epoch_manager, chain_store, shard_id)?; Ok(apply_result) }, @@ -239,8 +233,7 @@ fn apply_tx_in_chunk( store: Store, chain_store: &mut ChainStore, tx_hash: &CryptoHash, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> anyhow::Result> { if chain_store.get_transaction(tx_hash)?.is_none() { return Err(anyhow!("tx with hash {} not known", tx_hash)); @@ -285,16 +278,8 @@ fn apply_tx_in_chunk( let mut results = Vec::new(); for chunk_hash in chunk_hashes { println!("found tx in chunk {}. Equivalent command (which will run faster than apply_tx):\nview_state apply_chunk --chunk_hash {}\n", &chunk_hash.0, &chunk_hash.0); - let (apply_result, gas_limit) = apply_chunk( - epoch_manager, - runtime, - chain_store, - chunk_hash, - None, - None, - use_flat_storage, - use_trie_for_free, - )?; + let (apply_result, gas_limit) = + apply_chunk(epoch_manager, runtime, chain_store, chunk_hash, None, None, storage)?; println!( "resulting chunk extra:\n{:?}", crate::commands::resulting_chunk_extra(&apply_result, gas_limit, protocol_version) @@ -310,8 +295,7 @@ pub(crate) fn apply_tx( runtime: &dyn RuntimeAdapter, store: Store, tx_hash: CryptoHash, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> anyhow::Result> { let mut chain_store = ChainStore::new(store.clone(), genesis_height, false); let outcomes = chain_store.get_outcomes_by_id(&tx_hash)?; @@ -323,19 +307,10 @@ pub(crate) fn apply_tx( &mut chain_store, &tx_hash, outcome.block_hash, - use_flat_storage, - use_trie_for_free, + storage, )?]) } else { - apply_tx_in_chunk( - epoch_manager, - runtime, - store, - &mut chain_store, - &tx_hash, - use_flat_storage, - use_trie_for_free, - ) + apply_tx_in_chunk(epoch_manager, runtime, store, &mut chain_store, &tx_hash, storage) } } @@ -345,8 +320,7 @@ fn apply_receipt_in_block( chain_store: &mut ChainStore, id: &CryptoHash, block_hash: CryptoHash, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> anyhow::Result { match find_tx_or_receipt(id, &block_hash, epoch_manager, chain_store)? { Some((hash_type, shard_id)) => { @@ -357,7 +331,7 @@ fn apply_receipt_in_block( HashType::Receipt => { println!("Found receipt in block {}. Receiver is in shard {}. equivalent command:\nview_state apply --height {} --shard-id {}\n", &block_hash, shard_id, chain_store.get_block_header(&block_hash)?.height(), shard_id); - let (block, apply_result) = crate::commands::apply_block(block_hash, shard_id, epoch_manager, runtime, chain_store, use_flat_storage, use_trie_for_free); + let (block, apply_result) = crate::commands::apply_block(block_hash, shard_id, epoch_manager, runtime, chain_store, storage); crate::commands::check_apply_block_result(&block, &apply_result, epoch_manager, chain_store, shard_id)?; Ok(apply_result) }, @@ -376,8 +350,7 @@ fn apply_receipt_in_chunk( store: Store, chain_store: &mut ChainStore, id: &CryptoHash, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> anyhow::Result> { if chain_store.get_receipt(id)?.is_none() { // TODO: handle local/delayed receipts @@ -456,8 +429,7 @@ fn apply_receipt_in_chunk( chunk_hash.clone(), None, None, - use_flat_storage, - use_trie_for_free, + storage, )?; let chunk_extra = crate::commands::resulting_chunk_extra(&apply_result, gas_limit, protocol_version); @@ -473,8 +445,7 @@ pub(crate) fn apply_receipt( runtime: &dyn RuntimeAdapter, store: Store, id: CryptoHash, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> anyhow::Result> { let mut chain_store = ChainStore::new(store.clone(), genesis_height, false); let outcomes = chain_store.get_outcomes_by_id(&id)?; @@ -485,19 +456,10 @@ pub(crate) fn apply_receipt( &mut chain_store, &id, outcome.block_hash, - use_flat_storage, - use_trie_for_free, + storage, )?]) } else { - apply_receipt_in_chunk( - epoch_manager, - runtime, - store, - &mut chain_store, - &id, - use_flat_storage, - use_trie_for_free, - ) + apply_receipt_in_chunk(epoch_manager, runtime, store, &mut chain_store, &id, storage) } } @@ -521,6 +483,8 @@ mod test { use rand::SeedableRng; use std::path::Path; + use crate::cli::StorageSource; + fn send_txs(env: &mut TestEnv, signers: &[InMemorySigner], height: u64, hash: CryptoHash) { for (i, signer) in signers.iter().enumerate() { let from = format!("test{}", i); @@ -613,8 +577,7 @@ mod test { chunk_hash.clone(), None, Some(rng), - false, - false, + StorageSource::Trie, ) .unwrap(); assert_eq!(apply_result.new_root, new_root); @@ -697,8 +660,7 @@ mod test { runtime.as_ref(), store.clone(), tx.get_hash(), - false, - false, + StorageSource::Trie, ) .unwrap(); assert_eq!(results.len(), 1); @@ -717,8 +679,7 @@ mod test { runtime.as_ref(), store.clone(), receipt.get_hash(), - false, - false, + StorageSource::Trie, ) .unwrap(); assert_eq!(results.len(), 1); @@ -748,8 +709,7 @@ mod test { runtime.as_ref(), store.clone(), tx.get_hash(), - false, - false, + StorageSource::Trie, ) .unwrap(); for result in results { @@ -770,8 +730,7 @@ mod test { runtime.as_ref(), store.clone(), receipt.get_hash(), - false, - false, + StorageSource::Trie, ) .unwrap(); for result in results { diff --git a/tools/state-viewer/src/cli.rs b/tools/state-viewer/src/cli.rs index 00a73a885ba..0803bb0c359 100644 --- a/tools/state-viewer/src/cli.rs +++ b/tools/state-viewer/src/cli.rs @@ -5,12 +5,13 @@ use crate::rocksdb_stats::get_rocksdb_stats; use crate::trie_iteration_benchmark::TrieIterationBenchmarkCmd; use crate::latest_witnesses::StateWitnessCmd; +use near_chain::types::RuntimeStorageConfig; use near_chain_configs::{GenesisChangeConfig, GenesisValidationMode}; use near_primitives::account::id::AccountId; use near_primitives::hash::CryptoHash; use near_primitives::sharding::ChunkHash; use near_primitives::trie_key::col; -use near_primitives::types::{BlockHeight, ShardId}; +use near_primitives::types::{BlockHeight, ShardId, StateRoot}; use near_primitives_core::types::EpochHeight; use near_store::{Mode, NodeStorage, Store, Temperature}; use nearcore::{load_config, NearConfig}; @@ -180,18 +181,34 @@ impl StateViewerSubCommand { } } +#[derive(clap::ValueEnum, Debug, Clone, Copy)] +#[clap(rename_all = "kebab_case")] +pub enum StorageSource { + Trie, + /// Use the data stored in trie, but without paying extra gas costs. + /// This could be used to simulate flat storage when the latter is not present. + TrieFree, + FlatStorage, +} + +impl StorageSource { + pub fn create_runtime_storage(&self, state_root: StateRoot) -> RuntimeStorageConfig { + match self { + StorageSource::Trie => RuntimeStorageConfig::new(state_root, false), + StorageSource::TrieFree => RuntimeStorageConfig::with_db_trie_only(state_root), + StorageSource::FlatStorage => RuntimeStorageConfig::new(state_root, true), + } + } +} + #[derive(clap::Parser)] pub struct ApplyCmd { #[clap(long)] height: BlockHeight, #[clap(long)] shard_id: ShardId, - #[clap(long)] - use_flat_storage: bool, - /// Use the data stored in trie, but without paying extra gas costs. - /// This could be used to simulate flat storage when the latter is not present. - #[clap(long)] - use_trie_for_free: bool, + #[clap(long, default_value = "trie")] + storage: StorageSource, } impl ApplyCmd { @@ -199,8 +216,7 @@ impl ApplyCmd { apply_block_at_height( self.height, self.shard_id, - self.use_flat_storage, - self.use_trie_for_free, + self.storage, home_dir, near_config, store, @@ -215,27 +231,14 @@ pub struct ApplyChunkCmd { chunk_hash: String, #[clap(long)] target_height: Option, - #[clap(long)] - use_flat_storage: bool, - /// Use the data stored in trie, but without paying extra gas costs. - /// This could be used to simulate flat storage when the latter is not present. - #[clap(long)] - use_trie_for_free: bool, + #[clap(long, default_value = "trie")] + storage: StorageSource, } impl ApplyChunkCmd { pub fn run(self, home_dir: &Path, near_config: NearConfig, store: Store) { let hash = ChunkHash::from(CryptoHash::from_str(&self.chunk_hash).unwrap()); - apply_chunk( - home_dir, - near_config, - store, - hash, - self.target_height, - self.use_flat_storage, - self.use_trie_for_free, - ) - .unwrap() + apply_chunk(home_dir, near_config, store, hash, self.target_height, self.storage).unwrap() } } @@ -268,12 +271,8 @@ pub struct ApplyRangeCmd { csv_file: Option, #[clap(long)] only_contracts: bool, - #[clap(long)] - use_flat_storage: bool, - /// Use the data stored in trie, but without paying extra gas costs. - /// This could be used to simulate flat storage when the latter is not present. - #[clap(long)] - use_trie_for_free: bool, + #[clap(long, default_value = "trie")] + storage: StorageSource, #[clap(subcommand)] mode: ApplyRangeMode, } @@ -291,8 +290,7 @@ impl ApplyRangeCmd { near_config, store, self.only_contracts, - self.use_flat_storage, - self.use_trie_for_free, + self.storage, ); } } @@ -301,26 +299,14 @@ impl ApplyRangeCmd { pub struct ApplyReceiptCmd { #[clap(long)] hash: String, - #[clap(long)] - use_flat_storage: bool, - /// Use the data stored in trie, but without paying extra gas costs. - /// This could be used to simulate flat storage when the latter is not present. - #[clap(long)] - use_trie_for_free: bool, + #[clap(long, default_value = "trie")] + storage: StorageSource, } impl ApplyReceiptCmd { pub fn run(self, home_dir: &Path, near_config: NearConfig, store: Store) { let hash = CryptoHash::from_str(&self.hash).unwrap(); - apply_receipt( - home_dir, - near_config, - store, - hash, - self.use_flat_storage, - self.use_trie_for_free, - ) - .unwrap(); + apply_receipt(home_dir, near_config, store, hash, self.storage).unwrap(); } } @@ -328,19 +314,14 @@ impl ApplyReceiptCmd { pub struct ApplyTxCmd { #[clap(long)] hash: String, - #[clap(long)] - use_flat_storage: bool, - /// Use the data stored in trie, but without paying extra gas costs. - /// This could be used to simulate flat storage when the latter is not present. - #[clap(long)] - use_trie_for_free: bool, + #[clap(long, default_value = "trie")] + storage: StorageSource, } impl ApplyTxCmd { pub fn run(self, home_dir: &Path, near_config: NearConfig, store: Store) { let hash = CryptoHash::from_str(&self.hash).unwrap(); - apply_tx(home_dir, near_config, store, hash, self.use_flat_storage, self.use_trie_for_free) - .unwrap(); + apply_tx(home_dir, near_config, store, hash, self.storage).unwrap(); } } diff --git a/tools/state-viewer/src/commands.rs b/tools/state-viewer/src/commands.rs index d1cdc887556..1f9d8d36547 100644 --- a/tools/state-viewer/src/commands.rs +++ b/tools/state-viewer/src/commands.rs @@ -1,5 +1,5 @@ use crate::apply_chain_range::apply_chain_range; -use crate::cli::{ApplyRangeMode, EpochAnalysisMode}; +use crate::cli::{ApplyRangeMode, EpochAnalysisMode, StorageSource}; use crate::contract_accounts::ContractAccount; use crate::contract_accounts::ContractAccountFilter; use crate::contract_accounts::Summary; @@ -17,7 +17,6 @@ use near_chain::chain::collect_receipts_from_response; use near_chain::migrations::check_if_block_is_first_with_chunk_of_version; use near_chain::types::{ ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, RuntimeAdapter, - RuntimeStorageConfig, StorageDataSource, }; use near_chain::{ChainStore, ChainStoreAccess, ChainStoreUpdate, Error}; use near_chain_configs::GenesisChangeConfig; @@ -64,13 +63,12 @@ pub(crate) fn apply_block( epoch_manager: &dyn EpochManagerAdapter, runtime: &dyn RuntimeAdapter, chain_store: &mut ChainStore, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> (Block, ApplyChunkResult) { let block = chain_store.get_block(&block_hash).unwrap(); let height = block.header().height(); let shard_uid = epoch_manager.shard_id_to_uid(shard_id, block.header().epoch_id()).unwrap(); - if use_flat_storage { + if matches!(storage, StorageSource::FlatStorage) { runtime.get_flat_storage_manager().create_flat_storage_for_shard(shard_uid).unwrap(); } let apply_result = if block.chunks()[shard_id as usize].height_included() == height { @@ -96,15 +94,9 @@ pub(crate) fn apply_block( ) .unwrap(); - let mut storage = - RuntimeStorageConfig::new(*chunk_inner.prev_state_root(), use_flat_storage); - if use_trie_for_free { - storage.source = StorageDataSource::DbTrieOnly; - } - runtime .apply_chunk( - storage, + storage.create_runtime_storage(*chunk_inner.prev_state_root()), ApplyChunkReason::UpdateTrackedShard, ApplyChunkShardContext { shard_id, @@ -127,14 +119,9 @@ pub(crate) fn apply_block( chain_store.get_chunk_extra(block.header().prev_hash(), &shard_uid).unwrap(); let prev_block = chain_store.get_block(block.header().prev_hash()).unwrap(); - let mut storage = RuntimeStorageConfig::new(*chunk_extra.state_root(), use_flat_storage); - if use_trie_for_free { - storage.source = StorageDataSource::DbTrieOnly; - } - runtime .apply_chunk( - storage, + storage.create_runtime_storage(*chunk_extra.state_root()), ApplyChunkReason::UpdateTrackedShard, ApplyChunkShardContext { shard_id, @@ -159,8 +146,7 @@ pub(crate) fn apply_block( pub(crate) fn apply_block_at_height( height: BlockHeight, shard_id: ShardId, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, home_dir: &Path, near_config: NearConfig, store: Store, @@ -181,8 +167,7 @@ pub(crate) fn apply_block_at_height( epoch_manager.as_ref(), runtime.as_ref(), &mut chain_store, - use_flat_storage, - use_trie_for_free, + storage, ); check_apply_block_result( &block, @@ -199,8 +184,7 @@ pub(crate) fn apply_chunk( store: Store, chunk_hash: ChunkHash, target_height: Option, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> anyhow::Result<()> { let epoch_manager = EpochManager::new_arc_handle(store.clone(), &near_config.genesis.config); let runtime = NightshadeRuntime::from_config( @@ -222,8 +206,7 @@ pub(crate) fn apply_chunk( chunk_hash, target_height, None, - use_flat_storage, - use_trie_for_free, + storage, )?; let protocol_version = if let Some(height) = target_height { // Retrieve the protocol version at the given height. @@ -253,8 +236,7 @@ pub(crate) fn apply_range( near_config: NearConfig, store: Store, only_contracts: bool, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) { let mut csv_file = csv_file.map(|filename| std::fs::File::create(filename).unwrap()); @@ -278,8 +260,7 @@ pub(crate) fn apply_range( verbose_output, csv_file.as_mut(), only_contracts, - use_flat_storage, - use_trie_for_free, + storage, ); } @@ -288,8 +269,7 @@ pub(crate) fn apply_receipt( near_config: NearConfig, store: Store, hash: CryptoHash, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> anyhow::Result<()> { let epoch_manager = EpochManager::new_arc_handle(store.clone(), &near_config.genesis.config); let runtime = NightshadeRuntime::from_config( @@ -305,8 +285,7 @@ pub(crate) fn apply_receipt( runtime.as_ref(), store, hash, - use_flat_storage, - use_trie_for_free, + storage, ) .map(|_| ()) } @@ -316,8 +295,7 @@ pub(crate) fn apply_tx( near_config: NearConfig, store: Store, hash: CryptoHash, - use_flat_storage: bool, - use_trie_for_free: bool, + storage: StorageSource, ) -> anyhow::Result<()> { let epoch_manager = EpochManager::new_arc_handle(store.clone(), &near_config.genesis.config); let runtime = NightshadeRuntime::from_config( @@ -333,8 +311,7 @@ pub(crate) fn apply_tx( runtime.as_ref(), store, hash, - use_flat_storage, - use_trie_for_free, + storage, ) .map(|_| ()) } From 5b8d3ca5dc223a5d2d3a41031f1c1ade2ebf04b4 Mon Sep 17 00:00:00 2001 From: Trisfald Date: Thu, 11 Jul 2024 18:45:44 +0200 Subject: [PATCH 6/6] rename with_db_trie_only -> new_with_db_trie_only --- chain/chain/src/types.rs | 2 +- tools/state-viewer/src/cli.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/chain/src/types.rs b/chain/chain/src/types.rs index fe45a2928bd..6c11e99b1d5 100644 --- a/chain/chain/src/types.rs +++ b/chain/chain/src/types.rs @@ -292,7 +292,7 @@ impl RuntimeStorageConfig { /// Flat storage is disabled because it is implied to be missing. /// /// This's meant to be used only to replay blocks. - pub fn with_db_trie_only(state_root: StateRoot) -> Self { + pub fn new_with_db_trie_only(state_root: StateRoot) -> Self { Self { state_root, use_flat_storage: false, diff --git a/tools/state-viewer/src/cli.rs b/tools/state-viewer/src/cli.rs index 0803bb0c359..2be4de16e75 100644 --- a/tools/state-viewer/src/cli.rs +++ b/tools/state-viewer/src/cli.rs @@ -195,7 +195,7 @@ impl StorageSource { pub fn create_runtime_storage(&self, state_root: StateRoot) -> RuntimeStorageConfig { match self { StorageSource::Trie => RuntimeStorageConfig::new(state_root, false), - StorageSource::TrieFree => RuntimeStorageConfig::with_db_trie_only(state_root), + StorageSource::TrieFree => RuntimeStorageConfig::new_with_db_trie_only(state_root), StorageSource::FlatStorage => RuntimeStorageConfig::new(state_root, true), } }