diff --git a/Cargo.lock b/Cargo.lock index 004ddeef0..cdfaa0d11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6548,6 +6548,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick_cache" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d7c94f8935a9df96bb6380e8592c70edf497a643f94bd23b2f76b399385dbf4" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.14.5", + "parking_lot 0.12.3", +] + [[package]] name = "quinn" version = "0.11.5" @@ -7315,15 +7327,19 @@ dependencies = [ "alloy-signer-local", "auto_impl", "derive_more 1.0.0", + "lazy_static", "metrics", "parking_lot 0.12.3", "pin-project", + "quick_cache", "rand 0.8.5", "reth-chainspec", "reth-errors", "reth-execution-types", "reth-metrics", "reth-primitives", + "reth-provider", + "reth-revm", "reth-storage-api", "reth-trie", "revm", diff --git a/crates/chain-state/Cargo.toml b/crates/chain-state/Cargo.toml index 078fe7d0c..bfd005064 100644 --- a/crates/chain-state/Cargo.toml +++ b/crates/chain-state/Cargo.toml @@ -20,6 +20,7 @@ reth-metrics.workspace = true reth-primitives.workspace = true reth-storage-api.workspace = true reth-trie.workspace = true +reth-revm.workspace = true # alloy alloy-primitives.workspace = true @@ -31,6 +32,10 @@ tokio-stream = { workspace = true, features = ["sync"] } # tracing tracing.workspace = true +# cache +lazy_static = "1.5.0" +quick_cache = "0.6.6" + # misc auto_impl.workspace = true derive_more.workspace = true @@ -49,11 +54,12 @@ alloy-signer.workspace = true alloy-signer-local.workspace = true rand.workspace = true revm.workspace = true +reth-provider = { workspace = true, features = ["test-utils"] } [features] test-utils = [ - "alloy-signer", - "alloy-signer-local", - "rand", - "revm" + "alloy-signer", + "alloy-signer-local", + "rand", + "revm" ] diff --git a/crates/chain-state/src/cache/cached_provider.rs b/crates/chain-state/src/cache/cached_provider.rs new file mode 100644 index 000000000..2a1914e0f --- /dev/null +++ b/crates/chain-state/src/cache/cached_provider.rs @@ -0,0 +1,329 @@ +use reth_errors::ProviderResult; +use reth_primitives::{ + Account, Address, BlockNumber, Bytecode, Bytes, StorageKey, StorageValue, B256, +}; +use reth_storage_api::{ + AccountReader, BlockHashReader, StateProofProvider, StateProvider, StateProviderBox, + StateRootProvider, StorageRootProvider, +}; +use reth_trie::{ + updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, TrieInput, +}; +use std::collections::{HashMap, HashSet}; + +/// Cached state provider struct +#[allow(missing_debug_implementations)] +pub struct CachedStateProvider { + pub(crate) underlying: Box, +} + +impl CachedStateProvider { + /// Create a new `CachedStateProvider` + pub fn new(underlying: Box) -> Self { + Self { underlying } + } + + /// Turn this state provider into a [`StateProviderBox`] + pub fn boxed(self) -> StateProviderBox { + Box::new(self) + } +} + +impl BlockHashReader for CachedStateProvider { + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + BlockHashReader::block_hash(&self.underlying, number) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + let hashes = self.underlying.canonical_hashes_range(start, end)?; + Ok(hashes) + } +} + +impl AccountReader for CachedStateProvider { + fn basic_account(&self, address: Address) -> ProviderResult> { + // Check cache first + if let Some(v) = crate::cache::get_account(&address) { + return Ok(Some(v)) + } + // Fallback to underlying provider + if let Some(value) = AccountReader::basic_account(&self.underlying, address)? { + crate::cache::insert_account(address, value); + return Ok(Some(value)) + } + Ok(None) + } +} + +impl StateRootProvider for CachedStateProvider { + fn state_root(&self, state: HashedPostState) -> ProviderResult { + self.state_root_from_nodes(TrieInput::from_state(state)) + } + + fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { + self.underlying.state_root_from_nodes(input) + } + + fn state_root_with_updates( + &self, + state: HashedPostState, + ) -> ProviderResult<(B256, TrieUpdates)> { + self.state_root_from_nodes_with_updates(TrieInput::from_state(state)) + } + + fn state_root_from_nodes_with_updates( + &self, + input: TrieInput, + ) -> ProviderResult<(B256, TrieUpdates)> { + self.underlying.state_root_from_nodes_with_updates(input) + } +} + +impl StorageRootProvider for CachedStateProvider { + fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + self.underlying.storage_root(address, storage) + } +} + +impl StateProofProvider for CachedStateProvider { + fn proof( + &self, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> ProviderResult { + self.underlying.proof(input, address, slots) + } + + fn multiproof( + &self, + input: TrieInput, + targets: HashMap>, + ) -> ProviderResult { + self.underlying.multiproof(input, targets) + } + + fn witness( + &self, + input: TrieInput, + target: HashedPostState, + ) -> ProviderResult> { + self.underlying.witness(input, target) + } +} + +impl StateProvider for CachedStateProvider { + fn storage( + &self, + address: Address, + storage_key: StorageKey, + ) -> ProviderResult> { + let key = (address, storage_key); + // Check cache first + if let Some(v) = crate::cache::get_storage(&key) { + return Ok(Some(v)) + } + // Fallback to underlying provider + if let Some(value) = StateProvider::storage(&self.underlying, address, storage_key)? { + crate::cache::insert_storage(key, value); + return Ok(Some(value)) + } + Ok(None) + } + + fn bytecode_by_hash(&self, code_hash: B256) -> ProviderResult> { + // Check cache first + if let Some(v) = crate::cache::get_code(&code_hash) { + return Ok(Some(v)) + } + // Fallback to underlying provider + if let Some(value) = StateProvider::bytecode_by_hash(&self.underlying, code_hash)? { + crate::cache::insert_code(code_hash, value.clone()); + return Ok(Some(value)) + } + Ok(None) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cache::plain_state::{clear_plain_state, write_plain_state, PLAIN_ACCOUNTS}; + use reth_primitives::revm_primitives::{AccountInfo, KECCAK_EMPTY}; + use reth_provider::{ + providers::ConsistentDbView, test_utils, test_utils::create_test_provider_factory, + }; + use reth_revm::{ + db::{AccountStatus, BundleState}, + primitives::U256, + }; + use reth_storage_api::TryIntoHistoricalStateProvider; + use std::collections::HashMap; + + #[test] + fn test_basic_account_and_clear() { + let factory = create_test_provider_factory(); + + let consistent_view = ConsistentDbView::new_with_latest_tip(factory.clone()).unwrap(); + let state_provider = consistent_view + .provider_ro() + .unwrap() + .disable_long_read_transaction_safety() + .try_into_history_at_block(1); + let cached_state_provider = CachedStateProvider::new(state_provider.unwrap()); + + let account = Address::random(); + let result = + reth_storage_api::AccountReader::basic_account(&cached_state_provider, account) + .unwrap(); + assert_eq!(result.is_none(), true); + + PLAIN_ACCOUNTS + .insert(account, Account { nonce: 100, balance: U256::ZERO, bytecode_hash: None }); + let result = + reth_storage_api::AccountReader::basic_account(&cached_state_provider, account) + .unwrap(); + assert_eq!(result.unwrap().nonce, 100); + + // clear account + clear_plain_state(); + let result = + reth_storage_api::AccountReader::basic_account(&cached_state_provider, account) + .unwrap(); + assert_eq!(result.is_none(), true); + } + + #[test] + fn test_apply_bundle_state() { + let factory = test_utils::create_test_provider_factory(); + let consistent_view = ConsistentDbView::new_with_latest_tip(factory.clone()).unwrap(); + let state_provider = consistent_view + .provider_ro() + .unwrap() + .disable_long_read_transaction_safety() + .try_into_history_at_block(1); + let cached_state_provider = CachedStateProvider::new(state_provider.unwrap()); + + // apply bundle state to set cache + let account1 = Address::random(); + let account2 = Address::random(); + let bundle_state = BundleState::new( + vec![ + ( + account1, + None, + Some(AccountInfo { + nonce: 1, + balance: U256::from(10), + code_hash: KECCAK_EMPTY, + code: None, + }), + HashMap::from([ + (U256::from(2), (U256::from(0), U256::from(10))), + (U256::from(5), (U256::from(0), U256::from(15))), + ]), + ), + ( + account2, + None, + Some(AccountInfo { + nonce: 1, + balance: U256::from(10), + code_hash: KECCAK_EMPTY, + code: None, + }), + HashMap::from([]), + ), + ], + vec![vec![ + ( + account1, + Some(None), + vec![(U256::from(2), U256::from(0)), (U256::from(5), U256::from(0))], + ), + (account2, Some(None), vec![]), + ]], + vec![], + ); + write_plain_state(bundle_state); + + let account1_result = + reth_storage_api::AccountReader::basic_account(&cached_state_provider, account1) + .unwrap(); + assert_eq!(account1_result.unwrap().nonce, 1); + let storage1_result = reth_storage_api::StateProvider::storage( + &cached_state_provider, + account1, + B256::with_last_byte(2), + ) + .unwrap(); + assert_eq!(storage1_result.unwrap(), U256::from(10)); + let storage2_result = reth_storage_api::StateProvider::storage( + &cached_state_provider, + account1, + B256::with_last_byte(5), + ) + .unwrap(); + assert_eq!(storage2_result.unwrap(), U256::from(15)); + + let account2_result = + reth_storage_api::AccountReader::basic_account(&cached_state_provider, account2) + .unwrap(); + assert_eq!(account2_result.unwrap().nonce, 1); + + // apply bundle state to set clear cache + let account3 = Address::random(); + let mut bundle_state = BundleState::new( + vec![( + account3, + Some(AccountInfo { + nonce: 3, + balance: U256::from(10), + code_hash: KECCAK_EMPTY, + code: None, + }), + None, + HashMap::from([ + (U256::from(2), (U256::from(0), U256::from(10))), + (U256::from(5), (U256::from(0), U256::from(15))), + ]), + )], + vec![vec![( + account3, + Some(None), + vec![(U256::from(2), U256::from(0)), (U256::from(5), U256::from(0))], + )]], + vec![], + ); + bundle_state.state.get_mut(&account3).unwrap().status = AccountStatus::Destroyed; + write_plain_state(bundle_state); + + let account1_result = + reth_storage_api::AccountReader::basic_account(&cached_state_provider, account1) + .unwrap(); + assert_eq!(account1_result.unwrap().nonce, 1); + let storage1_result = reth_storage_api::StateProvider::storage( + &cached_state_provider, + account1, + B256::with_last_byte(2), + ) + .unwrap(); + assert_eq!(storage1_result.is_none(), true); + let storage2_result = reth_storage_api::StateProvider::storage( + &cached_state_provider, + account1, + B256::with_last_byte(5), + ) + .unwrap(); + assert_eq!(storage2_result.is_none(), true); + + let account2_result = + reth_storage_api::AccountReader::basic_account(&cached_state_provider, account2) + .unwrap(); + assert_eq!(account2_result.unwrap().nonce, 1); + } +} diff --git a/crates/chain-state/src/cache/mod.rs b/crates/chain-state/src/cache/mod.rs new file mode 100644 index 000000000..5ce255fbe --- /dev/null +++ b/crates/chain-state/src/cache/mod.rs @@ -0,0 +1,26 @@ +/// State provider with cached states for execution. +pub mod cached_provider; +mod plain_state; + +use crate::ExecutedBlock; +use tracing::debug; + +use crate::cache::plain_state::{ + clear_plain_state, get_account, get_code, get_storage, insert_account, insert_code, + insert_storage, write_plain_state, +}; + +/// Writes the execution outcomes of the given blocks to the cache. +pub fn write_to_cache(blocks: Vec) { + for block in blocks { + debug!("Start to write block {} to cache", block.block.header.number); + let bundle_state = block.execution_outcome().clone().bundle; + write_plain_state(bundle_state); + debug!("Finish to write block {} to cache", block.block.header.number); + } +} + +/// Clears all cached states. +pub fn clear_cache() { + clear_plain_state(); +} diff --git a/crates/chain-state/src/cache/plain_state.rs b/crates/chain-state/src/cache/plain_state.rs new file mode 100644 index 000000000..b31d4a27b --- /dev/null +++ b/crates/chain-state/src/cache/plain_state.rs @@ -0,0 +1,99 @@ +use lazy_static::lazy_static; +use quick_cache::sync::Cache; +use reth_primitives::{Account, Address, Bytecode, StorageKey, StorageValue, B256, U256}; +use reth_revm::db::{BundleState, OriginalValuesKnown}; + +// Cache sizes +const ACCOUNT_CACHE_SIZE: usize = 1000000; +const STORAGE_CACHE_SIZE: usize = ACCOUNT_CACHE_SIZE * 10; +const CONTRACT_CACHE_SIZE: usize = ACCOUNT_CACHE_SIZE / 10; + +// Type alias for address and storage key tuple +type AddressStorageKey = (Address, StorageKey); + +lazy_static! { + /// Account cache + pub(crate) static ref PLAIN_ACCOUNTS: Cache = Cache::new(ACCOUNT_CACHE_SIZE); + + /// Storage cache + pub(crate) static ref PLAIN_STORAGES: Cache = Cache::new(STORAGE_CACHE_SIZE); + + /// Contract cache + /// The size of contract is large and the hot contracts should be limited. + pub(crate) static ref CONTRACT_CODES: Cache = Cache::new(CONTRACT_CACHE_SIZE); +} + +pub(crate) fn insert_account(k: Address, v: Account) { + PLAIN_ACCOUNTS.insert(k, v); +} + +/// Insert storage into the cache +pub(crate) fn insert_storage(k: AddressStorageKey, v: U256) { + PLAIN_STORAGES.insert(k, v); +} + +// Get account from cache +pub(crate) fn get_account(k: &Address) -> Option { + PLAIN_ACCOUNTS.get(k) +} + +// Get storage from cache +pub(crate) fn get_storage(k: &AddressStorageKey) -> Option { + PLAIN_STORAGES.get(k) +} + +// Get code from cache +pub(crate) fn get_code(k: &B256) -> Option { + CONTRACT_CODES.get(k) +} + +// Insert code into cache +pub(crate) fn insert_code(k: B256, v: Bytecode) { + CONTRACT_CODES.insert(k, v); +} + +/// Write committed state to cache. +pub(crate) fn write_plain_state(bundle: BundleState) { + let change_set = bundle.into_plain_state(OriginalValuesKnown::Yes); + + // Update account cache + for (address, account_info) in &change_set.accounts { + match account_info { + None => { + PLAIN_ACCOUNTS.remove(address); + } + Some(acc) => { + PLAIN_ACCOUNTS.insert( + *address, + Account { + nonce: acc.nonce, + balance: acc.balance, + bytecode_hash: Some(acc.code_hash), + }, + ); + } + } + } + + // Update storage cache + let mut should_wipe = false; + for storage in &change_set.storage { + if storage.wipe_storage { + should_wipe = true; + break + } + + for (k, v) in storage.storage.clone() { + insert_storage((storage.address, StorageKey::from(k)), v); + } + } + if should_wipe { + PLAIN_STORAGES.clear(); + } +} + +/// Clear cached accounts and storages. +pub(crate) fn clear_plain_state() { + PLAIN_ACCOUNTS.clear(); + PLAIN_STORAGES.clear(); +} diff --git a/crates/chain-state/src/lib.rs b/crates/chain-state/src/lib.rs index 50a103111..0f05ef021 100644 --- a/crates/chain-state/src/lib.rs +++ b/crates/chain-state/src/lib.rs @@ -24,6 +24,11 @@ pub use notifications::{ mod memory_overlay; pub use memory_overlay::MemoryOverlayStateProvider; +pub use cache::cached_provider::CachedStateProvider; + +/// Cache layer for plain states. +pub mod cache; + #[cfg(any(test, feature = "test-utils"))] /// Common test helpers pub mod test_utils; diff --git a/crates/engine/tree/Cargo.toml b/crates/engine/tree/Cargo.toml index 486f103c8..48368b929 100644 --- a/crates/engine/tree/Cargo.toml +++ b/crates/engine/tree/Cargo.toml @@ -77,14 +77,14 @@ rand.workspace = true [features] test-utils = [ - "reth-db/test-utils", - "reth-chain-state/test-utils", - "reth-network-p2p/test-utils", - "reth-prune-types", - "reth-stages/test-utils", - "reth-static-file", - "reth-tracing", - "reth-chainspec" + "reth-db/test-utils", + "reth-chain-state/test-utils", + "reth-network-p2p/test-utils", + "reth-prune-types", + "reth-stages/test-utils", + "reth-static-file", + "reth-tracing", + "reth-chainspec" ] bsc = [] \ No newline at end of file diff --git a/crates/engine/tree/src/lib.rs b/crates/engine/tree/src/lib.rs index 100b71604..040a1b29d 100644 --- a/crates/engine/tree/src/lib.rs +++ b/crates/engine/tree/src/lib.rs @@ -97,6 +97,7 @@ pub use reth_blockchain_tree_api::*; /// Support for backfill sync mode. pub mod backfill; + /// The type that drives the chain forward. pub mod chain; /// Support for downloading blocks on demand for live sync. @@ -107,9 +108,8 @@ pub mod engine; pub mod metrics; /// The background writer service, coordinating write operations on static files and the database. pub mod persistence; -/// Support for interacting with the blockchain tree. -pub mod tree; - /// Test utilities. #[cfg(any(test, feature = "test-utils"))] pub mod test_utils; +/// Support for interacting with the blockchain tree. +pub mod tree; diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index b1ade6209..c2e532c04 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -110,6 +110,9 @@ impl PersistenceService { UnifiedStorageWriter::from(&provider_rw, &sf_provider).remove_blocks_above(new_tip_num)?; UnifiedStorageWriter::commit_unwind(provider_rw, sf_provider)?; + reth_chain_state::cache::clear_cache(); + debug!(target: "tree::persistence", "Finish to clear state cache"); + debug!(target: "engine::persistence", ?new_tip_num, ?new_tip_hash, "Removed blocks from disk"); self.metrics.remove_blocks_above_duration_seconds.record(start_time.elapsed()); Ok(new_tip_hash.map(|hash| BlockNumHash { hash, number: new_tip_num })) @@ -126,6 +129,10 @@ impl PersistenceService { .map(|block| BlockNumHash { hash: block.block().hash(), number: block.block().number }); if last_block_hash_num.is_some() { + // update plain state cache + reth_chain_state::cache::write_to_cache(blocks.clone()); + debug!(target: "tree::persistence", "Finish to write state cache"); + let provider_rw = self.provider.provider_rw()?; let static_file_provider = self.provider.static_file_provider(); diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 7cbc89746..12c3ff3e0 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -1292,6 +1292,9 @@ where // the canonical chain self.canonical_in_memory_state.clear_state(); + // clear finalized state/hashed/trie caches + reth_chain_state::cache::clear_cache(); + if let Ok(Some(new_head)) = self.provider.sealed_header(backfill_height) { // update the tracked chain height, after backfill sync both the canonical height and // persisted height are the same diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index b363f3b06..573936f56 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -10,8 +10,8 @@ use crate::{ use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256}; use alloy_rpc_types_engine::ForkchoiceState; use reth_chain_state::{ - BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, - MemoryOverlayStateProvider, + BlockState, CachedStateProvider, CanonicalInMemoryState, ForkChoiceNotifications, + ForkChoiceSubscriptions, MemoryOverlayStateProvider, }; use reth_chainspec::ChainInfo; use reth_db::Database; @@ -1118,7 +1118,8 @@ impl StateProviderFactory for BlockchainProvider2 { trace!(target: "providers::blockchain", ?block_hash, "Getting history by block hash"); if let Ok(state) = self.database.history_by_block_hash(block_hash) { // This could be tracked by a block in the database block - Ok(state) + + Ok(CachedStateProvider::new(state).boxed()) } else if let Some(state) = self.canonical_in_memory_state.state_by_hash(block_hash) { // ... or this could be tracked by the in memory state let state_provider = self.block_state_provider(state)?;