From 268cc368ed43abe7adbe701a694bef39133a542f Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:22:23 +0100 Subject: [PATCH 1/2] use Provider generic instead of DB --- crates/cli/commands/src/stage/drop.rs | 10 +- crates/storage/db-common/src/init.rs | 132 ++++++++++++++++++-------- 2 files changed, 98 insertions(+), 44 deletions(-) diff --git a/crates/cli/commands/src/stage/drop.rs b/crates/cli/commands/src/stage/drop.rs index d62a340bfcdc..8fcbaf044009 100644 --- a/crates/cli/commands/src/stage/drop.rs +++ b/crates/cli/commands/src/stage/drop.rs @@ -73,7 +73,7 @@ impl> Command { StageId::Headers.to_string(), Default::default(), )?; - insert_genesis_header(&provider_rw, &static_file_provider, &self.env.chain)?; + insert_genesis_header(&provider_rw.0, &static_file_provider, &self.env.chain)?; } StageEnum::Bodies => { tx.clear::()?; @@ -86,7 +86,7 @@ impl> Command { StageId::Bodies.to_string(), Default::default(), )?; - insert_genesis_header(&provider_rw, &static_file_provider, &self.env.chain)?; + insert_genesis_header(&provider_rw.0, &static_file_provider, &self.env.chain)?; } StageEnum::Senders => { tx.clear::()?; @@ -107,7 +107,7 @@ impl> Command { Default::default(), )?; let alloc = &self.env.chain.genesis().alloc; - insert_genesis_state(&provider_rw, alloc.iter())?; + insert_genesis_state(&provider_rw.0, alloc.iter())?; } StageEnum::AccountHashing => { tx.clear::()?; @@ -165,7 +165,7 @@ impl> Command { StageId::IndexStorageHistory.to_string(), Default::default(), )?; - insert_genesis_history(&provider_rw, self.env.chain.genesis.alloc.iter())?; + insert_genesis_history(&provider_rw.0, self.env.chain.genesis.alloc.iter())?; } StageEnum::TxLookup => { tx.clear::()?; @@ -173,7 +173,7 @@ impl> Command { StageId::TransactionLookup.to_string(), Default::default(), )?; - insert_genesis_header(&provider_rw, &static_file_provider, &self.env.chain)?; + insert_genesis_header(&provider_rw.0, &static_file_provider, &self.env.chain)?; } } diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index 2fa2026b849c..e1f483d55c22 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -6,18 +6,17 @@ use reth_chainspec::ChainSpec; use reth_codecs::Compact; use reth_config::config::EtlConfig; use reth_db::tables; -use reth_db_api::{database::Database, transaction::DbTxMut, DatabaseError}; +use reth_db_api::{transaction::DbTxMut, DatabaseError}; use reth_etl::Collector; -use reth_node_types::NodeTypesWithDB; use reth_primitives::{Account, Bytecode, GotExpected, Receipts, StaticFileSegment, StorageEntry}; use reth_provider::{ errors::provider::ProviderResult, providers::{StaticFileProvider, StaticFileWriter}, writer::UnifiedStorageWriter, - BlockHashReader, BlockNumReader, BundleStateInit, ChainSpecProvider, DatabaseProviderRW, - ExecutionOutcome, HashingWriter, HeaderProvider, HistoryWriter, OriginalValuesKnown, - ProviderError, ProviderFactory, RevertsInit, StageCheckpointWriter, StateWriter, - StaticFileProviderFactory, TrieWriter, + BlockHashReader, BlockNumReader, BundleStateInit, ChainSpecProvider, DBProvider, + DatabaseProviderFactory, ExecutionOutcome, HashingWriter, HeaderProvider, HistoryWriter, + OriginalValuesKnown, ProviderError, RevertsInit, StageCheckpointWriter, StateChangeWriter, + StateWriter, StaticFileProviderFactory, TrieWriter, }; use reth_stages_types::{StageCheckpoint, StageId}; use reth_trie::{IntermediateStateRootState, StateRoot as StateRootComputer, StateRootProgress}; @@ -70,9 +69,19 @@ impl From for InitDatabaseError { } /// Write the genesis block if it has not already been written -pub fn init_genesis>( - factory: &ProviderFactory, -) -> Result { +pub fn init_genesis(factory: &PF) -> Result +where + PF: DatabaseProviderFactory + + StaticFileProviderFactory + + ChainSpecProvider + + BlockHashReader, + PF::ProviderRW: StageCheckpointWriter + + HistoryWriter + + HeaderProvider + + HashingWriter + + StateChangeWriter + + AsRef, +{ let chain = factory.chain_spec(); let genesis = chain.genesis(); @@ -100,7 +109,7 @@ pub fn init_genesis>( let alloc = &genesis.alloc; // use transaction to insert genesis header - let provider_rw = factory.provider_rw()?; + let provider_rw = factory.database_provider_rw()?; insert_genesis_hashes(&provider_rw, alloc.iter())?; insert_genesis_history(&provider_rw, alloc.iter())?; @@ -130,19 +139,25 @@ pub fn init_genesis>( } /// Inserts the genesis state into the database. -pub fn insert_genesis_state<'a, 'b, DB: Database>( - provider: &DatabaseProviderRW, +pub fn insert_genesis_state<'a, 'b, Provider>( + provider: &Provider, alloc: impl Iterator, -) -> ProviderResult<()> { - insert_state::(provider, alloc, 0) +) -> ProviderResult<()> +where + Provider: DBProvider + StateChangeWriter + HeaderProvider + AsRef, +{ + insert_state(provider, alloc, 0) } /// Inserts state at given block into database. -pub fn insert_state<'a, 'b, DB: Database>( - provider: &DatabaseProviderRW, +pub fn insert_state<'a, 'b, Provider>( + provider: &Provider, alloc: impl Iterator, block: u64, -) -> ProviderResult<()> { +) -> ProviderResult<()> +where + Provider: DBProvider + StateChangeWriter + HeaderProvider + AsRef, +{ let capacity = alloc.size_hint().1.unwrap_or(0); let mut state_init: BundleStateInit = HashMap::with_capacity(capacity); let mut reverts_init = HashMap::with_capacity(capacity); @@ -217,10 +232,13 @@ pub fn insert_state<'a, 'b, DB: Database>( } /// Inserts hashes for the genesis state. -pub fn insert_genesis_hashes<'a, 'b, DB: Database>( - provider: &DatabaseProviderRW, +pub fn insert_genesis_hashes<'a, 'b, Provider>( + provider: &Provider, alloc: impl Iterator + Clone, -) -> ProviderResult<()> { +) -> ProviderResult<()> +where + Provider: DBProvider + HashingWriter, +{ // insert and hash accounts to hashing table let alloc_accounts = alloc.clone().map(|(addr, account)| (*addr, Some(Account::from(account)))); provider.insert_account_for_hashing(alloc_accounts)?; @@ -241,19 +259,25 @@ pub fn insert_genesis_hashes<'a, 'b, DB: Database>( } /// Inserts history indices for genesis accounts and storage. -pub fn insert_genesis_history<'a, 'b, DB: Database>( - provider: &DatabaseProviderRW, +pub fn insert_genesis_history<'a, 'b, Provider>( + provider: &Provider, alloc: impl Iterator + Clone, -) -> ProviderResult<()> { - insert_history::(provider, alloc, 0) +) -> ProviderResult<()> +where + Provider: DBProvider + HistoryWriter, +{ + insert_history(provider, alloc, 0) } /// Inserts history indices for genesis accounts and storage. -pub fn insert_history<'a, 'b, DB: Database>( - provider: &DatabaseProviderRW, +pub fn insert_history<'a, 'b, Provider>( + provider: &Provider, alloc: impl Iterator + Clone, block: u64, -) -> ProviderResult<()> { +) -> ProviderResult<()> +where + Provider: DBProvider + HistoryWriter, +{ let account_transitions = alloc.clone().map(|(addr, _)| (*addr, [block])); provider.insert_account_history_index(account_transitions)?; @@ -270,11 +294,14 @@ pub fn insert_history<'a, 'b, DB: Database>( } /// Inserts header for the genesis state. -pub fn insert_genesis_header( - provider: &DatabaseProviderRW, +pub fn insert_genesis_header( + provider: &Provider, static_file_provider: &StaticFileProvider, chain: &ChainSpec, -) -> ProviderResult<()> { +) -> ProviderResult<()> +where + Provider: DBProvider, +{ let (header, block_hash) = (chain.genesis_header(), chain.genesis_hash()); match static_file_provider.block_hash(0) { @@ -299,11 +326,26 @@ pub fn insert_genesis_header( /// It's similar to [`init_genesis`] but supports importing state too big to fit in memory, and can /// be set to the highest block present. One practical usecase is to import OP mainnet state at /// bedrock transition block. -pub fn init_from_state_dump>( +pub fn init_from_state_dump( mut reader: impl BufRead, - factory: ProviderFactory, + factory: PF, etl_config: EtlConfig, -) -> eyre::Result { +) -> eyre::Result +where + PF: DatabaseProviderFactory + + StaticFileProviderFactory + + ChainSpecProvider + + BlockHashReader + + BlockNumReader + + HeaderProvider, + PF::ProviderRW: StageCheckpointWriter + + HistoryWriter + + HeaderProvider + + HashingWriter + + StateChangeWriter + + TrieWriter + + AsRef, +{ let block = factory.last_block_number()?; let hash = factory.block_hash(block)?.unwrap(); let expected_state_root = factory @@ -336,7 +378,7 @@ pub fn init_from_state_dump>( let collector = parse_accounts(&mut reader, etl_config)?; // write state to db - let provider_rw = factory.provider_rw()?; + let provider_rw = factory.database_provider_rw()?; dump_state(collector, &provider_rw, block)?; // compute and compare state root. this advances the stage checkpoints. @@ -413,11 +455,19 @@ fn parse_accounts( } /// Takes a [`Collector`] and processes all accounts. -fn dump_state( +fn dump_state( mut collector: Collector, - provider_rw: &DatabaseProviderRW, + provider_rw: &Provider, block: u64, -) -> Result<(), eyre::Error> { +) -> Result<(), eyre::Error> +where + Provider: DBProvider + + HeaderProvider + + HashingWriter + + HistoryWriter + + StateChangeWriter + + AsRef, +{ let accounts_len = collector.len(); let mut accounts = Vec::with_capacity(AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP); let mut total_inserted_accounts = 0; @@ -452,7 +502,7 @@ fn dump_state( )?; // block is already written to static files - insert_state::( + insert_state( provider_rw, accounts.iter().map(|(address, account)| (address, account)), block, @@ -466,7 +516,10 @@ fn dump_state( /// Computes the state root (from scratch) based on the accounts and storages present in the /// database. -fn compute_state_root(provider: &DatabaseProviderRW) -> eyre::Result { +fn compute_state_root(provider: &Provider) -> eyre::Result +where + Provider: DBProvider + TrieWriter, +{ trace!(target: "reth::cli", "Computing state root"); let tx = provider.tx_ref(); @@ -543,6 +596,7 @@ mod tests { models::{storage_sharded_key::StorageShardedKey, ShardedKey}, table::{Table, TableRow}, transaction::DbTx, + Database, }; use reth_primitives::{HOLESKY_GENESIS_HASH, MAINNET_GENESIS_HASH, SEPOLIA_GENESIS_HASH}; use reth_primitives_traits::IntegerList; From 4def26f38bc3e5836f023ec2fc489616589f1f30 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:50:42 +0100 Subject: [PATCH 2/2] add missing test import --- crates/storage/db-common/src/init.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index e1f483d55c22..864849ebe5fe 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -600,8 +600,9 @@ mod tests { }; use reth_primitives::{HOLESKY_GENESIS_HASH, MAINNET_GENESIS_HASH, SEPOLIA_GENESIS_HASH}; use reth_primitives_traits::IntegerList; - use reth_provider::test_utils::{ - create_test_provider_factory_with_chain_spec, MockNodeTypesWithDB, + use reth_provider::{ + test_utils::{create_test_provider_factory_with_chain_spec, MockNodeTypesWithDB}, + ProviderFactory, }; use std::{collections::BTreeMap, sync::Arc};