diff --git a/CODEOWNERS b/CODEOWNERS index d5161e89cd0a..6a9c470281f3 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -19,5 +19,5 @@ crates/metrics @onbjerg crates/tracing @onbjerg crates/tasks @mattsse crates/prune @shekhirin @joshieDo -crates/snapshot @joshieDo +crates/static-file @joshieDo @shekhirin .github/ @onbjerg @gakonst @DaniPopes diff --git a/Cargo.lock b/Cargo.lock index b067d9fbb843..5a098ade5e40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5745,8 +5745,8 @@ dependencies = [ "reth-rpc-engine-api", "reth-rpc-types", "reth-rpc-types-compat", - "reth-snapshot", "reth-stages", + "reth-static-file", "reth-tasks", "reth-tracing", "reth-transaction-pool", @@ -5834,8 +5834,8 @@ dependencies = [ "reth-revm", "reth-rpc-types", "reth-rpc-types-compat", - "reth-snapshot", "reth-stages", + "reth-static-file", "reth-tasks", "reth-tokio-util", "reth-tracing", @@ -6390,8 +6390,8 @@ dependencies = [ "reth-revm", "reth-rpc", "reth-rpc-engine-api", - "reth-snapshot", "reth-stages", + "reth-static-file", "reth-tasks", "reth-tracing", "reth-transaction-pool", @@ -6452,8 +6452,8 @@ dependencies = [ "reth-rpc-engine-api", "reth-rpc-types", "reth-rpc-types-compat", - "reth-snapshot", "reth-stages", + "reth-static-file", "reth-tasks", "reth-tracing", "reth-transaction-pool", @@ -6658,8 +6658,8 @@ dependencies = [ "reth-metrics", "reth-primitives", "reth-provider", - "reth-snapshot", "reth-stages", + "reth-static-file", "reth-tokio-util", "thiserror", "tokio", @@ -6871,27 +6871,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "reth-snapshot" -version = "0.1.0-alpha.19" -dependencies = [ - "assert_matches", - "clap", - "rayon", - "reth-db", - "reth-interfaces", - "reth-nippy-jar", - "reth-primitives", - "reth-provider", - "reth-stages", - "reth-tokio-util", - "tempfile", - "thiserror", - "tokio", - "tokio-stream", - "tracing", -] - [[package]] name = "reth-stages" version = "0.1.0-alpha.19" @@ -6924,7 +6903,7 @@ dependencies = [ "reth-primitives", "reth-provider", "reth-revm", - "reth-snapshot", + "reth-static-file", "reth-tokio-util", "reth-trie", "revm", @@ -6937,6 +6916,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "reth-static-file" +version = "0.1.0-alpha.19" +dependencies = [ + "assert_matches", + "clap", + "rayon", + "reth-db", + "reth-interfaces", + "reth-nippy-jar", + "reth-primitives", + "reth-provider", + "reth-stages", + "reth-tokio-util", + "tempfile", + "thiserror", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "reth-tasks" version = "0.1.0-alpha.19" diff --git a/Cargo.toml b/Cargo.toml index 860892c59262..ae739ef7d48d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,8 +42,8 @@ members = [ "crates/node-optimism/", "crates/node-core/", "crates/node-api/", - "crates/snapshot/", "crates/stages/", + "crates/static-file/", "crates/storage/codecs/", "crates/storage/codecs/derive/", "crates/storage/db/", @@ -163,8 +163,8 @@ reth-rpc-builder = { path = "crates/rpc/rpc-builder" } reth-rpc-engine-api = { path = "crates/rpc/rpc-engine-api" } reth-rpc-types = { path = "crates/rpc/rpc-types" } reth-rpc-types-compat = { path = "crates/rpc/rpc-types-compat" } -reth-snapshot = { path = "crates/snapshot" } reth-stages = { path = "crates/stages" } +reth-static-file = { path = "crates/static-file" } reth-tasks = { path = "crates/tasks" } reth-tokio-util = { path = "crates/tokio-util" } reth-tracing = { path = "crates/tracing" } diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 1d1d4639ff7d..afc71932e235 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -50,7 +50,7 @@ reth-payload-validator.workspace = true reth-basic-payload-builder.workspace = true reth-discv4.workspace = true reth-prune.workspace = true -reth-snapshot = { workspace = true, features = ["clap"] } +reth-static-file = { workspace = true, features = ["clap"] } reth-trie.workspace = true reth-nippy-jar.workspace = true reth-node-api.workspace = true diff --git a/bin/reth/src/builder.rs b/bin/reth/src/builder.rs index d1639557c0cb..04d328b2dd42 100644 --- a/bin/reth/src/builder.rs +++ b/bin/reth/src/builder.rs @@ -6,7 +6,7 @@ use fdlimit::raise_fd_limit; use futures::{future::Either, stream, stream_select, StreamExt}; use reth_auto_seal_consensus::AutoSealBuilder; use reth_beacon_consensus::{ - hooks::{EngineHooks, PruneHook, SnapshotHook}, + hooks::{EngineHooks, PruneHook, StaticFileHook}, BeaconConsensusEngine, MIN_BLOCKS_FOR_PIPELINE_RUN, }; use reth_blockchain_tree::{config::BlockchainTreeConfig, ShareableBlockchainTree}; @@ -39,7 +39,7 @@ use reth_payload_builder::PayloadBuilderHandle; use reth_provider::{providers::BlockchainProvider, ProviderFactory}; use reth_prune::PrunerBuilder; use reth_rpc_engine_api::EngineApi; -use reth_snapshot::Snapshotter; +use reth_static_file::StaticFileProducer; use reth_tasks::{TaskExecutor, TaskManager}; use reth_transaction_pool::TransactionPool; use std::{path::PathBuf, sync::Arc}; @@ -130,9 +130,9 @@ impl NodeBuilderWit let provider_factory = ProviderFactory::new( Arc::clone(&self.db), Arc::clone(&self.config.chain), - self.data_dir.snapshots_path(), + self.data_dir.static_files_path(), )? - .with_snapshots_metrics(); + .with_static_files_metrics(); self.config.start_metrics_endpoint(prometheus_handle, Arc::clone(&self.db)).await?; @@ -264,14 +264,14 @@ impl NodeBuilderWit let mut hooks = EngineHooks::new(); - let mut snapshotter = Snapshotter::new( + let mut static_file_producer = StaticFileProducer::new( provider_factory.clone(), - provider_factory.snapshot_provider(), + provider_factory.static_file_provider(), prune_config.clone().unwrap_or_default().segments, ); - let snapshotter_events = snapshotter.events(); - hooks.add(SnapshotHook::new(snapshotter.clone(), Box::new(executor.clone()))); - info!(target: "reth::cli", "Snapshotter initialized"); + let static_file_producer_events = static_file_producer.events(); + hooks.add(StaticFileHook::new(static_file_producer.clone(), Box::new(executor.clone()))); + info!(target: "reth::cli", "StaticFileProducer initialized"); // Configure the pipeline let (mut pipeline, client) = if self.config.dev.dev { @@ -301,7 +301,7 @@ impl NodeBuilderWit sync_metrics_tx, prune_config.clone(), max_block, - snapshotter, + static_file_producer, evm_config, ) .await?; @@ -324,7 +324,7 @@ impl NodeBuilderWit sync_metrics_tx, prune_config.clone(), max_block, - snapshotter, + static_file_producer, evm_config, ) .await?; @@ -377,7 +377,7 @@ impl NodeBuilderWit Either::Right(stream::empty()) }, pruner_events.map(Into::into), - snapshotter_events.map(Into::into), + static_file_producer_events.map(Into::into), ); executor.spawn_critical( "events task", diff --git a/bin/reth/src/commands/db/clear.rs b/bin/reth/src/commands/db/clear.rs index 1e06ed5d004a..a7c32cac1b4e 100644 --- a/bin/reth/src/commands/db/clear.rs +++ b/bin/reth/src/commands/db/clear.rs @@ -1,12 +1,12 @@ use clap::{Parser, Subcommand}; use reth_db::{ database::Database, - snapshot::iter_snapshots, + static_file::iter_static_files, table::Table, transaction::{DbTx, DbTxMut}, TableViewer, Tables, }; -use reth_primitives::{snapshot::find_fixed_range, SnapshotSegment}; +use reth_primitives::{static_file::find_fixed_range, StaticFileSegment}; use reth_provider::ProviderFactory; /// The arguments for the `reth db clear` command @@ -23,13 +23,13 @@ impl Command { Subcommands::Mdbx { table } => { table.view(&ClearViewer { db: provider_factory.db_ref() })? } - Subcommands::Snapshot { segment } => { - let snapshot_provider = provider_factory.snapshot_provider(); - let snapshots = iter_snapshots(snapshot_provider.directory())?; + Subcommands::StaticFile { segment } => { + let static_file_provider = provider_factory.static_file_provider(); + let static_files = iter_static_files(static_file_provider.directory())?; - if let Some(segment_snapshots) = snapshots.get(&segment) { - for (block_range, _) in segment_snapshots { - snapshot_provider + if let Some(segment_static_files) = static_files.get(&segment) { + for (block_range, _) in segment_static_files { + static_file_provider .delete_jar(segment, find_fixed_range(block_range.start()))?; } } @@ -44,8 +44,8 @@ impl Command { enum Subcommands { /// Deletes all database table entries Mdbx { table: Tables }, - /// Deletes all snapshot segment entries - Snapshot { segment: SnapshotSegment }, + /// Deletes all static file segment entries + StaticFile { segment: StaticFileSegment }, } struct ClearViewer<'a, DB: Database> { diff --git a/bin/reth/src/commands/db/get.rs b/bin/reth/src/commands/db/get.rs index b785892cdc14..1239b1483079 100644 --- a/bin/reth/src/commands/db/get.rs +++ b/bin/reth/src/commands/db/get.rs @@ -2,11 +2,11 @@ use crate::utils::DbTool; use clap::Parser; use reth_db::{ database::Database, - snapshot::{ColumnSelectorOne, ColumnSelectorTwo, HeaderMask, ReceiptMask, TransactionMask}, + static_file::{ColumnSelectorOne, ColumnSelectorTwo, HeaderMask, ReceiptMask, TransactionMask}, table::{Decompress, DupSort, Table}, tables, RawKey, RawTable, Receipts, TableViewer, Transactions, }; -use reth_primitives::{BlockHash, Header, SnapshotSegment}; +use reth_primitives::{BlockHash, Header, StaticFileSegment}; use tracing::error; /// The arguments for the `reth db get` command @@ -34,9 +34,9 @@ enum Subcommand { #[clap(long)] raw: bool, }, - /// Gets the content of a snapshot segment for the given key - Snapshot { - segment: SnapshotSegment, + /// Gets the content of a static file segment for the given key + StaticFile { + segment: StaticFileSegment, /// The key to get content for #[arg(value_parser = maybe_json_value_parser)] @@ -55,22 +55,22 @@ impl Command { Subcommand::Mdbx { table, key, subkey, raw } => { table.view(&GetValueViewer { tool, key, subkey, raw })? } - Subcommand::Snapshot { segment, key, raw } => { + Subcommand::StaticFile { segment, key, raw } => { let (key, mask): (u64, _) = match segment { - SnapshotSegment::Headers => { + StaticFileSegment::Headers => { (table_key::(&key)?, >::MASK) } - SnapshotSegment::Transactions => ( + StaticFileSegment::Transactions => ( table_key::(&key)?, ::Value>>::MASK, ), - SnapshotSegment::Receipts => ( + StaticFileSegment::Receipts => ( table_key::(&key)?, ::Value>>::MASK, ), }; - let content = tool.provider_factory.snapshot_provider().find_snapshot( + let content = tool.provider_factory.static_file_provider().find_static_file( segment, |provider| { let mut cursor = provider.cursor()?; @@ -88,7 +88,7 @@ impl Command { println!("{:?}", content); } else { match segment { - SnapshotSegment::Headers => { + StaticFileSegment::Headers => { let header = Header::decompress(content[0].as_slice())?; let block_hash = BlockHash::decompress(content[1].as_slice())?; println!( @@ -97,13 +97,13 @@ impl Command { serde_json::to_string_pretty(&block_hash)? ); } - SnapshotSegment::Transactions => { + StaticFileSegment::Transactions => { let transaction = <::Value>::decompress( content[0].as_slice(), )?; println!("{}", serde_json::to_string_pretty(&transaction)?); } - SnapshotSegment::Receipts => { + StaticFileSegment::Receipts => { let receipt = <::Value>::decompress( content[0].as_slice(), )?; diff --git a/bin/reth/src/commands/db/mod.rs b/bin/reth/src/commands/db/mod.rs index 3343a61402ad..66ccd24e2ed3 100644 --- a/bin/reth/src/commands/db/mod.rs +++ b/bin/reth/src/commands/db/mod.rs @@ -25,7 +25,7 @@ mod clear; mod diff; mod get; mod list; -mod snapshots; +mod static_files; mod stats; /// DB List TUI mod tui; @@ -82,8 +82,8 @@ pub enum Subcommands { }, /// Deletes all table entries Clear(clear::Command), - /// Snapshots tables from database - Snapshot(snapshots::Command), + /// Creates static files from database tables + CreateStaticFiles(static_files::Command), /// Lists current and local database versions Version, /// Returns the full database path @@ -105,7 +105,7 @@ impl Command { DatabaseArguments::default().log_level(self.db.log_level), )?; let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db, self.chain.clone(), data_dir.static_files_path())?; let tool = DbTool::new(provider_factory, self.chain.clone())?; command.execute(data_dir, &tool)?; @@ -116,7 +116,7 @@ impl Command { DatabaseArguments::default().log_level(self.db.log_level), )?; let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db, self.chain.clone(), data_dir.static_files_path())?; let tool = DbTool::new(provider_factory, self.chain.clone())?; command.execute(&tool)?; @@ -127,7 +127,7 @@ impl Command { DatabaseArguments::default().log_level(self.db.log_level), )?; let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db, self.chain.clone(), data_dir.static_files_path())?; let tool = DbTool::new(provider_factory, self.chain.clone())?; command.execute(&tool)?; @@ -138,7 +138,7 @@ impl Command { DatabaseArguments::default().log_level(self.db.log_level), )?; let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db, self.chain.clone(), data_dir.static_files_path())?; let tool = DbTool::new(provider_factory, self.chain.clone())?; command.execute(&tool)?; @@ -162,7 +162,7 @@ impl Command { let db = open_db(&db_path, DatabaseArguments::default().log_level(self.db.log_level))?; let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db, self.chain.clone(), data_dir.static_files_path())?; let mut tool = DbTool::new(provider_factory, self.chain.clone())?; tool.drop(db_path)?; @@ -171,11 +171,11 @@ impl Command { let db = open_db(&db_path, DatabaseArguments::default().log_level(self.db.log_level))?; let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db, self.chain.clone(), data_dir.static_files_path())?; command.execute(provider_factory)?; } - Subcommands::Snapshot(command) => { + Subcommands::CreateStaticFiles(command) => { command.execute(data_dir, self.db.log_level, self.chain.clone())?; } Subcommands::Version => { diff --git a/bin/reth/src/commands/db/snapshots/bench.rs b/bin/reth/src/commands/db/static_files/bench.rs similarity index 78% rename from bin/reth/src/commands/db/snapshots/bench.rs rename to bin/reth/src/commands/db/static_files/bench.rs index aca76280e65b..d3de628e2b43 100644 --- a/bin/reth/src/commands/db/snapshots/bench.rs +++ b/bin/reth/src/commands/db/static_files/bench.rs @@ -1,7 +1,7 @@ use reth_db::DatabaseEnv; use reth_primitives::{ - snapshot::{Compression, Filters}, - SnapshotSegment, + static_file::{Compression, Filters}, + StaticFileSegment, }; use reth_provider::{DatabaseProviderRO, ProviderFactory}; use std::{fmt::Debug, sync::Arc, time::Instant}; @@ -17,10 +17,10 @@ pub(crate) enum BenchKind { pub(crate) fn bench( bench_kind: BenchKind, provider_factory: Arc>, - segment: SnapshotSegment, + segment: StaticFileSegment, filters: Filters, compression: Compression, - mut snapshot_method: F1, + mut static_file_method: F1, database_method: F2, ) -> eyre::Result<()> where @@ -31,11 +31,11 @@ where println!(); println!("############"); println!("## [{segment:?}] [{compression:?}] [{filters:?}] [{bench_kind:?}]"); - let snap_result = { + let static_file_result = { let start = Instant::now(); - let result = snapshot_method()?; + let result = static_file_method()?; let end = start.elapsed().as_micros(); - println!("# snapshot {bench_kind:?} | {end} μs"); + println!("# static file {bench_kind:?} | {end} μs"); result }; @@ -48,7 +48,7 @@ where result }; - assert_eq!(snap_result, db_result); + assert_eq!(static_file_result, db_result); Ok(()) } diff --git a/bin/reth/src/commands/db/snapshots/headers.rs b/bin/reth/src/commands/db/static_files/headers.rs similarity index 87% rename from bin/reth/src/commands/db/snapshots/headers.rs rename to bin/reth/src/commands/db/static_files/headers.rs index 65ca309dd9c3..452063a89a19 100644 --- a/bin/reth/src/commands/db/snapshots/headers.rs +++ b/bin/reth/src/commands/db/static_files/headers.rs @@ -3,18 +3,18 @@ use super::{ Command, }; use rand::{seq::SliceRandom, Rng}; -use reth_db::{snapshot::HeaderMask, DatabaseEnv}; +use reth_db::{static_file::HeaderMask, DatabaseEnv}; use reth_primitives::{ - snapshot::{Compression, Filters, InclusionFilter, PerfectHashingFunction}, - BlockHash, Header, SnapshotSegment, + static_file::{Compression, Filters, InclusionFilter, PerfectHashingFunction}, + BlockHash, Header, StaticFileSegment, }; use reth_provider::{ - providers::SnapshotProvider, BlockNumReader, HeaderProvider, ProviderError, ProviderFactory, + providers::StaticFileProvider, BlockNumReader, HeaderProvider, ProviderError, ProviderFactory, }; use std::{ops::RangeInclusive, path::PathBuf, sync::Arc}; impl Command { - pub(crate) fn bench_headers_snapshot( + pub(crate) fn bench_headers_static_file( &self, provider_factory: Arc>, compression: Compression, @@ -35,12 +35,12 @@ impl Command { let mut row_indexes = range.collect::>(); let mut rng = rand::thread_rng(); - let path: PathBuf = SnapshotSegment::Headers + let path: PathBuf = StaticFileSegment::Headers .filename_with_configuration(filters, compression, &block_range) .into(); - let provider = SnapshotProvider::new(PathBuf::default())?; + let provider = StaticFileProvider::new(PathBuf::default())?; let jar_provider = provider.get_segment_provider_from_block( - SnapshotSegment::Headers, + StaticFileSegment::Headers, self.from, Some(&path), )?; @@ -50,7 +50,7 @@ impl Command { bench( bench_kind, provider_factory.clone(), - SnapshotSegment::Headers, + StaticFileSegment::Headers, filters, compression, || { @@ -81,7 +81,7 @@ impl Command { bench( BenchKind::RandomOne, provider_factory.clone(), - SnapshotSegment::Headers, + StaticFileSegment::Headers, filters, compression, || { @@ -108,7 +108,7 @@ impl Command { bench( BenchKind::RandomHash, provider_factory.clone(), - SnapshotSegment::Headers, + StaticFileSegment::Headers, filters, compression, || { diff --git a/bin/reth/src/commands/db/snapshots/mod.rs b/bin/reth/src/commands/db/static_files/mod.rs similarity index 74% rename from bin/reth/src/commands/db/snapshots/mod.rs rename to bin/reth/src/commands/db/static_files/mod.rs index 76b17cfc817f..eb433ac6143c 100644 --- a/bin/reth/src/commands/db/snapshots/mod.rs +++ b/bin/reth/src/commands/db/static_files/mod.rs @@ -11,14 +11,14 @@ use reth_interfaces::db::LogLevel; use reth_nippy_jar::{NippyJar, NippyJarCursor}; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_primitives::{ - snapshot::{ + static_file::{ Compression, Filters, InclusionFilter, PerfectHashingFunction, SegmentConfig, SegmentHeader, SegmentRangeInclusive, }, - BlockNumber, ChainSpec, SnapshotSegment, + BlockNumber, ChainSpec, StaticFileSegment, }; use reth_provider::{BlockNumReader, ProviderFactory}; -use reth_snapshot::{segments as snap_segments, segments::Segment}; +use reth_static_file::{segments as static_file_segments, segments::Segment}; use std::{ path::{Path, PathBuf}, sync::Arc, @@ -31,20 +31,20 @@ mod receipts; mod transactions; #[derive(Parser, Debug)] -/// Arguments for the `reth db snapshot` command. +/// Arguments for the `reth db create-static-files` command. pub struct Command { - /// Snapshot segments to generate. - segments: Vec, + /// Static File segments to generate. + segments: Vec, - /// Starting block for the snapshot. + /// Starting block for the static file. #[arg(long, short, default_value = "0")] from: BlockNumber, - /// Number of blocks in the snapshot. + /// Number of blocks in the static file. #[arg(long, short, default_value = "500000")] block_interval: u64, - /// Sets the number of snapshots built in parallel. Note: Each parallel build is + /// Sets the number of static files built in parallel. Note: Each parallel build is /// memory-intensive. #[arg( long, short, @@ -53,15 +53,15 @@ pub struct Command { )] parallel: u64, - /// Flag to skip snapshot creation and print snapshot files stats. + /// Flag to skip static file creation and print static files stats. #[arg(long, default_value = "false")] only_stats: bool, - /// Flag to enable database-to-snapshot benchmarking. + /// Flag to enable database-to-static file benchmarking. #[arg(long, default_value = "false")] bench: bool, - /// Flag to skip snapshot creation and only run benchmarks on existing snapshots. + /// Flag to skip static file creation and only run benchmarks on existing static files. #[arg(long, default_value = "false")] only_bench: bool, @@ -79,7 +79,7 @@ pub struct Command { } impl Command { - /// Execute `db snapshot` command + /// Execute `db create-static-files` command pub fn execute( self, data_dir: ChainPath, @@ -103,7 +103,7 @@ impl Command { .max_read_transaction_duration(Some(MaxReadTransactionDuration::Unbounded)), )?; let provider_factory = - Arc::new(ProviderFactory::new(db, chain.clone(), data_dir.snapshots_path())?); + Arc::new(ProviderFactory::new(db, chain.clone(), data_dir.static_files_path())?); { if !self.only_bench { @@ -115,19 +115,20 @@ impl Command { }; match mode { - SnapshotSegment::Headers => self.generate_snapshot::( + StaticFileSegment::Headers => self.generate_static_file::( provider_factory.clone(), - snap_segments::Headers, + static_file_segments::Headers, SegmentConfig { filters, compression }, )?, - SnapshotSegment::Transactions => self.generate_snapshot::( + StaticFileSegment::Transactions => self + .generate_static_file::( + provider_factory.clone(), + static_file_segments::Transactions, + SegmentConfig { filters, compression }, + )?, + StaticFileSegment::Receipts => self.generate_static_file::( provider_factory.clone(), - snap_segments::Transactions, - SegmentConfig { filters, compression }, - )?, - SnapshotSegment::Receipts => self.generate_snapshot::( - provider_factory.clone(), - snap_segments::Receipts, + static_file_segments::Receipts, SegmentConfig { filters, compression }, )?, } @@ -138,19 +139,19 @@ impl Command { if self.only_bench || self.bench { for ((mode, compression), phf) in all_combinations { match mode { - SnapshotSegment::Headers => self.bench_headers_snapshot( + StaticFileSegment::Headers => self.bench_headers_static_file( provider_factory.clone(), compression, InclusionFilter::Cuckoo, phf, )?, - SnapshotSegment::Transactions => self.bench_transactions_snapshot( + StaticFileSegment::Transactions => self.bench_transactions_static_file( provider_factory.clone(), compression, InclusionFilter::Cuckoo, phf, )?, - SnapshotSegment::Receipts => self.bench_receipts_snapshot( + StaticFileSegment::Receipts => self.bench_receipts_static_file( provider_factory.clone(), compression, InclusionFilter::Cuckoo, @@ -177,9 +178,9 @@ impl Command { ranges } - /// Generates snapshots from `self.from` with a `self.block_interval`. Generates them in + /// Generates static files from `self.from` with a `self.block_interval`. Generates them in /// parallel if specified. - fn generate_snapshot( + fn generate_static_file( &self, factory: Arc>, segment: impl Segment + Send + Sync, @@ -188,7 +189,7 @@ impl Command { let dir = PathBuf::default(); let ranges = self.block_ranges(factory.best_block_number()?); - let mut created_snapshots = vec![]; + let mut created_static_files = vec![]; // Filter/PHF is memory intensive, so we have to limit the parallelism. for block_ranges in ranges.chunks(self.parallel as usize) { @@ -198,7 +199,7 @@ impl Command { let provider = factory.provider()?; if !self.only_stats { - segment.create_snapshot_file( + segment.create_static_file_file( &provider, dir.as_path(), config, @@ -210,24 +211,24 @@ impl Command { }) .collect::, eyre::Report>>()?; - created_snapshots.extend(created_files); + created_static_files.extend(created_files); } - self.stats(created_snapshots) + self.stats(created_static_files) } - /// Prints detailed statistics for each snapshot, including loading time. + /// Prints detailed statistics for each static file, including loading time. /// - /// This function loads each snapshot from the provided paths and prints - /// statistics about various aspects of each snapshot, such as filters size, + /// This function loads each static file from the provided paths and prints + /// statistics about various aspects of each static file, such as filters size, /// offset index size, offset list size, and loading time. - fn stats(&self, snapshots: Vec>) -> eyre::Result<()> { + fn stats(&self, static_files: Vec>) -> eyre::Result<()> { let mut total_filters_size = 0; let mut total_index_size = 0; let mut total_duration = Duration::new(0, 0); let mut total_file_size = 0; - for snap in &snapshots { + for snap in &static_files { let start_time = Instant::now(); let jar = NippyJar::::load(snap.as_ref())?; let _cursor = NippyJarCursor::new(&jar)?; @@ -239,7 +240,7 @@ impl Command { total_duration += duration; total_file_size += file_size; - println!("Snapshot: {:?}", snap.as_ref().file_name()); + println!("StaticFile: {:?}", snap.as_ref().file_name()); println!(" File Size: {:>7}", human_bytes(file_size as f64)); println!(" Filters Size: {:>7}", human_bytes(jar.filter_size() as f64)); println!(" Offset Index Size: {:>7}", human_bytes(jar.offsets_index_size() as f64)); @@ -250,7 +251,7 @@ impl Command { ); } - let avg_duration = total_duration / snapshots.len() as u32; + let avg_duration = total_duration / static_files.len() as u32; println!("Total Filters Size: {:>7}", human_bytes(total_filters_size as f64)); println!("Total Offset Index Size: {:>7}", human_bytes(total_index_size as f64)); diff --git a/bin/reth/src/commands/db/snapshots/receipts.rs b/bin/reth/src/commands/db/static_files/receipts.rs similarity index 87% rename from bin/reth/src/commands/db/snapshots/receipts.rs rename to bin/reth/src/commands/db/static_files/receipts.rs index e3c27de04205..ec9f580246cb 100644 --- a/bin/reth/src/commands/db/snapshots/receipts.rs +++ b/bin/reth/src/commands/db/static_files/receipts.rs @@ -3,19 +3,19 @@ use super::{ Command, Compression, PerfectHashingFunction, }; use rand::{seq::SliceRandom, Rng}; -use reth_db::{snapshot::ReceiptMask, DatabaseEnv}; +use reth_db::{static_file::ReceiptMask, DatabaseEnv}; use reth_primitives::{ - snapshot::{Filters, InclusionFilter}, - Receipt, SnapshotSegment, + static_file::{Filters, InclusionFilter}, + Receipt, StaticFileSegment, }; use reth_provider::{ - providers::SnapshotProvider, BlockNumReader, ProviderError, ProviderFactory, ReceiptProvider, + providers::StaticFileProvider, BlockNumReader, ProviderError, ProviderFactory, ReceiptProvider, TransactionsProvider, TransactionsProviderExt, }; use std::{path::PathBuf, sync::Arc}; impl Command { - pub(crate) fn bench_receipts_snapshot( + pub(crate) fn bench_receipts_static_file( &self, provider_factory: Arc>, compression: Compression, @@ -39,13 +39,13 @@ impl Command { let mut row_indexes = tx_range.clone().collect::>(); - let path: PathBuf = SnapshotSegment::Receipts + let path: PathBuf = StaticFileSegment::Receipts .filename_with_configuration(filters, compression, &block_range) .into(); - let provider = SnapshotProvider::new(PathBuf::default())?; + let provider = StaticFileProvider::new(PathBuf::default())?; let jar_provider = provider.get_segment_provider_from_block( - SnapshotSegment::Receipts, + StaticFileSegment::Receipts, self.from, Some(&path), )?; @@ -55,7 +55,7 @@ impl Command { bench( bench_kind, provider_factory.clone(), - SnapshotSegment::Receipts, + StaticFileSegment::Receipts, filters, compression, || { @@ -86,7 +86,7 @@ impl Command { bench( BenchKind::RandomOne, provider_factory.clone(), - SnapshotSegment::Receipts, + StaticFileSegment::Receipts, filters, compression, || { @@ -113,7 +113,7 @@ impl Command { bench( BenchKind::RandomHash, provider_factory, - SnapshotSegment::Receipts, + StaticFileSegment::Receipts, filters, compression, || { diff --git a/bin/reth/src/commands/db/snapshots/transactions.rs b/bin/reth/src/commands/db/static_files/transactions.rs similarity index 87% rename from bin/reth/src/commands/db/snapshots/transactions.rs rename to bin/reth/src/commands/db/static_files/transactions.rs index e29c36707e86..79758c09325e 100644 --- a/bin/reth/src/commands/db/snapshots/transactions.rs +++ b/bin/reth/src/commands/db/static_files/transactions.rs @@ -3,20 +3,20 @@ use super::{ Command, Compression, PerfectHashingFunction, }; use rand::{seq::SliceRandom, Rng}; -use reth_db::{snapshot::TransactionMask, DatabaseEnv}; +use reth_db::{static_file::TransactionMask, DatabaseEnv}; use reth_primitives::{ - snapshot::{Filters, InclusionFilter}, - SnapshotSegment, TransactionSignedNoHash, + static_file::{Filters, InclusionFilter}, + StaticFileSegment, TransactionSignedNoHash, }; use reth_provider::{ - providers::SnapshotProvider, BlockNumReader, ProviderError, ProviderFactory, + providers::StaticFileProvider, BlockNumReader, ProviderError, ProviderFactory, TransactionsProvider, TransactionsProviderExt, }; use std::{path::PathBuf, sync::Arc}; impl Command { - pub(crate) fn bench_transactions_snapshot( + pub(crate) fn bench_transactions_static_file( &self, provider_factory: Arc>, compression: Compression, @@ -39,12 +39,12 @@ impl Command { let mut row_indexes = tx_range.clone().collect::>(); - let path: PathBuf = SnapshotSegment::Transactions + let path: PathBuf = StaticFileSegment::Transactions .filename_with_configuration(filters, compression, &block_range) .into(); - let provider = SnapshotProvider::new(PathBuf::default())?; + let provider = StaticFileProvider::new(PathBuf::default())?; let jar_provider = provider.get_segment_provider_from_block( - SnapshotSegment::Transactions, + StaticFileSegment::Transactions, self.from, Some(&path), )?; @@ -54,7 +54,7 @@ impl Command { bench( bench_kind, provider_factory.clone(), - SnapshotSegment::Transactions, + StaticFileSegment::Transactions, filters, compression, || { @@ -86,7 +86,7 @@ impl Command { bench( BenchKind::RandomOne, provider_factory.clone(), - SnapshotSegment::Transactions, + StaticFileSegment::Transactions, filters, compression, || { @@ -114,7 +114,7 @@ impl Command { bench( BenchKind::RandomHash, provider_factory, - SnapshotSegment::Transactions, + StaticFileSegment::Transactions, filters, compression, || { diff --git a/bin/reth/src/commands/db/stats.rs b/bin/reth/src/commands/db/stats.rs index 490c0b05ab60..09ad3a61b53e 100644 --- a/bin/reth/src/commands/db/stats.rs +++ b/bin/reth/src/commands/db/stats.rs @@ -4,19 +4,19 @@ use comfy_table::{Cell, Row, Table as ComfyTable}; use eyre::WrapErr; use human_bytes::human_bytes; use itertools::Itertools; -use reth_db::{database::Database, mdbx, snapshot::iter_snapshots, DatabaseEnv, Tables}; +use reth_db::{database::Database, mdbx, static_file::iter_static_files, DatabaseEnv, Tables}; use reth_node_core::dirs::{ChainPath, DataDirPath}; -use reth_primitives::snapshot::{find_fixed_range, SegmentRangeInclusive}; -use reth_provider::providers::SnapshotProvider; +use reth_primitives::static_file::{find_fixed_range, SegmentRangeInclusive}; +use reth_provider::providers::StaticFileProvider; use std::fs::File; #[derive(Parser, Debug)] /// The arguments for the `reth db stats` command pub struct Command { - /// Show only the total size for snapshot files. + /// Show only the total size for static files. #[arg(long, default_value_t = false)] only_total_size: bool, - /// Show only the summary per snapshot segment. + /// Show only the summary per static file segment. #[arg(long, default_value_t = false)] summary: bool, } @@ -28,8 +28,8 @@ impl Command { data_dir: ChainPath, tool: &DbTool, ) -> eyre::Result<()> { - let snapshots_stats_table = self.snapshots_stats_table(data_dir)?; - println!("{snapshots_stats_table}"); + let static_files_stats_table = self.static_files_stats_table(data_dir)?; + println!("{static_files_stats_table}"); println!("\n"); @@ -119,7 +119,10 @@ impl Command { Ok(table) } - fn snapshots_stats_table(&self, data_dir: ChainPath) -> eyre::Result { + fn static_files_stats_table( + &self, + data_dir: ChainPath, + ) -> eyre::Result { let mut table = ComfyTable::new(); table.load_preset(comfy_table::presets::ASCII_MARKDOWN); @@ -145,15 +148,15 @@ impl Command { ]); } - let snapshots = iter_snapshots(data_dir.snapshots_path())?; - let snapshot_provider = SnapshotProvider::new(data_dir.snapshots_path())?; + let static_files = iter_static_files(data_dir.static_files_path())?; + let static_file_provider = StaticFileProvider::new(data_dir.static_files_path())?; let mut total_data_size = 0; let mut total_index_size = 0; let mut total_offsets_size = 0; let mut total_config_size = 0; - for (segment, ranges) in snapshots.into_iter().sorted_by_key(|(segment, _)| *segment) { + for (segment, ranges) in static_files.into_iter().sorted_by_key(|(segment, _)| *segment) { let ( mut segment_columns, mut segment_rows, @@ -165,7 +168,7 @@ impl Command { for (block_range, tx_range) in &ranges { let fixed_block_range = find_fixed_range(block_range.start()); - let jar_provider = snapshot_provider + let jar_provider = static_file_provider .get_segment_provider(segment, || Some(fixed_block_range), None)? .expect("something went wrong"); diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs index e64ac4c8fde6..67215aa54857 100644 --- a/bin/reth/src/commands/debug_cmd/build_block.rs +++ b/bin/reth/src/commands/debug_cmd/build_block.rs @@ -119,7 +119,7 @@ impl Command { let factory = ProviderFactory::new( db, self.chain.clone(), - self.datadir.unwrap_or_chain_default(self.chain.chain).snapshots_path(), + self.datadir.unwrap_or_chain_default(self.chain.chain).static_files_path(), )?; let provider = factory.provider()?; @@ -162,7 +162,7 @@ impl Command { let provider_factory = ProviderFactory::new( Arc::clone(&db), Arc::clone(&self.chain), - data_dir.snapshots_path(), + data_dir.static_files_path(), )?; let consensus: Arc = Arc::new(BeaconConsensus::new(Arc::clone(&self.chain))); diff --git a/bin/reth/src/commands/debug_cmd/execution.rs b/bin/reth/src/commands/debug_cmd/execution.rs index 616038ec19cf..e6b8c489d1ef 100644 --- a/bin/reth/src/commands/debug_cmd/execution.rs +++ b/bin/reth/src/commands/debug_cmd/execution.rs @@ -31,12 +31,12 @@ use reth_primitives::{ fs, stage::StageId, BlockHashOrNumber, BlockNumber, ChainSpec, PruneModes, B256, }; use reth_provider::{BlockExecutionWriter, HeaderSyncMode, ProviderFactory, StageCheckpointReader}; -use reth_snapshot::Snapshotter; use reth_stages::{ sets::DefaultStages, stages::{ExecutionStage, ExecutionStageThresholds, SenderRecoveryStage}, Pipeline, StageSet, }; +use reth_static_file::StaticFileProducer; use reth_tasks::TaskExecutor; use std::{ net::{SocketAddr, SocketAddrV4}, @@ -95,7 +95,7 @@ impl Command { consensus: Arc, provider_factory: ProviderFactory, task_executor: &TaskExecutor, - snapshotter: Snapshotter, + static_file_producer: StaticFileProducer, ) -> eyre::Result> where DB: Database + Unpin + Clone + 'static, @@ -127,7 +127,7 @@ impl Command { header_downloader, body_downloader, factory.clone(), - snapshotter, + static_file_producer, )? .set(SenderRecoveryStage { commit_threshold: stage_conf.sender_recovery.commit_threshold, @@ -174,7 +174,7 @@ impl Command { .build(ProviderFactory::new( db, self.chain.clone(), - self.datadir.unwrap_or_chain_default(self.chain.chain).snapshots_path(), + self.datadir.unwrap_or_chain_default(self.chain.chain).static_files_path(), )?) .start_network() .await?; @@ -212,7 +212,7 @@ impl Command { let db = Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?); let provider_factory = - ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files_path())?; debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis"); init_genesis(provider_factory.clone())?; @@ -232,9 +232,9 @@ impl Command { ) .await?; - let snapshotter = Snapshotter::new( + let static_file_producer = StaticFileProducer::new( provider_factory.clone(), - provider_factory.snapshot_provider(), + provider_factory.static_file_provider(), PruneModes::default(), ); @@ -246,7 +246,7 @@ impl Command { Arc::clone(&consensus), provider_factory.clone(), &ctx.task_executor, - snapshotter, + static_file_producer, )?; let provider = provider_factory.provider()?; diff --git a/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs b/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs index 11013abebe0d..54af17e97be1 100644 --- a/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs +++ b/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs @@ -97,7 +97,7 @@ impl Command { .build(ProviderFactory::new( db, self.chain.clone(), - self.datadir.unwrap_or_chain_default(self.chain.chain).snapshots_path(), + self.datadir.unwrap_or_chain_default(self.chain.chain).static_files_path(), )?) .start_network() .await?; @@ -118,7 +118,7 @@ impl Command { // initialize the database let db = Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?); - let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.snapshots_path())?; + let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files_path())?; let provider = factory.provider()?; // Look up merkle checkpoint @@ -170,7 +170,7 @@ impl Command { reth_revm::EvmProcessorFactory::new(self.chain.clone(), EthEvmConfig::default()); let mut executor = executor_factory.with_state(LatestStateProviderRef::new( provider.tx_ref(), - factory.snapshot_provider(), + factory.static_file_provider(), )); let merkle_block_td = diff --git a/bin/reth/src/commands/debug_cmd/merkle.rs b/bin/reth/src/commands/debug_cmd/merkle.rs index 9ab66d00a37c..8a5cce295a26 100644 --- a/bin/reth/src/commands/debug_cmd/merkle.rs +++ b/bin/reth/src/commands/debug_cmd/merkle.rs @@ -108,7 +108,7 @@ impl Command { .build(ProviderFactory::new( db, self.chain.clone(), - self.datadir.unwrap_or_chain_default(self.chain.chain).snapshots_path(), + self.datadir.unwrap_or_chain_default(self.chain.chain).static_files_path(), )?) .start_network() .await?; @@ -129,7 +129,7 @@ impl Command { // initialize the database let db = Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?); - let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.snapshots_path())?; + let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files_path())?; let provider_rw = factory.provider_rw()?; // Configure and build network diff --git a/bin/reth/src/commands/debug_cmd/replay_engine.rs b/bin/reth/src/commands/debug_cmd/replay_engine.rs index fdb42c191125..4c06f1ef2cd7 100644 --- a/bin/reth/src/commands/debug_cmd/replay_engine.rs +++ b/bin/reth/src/commands/debug_cmd/replay_engine.rs @@ -104,7 +104,7 @@ impl Command { .build(ProviderFactory::new( db, self.chain.clone(), - self.datadir.unwrap_or_chain_default(self.chain.chain).snapshots_path(), + self.datadir.unwrap_or_chain_default(self.chain.chain).static_files_path(), )?) .start_network() .await?; @@ -126,7 +126,7 @@ impl Command { let db = Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?); let provider_factory = - ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files_path())?; let consensus: Arc = Arc::new(BeaconConsensus::new(Arc::clone(&self.chain))); diff --git a/bin/reth/src/commands/import.rs b/bin/reth/src/commands/import.rs index 0a8ffeffb0ba..6bf89793fd26 100644 --- a/bin/reth/src/commands/import.rs +++ b/bin/reth/src/commands/import.rs @@ -23,11 +23,11 @@ use reth_node_core::{events::node::NodeEvent, init::init_genesis}; use reth_node_ethereum::EthEvmConfig; use reth_primitives::{stage::StageId, ChainSpec, PruneModes, B256}; use reth_provider::{HeaderSyncMode, ProviderFactory, StageCheckpointReader}; -use reth_snapshot::Snapshotter; use reth_stages::{ prelude::*, stages::{ExecutionStage, ExecutionStageThresholds, SenderRecoveryStage}, }; +use reth_static_file::StaticFileProducer; use std::{path::PathBuf, sync::Arc}; use tokio::sync::watch; use tracing::{debug, info}; @@ -91,7 +91,7 @@ impl ImportCommand { Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?); info!(target: "reth::cli", "Database opened"); let provider_factory = - ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files_path())?; debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis"); @@ -108,9 +108,9 @@ impl ImportCommand { let tip = file_client.tip().expect("file client has no tip"); info!(target: "reth::cli", "Chain file imported"); - let snapshotter = Snapshotter::new( + let static_file_producer = StaticFileProducer::new( provider_factory.clone(), - provider_factory.snapshot_provider(), + provider_factory.static_file_provider(), PruneModes::default(), ); @@ -120,7 +120,7 @@ impl ImportCommand { provider_factory.clone(), &consensus, file_client, - snapshotter, + static_file_producer, ) .await?; @@ -156,7 +156,7 @@ impl ImportCommand { provider_factory: ProviderFactory, consensus: &Arc, file_client: Arc, - snapshotter: Snapshotter, + static_file_producer: StaticFileProducer, ) -> eyre::Result<(Pipeline, impl Stream)> where DB: Database + Clone + Unpin + 'static, @@ -191,7 +191,7 @@ impl ImportCommand { header_downloader, body_downloader, factory.clone(), - snapshotter, + static_file_producer, )? .set(SenderRecoveryStage { commit_threshold: config.stages.sender_recovery.commit_threshold, diff --git a/bin/reth/src/commands/init_cmd.rs b/bin/reth/src/commands/init_cmd.rs index 40bf332f5532..abd3edc5bddf 100644 --- a/bin/reth/src/commands/init_cmd.rs +++ b/bin/reth/src/commands/init_cmd.rs @@ -58,7 +58,7 @@ impl InitCommand { info!(target: "reth::cli", "Database opened"); let provider_factory = - ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.static_files_path())?; info!(target: "reth::cli", "Writing genesis block"); diff --git a/bin/reth/src/commands/p2p/mod.rs b/bin/reth/src/commands/p2p/mod.rs index eb04e126d8b7..0dbcd6c270bf 100644 --- a/bin/reth/src/commands/p2p/mod.rs +++ b/bin/reth/src/commands/p2p/mod.rs @@ -134,7 +134,7 @@ impl Command { .build(Arc::new(ProviderFactory::new( noop_db, self.chain.clone(), - data_dir.snapshots_path(), + data_dir.static_files_path(), )?)) .start_network() .await?; diff --git a/bin/reth/src/commands/recover/storage_tries.rs b/bin/reth/src/commands/recover/storage_tries.rs index 61058ace9ef4..007ff3216095 100644 --- a/bin/reth/src/commands/recover/storage_tries.rs +++ b/bin/reth/src/commands/recover/storage_tries.rs @@ -50,7 +50,7 @@ impl Command { fs::create_dir_all(&db_path)?; let db = Arc::new(init_db(db_path, Default::default())?); - let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.snapshots_path())?; + let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files_path())?; debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis"); init_genesis(factory.clone())?; diff --git a/bin/reth/src/commands/stage/drop.rs b/bin/reth/src/commands/stage/drop.rs index b2856f4f4c24..b0b0aa95127b 100644 --- a/bin/reth/src/commands/stage/drop.rs +++ b/bin/reth/src/commands/stage/drop.rs @@ -10,11 +10,13 @@ use crate::{ }; use clap::Parser; use reth_db::{ - database::Database, mdbx::DatabaseArguments, open_db, snapshot::iter_snapshots, tables, + database::Database, mdbx::DatabaseArguments, open_db, static_file::iter_static_files, tables, transaction::DbTxMut, DatabaseEnv, }; use reth_node_core::init::{insert_genesis_header, insert_genesis_state}; -use reth_primitives::{fs, snapshot::find_fixed_range, stage::StageId, ChainSpec, SnapshotSegment}; +use reth_primitives::{ + fs, stage::StageId, static_file::find_fixed_range, ChainSpec, StaticFileSegment, +}; use reth_provider::ProviderFactory; use std::sync::Arc; @@ -60,26 +62,26 @@ impl Command { let db = open_db(db_path.as_ref(), DatabaseArguments::default().log_level(self.db.log_level))?; let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.snapshots_path())?; - let snapshot_provider = provider_factory.snapshot_provider(); + ProviderFactory::new(db, self.chain.clone(), data_dir.static_files_path())?; + let static_file_provider = provider_factory.static_file_provider(); let tool = DbTool::new(provider_factory, self.chain.clone())?; - let snapshot_segment = match self.stage { - StageEnum::Headers => Some(SnapshotSegment::Headers), - StageEnum::Bodies => Some(SnapshotSegment::Transactions), - StageEnum::Execution => Some(SnapshotSegment::Receipts), + let static_file_segment = match self.stage { + StageEnum::Headers => Some(StaticFileSegment::Headers), + StageEnum::Bodies => Some(StaticFileSegment::Transactions), + StageEnum::Execution => Some(StaticFileSegment::Receipts), _ => None, }; - // Delete snapshot segment data before inserting the genesis header below - if let Some(snapshot_segment) = snapshot_segment { - let snapshot_provider = tool.provider_factory.snapshot_provider(); - let snapshots = iter_snapshots(snapshot_provider.directory())?; - if let Some(segment_snapshots) = snapshots.get(&snapshot_segment) { - for (block_range, _) in segment_snapshots { - snapshot_provider - .delete_jar(snapshot_segment, find_fixed_range(block_range.start()))?; + // Delete static file segment data before inserting the genesis header below + if let Some(static_file_segment) = static_file_segment { + let static_file_provider = tool.provider_factory.static_file_provider(); + let static_files = iter_static_files(static_file_provider.directory())?; + if let Some(segment_static_files) = static_files.get(&static_file_segment) { + for (block_range, _) in segment_static_files { + static_file_provider + .delete_jar(static_file_segment, find_fixed_range(block_range.start()))?; } } } @@ -92,7 +94,7 @@ impl Command { tx.clear::()?; tx.clear::()?; tx.put::(StageId::Headers.to_string(), Default::default())?; - insert_genesis_header::(tx, snapshot_provider, self.chain)?; + insert_genesis_header::(tx, static_file_provider, self.chain)?; } StageEnum::Bodies => { tx.clear::()?; @@ -101,7 +103,7 @@ impl Command { tx.clear::()?; tx.clear::()?; tx.put::(StageId::Bodies.to_string(), Default::default())?; - insert_genesis_header::(tx, snapshot_provider, self.chain)?; + insert_genesis_header::(tx, static_file_provider, self.chain)?; } StageEnum::Senders => { tx.clear::()?; @@ -186,7 +188,7 @@ impl Command { StageId::TransactionLookup.to_string(), Default::default(), )?; - insert_genesis_header::(tx, snapshot_provider, self.chain)?; + insert_genesis_header::(tx, static_file_provider, self.chain)?; } } diff --git a/bin/reth/src/commands/stage/dump/execution.rs b/bin/reth/src/commands/stage/dump/execution.rs index 42a7e1766aa9..336846b8e666 100644 --- a/bin/reth/src/commands/stage/dump/execution.rs +++ b/bin/reth/src/commands/stage/dump/execution.rs @@ -31,7 +31,7 @@ pub(crate) async fn dump_execution_stage( ProviderFactory::new( output_db, db_tool.chain.clone(), - output_datadir.snapshots_path(), + output_datadir.static_files_path(), )?, to, from, diff --git a/bin/reth/src/commands/stage/dump/hashing_account.rs b/bin/reth/src/commands/stage/dump/hashing_account.rs index 3bc3c91e58c5..84bc074c2937 100644 --- a/bin/reth/src/commands/stage/dump/hashing_account.rs +++ b/bin/reth/src/commands/stage/dump/hashing_account.rs @@ -33,7 +33,7 @@ pub(crate) async fn dump_hashing_account_stage( ProviderFactory::new( output_db, db_tool.chain.clone(), - output_datadir.snapshots_path(), + output_datadir.static_files_path(), )?, to, from, diff --git a/bin/reth/src/commands/stage/dump/hashing_storage.rs b/bin/reth/src/commands/stage/dump/hashing_storage.rs index e739539caacb..0bb2ddd28fe0 100644 --- a/bin/reth/src/commands/stage/dump/hashing_storage.rs +++ b/bin/reth/src/commands/stage/dump/hashing_storage.rs @@ -24,7 +24,7 @@ pub(crate) async fn dump_hashing_storage_stage( ProviderFactory::new( output_db, db_tool.chain.clone(), - output_datadir.snapshots_path(), + output_datadir.static_files_path(), )?, to, from, diff --git a/bin/reth/src/commands/stage/dump/merkle.rs b/bin/reth/src/commands/stage/dump/merkle.rs index c69cbf36aece..90f6088fd43b 100644 --- a/bin/reth/src/commands/stage/dump/merkle.rs +++ b/bin/reth/src/commands/stage/dump/merkle.rs @@ -47,7 +47,7 @@ pub(crate) async fn dump_merkle_stage( ProviderFactory::new( output_db, db_tool.chain.clone(), - output_datadir.snapshots_path(), + output_datadir.static_files_path(), )?, to, from, diff --git a/bin/reth/src/commands/stage/dump/mod.rs b/bin/reth/src/commands/stage/dump/mod.rs index 78d1744b508d..f85ce0e8b399 100644 --- a/bin/reth/src/commands/stage/dump/mod.rs +++ b/bin/reth/src/commands/stage/dump/mod.rs @@ -107,7 +107,7 @@ impl Command { let db = Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?); let provider_factory = - ProviderFactory::new(db, self.chain.clone(), data_dir.snapshots_path())?; + ProviderFactory::new(db, self.chain.clone(), data_dir.static_files_path())?; info!(target: "reth::cli", "Database opened"); diff --git a/bin/reth/src/commands/stage/run.rs b/bin/reth/src/commands/stage/run.rs index 114606e0430a..370c29aad78f 100644 --- a/bin/reth/src/commands/stage/run.rs +++ b/bin/reth/src/commands/stage/run.rs @@ -127,8 +127,11 @@ impl Command { Arc::new(init_db(db_path, DatabaseArguments::default().log_level(self.db.log_level))?); info!(target: "reth::cli", "Database opened"); - let factory = - ProviderFactory::new(Arc::clone(&db), self.chain.clone(), data_dir.snapshots_path())?; + let factory = ProviderFactory::new( + Arc::clone(&db), + self.chain.clone(), + data_dir.static_files_path(), + )?; let mut provider_rw = factory.provider_rw()?; if let Some(listen_addr) = self.metrics { @@ -169,7 +172,7 @@ impl Command { let provider_factory = Arc::new(ProviderFactory::new( db.clone(), self.chain.clone(), - data_dir.snapshots_path(), + data_dir.static_files_path(), )?); let network = self diff --git a/bin/reth/src/commands/stage/unwind.rs b/bin/reth/src/commands/stage/unwind.rs index e6c44d2369f7..44d4c2845fa1 100644 --- a/bin/reth/src/commands/stage/unwind.rs +++ b/bin/reth/src/commands/stage/unwind.rs @@ -68,7 +68,7 @@ impl Command { eyre::bail!("Cannot unwind genesis block") } - let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.snapshots_path())?; + let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.static_files_path())?; let provider = factory.provider_rw()?; let blocks_and_execution = provider diff --git a/book/SUMMARY.md b/book/SUMMARY.md index 97569eeebdb7..42065930d04f 100644 --- a/book/SUMMARY.md +++ b/book/SUMMARY.md @@ -37,12 +37,12 @@ - [`reth db diff`](./cli/reth/db/diff.md) - [`reth db get`](./cli/reth/db/get.md) - [`reth db get mdbx`](./cli/reth/db/get/mdbx.md) - - [`reth db get snapshot`](./cli/reth/db/get/snapshot.md) + - [`reth db get static-file`](./cli/reth/db/get/static-file.md) - [`reth db drop`](./cli/reth/db/drop.md) - [`reth db clear`](./cli/reth/db/clear.md) - [`reth db clear mdbx`](./cli/reth/db/clear/mdbx.md) - - [`reth db clear snapshot`](./cli/reth/db/clear/snapshot.md) - - [`reth db snapshot`](./cli/reth/db/snapshot.md) + - [`reth db clear static-file`](./cli/reth/db/clear/static-file.md) + - [`reth db create-static-files`](./cli/reth/db/create-static-files.md) - [`reth db version`](./cli/reth/db/version.md) - [`reth db path`](./cli/reth/db/path.md) - [`reth stage`](./cli/reth/stage.md) diff --git a/book/cli/SUMMARY.md b/book/cli/SUMMARY.md index b5f8bd06f7a2..be6a2ce09097 100644 --- a/book/cli/SUMMARY.md +++ b/book/cli/SUMMARY.md @@ -8,12 +8,12 @@ - [`reth db diff`](./reth/db/diff.md) - [`reth db get`](./reth/db/get.md) - [`reth db get mdbx`](./reth/db/get/mdbx.md) - - [`reth db get snapshot`](./reth/db/get/snapshot.md) + - [`reth db get static-file`](./reth/db/get/static-file.md) - [`reth db drop`](./reth/db/drop.md) - [`reth db clear`](./reth/db/clear.md) - [`reth db clear mdbx`](./reth/db/clear/mdbx.md) - - [`reth db clear snapshot`](./reth/db/clear/snapshot.md) - - [`reth db snapshot`](./reth/db/snapshot.md) + - [`reth db clear static-file`](./reth/db/clear/static-file.md) + - [`reth db create-static-files`](./reth/db/create-static-files.md) - [`reth db version`](./reth/db/version.md) - [`reth db path`](./reth/db/path.md) - [`reth stage`](./reth/stage.md) diff --git a/book/cli/reth/db.md b/book/cli/reth/db.md index 158b4b726a4a..7c72730ae16c 100644 --- a/book/cli/reth/db.md +++ b/book/cli/reth/db.md @@ -7,16 +7,16 @@ $ reth db --help Usage: reth db [OPTIONS] Commands: - stats Lists all the tables, their entry count and their size - list Lists the contents of a table - diff Create a diff between two database tables or two entire databases - get Gets the content of a table for the given key - drop Deletes all database entries - clear Deletes all table entries - snapshot Snapshots tables from database - version Lists current and local database versions - path Returns the full database path - help Print this message or the help of the given subcommand(s) + stats Lists all the tables, their entry count and their size + list Lists the contents of a table + diff Create a diff between two database tables or two entire databases + get Gets the content of a table for the given key + drop Deletes all database entries + clear Deletes all table entries + create-static-files Creates static files from database tables + version Lists current and local database versions + path Returns the full database path + help Print this message or the help of the given subcommand(s) Options: --datadir diff --git a/book/cli/reth/db/clear.md b/book/cli/reth/db/clear.md index ccc459c76a2c..f69e29b60622 100644 --- a/book/cli/reth/db/clear.md +++ b/book/cli/reth/db/clear.md @@ -7,9 +7,9 @@ $ reth db clear --help Usage: reth db clear [OPTIONS] Commands: - mdbx Deletes all database table entries - snapshot Deletes all snapshot segment entries - help Print this message or the help of the given subcommand(s) + mdbx Deletes all database table entries + static-file Deletes all static file segment entries + help Print this message or the help of the given subcommand(s) Options: --datadir diff --git a/book/cli/reth/db/clear/static-file.md b/book/cli/reth/db/clear/static-file.md new file mode 100644 index 000000000000..01abfb9a6163 --- /dev/null +++ b/book/cli/reth/db/clear/static-file.md @@ -0,0 +1,127 @@ +# reth db clear static-file + +Deletes all static file segment entries + +```bash +$ reth db clear static-file --help +Usage: reth db clear static-file [OPTIONS] + +Arguments: + + Possible values: + - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables + - transactions: Static File segment responsible for the `Transactions` table + - receipts: Static File segment responsible for the `Receipts` table + +Options: + --datadir + The path to the data dir for all reth files and subdirectories. + + Defaults to the OS-specific data directory: + + - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` + - Windows: `{FOLDERID_RoamingAppData}/reth/` + - macOS: `$HOME/Library/Application Support/reth/` + + [default: default] + + --chain + The chain this node is running. + Possible values are either a built-in chain or the path to a chain specification file. + + Built-in chains: + mainnet, sepolia, goerli, holesky, dev + + [default: mainnet] + + --instance + Add a new instance of a node. + + Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine. + + Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other. + + Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2 + + [default: 1] + + -h, --help + Print help (see a summary with '-h') + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ] + + --log.file.format + The format to use for logs written to the log file + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled + + [default: 5] + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + [default: always] + + Possible values: + - always: Colors on + - auto: Colors on + - never: Colors off + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output +``` \ No newline at end of file diff --git a/book/cli/reth/db/clear/snapshot.md b/book/cli/reth/db/clear/static_file.md similarity index 90% rename from book/cli/reth/db/clear/snapshot.md rename to book/cli/reth/db/clear/static_file.md index e8cb9063d6d4..c8b3ab639c84 100644 --- a/book/cli/reth/db/clear/snapshot.md +++ b/book/cli/reth/db/clear/static_file.md @@ -1,17 +1,17 @@ -# reth db clear snapshot +# reth db clear static-file -Deletes all snapshot segment entries +Deletes all static_file segment entries ```bash -$ reth db clear snapshot --help -Usage: reth db clear snapshot [OPTIONS] +$ reth db clear static-file --help +Usage: reth db clear static-file [OPTIONS] Arguments: Possible values: - - headers: Snapshot segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables - - transactions: Snapshot segment responsible for the `Transactions` table - - receipts: Snapshot segment responsible for the `Receipts` table + - headers: StaticFile segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables + - transactions: StaticFile segment responsible for the `Transactions` table + - receipts: StaticFile segment responsible for the `Receipts` table Options: --datadir diff --git a/book/cli/reth/db/create-static-files.md b/book/cli/reth/db/create-static-files.md new file mode 100644 index 000000000000..dee83e657a7a --- /dev/null +++ b/book/cli/reth/db/create-static-files.md @@ -0,0 +1,174 @@ +# reth db create-static-files + +Creates static files from database tables + +```bash +$ reth db create-static-files --help +Usage: reth db create-static-files [OPTIONS] [SEGMENTS]... + +Arguments: + [SEGMENTS]... + Static File segments to generate + + Possible values: + - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables + - transactions: Static File segment responsible for the `Transactions` table + - receipts: Static File segment responsible for the `Receipts` table + +Options: + --datadir + The path to the data dir for all reth files and subdirectories. + + Defaults to the OS-specific data directory: + + - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` + - Windows: `{FOLDERID_RoamingAppData}/reth/` + - macOS: `$HOME/Library/Application Support/reth/` + + [default: default] + + -f, --from + Starting block for the static file + + [default: 0] + + -b, --block-interval + Number of blocks in the static file + + [default: 500000] + + --chain + The chain this node is running. + Possible values are either a built-in chain or the path to a chain specification file. + + Built-in chains: + mainnet, sepolia, goerli, holesky, dev + + [default: mainnet] + + -p, --parallel + Sets the number of static files built in parallel. Note: Each parallel build is memory-intensive + + [default: 1] + + --only-stats + Flag to skip static file creation and print static files stats + + --bench + Flag to enable database-to-static file benchmarking + + --only-bench + Flag to skip static file creation and only run benchmarks on existing static files + + -c, --compression + Compression algorithms to use + + [default: uncompressed] + + Possible values: + - lz4: LZ4 compression algorithm + - zstd: Zstandard (Zstd) compression algorithm + - zstd-with-dictionary: Zstandard (Zstd) compression algorithm with a dictionary + - uncompressed: No compression + + --with-filters + Flag to enable inclusion list filters and PHFs + + --phf + Specifies the perfect hashing function to use + + Possible values: + - fmph: Fingerprint-Based Minimal Perfect Hash Function + - go-fmph: Fingerprint-Based Minimal Perfect Hash Function with Group Optimization + + --instance + Add a new instance of a node. + + Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine. + + Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other. + + Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2 + + [default: 1] + + -h, --help + Print help (see a summary with '-h') + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ] + + --log.file.format + The format to use for logs written to the log file + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled + + [default: 5] + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + [default: always] + + Possible values: + - always: Colors on + - auto: Colors on + - never: Colors off + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output +``` \ No newline at end of file diff --git a/book/cli/reth/db/get.md b/book/cli/reth/db/get.md index bc539f6dc0e9..de2f83b56c37 100644 --- a/book/cli/reth/db/get.md +++ b/book/cli/reth/db/get.md @@ -7,9 +7,9 @@ $ reth db get --help Usage: reth db get [OPTIONS] Commands: - mdbx Gets the content of a database table for the given key - snapshot Gets the content of a snapshot segment for the given key - help Print this message or the help of the given subcommand(s) + mdbx Gets the content of a database table for the given key + static-file Gets the content of a static file segment for the given key + help Print this message or the help of the given subcommand(s) Options: --datadir diff --git a/book/cli/reth/db/get/static-file.md b/book/cli/reth/db/get/static-file.md new file mode 100644 index 000000000000..26de090a3aa5 --- /dev/null +++ b/book/cli/reth/db/get/static-file.md @@ -0,0 +1,133 @@ +# reth db get static-file + +Gets the content of a static file segment for the given key + +```bash +$ reth db get static-file --help +Usage: reth db get static-file [OPTIONS] + +Arguments: + + Possible values: + - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables + - transactions: Static File segment responsible for the `Transactions` table + - receipts: Static File segment responsible for the `Receipts` table + + + The key to get content for + +Options: + --datadir + The path to the data dir for all reth files and subdirectories. + + Defaults to the OS-specific data directory: + + - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` + - Windows: `{FOLDERID_RoamingAppData}/reth/` + - macOS: `$HOME/Library/Application Support/reth/` + + [default: default] + + --raw + Output bytes instead of human-readable decoded value + + --chain + The chain this node is running. + Possible values are either a built-in chain or the path to a chain specification file. + + Built-in chains: + mainnet, sepolia, goerli, holesky, dev + + [default: mainnet] + + --instance + Add a new instance of a node. + + Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine. + + Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other. + + Changes to the following port numbers: - DISCOVERY_PORT: default + `instance` - 1 - AUTH_PORT: default + `instance` * 100 - 100 - HTTP_RPC_PORT: default - `instance` + 1 - WS_RPC_PORT: default + `instance` * 2 - 2 + + [default: 1] + + -h, --help + Print help (see a summary with '-h') + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ] + + --log.file.format + The format to use for logs written to the log file + + [default: terminal] + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled + + [default: 5] + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + [default: always] + + Possible values: + - always: Colors on + - auto: Colors on + - never: Colors off + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output +``` \ No newline at end of file diff --git a/book/cli/reth/db/get/snapshot.md b/book/cli/reth/db/get/static_file.md similarity index 90% rename from book/cli/reth/db/get/snapshot.md rename to book/cli/reth/db/get/static_file.md index d13742ee017f..3f861b788b0f 100644 --- a/book/cli/reth/db/get/snapshot.md +++ b/book/cli/reth/db/get/static_file.md @@ -1,17 +1,17 @@ -# reth db get snapshot +# reth db get static-file -Gets the content of a snapshot segment for the given key +Gets the content of a static_file segment for the given key ```bash -$ reth db get snapshot --help -Usage: reth db get snapshot [OPTIONS] +$ reth db get static-file --help +Usage: reth db get static-file [OPTIONS] Arguments: Possible values: - - headers: Snapshot segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables - - transactions: Snapshot segment responsible for the `Transactions` table - - receipts: Snapshot segment responsible for the `Receipts` table + - headers: StaticFile segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables + - transactions: StaticFile segment responsible for the `Transactions` table + - receipts: StaticFile segment responsible for the `Receipts` table The key to get content for diff --git a/book/cli/reth/db/snapshot.md b/book/cli/reth/db/static_file.md similarity index 84% rename from book/cli/reth/db/snapshot.md rename to book/cli/reth/db/static_file.md index 8e4aef50126b..f47dd7cc5e27 100644 --- a/book/cli/reth/db/snapshot.md +++ b/book/cli/reth/db/static_file.md @@ -1,19 +1,19 @@ -# reth db snapshot +# reth db static-file -Snapshots tables from database +StaticFiles tables from database ```bash -$ reth db snapshot --help -Usage: reth db snapshot [OPTIONS] [SEGMENTS]... +$ reth db static-file --help +Usage: reth db static-file [OPTIONS] [SEGMENTS]... Arguments: [SEGMENTS]... - Snapshot segments to generate + StaticFile segments to generate Possible values: - - headers: Snapshot segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables - - transactions: Snapshot segment responsible for the `Transactions` table - - receipts: Snapshot segment responsible for the `Receipts` table + - headers: StaticFile segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables + - transactions: StaticFile segment responsible for the `Transactions` table + - receipts: StaticFile segment responsible for the `Receipts` table Options: --datadir @@ -28,12 +28,12 @@ Options: [default: default] -f, --from - Starting block for the snapshot + Starting block for the static_file [default: 0] -b, --block-interval - Number of blocks in the snapshot + Number of blocks in the static_file [default: 500000] @@ -47,18 +47,18 @@ Options: [default: mainnet] -p, --parallel - Sets the number of snapshots built in parallel. Note: Each parallel build is memory-intensive + Sets the number of static files built in parallel. Note: Each parallel build is memory-intensive [default: 1] --only-stats - Flag to skip snapshot creation and print snapshot files stats + Flag to skip static_file creation and print static_file files stats --bench - Flag to enable database-to-snapshot benchmarking + Flag to enable database-to-static_file benchmarking --only-bench - Flag to skip snapshot creation and only run benchmarks on existing snapshots + Flag to skip static_file creation and only run benchmarks on existing static files -c, --compression Compression algorithms to use @@ -69,7 +69,7 @@ Options: - lz4: LZ4 compression algorithm - zstd: Zstandard (Zstd) compression algorithm - zstd-with-dictionary: Zstandard (Zstd) compression algorithm with a dictionary - - uncompressed: No compression, uncompressed snapshot + - uncompressed: No compression, uncompressed static_file --with-filters Flag to enable inclusion list filters and PHFs diff --git a/book/cli/reth/db/stats.md b/book/cli/reth/db/stats.md index 275834bbb715..dea5e3d058c0 100644 --- a/book/cli/reth/db/stats.md +++ b/book/cli/reth/db/stats.md @@ -19,7 +19,7 @@ Options: [default: default] --only-total-size - Show only the total size for snapshot files + Show only the total size for static files --chain The chain this node is running. @@ -31,7 +31,7 @@ Options: [default: mainnet] --summary - Show only the summary per snapshot segment + Show only the summary per static file segment --instance Add a new instance of a node. diff --git a/book/cli/reth/stage/drop.md b/book/cli/reth/stage/drop.md index d6c912b4cc29..88a0197990d3 100644 --- a/book/cli/reth/stage/drop.md +++ b/book/cli/reth/stage/drop.md @@ -57,18 +57,17 @@ Database: Possible values: - - headers: The headers stage within the pipeline - - bodies: The bodies stage within the pipeline - - senders: The senders stage within the pipeline - - execution: The execution stage within the pipeline - - account-hashing: The account hashing stage within the pipeline - - storage-hashing: The storage hashing stage within the pipeline - - hashing: The hashing stage within the pipeline - - merkle: The Merkle stage within the pipeline - - tx-lookup: The transaction lookup stage within the pipeline - - account-history: The account history stage within the pipeline - - storage-history: The storage history stage within the pipeline - - total-difficulty: The total difficulty stage within the pipeline + - headers: The headers stage within the pipeline + - bodies: The bodies stage within the pipeline + - senders: The senders stage within the pipeline + - execution: The execution stage within the pipeline + - account-hashing: The account hashing stage within the pipeline + - storage-hashing: The storage hashing stage within the pipeline + - hashing: The hashing stage within the pipeline + - merkle: The Merkle stage within the pipeline + - tx-lookup: The transaction lookup stage within the pipeline + - account-history: The account history stage within the pipeline + - storage-history: The storage history stage within the pipeline Logging: --log.stdout.format diff --git a/book/cli/reth/stage/run.md b/book/cli/reth/stage/run.md index 3657059d5387..a7d8d61bcd22 100644 --- a/book/cli/reth/stage/run.md +++ b/book/cli/reth/stage/run.md @@ -11,18 +11,17 @@ Arguments: The name of the stage to run Possible values: - - headers: The headers stage within the pipeline - - bodies: The bodies stage within the pipeline - - senders: The senders stage within the pipeline - - execution: The execution stage within the pipeline - - account-hashing: The account hashing stage within the pipeline - - storage-hashing: The storage hashing stage within the pipeline - - hashing: The hashing stage within the pipeline - - merkle: The Merkle stage within the pipeline - - tx-lookup: The transaction lookup stage within the pipeline - - account-history: The account history stage within the pipeline - - storage-history: The storage history stage within the pipeline - - total-difficulty: The total difficulty stage within the pipeline + - headers: The headers stage within the pipeline + - bodies: The bodies stage within the pipeline + - senders: The senders stage within the pipeline + - execution: The execution stage within the pipeline + - account-hashing: The account hashing stage within the pipeline + - storage-hashing: The storage hashing stage within the pipeline + - hashing: The hashing stage within the pipeline + - merkle: The Merkle stage within the pipeline + - tx-lookup: The transaction lookup stage within the pipeline + - account-history: The account history stage within the pipeline + - storage-history: The storage history stage within the pipeline Options: --config diff --git a/book/run/observability.md b/book/run/observability.md index 4ab2951805e4..39d485e1f362 100644 --- a/book/run/observability.md +++ b/book/run/observability.md @@ -12,7 +12,7 @@ Now, as the node is running, you can `curl` the endpoint you provided to the `-- curl 127.0.0.1:9001 ``` -The response from this is quite descriptive, but it can be a bit verbose. Plus, it's just a snapshot of the metrics at the time that you `curl`ed the endpoint. +The response from this is quite descriptive, but it can be a bit verbose. Plus, it's just a static_file of the metrics at the time that you `curl`ed the endpoint. You can run the following command in a separate terminal to periodically poll the endpoint, and just print the values (without the header text) to the terminal: diff --git a/crates/blockchain-tree/src/externals.rs b/crates/blockchain-tree/src/externals.rs index 2cc6d677d53c..0fd616876346 100644 --- a/crates/blockchain-tree/src/externals.rs +++ b/crates/blockchain-tree/src/externals.rs @@ -1,10 +1,10 @@ //! Blockchain tree externals. use reth_db::{ - cursor::DbCursorRO, database::Database, snapshot::HeaderMask, tables, transaction::DbTx, + cursor::DbCursorRO, database::Database, static_file::HeaderMask, tables, transaction::DbTx, }; use reth_interfaces::{consensus::Consensus, RethResult}; -use reth_primitives::{BlockHash, BlockNumber, SnapshotSegment}; +use reth_primitives::{BlockHash, BlockNumber, StaticFileSegment}; use reth_provider::{ProviderFactory, StatsReader}; use std::{collections::BTreeMap, sync::Arc}; @@ -56,20 +56,20 @@ impl TreeExternals { .take(num_hashes) .collect::, _>>()?; - // Fetch the same number of latest canonical hashes from the snapshots and merge them with - // the database hashes. It is needed due to the fact that we're writing directly to - // snapshots in pipeline sync, but to the database in live sync, which means that the latest - // canonical hashes in the snapshot might be more recent than in the database, and vice - // versa, or even some ranges of the latest `num_hashes` blocks may be in database, and some - // ranges in snapshots. - let snapshot_provider = self.provider_factory.snapshot_provider(); - let total_headers = snapshot_provider.count_entries::()? as u64; + // Fetch the same number of latest canonical hashes from the static_files and merge them + // with the database hashes. It is needed due to the fact that we're writing + // directly to static_files in pipeline sync, but to the database in live sync, + // which means that the latest canonical hashes in the static file might be more recent + // than in the database, and vice versa, or even some ranges of the latest + // `num_hashes` blocks may be in database, and some ranges in static_files. + let static_file_provider = self.provider_factory.static_file_provider(); + let total_headers = static_file_provider.count_entries::()? as u64; if total_headers > 0 { let range = total_headers.saturating_sub(1).saturating_sub(num_hashes as u64)..total_headers; - hashes.extend(range.clone().zip(snapshot_provider.fetch_range_with_predicate( - SnapshotSegment::Headers, + hashes.extend(range.clone().zip(static_file_provider.fetch_range_with_predicate( + StaticFileSegment::Headers, range, |cursor, number| cursor.get_one::>(number.into()), |_| true, diff --git a/crates/consensus/beacon/Cargo.toml b/crates/consensus/beacon/Cargo.toml index 2109857893c0..a6407240ba09 100644 --- a/crates/consensus/beacon/Cargo.toml +++ b/crates/consensus/beacon/Cargo.toml @@ -23,7 +23,7 @@ reth-tasks.workspace = true reth-payload-builder.workspace = true reth-payload-validator.workspace = true reth-prune.workspace = true -reth-snapshot.workspace = true +reth-static-file.workspace = true reth-tokio-util.workspace = true reth-node-api.workspace = true diff --git a/crates/consensus/beacon/src/engine/hooks/mod.rs b/crates/consensus/beacon/src/engine/hooks/mod.rs index 69101db1f698..ce149717ada0 100644 --- a/crates/consensus/beacon/src/engine/hooks/mod.rs +++ b/crates/consensus/beacon/src/engine/hooks/mod.rs @@ -11,8 +11,8 @@ pub(crate) use controller::{EngineHooksController, PolledHook}; mod prune; pub use prune::PruneHook; -mod snapshot; -pub use snapshot::SnapshotHook; +mod static_file; +pub use static_file::StaticFileHook; /// Collection of [engine hooks][`EngineHook`]. #[derive(Default)] diff --git a/crates/consensus/beacon/src/engine/hooks/snapshot.rs b/crates/consensus/beacon/src/engine/hooks/snapshot.rs deleted file mode 100644 index 11d926b381d9..000000000000 --- a/crates/consensus/beacon/src/engine/hooks/snapshot.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! Snapshot hook for the engine implementation. - -use crate::{ - engine::hooks::{EngineContext, EngineHook, EngineHookError, EngineHookEvent}, - hooks::EngineHookDBAccessLevel, -}; -use futures::FutureExt; -use reth_db::database::Database; -use reth_interfaces::RethResult; -use reth_primitives::{snapshot::HighestSnapshots, BlockNumber}; -use reth_snapshot::{Snapshotter, SnapshotterWithResult}; -use reth_tasks::TaskSpawner; -use std::task::{ready, Context, Poll}; -use tokio::sync::oneshot; -use tracing::trace; - -/// Manages snapshotting under the control of the engine. -/// -/// This type controls the [Snapshotter]. -#[derive(Debug)] -pub struct SnapshotHook { - /// The current state of the snapshotter. - state: SnapshotterState, - /// The type that can spawn the snapshotter task. - task_spawner: Box, -} - -impl SnapshotHook { - /// Create a new instance - pub fn new(snapshotter: Snapshotter, task_spawner: Box) -> Self { - Self { state: SnapshotterState::Idle(Some(snapshotter)), task_spawner } - } - - /// Advances the snapshotter state. - /// - /// This checks for the result in the channel, or returns pending if the snapshotter is idle. - fn poll_snapshotter(&mut self, cx: &mut Context<'_>) -> Poll> { - let result = match self.state { - SnapshotterState::Idle(_) => return Poll::Pending, - SnapshotterState::Running(ref mut fut) => { - ready!(fut.poll_unpin(cx)) - } - }; - - let event = match result { - Ok((snapshotter, result)) => { - self.state = SnapshotterState::Idle(Some(snapshotter)); - - match result { - Ok(_) => EngineHookEvent::Finished(Ok(())), - Err(err) => EngineHookEvent::Finished(Err(err.into())), - } - } - Err(_) => { - // failed to receive the snapshotter - EngineHookEvent::Finished(Err(EngineHookError::ChannelClosed)) - } - }; - - Poll::Ready(Ok(event)) - } - - /// This will try to spawn the snapshotter if it is idle: - /// 1. Check if snapshotting is needed through [Snapshotter::get_snapshot_targets] and then - /// [SnapshotTargets::any](reth_snapshot::SnapshotTargets::any). - /// 2. - /// 1. If snapshotting is needed, pass snapshot request to the [Snapshotter::run] and spawn - /// it in a separate task. Set snapshotter state to [SnapshotterState::Running]. - /// 2. If snapshotting is not needed, set snapshotter state back to - /// [SnapshotterState::Idle]. - /// - /// If snapshotter is already running, do nothing. - fn try_spawn_snapshotter( - &mut self, - finalized_block_number: BlockNumber, - ) -> RethResult> { - Ok(match &mut self.state { - SnapshotterState::Idle(snapshotter) => { - let Some(mut snapshotter) = snapshotter.take() else { - trace!(target: "consensus::engine::hooks::snapshot", "Snapshotter is already running but the state is idle"); - return Ok(None) - }; - - let targets = snapshotter.get_snapshot_targets(HighestSnapshots { - headers: Some(finalized_block_number), - receipts: Some(finalized_block_number), - transactions: Some(finalized_block_number), - })?; - - // Check if the snapshotting of any data has been requested. - if targets.any() { - let (tx, rx) = oneshot::channel(); - self.task_spawner.spawn_critical_blocking( - "snapshotter task", - Box::pin(async move { - let result = snapshotter.run(targets); - let _ = tx.send((snapshotter, result)); - }), - ); - self.state = SnapshotterState::Running(rx); - - Some(EngineHookEvent::Started) - } else { - self.state = SnapshotterState::Idle(Some(snapshotter)); - Some(EngineHookEvent::NotReady) - } - } - SnapshotterState::Running(_) => None, - }) - } -} - -impl EngineHook for SnapshotHook { - fn name(&self) -> &'static str { - "Snapshot" - } - - fn poll( - &mut self, - cx: &mut Context<'_>, - ctx: EngineContext, - ) -> Poll> { - let Some(finalized_block_number) = ctx.finalized_block_number else { - trace!(target: "consensus::engine::hooks::snapshot", ?ctx, "Finalized block number is not available"); - return Poll::Pending - }; - - // Try to spawn a snapshotter - match self.try_spawn_snapshotter(finalized_block_number)? { - Some(EngineHookEvent::NotReady) => return Poll::Pending, - Some(event) => return Poll::Ready(Ok(event)), - None => (), - } - - // Poll snapshotter and check its status - self.poll_snapshotter(cx) - } - - fn db_access_level(&self) -> EngineHookDBAccessLevel { - EngineHookDBAccessLevel::ReadOnly - } -} - -/// The possible snapshotter states within the sync controller. -/// -/// [SnapshotterState::Idle] means that the snapshotter is currently idle. -/// [SnapshotterState::Running] means that the snapshotter is currently running. -#[derive(Debug)] -enum SnapshotterState { - /// Snapshotter is idle. - Idle(Option>), - /// Snapshotter is running and waiting for a response - Running(oneshot::Receiver>), -} diff --git a/crates/consensus/beacon/src/engine/hooks/static_file.rs b/crates/consensus/beacon/src/engine/hooks/static_file.rs new file mode 100644 index 000000000000..ee4309cc0b72 --- /dev/null +++ b/crates/consensus/beacon/src/engine/hooks/static_file.rs @@ -0,0 +1,163 @@ +//! StaticFile hook for the engine implementation. + +use crate::{ + engine::hooks::{EngineContext, EngineHook, EngineHookError, EngineHookEvent}, + hooks::EngineHookDBAccessLevel, +}; +use futures::FutureExt; +use reth_db::database::Database; +use reth_interfaces::RethResult; +use reth_primitives::{static_file::HighestStaticFiles, BlockNumber}; +use reth_static_file::{StaticFileProducer, StaticFileProducerWithResult}; +use reth_tasks::TaskSpawner; +use std::task::{ready, Context, Poll}; +use tokio::sync::oneshot; +use tracing::trace; + +/// Manages producing static files under the control of the engine. +/// +/// This type controls the [StaticFileProducer]. +#[derive(Debug)] +pub struct StaticFileHook { + /// The current state of the static_file_producer. + state: StaticFileProducerState, + /// The type that can spawn the static_file_producer task. + task_spawner: Box, +} + +impl StaticFileHook { + /// Create a new instance + pub fn new( + static_file_producer: StaticFileProducer, + task_spawner: Box, + ) -> Self { + Self { state: StaticFileProducerState::Idle(Some(static_file_producer)), task_spawner } + } + + /// Advances the static_file_producer state. + /// + /// This checks for the result in the channel, or returns pending if the static_file_producer is + /// idle. + fn poll_static_file_producer( + &mut self, + cx: &mut Context<'_>, + ) -> Poll> { + let result = match self.state { + StaticFileProducerState::Idle(_) => return Poll::Pending, + StaticFileProducerState::Running(ref mut fut) => { + ready!(fut.poll_unpin(cx)) + } + }; + + let event = match result { + Ok((static_file_producer, result)) => { + self.state = StaticFileProducerState::Idle(Some(static_file_producer)); + + match result { + Ok(_) => EngineHookEvent::Finished(Ok(())), + Err(err) => EngineHookEvent::Finished(Err(err.into())), + } + } + Err(_) => { + // failed to receive the static_file_producer + EngineHookEvent::Finished(Err(EngineHookError::ChannelClosed)) + } + }; + + Poll::Ready(Ok(event)) + } + + /// This will try to spawn the static_file_producer if it is idle: + /// 1. Check if producing static files is needed through + /// [StaticFileProducer::get_static_file_targets] and then + /// [StaticFileTargets::any](reth_static_file::StaticFileTargets::any). + /// 2. + /// 1. If producing static files is needed, pass static file request to the + /// [StaticFileProducer::run] and spawn it in a separate task. Set static file producer + /// state to [StaticFileProducerState::Running]. + /// 2. If producing static files is not needed, set static file producer state back to + /// [StaticFileProducerState::Idle]. + /// + /// If static_file_producer is already running, do nothing. + fn try_spawn_static_file_producer( + &mut self, + finalized_block_number: BlockNumber, + ) -> RethResult> { + Ok(match &mut self.state { + StaticFileProducerState::Idle(static_file_producer) => { + let Some(mut static_file_producer) = static_file_producer.take() else { + trace!(target: "consensus::engine::hooks::static_file", "StaticFileProducer is already running but the state is idle"); + return Ok(None); + }; + + let targets = static_file_producer.get_static_file_targets(HighestStaticFiles { + headers: Some(finalized_block_number), + receipts: Some(finalized_block_number), + transactions: Some(finalized_block_number), + })?; + + // Check if the moving data to static files has been requested. + if targets.any() { + let (tx, rx) = oneshot::channel(); + self.task_spawner.spawn_critical_blocking( + "static_file_producer task", + Box::pin(async move { + let result = static_file_producer.run(targets); + let _ = tx.send((static_file_producer, result)); + }), + ); + self.state = StaticFileProducerState::Running(rx); + + Some(EngineHookEvent::Started) + } else { + self.state = StaticFileProducerState::Idle(Some(static_file_producer)); + Some(EngineHookEvent::NotReady) + } + } + StaticFileProducerState::Running(_) => None, + }) + } +} + +impl EngineHook for StaticFileHook { + fn name(&self) -> &'static str { + "StaticFile" + } + + fn poll( + &mut self, + cx: &mut Context<'_>, + ctx: EngineContext, + ) -> Poll> { + let Some(finalized_block_number) = ctx.finalized_block_number else { + trace!(target: "consensus::engine::hooks::static_file", ?ctx, "Finalized block number is not available"); + return Poll::Pending; + }; + + // Try to spawn a static_file_producer + match self.try_spawn_static_file_producer(finalized_block_number)? { + Some(EngineHookEvent::NotReady) => return Poll::Pending, + Some(event) => return Poll::Ready(Ok(event)), + None => (), + } + + // Poll static_file_producer and check its status + self.poll_static_file_producer(cx) + } + + fn db_access_level(&self) -> EngineHookDBAccessLevel { + EngineHookDBAccessLevel::ReadOnly + } +} + +/// The possible static_file_producer states within the sync controller. +/// +/// [StaticFileProducerState::Idle] means that the static file producer is currently idle. +/// [StaticFileProducerState::Running] means that the static file producer is currently running. +#[derive(Debug)] +enum StaticFileProducerState { + /// [StaticFileProducer] is idle. + Idle(Option>), + /// [StaticFileProducer] is running and waiting for a response + Running(oneshot::Receiver>), +} diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index efa858782ff3..168a8601477f 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -2096,7 +2096,7 @@ mod tests { mod fork_choice_updated { use super::*; - use reth_db::{tables, test_utils::create_test_snapshots_dir, transaction::DbTxMut}; + use reth_db::{tables, test_utils::create_test_static_files_dir, transaction::DbTxMut}; use reth_interfaces::test_utils::generators::random_block; use reth_primitives::U256; use reth_rpc_types::engine::ForkchoiceUpdateError; @@ -2155,9 +2155,9 @@ mod tests { ProviderFactory::new( env.db.as_ref(), chain_spec.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) - .expect("create provider factory with snapshots"), + .expect("create provider factory with static_files"), [&genesis, &block1].into_iter(), ); env.db @@ -2213,9 +2213,9 @@ mod tests { ProviderFactory::new( env.db.as_ref(), chain_spec.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) - .expect("create provider factory with snapshots"), + .expect("create provider factory with static_files"), [&genesis, &block1].into_iter(), ); @@ -2237,9 +2237,9 @@ mod tests { ProviderFactory::new( env.db.as_ref(), chain_spec.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) - .expect("create provider factory with snapshots"), + .expect("create provider factory with static_files"), [&next_head].into_iter(), ); @@ -2279,9 +2279,9 @@ mod tests { ProviderFactory::new( env.db.as_ref(), chain_spec.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) - .expect("create provider factory with snapshots"), + .expect("create provider factory with static_files"), [&genesis, &block1].into_iter(), ); @@ -2334,9 +2334,9 @@ mod tests { ProviderFactory::new( env.db.as_ref(), chain_spec.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) - .expect("create provider factory with snapshots"), + .expect("create provider factory with static_files"), [&genesis, &block1, &block2, &block3].into_iter(), ); @@ -2382,9 +2382,9 @@ mod tests { ProviderFactory::new( env.db.as_ref(), chain_spec.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) - .expect("create provider factory with snapshots"), + .expect("create provider factory with static_files"), [&genesis, &block1].into_iter(), ); @@ -2408,7 +2408,7 @@ mod tests { mod new_payload { use super::*; - use reth_db::test_utils::create_test_snapshots_dir; + use reth_db::test_utils::create_test_static_files_dir; use reth_interfaces::test_utils::generators::random_block; use reth_primitives::{ genesis::{Genesis, GenesisAllocator}, @@ -2486,9 +2486,9 @@ mod tests { ProviderFactory::new( env.db.as_ref(), chain_spec.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) - .expect("create provider factory with snapshots"), + .expect("create provider factory with static_files"), [&genesis, &block1, &block2].into_iter(), ); @@ -2557,9 +2557,9 @@ mod tests { ProviderFactory::new( env.db.as_ref(), chain_spec.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) - .expect("create provider factory with snapshots"), + .expect("create provider factory with static_files"), [&genesis, &block1].into_iter(), ); @@ -2603,9 +2603,9 @@ mod tests { ProviderFactory::new( env.db.as_ref(), chain_spec.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) - .expect("create provider factory with snapshots"), + .expect("create provider factory with static_files"), [&genesis].into_iter(), ); @@ -2669,9 +2669,9 @@ mod tests { ProviderFactory::new( env.db.as_ref(), chain_spec.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) - .expect("create provider factory with snapshots"), + .expect("create provider factory with static_files"), [&data.genesis, &block1].into_iter(), ); diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs index 157a8866b1f7..de6280807d12 100644 --- a/crates/consensus/beacon/src/engine/test_utils.rs +++ b/crates/consensus/beacon/src/engine/test_utils.rs @@ -31,8 +31,8 @@ use reth_revm::EvmProcessorFactory; use reth_rpc_types::engine::{ CancunPayloadFields, ExecutionPayload, ForkchoiceState, ForkchoiceUpdated, PayloadStatus, }; -use reth_snapshot::Snapshotter; use reth_stages::{sets::DefaultStages, test_utils::TestStages, ExecOutput, Pipeline, StageError}; +use reth_static_file::StaticFileProducer; use reth_tasks::TokioTaskExecutor; use std::{collections::VecDeque, sync::Arc}; use tokio::sync::{oneshot, watch}; @@ -377,9 +377,9 @@ where )), }; - let snapshotter = Snapshotter::new( + let static_file_producer = StaticFileProducer::new( provider_factory.clone(), - provider_factory.snapshot_provider(), + provider_factory.static_file_provider(), PruneModes::default(), ); @@ -406,7 +406,7 @@ where header_downloader, body_downloader, executor_factory.clone(), - snapshotter, + static_file_producer, ) .expect("should build"), ) diff --git a/crates/interfaces/src/provider.rs b/crates/interfaces/src/provider.rs index a3d928ba5e6f..513e3efe9d91 100644 --- a/crates/interfaces/src/provider.rs +++ b/crates/interfaces/src/provider.rs @@ -1,5 +1,5 @@ use reth_primitives::{ - Address, BlockHash, BlockHashOrNumber, BlockNumber, GotExpected, SnapshotSegment, + Address, BlockHash, BlockHashOrNumber, BlockNumber, GotExpected, StaticFileSegment, TxHashOrNumber, TxNumber, B256, U256, }; use std::path::PathBuf; @@ -113,18 +113,18 @@ pub enum ProviderError { /// Provider does not support this particular request. #[error("this provider does not support this request")] UnsupportedProvider, - /// Snapshot file is not found at specified path. - #[error("not able to find {0} snapshot file at {1}")] - MissingSnapshotPath(SnapshotSegment, PathBuf), - /// Snapshot file is not found for requested block. - #[error("not able to find {0} snapshot file for block number {1}")] - MissingSnapshotBlock(SnapshotSegment, BlockNumber), - /// Snapshot file is not found for requested transaction. - #[error("unable to find {0} snapshot file for transaction id {1}")] - MissingSnapshotTx(SnapshotSegment, TxNumber), - /// Snapshot is finalized and cannot be written to. - #[error("unable to write block #{1} to finalized snapshot {0}")] - FinalizedSnapshot(SnapshotSegment, BlockNumber), + /// Static File is not found at specified path. + #[error("not able to find {0} static file at {1}")] + MissingStaticFilePath(StaticFileSegment, PathBuf), + /// Static File is not found for requested block. + #[error("not able to find {0} static file for block number {1}")] + MissingStaticFileBlock(StaticFileSegment, BlockNumber), + /// Static File is not found for requested transaction. + #[error("unable to find {0} static file for transaction id {1}")] + MissingStaticFileTx(StaticFileSegment, TxNumber), + /// Static File is finalized and cannot be written to. + #[error("unable to write block #{1} to finalized static file {0}")] + FinalizedStaticFile(StaticFileSegment, BlockNumber), /// Error encountered when the block number conversion from U256 to u64 causes an overflow. #[error("failed to convert block number U256 to u64: {0}")] BlockNumberOverflow(U256), diff --git a/crates/net/downloaders/src/bodies/bodies.rs b/crates/net/downloaders/src/bodies/bodies.rs index 235f6cbfb8bb..33d1303599d1 100644 --- a/crates/net/downloaders/src/bodies/bodies.rs +++ b/crates/net/downloaders/src/bodies/bodies.rs @@ -596,7 +596,7 @@ mod tests { test_utils::{generate_bodies, TestBodiesClient}, }; use assert_matches::assert_matches; - use reth_db::test_utils::{create_test_rw_db, create_test_snapshots_dir}; + use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; use reth_interfaces::test_utils::{generators, generators::random_block_range, TestConsensus}; use reth_primitives::{BlockBody, B256, MAINNET}; use reth_provider::ProviderFactory; @@ -618,7 +618,7 @@ mod tests { let mut downloader = BodiesDownloaderBuilder::default().build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), create_test_snapshots_dir()).unwrap(), + ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(), ); downloader.set_download_range(0..=19).expect("failed to set download range"); @@ -657,7 +657,7 @@ mod tests { BodiesDownloaderBuilder::default().with_request_limit(request_limit).build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), create_test_snapshots_dir()).unwrap(), + ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(), ); downloader.set_download_range(0..=199).expect("failed to set download range"); @@ -686,7 +686,7 @@ mod tests { .build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), create_test_snapshots_dir()).unwrap(), + ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(), ); let mut range_start = 0; @@ -716,7 +716,7 @@ mod tests { let mut downloader = BodiesDownloaderBuilder::default().with_stream_batch_size(100).build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), create_test_snapshots_dir()).unwrap(), + ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(), ); // Set and download the first range @@ -756,7 +756,7 @@ mod tests { .build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), create_test_snapshots_dir()).unwrap(), + ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(), ); // Set and download the entire range @@ -787,7 +787,7 @@ mod tests { .build( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::new(db, MAINNET.clone(), create_test_snapshots_dir()).unwrap(), + ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()).unwrap(), ); // Download the requested range diff --git a/crates/node-builder/Cargo.toml b/crates/node-builder/Cargo.toml index 8c63d540f88e..6a7942b29a54 100644 --- a/crates/node-builder/Cargo.toml +++ b/crates/node-builder/Cargo.toml @@ -29,7 +29,7 @@ reth-transaction-pool.workspace = true reth-tasks.workspace = true reth-tracing.workspace = true reth-interfaces.workspace = true -reth-snapshot.workspace = true +reth-static-file.workspace = true reth-prune.workspace = true reth-stages.workspace = true reth-config.workspace = true diff --git a/crates/node-builder/src/builder.rs b/crates/node-builder/src/builder.rs index ca37a582f4b4..2a7aa8c0935a 100644 --- a/crates/node-builder/src/builder.rs +++ b/crates/node-builder/src/builder.rs @@ -353,13 +353,13 @@ where let provider_factory = ProviderFactory::new( database.clone(), Arc::clone(&config.chain), - data_dir.snapshots_path(), + data_dir.static_files_path(), )?; - // configure snapshotter - let snapshotter = reth_snapshot::Snapshotter::new( + // configure static_file_producer + let static_file_producer = reth_static_file::StaticFileProducer::new( provider_factory.clone(), - provider_factory.snapshot_provider(), + provider_factory.static_file_provider(), config.prune_config()?.unwrap_or_default().segments, ); @@ -467,7 +467,7 @@ where sync_metrics_tx, prune_config.clone(), max_block, - snapshotter, + static_file_producer, evm_config, ) .await?; @@ -489,7 +489,7 @@ where sync_metrics_tx, prune_config.clone(), max_block, - snapshotter, + static_file_producer, evm_config, ) .await?; diff --git a/crates/node-core/Cargo.toml b/crates/node-core/Cargo.toml index 49483632e1c9..308ae00b15bc 100644 --- a/crates/node-core/Cargo.toml +++ b/crates/node-core/Cargo.toml @@ -44,7 +44,7 @@ reth-stages.workspace = true reth-prune.workspace = true reth-blockchain-tree.workspace = true revm-inspectors.workspace = true -reth-snapshot.workspace = true +reth-static-file.workspace = true reth-eth-wire.workspace = true # `optimism` feature diff --git a/crates/node-core/src/dirs.rs b/crates/node-core/src/dirs.rs index b4e6d8955bb6..affcb7f8a6d3 100644 --- a/crates/node-core/src/dirs.rs +++ b/crates/node-core/src/dirs.rs @@ -282,9 +282,9 @@ impl ChainPath { self.0.join("db").into() } - /// Returns the path to the snapshots directory for this chain. - pub fn snapshots_path(&self) -> PathBuf { - self.0.join("snapshots").into() + /// Returns the path to the static_files directory for this chain. + pub fn static_files_path(&self) -> PathBuf { + self.0.join("static_files").into() } /// Returns the path to the reth p2p secret key for this chain. diff --git a/crates/node-core/src/events/node.rs b/crates/node-core/src/events/node.rs index 156a04c2ce02..25eab823b2ab 100644 --- a/crates/node-core/src/events/node.rs +++ b/crates/node-core/src/events/node.rs @@ -13,8 +13,8 @@ use reth_primitives::{ BlockNumber, B256, }; use reth_prune::PrunerEvent; -use reth_snapshot::SnapshotterEvent; use reth_stages::{ExecOutput, PipelineEvent}; +use reth_static_file::StaticFileProducerEvent; use std::{ fmt::{Display, Formatter}, future::Future, @@ -240,10 +240,10 @@ impl NodeState { } } - fn handle_snapshotter_event(&self, event: SnapshotterEvent) { + fn handle_static_file_producer_event(&self, event: StaticFileProducerEvent) { match event { - SnapshotterEvent::Finished { targets, elapsed } => { - info!(?targets, ?elapsed, "Snapshotter finished"); + StaticFileProducerEvent::Finished { targets, elapsed } => { + info!(?targets, ?elapsed, "StaticFileProducer finished"); } } } @@ -291,8 +291,8 @@ pub enum NodeEvent { ConsensusLayerHealth(ConsensusLayerHealthEvent), /// A pruner event Pruner(PrunerEvent), - /// A snapshotter event - Snapshotter(SnapshotterEvent), + /// A static_file_producer event + StaticFileProducer(StaticFileProducerEvent), } impl From for NodeEvent { @@ -325,9 +325,9 @@ impl From for NodeEvent { } } -impl From for NodeEvent { - fn from(event: SnapshotterEvent) -> Self { - NodeEvent::Snapshotter(event) +impl From for NodeEvent { + fn from(event: StaticFileProducerEvent) -> Self { + NodeEvent::StaticFileProducer(event) } } @@ -447,8 +447,8 @@ where NodeEvent::Pruner(event) => { this.state.handle_pruner_event(event); } - NodeEvent::Snapshotter(event) => { - this.state.handle_snapshotter_event(event); + NodeEvent::StaticFileProducer(event) => { + this.state.handle_static_file_producer_event(event); } } } @@ -522,7 +522,7 @@ impl Display for Eta { f, "{}", humantime::format_duration(Duration::from_secs(remaining.as_secs())) - ) + ); } } diff --git a/crates/node-core/src/init.rs b/crates/node-core/src/init.rs index eb2808449ee1..c835a7a5c994 100644 --- a/crates/node-core/src/init.rs +++ b/crates/node-core/src/init.rs @@ -7,12 +7,12 @@ use reth_db::{ }; use reth_interfaces::{db::DatabaseError, provider::ProviderResult}; use reth_primitives::{ - stage::StageId, Account, Bytecode, ChainSpec, Receipts, SnapshotSegment, StorageEntry, B256, + stage::StageId, Account, Bytecode, ChainSpec, Receipts, StaticFileSegment, StorageEntry, B256, U256, }; use reth_provider::{ bundle_state::{BundleStateInit, RevertsInit}, - providers::{SnapshotProvider, SnapshotWriter}, + providers::{StaticFileProvider, StaticFileWriter}, BlockHashReader, BundleStateWithReceipts, ChainSpecProvider, DatabaseProviderRW, HashingWriter, HistoryWriter, OriginalValuesKnown, ProviderError, ProviderFactory, }; @@ -55,7 +55,7 @@ pub fn init_genesis(factory: ProviderFactory) -> Result {} + Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => {} Ok(Some(block_hash)) => { if block_hash == hash { debug!("Genesis already written, skipping."); @@ -79,7 +79,7 @@ pub fn init_genesis(factory: ProviderFactory) -> Result(&tx, factory.snapshot_provider(), chain.clone())?; + insert_genesis_header::(&tx, factory.static_file_provider(), chain.clone())?; insert_genesis_state::(&tx, genesis)?; @@ -209,15 +209,15 @@ pub fn insert_genesis_history( /// Inserts header for the genesis state. pub fn insert_genesis_header( tx: &::TXMut, - snapshot_provider: SnapshotProvider, + static_file_provider: StaticFileProvider, chain: Arc, ) -> ProviderResult<()> { let (header, block_hash) = chain.sealed_genesis_header().split(); - match snapshot_provider.block_hash(0) { - Ok(None) | Err(ProviderError::MissingSnapshotBlock(SnapshotSegment::Headers, 0)) => { + match static_file_provider.block_hash(0) { + Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => { let (difficulty, hash) = (header.difficulty, block_hash); - let mut writer = snapshot_provider.latest_writer(SnapshotSegment::Headers)?; + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; writer.append_header(header, difficulty, hash)?; writer.commit()?; } @@ -287,7 +287,7 @@ mod tests { #[test] fn fail_init_inconsistent_db() { let factory = create_test_provider_factory_with_chain_spec(SEPOLIA.clone()); - let snapshot_provider = factory.snapshot_provider(); + let static_file_provider = factory.static_file_provider(); init_genesis(factory.clone()).unwrap(); // Try to init db with a different genesis block @@ -295,7 +295,7 @@ mod tests { ProviderFactory::new( factory.into_db(), MAINNET.clone(), - snapshot_provider.path().into(), + static_file_provider.path().into(), ) .unwrap(), ); diff --git a/crates/node-core/src/node_config.rs b/crates/node-core/src/node_config.rs index e6d88bfd48d8..0c2c62068579 100644 --- a/crates/node-core/src/node_config.rs +++ b/crates/node-core/src/node_config.rs @@ -52,7 +52,6 @@ use reth_provider::{ ProviderFactory, StageCheckpointReader, }; use reth_revm::EvmProcessorFactory; -use reth_snapshot::Snapshotter; use reth_stages::{ prelude::*, stages::{ @@ -62,6 +61,7 @@ use reth_stages::{ }, MetricEvent, }; +use reth_static_file::StaticFileProducer; use reth_tasks::TaskExecutor; use reth_transaction_pool::{ blobstore::DiskFileBlobStore, EthTransactionPool, TransactionPool, @@ -543,7 +543,7 @@ impl NodeConfig { metrics_tx: reth_stages::MetricEventsSender, prune_config: Option, max_block: Option, - snapshotter: Snapshotter, + static_file_producer: StaticFileProducer, evm_config: EvmConfig, ) -> eyre::Result> where @@ -571,7 +571,7 @@ impl NodeConfig { self.debug.continuous, metrics_tx, prune_config, - snapshotter, + static_file_producer, evm_config, ) .await?; @@ -793,7 +793,7 @@ impl NodeConfig { continuous: bool, metrics_tx: reth_stages::MetricEventsSender, prune_config: Option, - snapshotter: Snapshotter, + static_file_producer: StaticFileProducer, evm_config: EvmConfig, ) -> eyre::Result> where @@ -843,7 +843,7 @@ impl NodeConfig { header_downloader, body_downloader, factory.clone(), - snapshotter, + static_file_producer, )? .set(SenderRecoveryStage { commit_threshold: stage_config.sender_recovery.commit_threshold, diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 703554978aa4..634a75e860ef 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -35,8 +35,8 @@ mod receipt; /// Helpers for working with revm pub mod revm; pub mod serde_helper; -pub mod snapshot; pub mod stage; +pub mod static_file; mod storage; /// Helpers for working with transactions pub mod transaction; @@ -73,7 +73,7 @@ pub use prune::{ PruneSegmentError, ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE, }; pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef, Receipts}; -pub use snapshot::SnapshotSegment; +pub use static_file::StaticFileSegment; pub use storage::StorageEntry; #[cfg(feature = "c-kzg")] diff --git a/crates/primitives/src/prune/segment.rs b/crates/primitives/src/prune/segment.rs index a0ed8f0f3e4c..1ec75f80a09f 100644 --- a/crates/primitives/src/prune/segment.rs +++ b/crates/primitives/src/prune/segment.rs @@ -32,7 +32,7 @@ impl PruneSegment { Self::SenderRecovery | Self::TransactionLookup | Self::Headers | Self::Transactions => { 0 } - Self::Receipts if purpose.is_snapshot() => 0, + Self::Receipts if purpose.is_static_file() => 0, Self::ContractLogs | Self::AccountHistory | Self::StorageHistory => { MINIMUM_PRUNING_DISTANCE } @@ -46,8 +46,8 @@ impl PruneSegment { pub enum PrunePurpose { /// Prune data according to user configuration. User, - /// Prune data according to highest snapshots to delete the data from database. - Snapshot, + /// Prune data according to highest static_files to delete the data from database. + StaticFile, } impl PrunePurpose { @@ -56,9 +56,9 @@ impl PrunePurpose { matches!(self, Self::User) } - /// Returns true if the purpose is [`PrunePurpose::Snapshot`]. - pub fn is_snapshot(self) -> bool { - matches!(self, Self::Snapshot) + /// Returns true if the purpose is [`PrunePurpose::StaticFile`]. + pub fn is_static_file(self) -> bool { + matches!(self, Self::StaticFile) } } diff --git a/crates/primitives/src/snapshot/mod.rs b/crates/primitives/src/snapshot/mod.rs deleted file mode 100644 index 697d6a01dea4..000000000000 --- a/crates/primitives/src/snapshot/mod.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! Snapshot primitives. - -mod compression; -mod filters; -mod segment; - -use alloy_primitives::BlockNumber; -pub use compression::Compression; -pub use filters::{Filters, InclusionFilter, PerfectHashingFunction}; -pub use segment::{SegmentConfig, SegmentHeader, SegmentRangeInclusive, SnapshotSegment}; - -/// Default snapshot block count. -pub const BLOCKS_PER_SNAPSHOT: u64 = 500_000; - -/// Highest snapshotted block numbers, per data part. -#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] -pub struct HighestSnapshots { - /// Highest snapshotted block of headers, inclusive. - /// If [`None`], no snapshot is available. - pub headers: Option, - /// Highest snapshotted block of receipts, inclusive. - /// If [`None`], no snapshot is available. - pub receipts: Option, - /// Highest snapshotted block of transactions, inclusive. - /// If [`None`], no snapshot is available. - pub transactions: Option, -} - -impl HighestSnapshots { - /// Returns the highest snapshot if it exists for a segment - pub fn highest(&self, segment: SnapshotSegment) -> Option { - match segment { - SnapshotSegment::Headers => self.headers, - SnapshotSegment::Transactions => self.transactions, - SnapshotSegment::Receipts => self.receipts, - } - } - - /// Returns a mutable reference to a snapshot segment - pub fn as_mut(&mut self, segment: SnapshotSegment) -> &mut Option { - match segment { - SnapshotSegment::Headers => &mut self.headers, - SnapshotSegment::Transactions => &mut self.transactions, - SnapshotSegment::Receipts => &mut self.receipts, - } - } -} - -/// Each snapshot has a fixed number of blocks. This gives out the range where the requested block -/// is positioned. Used for segment filename. -pub fn find_fixed_range(block: BlockNumber) -> SegmentRangeInclusive { - let start = (block / BLOCKS_PER_SNAPSHOT) * BLOCKS_PER_SNAPSHOT; - SegmentRangeInclusive::new(start, start + BLOCKS_PER_SNAPSHOT - 1) -} diff --git a/crates/primitives/src/stage/id.rs b/crates/primitives/src/stage/id.rs index 9fdb8454830e..df92bd112c71 100644 --- a/crates/primitives/src/stage/id.rs +++ b/crates/primitives/src/stage/id.rs @@ -3,8 +3,8 @@ /// For custom stages, use [`StageId::Other`] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum StageId { - /// Snapshot stage in the process. - Snapshot, + /// Static File stage in the process. + StaticFile, /// Header stage in the process. Headers, /// Bodies stage in the process. @@ -36,7 +36,7 @@ pub enum StageId { impl StageId { /// All supported Stages pub const ALL: [StageId; 13] = [ - StageId::Snapshot, + StageId::StaticFile, StageId::Headers, StageId::Bodies, StageId::SenderRecovery, @@ -54,7 +54,7 @@ impl StageId { /// Return stage id formatted as string. pub fn as_str(&self) -> &str { match self { - StageId::Snapshot => "Snapshot", + StageId::StaticFile => "StaticFile", StageId::Headers => "Headers", StageId::Bodies => "Bodies", StageId::SenderRecovery => "SenderRecovery", @@ -94,7 +94,7 @@ mod tests { #[test] fn stage_id_as_string() { - assert_eq!(StageId::Snapshot.to_string(), "Snapshot"); + assert_eq!(StageId::StaticFile.to_string(), "StaticFile"); assert_eq!(StageId::Headers.to_string(), "Headers"); assert_eq!(StageId::Bodies.to_string(), "Bodies"); assert_eq!(StageId::SenderRecovery.to_string(), "SenderRecovery"); diff --git a/crates/primitives/src/snapshot/compression.rs b/crates/primitives/src/static_file/compression.rs similarity index 86% rename from crates/primitives/src/snapshot/compression.rs rename to crates/primitives/src/static_file/compression.rs index 2d5599c2cda9..f1a64a501403 100644 --- a/crates/primitives/src/snapshot/compression.rs +++ b/crates/primitives/src/static_file/compression.rs @@ -1,6 +1,6 @@ use strum::AsRefStr; -/// Snapshot compression types. +/// Static File compression types. #[derive(Debug, Copy, Clone, Default, AsRefStr)] #[cfg_attr(feature = "clap", derive(clap::ValueEnum))] pub enum Compression { @@ -13,7 +13,7 @@ pub enum Compression { /// Zstandard (Zstd) compression algorithm with a dictionary. #[strum(serialize = "zstd-dict")] ZstdWithDictionary, - /// No compression, uncompressed snapshot. + /// No compression. #[strum(serialize = "uncompressed")] #[default] Uncompressed, diff --git a/crates/primitives/src/snapshot/filters.rs b/crates/primitives/src/static_file/filters.rs similarity index 71% rename from crates/primitives/src/snapshot/filters.rs rename to crates/primitives/src/static_file/filters.rs index 3443d474706e..cc844468e545 100644 --- a/crates/primitives/src/snapshot/filters.rs +++ b/crates/primitives/src/static_file/filters.rs @@ -1,16 +1,16 @@ use strum::AsRefStr; #[derive(Debug, Copy, Clone)] -/// Snapshot filters. +/// Static File filters. pub enum Filters { - /// Snapshot uses filters with [InclusionFilter] and [PerfectHashingFunction]. + /// Static File uses filters with [InclusionFilter] and [PerfectHashingFunction]. WithFilters(InclusionFilter, PerfectHashingFunction), - /// Snapshot doesn't use any filters. + /// Static File doesn't use any filters. WithoutFilters, } impl Filters { - /// Returns `true` if snapshot uses filters. + /// Returns `true` if static file uses filters. pub const fn has_filters(&self) -> bool { matches!(self, Self::WithFilters(_, _)) } @@ -18,7 +18,7 @@ impl Filters { #[derive(Debug, Copy, Clone, AsRefStr)] #[cfg_attr(feature = "clap", derive(clap::ValueEnum))] -/// Snapshot inclusion filter. Also see [Filters]. +/// Static File inclusion filter. Also see [Filters]. pub enum InclusionFilter { #[strum(serialize = "cuckoo")] /// Cuckoo filter @@ -27,7 +27,7 @@ pub enum InclusionFilter { #[derive(Debug, Copy, Clone, AsRefStr)] #[cfg_attr(feature = "clap", derive(clap::ValueEnum))] -/// Snapshot perfect hashing function. Also see [Filters]. +/// Static File perfect hashing function. Also see [Filters]. pub enum PerfectHashingFunction { #[strum(serialize = "fmph")] /// Fingerprint-Based Minimal Perfect Hash Function diff --git a/crates/primitives/src/static_file/mod.rs b/crates/primitives/src/static_file/mod.rs new file mode 100644 index 000000000000..fe15bd1c759a --- /dev/null +++ b/crates/primitives/src/static_file/mod.rs @@ -0,0 +1,54 @@ +//! StaticFile primitives. + +mod compression; +mod filters; +mod segment; + +use alloy_primitives::BlockNumber; +pub use compression::Compression; +pub use filters::{Filters, InclusionFilter, PerfectHashingFunction}; +pub use segment::{SegmentConfig, SegmentHeader, SegmentRangeInclusive, StaticFileSegment}; + +/// Default static file block count. +pub const BLOCKS_PER_STATIC_FILE: u64 = 500_000; + +/// Highest static file block numbers, per data part. +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub struct HighestStaticFiles { + /// Highest static file block of headers, inclusive. + /// If [`None`], no static file is available. + pub headers: Option, + /// Highest static file block of receipts, inclusive. + /// If [`None`], no static file is available. + pub receipts: Option, + /// Highest static file block of transactions, inclusive. + /// If [`None`], no static file is available. + pub transactions: Option, +} + +impl HighestStaticFiles { + /// Returns the highest static file if it exists for a segment + pub fn highest(&self, segment: StaticFileSegment) -> Option { + match segment { + StaticFileSegment::Headers => self.headers, + StaticFileSegment::Transactions => self.transactions, + StaticFileSegment::Receipts => self.receipts, + } + } + + /// Returns a mutable reference to a static file segment + pub fn as_mut(&mut self, segment: StaticFileSegment) -> &mut Option { + match segment { + StaticFileSegment::Headers => &mut self.headers, + StaticFileSegment::Transactions => &mut self.transactions, + StaticFileSegment::Receipts => &mut self.receipts, + } + } +} + +/// Each static file has a fixed number of blocks. This gives out the range where the requested +/// block is positioned. Used for segment filename. +pub fn find_fixed_range(block: BlockNumber) -> SegmentRangeInclusive { + let start = (block / BLOCKS_PER_STATIC_FILE) * BLOCKS_PER_STATIC_FILE; + SegmentRangeInclusive::new(start, start + BLOCKS_PER_STATIC_FILE - 1) +} diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/static_file/segment.rs similarity index 77% rename from crates/primitives/src/snapshot/segment.rs rename to crates/primitives/src/static_file/segment.rs index 6f8b0e316390..ed7d5353bc4e 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/static_file/segment.rs @@ -1,5 +1,5 @@ use crate::{ - snapshot::{Compression, Filters, InclusionFilter}, + static_file::{Compression, Filters, InclusionFilter}, BlockNumber, TxNumber, }; use derive_more::Display; @@ -24,26 +24,26 @@ use strum::{AsRefStr, EnumIter, EnumString}; Display, )] #[cfg_attr(feature = "clap", derive(clap::ValueEnum))] -/// Segment of the data that can be snapshotted. -pub enum SnapshotSegment { +/// Segment of the data that can be moved to static files. +pub enum StaticFileSegment { #[strum(serialize = "headers")] - /// Snapshot segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables. + /// Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables. Headers, #[strum(serialize = "transactions")] - /// Snapshot segment responsible for the `Transactions` table. + /// Static File segment responsible for the `Transactions` table. Transactions, #[strum(serialize = "receipts")] - /// Snapshot segment responsible for the `Receipts` table. + /// Static File segment responsible for the `Receipts` table. Receipts, } -impl SnapshotSegment { +impl StaticFileSegment { /// Returns the segment as a string. pub const fn as_str(&self) -> &'static str { match self { - SnapshotSegment::Headers => "headers", - SnapshotSegment::Transactions => "transactions", - SnapshotSegment::Receipts => "receipts", + StaticFileSegment::Headers => "headers", + StaticFileSegment::Transactions => "transactions", + StaticFileSegment::Receipts => "receipts", } } @@ -58,18 +58,18 @@ impl SnapshotSegment { }; match self { - SnapshotSegment::Headers => default_config, - SnapshotSegment::Transactions => default_config, - SnapshotSegment::Receipts => default_config, + StaticFileSegment::Headers => default_config, + StaticFileSegment::Transactions => default_config, + StaticFileSegment::Receipts => default_config, } } /// Returns the number of columns for the segment pub const fn columns(&self) -> usize { match self { - SnapshotSegment::Headers => 3, - SnapshotSegment::Transactions => 1, - SnapshotSegment::Receipts => 1, + StaticFileSegment::Headers => 3, + StaticFileSegment::Transactions => 1, + StaticFileSegment::Receipts => 1, } } @@ -77,7 +77,7 @@ impl SnapshotSegment { pub fn filename(&self, block_range: &SegmentRangeInclusive) -> String { // ATTENTION: if changing the name format, be sure to reflect those changes in // [`Self::parse_filename`]. - format!("snapshot_{}_{}_{}", self.as_ref(), block_range.start(), block_range.end()) + format!("static_file_{}_{}_{}", self.as_ref(), block_range.start(), block_range.end()) } /// Returns file name for the provided segment and range, alongisde filters, compression. @@ -101,11 +101,11 @@ impl SnapshotSegment { format!("{prefix}_{}_{}", filters_name, compression.as_ref()) } - /// Parses a filename into a `SnapshotSegment` and its expected block range. + /// Parses a filename into a `StaticFileSegment` and its expected block range. /// /// The filename is expected to follow the format: - /// "snapshot_{segment}_{block_start}_{block_end}". This function checks - /// for the correct prefix ("snapshot"), and then parses the segment and the inclusive + /// "static_file_{segment}_{block_start}_{block_end}". This function checks + /// for the correct prefix ("static_file"), and then parses the segment and the inclusive /// ranges for blocks. It ensures that the start of each range is less than or equal to the /// end. /// @@ -119,7 +119,7 @@ impl SnapshotSegment { /// Any changes in the filename format in `filename` should be reflected here. pub fn parse_filename(name: &str) -> Option<(Self, SegmentRangeInclusive)> { let mut parts = name.split('_'); - if parts.next() != Some("snapshot") { + if !(parts.next() == Some("static") && parts.next() == Some("file")) { return None } @@ -133,26 +133,26 @@ impl SnapshotSegment { Some((segment, SegmentRangeInclusive::new(block_start, block_end))) } - /// Returns `true` if the segment is `SnapshotSegment::Headers`. + /// Returns `true` if the segment is `StaticFileSegment::Headers`. pub fn is_headers(&self) -> bool { - matches!(self, SnapshotSegment::Headers) + matches!(self, StaticFileSegment::Headers) } } /// A segment header that contains information common to all segments. Used for storage. #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Hash, Clone)] pub struct SegmentHeader { - /// Defines the expected block range for a snapshot segment. This attribute is crucial for + /// Defines the expected block range for a static file segment. This attribute is crucial for /// scenarios where the file contains no data, allowing for a representation beyond a /// simple `start..=start` range. It ensures clarity in differentiating between an empty file /// and a file with a single block numbered 0. expected_block_range: SegmentRangeInclusive, - /// Block range of data on the snapshot segment + /// Block range of data on the static file segment block_range: Option, - /// Transaction range of data of the snapshot segment + /// Transaction range of data of the static file segment tx_range: Option, /// Segment type - segment: SnapshotSegment, + segment: StaticFileSegment, } impl SegmentHeader { @@ -161,13 +161,13 @@ impl SegmentHeader { expected_block_range: SegmentRangeInclusive, block_range: Option, tx_range: Option, - segment: SnapshotSegment, + segment: StaticFileSegment, ) -> Self { Self { expected_block_range, block_range, tx_range, segment } } - /// Returns the snapshot segment kind. - pub fn segment(&self) -> SnapshotSegment { + /// Returns the static file segment kind. + pub fn segment(&self) -> StaticFileSegment { self.segment } @@ -238,8 +238,8 @@ impl SegmentHeader { /// Increments tx end range depending on segment pub fn increment_tx(&mut self) { match self.segment { - SnapshotSegment::Headers => (), - SnapshotSegment::Transactions | SnapshotSegment::Receipts => { + StaticFileSegment::Headers => (), + StaticFileSegment::Transactions | StaticFileSegment::Receipts => { if let Some(tx_range) = &mut self.tx_range { tx_range.end += 1; } else { @@ -252,7 +252,7 @@ impl SegmentHeader { /// Removes `num` elements from end of tx or block range. pub fn prune(&mut self, num: u64) { match self.segment { - SnapshotSegment::Headers => { + StaticFileSegment::Headers => { if let Some(range) = &mut self.block_range { if num > range.end { self.block_range = None; @@ -261,7 +261,7 @@ impl SegmentHeader { } }; } - SnapshotSegment::Transactions | SnapshotSegment::Receipts => { + StaticFileSegment::Transactions | StaticFileSegment::Receipts => { if let Some(range) = &mut self.tx_range { if num > range.end { self.tx_range = None; @@ -296,8 +296,8 @@ impl SegmentHeader { /// Returns the row offset which depends on whether the segment is block or transaction based. pub fn start(&self) -> Option { match self.segment { - SnapshotSegment::Headers => self.block_start(), - SnapshotSegment::Transactions | SnapshotSegment::Receipts => self.tx_start(), + StaticFileSegment::Headers => self.block_start(), + StaticFileSegment::Transactions | StaticFileSegment::Receipts => self.tx_start(), } } } @@ -368,47 +368,47 @@ mod tests { #[test] fn test_filename() { let test_vectors = [ - (SnapshotSegment::Headers, 2..=30, "snapshot_headers_2_30", None), - (SnapshotSegment::Receipts, 30..=300, "snapshot_receipts_30_300", None), + (StaticFileSegment::Headers, 2..=30, "static_file_headers_2_30", None), + (StaticFileSegment::Receipts, 30..=300, "static_file_receipts_30_300", None), ( - SnapshotSegment::Transactions, + StaticFileSegment::Transactions, 1_123_233..=11_223_233, - "snapshot_transactions_1123233_11223233", + "static_file_transactions_1123233_11223233", None, ), ( - SnapshotSegment::Headers, + StaticFileSegment::Headers, 2..=30, - "snapshot_headers_2_30_cuckoo-fmph_lz4", + "static_file_headers_2_30_cuckoo-fmph_lz4", Some(( Compression::Lz4, Filters::WithFilters( InclusionFilter::Cuckoo, - crate::snapshot::PerfectHashingFunction::Fmph, + crate::static_file::PerfectHashingFunction::Fmph, ), )), ), ( - SnapshotSegment::Headers, + StaticFileSegment::Headers, 2..=30, - "snapshot_headers_2_30_cuckoo-fmph_zstd", + "static_file_headers_2_30_cuckoo-fmph_zstd", Some(( Compression::Zstd, Filters::WithFilters( InclusionFilter::Cuckoo, - crate::snapshot::PerfectHashingFunction::Fmph, + crate::static_file::PerfectHashingFunction::Fmph, ), )), ), ( - SnapshotSegment::Headers, + StaticFileSegment::Headers, 2..=30, - "snapshot_headers_2_30_cuckoo-fmph_zstd-dict", + "static_file_headers_2_30_cuckoo-fmph_zstd-dict", Some(( Compression::ZstdWithDictionary, Filters::WithFilters( InclusionFilter::Cuckoo, - crate::snapshot::PerfectHashingFunction::Fmph, + crate::static_file::PerfectHashingFunction::Fmph, ), )), ), @@ -425,10 +425,10 @@ mod tests { assert_eq!(segment.filename(&block_range), filename); } - assert_eq!(SnapshotSegment::parse_filename(filename), Some((segment, block_range))); + assert_eq!(StaticFileSegment::parse_filename(filename), Some((segment, block_range))); } - assert_eq!(SnapshotSegment::parse_filename("snapshot_headers_2"), None); - assert_eq!(SnapshotSegment::parse_filename("snapshot_headers_"), None); + assert_eq!(StaticFileSegment::parse_filename("static_file_headers_2"), None); + assert_eq!(StaticFileSegment::parse_filename("static_file_headers_"), None); } } diff --git a/crates/prune/Cargo.toml b/crates/prune/Cargo.toml index 750cd84d6bf0..6699c567bc29 100644 --- a/crates/prune/Cargo.toml +++ b/crates/prune/Cargo.toml @@ -17,7 +17,7 @@ reth-primitives.workspace = true reth-db.workspace = true reth-provider.workspace = true reth-interfaces.workspace = true -reth-snapshot.workspace = true +reth-static-file.workspace = true reth-tokio-util.workspace = true reth-config.workspace = true diff --git a/crates/prune/src/pruner.rs b/crates/prune/src/pruner.rs index eb9ac9505575..09ffba740f4b 100644 --- a/crates/prune/src/pruner.rs +++ b/crates/prune/src/pruner.rs @@ -7,7 +7,7 @@ use crate::{ }; use reth_db::database::Database; use reth_primitives::{ - BlockNumber, PruneMode, PruneProgress, PrunePurpose, PruneSegment, SnapshotSegment, + BlockNumber, PruneMode, PruneProgress, PrunePurpose, PruneSegment, StaticFileSegment, }; use reth_provider::{DatabaseProviderRW, ProviderFactory, PruneCheckpointReader}; use reth_tokio_util::EventListeners; @@ -76,7 +76,7 @@ impl Pruner { self.previous_tip_block_number = Some(tip_block_number); debug!(target: "pruner", %tip_block_number, "Nothing to prune yet"); - return Ok(PruneProgress::Finished) + return Ok(PruneProgress::Finished); } debug!(target: "pruner", %tip_block_number, "Pruner started"); @@ -125,7 +125,7 @@ impl Pruner { } /// Prunes the segments that the [Pruner] was initialized with, and the segments that needs to - /// be pruned according to the highest snapshots. + /// be pruned according to the highest static_files. /// /// Returns [PrunerStats], `delete_limit` that remained after pruning all segments, and /// [PruneProgress]. @@ -135,10 +135,10 @@ impl Pruner { tip_block_number: BlockNumber, mut delete_limit: usize, ) -> Result<(PrunerStats, usize, PruneProgress), PrunerError> { - let snapshot_segments = self.snapshot_segments(); - let segments = snapshot_segments + let static_file_segments = self.static_file_segments(); + let segments = static_file_segments .iter() - .map(|segment| (segment, PrunePurpose::Snapshot)) + .map(|segment| (segment, PrunePurpose::StaticFile)) .chain(self.segments.iter().map(|segment| (segment, PrunePurpose::User))); let mut done = true; @@ -146,7 +146,7 @@ impl Pruner { for (segment, purpose) in segments { if delete_limit == 0 { - break + break; } if let Some((to_block, prune_mode)) = segment @@ -196,28 +196,29 @@ impl Pruner { Ok((stats, delete_limit, PruneProgress::from_done(done))) } - /// Returns pre-configured segments that needs to be pruned according to the highest snapshots - /// for [PruneSegment::Transactions], [PruneSegment::Headers] and [PruneSegment::Receipts]. - fn snapshot_segments(&self) -> Vec>> { + /// Returns pre-configured segments that needs to be pruned according to the highest + /// static_files for [PruneSegment::Transactions], [PruneSegment::Headers] and + /// [PruneSegment::Receipts]. + fn static_file_segments(&self) -> Vec>> { let mut segments = Vec::>>::new(); - let snapshot_provider = self.provider_factory.snapshot_provider(); + let static_file_provider = self.provider_factory.static_file_provider(); if let Some(to_block) = - snapshot_provider.get_highest_snapshot_block(SnapshotSegment::Transactions) + static_file_provider.get_highest_static_file_block(StaticFileSegment::Transactions) { segments .push(Box::new(segments::Transactions::new(PruneMode::before_inclusive(to_block)))) } if let Some(to_block) = - snapshot_provider.get_highest_snapshot_block(SnapshotSegment::Headers) + static_file_provider.get_highest_static_file_block(StaticFileSegment::Headers) { segments.push(Box::new(segments::Headers::new(PruneMode::before_inclusive(to_block)))) } if let Some(to_block) = - snapshot_provider.get_highest_snapshot_block(SnapshotSegment::Receipts) + static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts) { segments.push(Box::new(segments::Receipts::new(PruneMode::before_inclusive(to_block)))) } @@ -251,7 +252,7 @@ impl Pruner { #[cfg(test)] mod tests { use crate::Pruner; - use reth_db::test_utils::{create_test_rw_db, create_test_snapshots_dir}; + use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; use reth_primitives::MAINNET; use reth_provider::ProviderFactory; @@ -259,8 +260,8 @@ mod tests { fn is_pruning_needed() { let db = create_test_rw_db(); let provider_factory = - ProviderFactory::new(db, MAINNET.clone(), create_test_snapshots_dir()) - .expect("create provide factory with snapshots"); + ProviderFactory::new(db, MAINNET.clone(), create_test_static_files_dir()) + .expect("create provide factory with static_files"); let mut pruner = Pruner::new(provider_factory, vec![], 5, 0, 5); // No last pruned block number was set before diff --git a/crates/snapshot/README.md b/crates/snapshot/README.md deleted file mode 100644 index 6056bbf9f0a0..000000000000 --- a/crates/snapshot/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# Snapshot - -## Overview - -Data that has reached a finalized state and won't undergo further changes (essentially frozen) should be read without concerns of modification. This makes it unsuitable for traditional databases. - -This crate aims to copy this data from the current database to multiple static files, aggregated by block ranges. At every 500_000th block new static files are created. - -Below are two diagrams illustrating the processes of creating static files (custom format: `NippyJar`) and querying them. A glossary is also provided to explain the different (linked) components involved in these processes. - -
- Creation diagram (Snapshotter) - -```mermaid -graph TD; - I("BLOCK_HEIGHT % 500_000 == 0")--triggers-->SP(Snapshotter) - SP --> |triggers| SH["create_snapshot(block_range, SnapshotSegment::Headers)"] - SP --> |triggers| ST["create_snapshot(block_range, SnapshotSegment::Transactions)"] - SP --> |triggers| SR["create_snapshot(block_range, SnapshotSegment::Receipts)"] - SP --> |triggers| ETC["create_snapshot(block_range, ...)"] - SH --> CS["create_snapshot::< T >(DatabaseCursor)"] - ST --> CS - SR --> CS - ETC --> CS - CS --> |create| IF(NippyJar::InclusionFilters) - CS -- iterates --> DC(DatabaseCursor) -->HN{HasNext} - HN --> |true| NJC(NippyJar::Compression) - NJC --> HN - NJC --store--> NJ - HN --> |false| NJ - IF --store--> NJ(NippyJar) - NJ --freeze--> F(File) - F--"on success"--> SP1(Snapshotter) - SP1 --"sends BLOCK_HEIGHT"--> HST(HighestSnapshotTracker) - HST --"read by"-->Pruner - HST --"read by"-->DatabaseProvider - HST --"read by"-->SnapsotProvider - HST --"read by"-->ProviderFactory - -``` -
- - -
- Query diagram (Provider) - -```mermaid -graph TD; - RPC-->P - P("Provider::header(block_number)")-->PF(ProviderFactory) - PF--shares-->SP1("Arc(SnapshotProvider)") - SP1--shares-->PD(DatabaseProvider) - PF--creates-->PD - PD--check `HighestSnapshotTracker`-->PD - PD-->DC1{block_number
>
highest snapshot block} - DC1 --> |true| PD1("DatabaseProvider::header(block_number)") - DC1 --> |false| ASP("SnapshotProvider::header(block_number)") - PD1 --> MDBX - ASP --find correct jar and creates--> JP("SnapshotJarProvider::header(block_number)") - JP --"creates"-->SC(SnapshotCursor) - SC --".get_one< HeaderMask< Header > >(number)"--->NJC("NippyJarCursor") - NJC--".row_by_number(row_index, mask)"-->NJ[NippyJar] - NJ--"&[u8]"-->NJC - NJC--"&[u8]"-->SC - SC--"Header"--> JP - JP--"Header"--> ASP -``` -
- - -### Glossary -In descending order of abstraction hierarchy: - -[`Snapshotter`](../../crates/snapshot/src/snapshotter.rs#L20): A `reth` background service that **copies** data from the database to new snapshot files when the block height reaches a certain threshold (e.g., `500_000th`). Upon completion, it dispatches a notification about the higher snapshotted block to `HighestSnapshotTracker` channel. **It DOES NOT remove data from the database.** - -[`HighestSnapshotTracker`](../../crates/snapshot/src/snapshotter.rs#L22): A channel utilized by `Snapshotter` to announce the newest snapshot block to all components with a listener: `Pruner` (to know which additional tables can be pruned) and `DatabaseProvider` (to know which data can be queried from the snapshots). - -[`SnapshotProvider`](../../crates/storage/provider/src/providers/snapshot/manager.rs#L15) A provider similar to `DatabaseProvider`, **managing all existing snapshot files** and selecting the optimal one (by range and segment type) to fulfill a request. **A single instance is shared across all components and should be instantiated only once within `ProviderFactory`**. An immutable reference is given everytime `ProviderFactory` creates a new `DatabaseProvider`. - -[`SnapshotJarProvider`](../../crates/storage/provider/src/providers/snapshot/jar.rs#L42) A provider similar to `DatabaseProvider` that provides access to a **single snapshot file**. - -[`SnapshotCursor`](../../crates/storage/db/src/snapshot/cursor.rs#L12) An elevated abstraction of `NippyJarCursor` for simplified access. It associates the bitmasks with type decoding. For instance, `cursor.get_two::>(tx_number)` would yield `Tx` and `Signature`, eliminating the need to manage masks or invoke a decoder/decompressor. - -[`SnapshotSegment`](../../crates/primitives/src/snapshot/segment.rs#L10) Each snapshot file only contains data of a specific segment, e.g., `Headers`, `Transactions`, or `Receipts`. - -[`NippyJarCursor`](../../crates/storage/nippy-jar/src/cursor.rs#L12) Accessor of data in a `NippyJar` file. It enables queries either by row number (e.g., block number 1) or by a predefined key not part of the file (e.g., transaction hashes). If a file has multiple columns (e.g., `Tx | TxSender | Signature`), and one wishes to access only one of the column values, this can be accomplished by bitmasks. (e.g., for `TxSender`, the mask would be `0b010`). - -[`NippyJar`](../../crates/storage/nippy-jar/src/lib.rs#57) A create-only file format. No data can be appended after creation. It supports multiple columns, compression (e.g., Zstd (with and without dictionaries), lz4, uncompressed) and inclusion filters (e.g., cuckoo filter: `is hash X part of this dataset`). Snapshots are organized by block ranges. (e.g., `TransactionSnapshot_499_999.jar` contains a transaction per row for all transactions from block `0` to block `499_999`). For more check the struct documentation. diff --git a/crates/snapshot/src/event.rs b/crates/snapshot/src/event.rs deleted file mode 100644 index b9e45a11118c..000000000000 --- a/crates/snapshot/src/event.rs +++ /dev/null @@ -1,14 +0,0 @@ -use crate::SnapshotTargets; -use std::time::Duration; - -/// An event emitted by a [Snapshotter][crate::Snapshotter]. -#[derive(Debug, PartialEq, Eq, Clone)] -pub enum SnapshotterEvent { - /// Emitted when snapshotter finished running. - Finished { - /// Targets that were snapshotted - targets: SnapshotTargets, - /// Time it took to run the snapshotter - elapsed: Duration, - }, -} diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs deleted file mode 100644 index bde716dc6767..000000000000 --- a/crates/snapshot/src/snapshotter.rs +++ /dev/null @@ -1,311 +0,0 @@ -//! Support for snapshotting. - -use crate::{segments, segments::Segment, SnapshotterEvent}; -use rayon::prelude::*; -use reth_db::database::Database; -use reth_interfaces::RethResult; -use reth_primitives::{snapshot::HighestSnapshots, BlockNumber, PruneModes}; -use reth_provider::{ - providers::{SnapshotProvider, SnapshotWriter}, - ProviderFactory, -}; -use reth_tokio_util::EventListeners; -use std::{ops::RangeInclusive, time::Instant}; -use tokio_stream::wrappers::UnboundedReceiverStream; -use tracing::{debug, trace}; - -/// Result of [Snapshotter::run] execution. -pub type SnapshotterResult = RethResult; - -/// The snapshotter type itself with the result of [Snapshotter::run] -pub type SnapshotterWithResult = (Snapshotter, SnapshotterResult); - -/// Snapshotting routine. See [Snapshotter::run] for more detailed description. -#[derive(Debug, Clone)] -pub struct Snapshotter { - /// Provider factory - provider_factory: ProviderFactory, - /// Snapshot provider - snapshot_provider: SnapshotProvider, - /// Pruning configuration for every part of the data that can be pruned. Set by user, and - /// needed in [Snapshotter] to prevent snapshotting the prunable data. - /// See [Snapshotter::get_snapshot_targets]. - prune_modes: PruneModes, - listeners: EventListeners, -} - -/// Snapshot targets, per data part, measured in [`BlockNumber`]. -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct SnapshotTargets { - headers: Option>, - receipts: Option>, - transactions: Option>, -} - -impl SnapshotTargets { - /// Returns `true` if any of the targets are [Some]. - pub fn any(&self) -> bool { - self.headers.is_some() || self.receipts.is_some() || self.transactions.is_some() - } - - // Returns `true` if all targets are either [`None`] or has beginning of the range equal to the - // highest snapshot. - fn is_contiguous_to_highest_snapshots(&self, snapshots: HighestSnapshots) -> bool { - [ - (self.headers.as_ref(), snapshots.headers), - (self.receipts.as_ref(), snapshots.receipts), - (self.transactions.as_ref(), snapshots.transactions), - ] - .iter() - .all(|(target_block_range, highest_snapshotted_block)| { - target_block_range.map_or(true, |target_block_range| { - *target_block_range.start() == - highest_snapshotted_block - .map_or(0, |highest_snapshotted_block| highest_snapshotted_block + 1) - }) - }) - } -} - -impl Snapshotter { - /// Creates a new [Snapshotter]. - pub fn new( - provider_factory: ProviderFactory, - snapshot_provider: SnapshotProvider, - prune_modes: PruneModes, - ) -> Self { - Self { provider_factory, snapshot_provider, prune_modes, listeners: Default::default() } - } - - /// Listen for events on the snapshotter. - pub fn events(&mut self) -> UnboundedReceiverStream { - self.listeners.new_listener() - } - - /// Run the snapshotter. - /// - /// For each [Some] target in [SnapshotTargets], initializes a corresponding [Segment] and runs - /// it with the provided block range using [SnapshotProvider] and a read-only database - /// transaction from [ProviderFactory]. All segments are run in parallel. - /// - /// NOTE: it doesn't delete the data from database, and the actual deleting (aka pruning) logic - /// lives in the `prune` crate. - pub fn run(&mut self, targets: SnapshotTargets) -> SnapshotterResult { - debug_assert!(targets - .is_contiguous_to_highest_snapshots(self.snapshot_provider.get_highest_snapshots())); - - debug!(target: "snapshot", ?targets, "Snapshotter started"); - let start = Instant::now(); - - let mut segments = Vec::<(Box>, RangeInclusive)>::new(); - - if let Some(block_range) = targets.transactions.clone() { - segments.push((Box::new(segments::Transactions), block_range)); - } - if let Some(block_range) = targets.headers.clone() { - segments.push((Box::new(segments::Headers), block_range)); - } - if let Some(block_range) = targets.receipts.clone() { - segments.push((Box::new(segments::Receipts), block_range)); - } - - segments.par_iter().try_for_each(|(segment, block_range)| -> RethResult<()> { - debug!(target: "snapshot", segment = %segment.segment(), ?block_range, "Snapshotting segment"); - let start = Instant::now(); - - // Create a new database transaction on every segment to prevent long-lived read-only - // transactions - let provider = self.provider_factory.provider()?.disable_long_read_transaction_safety(); - segment.snapshot(provider, self.snapshot_provider.clone(), block_range.clone())?; - - let elapsed = start.elapsed(); // TODO(alexey): track in metrics - debug!(target: "snapshot", segment = %segment.segment(), ?block_range, ?elapsed, "Finished snapshotting segment"); - - Ok(()) - })?; - - self.snapshot_provider.commit()?; - for (segment, block_range) in segments { - self.snapshot_provider.update_index(segment.segment(), Some(*block_range.end()))?; - } - - let elapsed = start.elapsed(); // TODO(alexey): track in metrics - debug!(target: "snapshot", ?targets, ?elapsed, "Snapshotter finished"); - - self.listeners.notify(SnapshotterEvent::Finished { targets: targets.clone(), elapsed }); - - Ok(targets) - } - - /// Returns a snapshot targets at the provided finalized block numbers per segment. - /// The target is determined by the check against highest snapshots using - /// [SnapshotProvider::get_highest_snapshots]. - pub fn get_snapshot_targets( - &self, - finalized_block_numbers: HighestSnapshots, - ) -> RethResult { - let highest_snapshots = self.snapshot_provider.get_highest_snapshots(); - - let targets = SnapshotTargets { - headers: finalized_block_numbers.headers.and_then(|finalized_block_number| { - self.get_snapshot_target(highest_snapshots.headers, finalized_block_number) - }), - // Snapshot receipts only if they're not pruned according to the user configuration - receipts: if self.prune_modes.receipts.is_none() && - self.prune_modes.receipts_log_filter.is_empty() - { - finalized_block_numbers.receipts.and_then(|finalized_block_number| { - self.get_snapshot_target(highest_snapshots.receipts, finalized_block_number) - }) - } else { - None - }, - transactions: finalized_block_numbers.transactions.and_then(|finalized_block_number| { - self.get_snapshot_target(highest_snapshots.transactions, finalized_block_number) - }), - }; - - trace!( - target: "snapshot", - ?finalized_block_numbers, - ?highest_snapshots, - ?targets, - any = %targets.any(), - "Snapshot targets" - ); - - Ok(targets) - } - - fn get_snapshot_target( - &self, - highest_snapshot: Option, - finalized_block_number: BlockNumber, - ) -> Option> { - let range = highest_snapshot.map_or(0, |block| block + 1)..=finalized_block_number; - (!range.is_empty()).then_some(range) - } -} - -#[cfg(test)] -mod tests { - use crate::{snapshotter::SnapshotTargets, Snapshotter}; - use assert_matches::assert_matches; - use reth_db::{database::Database, transaction::DbTx}; - use reth_interfaces::{ - provider::ProviderError, - test_utils::{ - generators, - generators::{random_block_range, random_receipt}, - }, - RethError, - }; - use reth_primitives::{snapshot::HighestSnapshots, PruneModes, SnapshotSegment, B256, U256}; - use reth_provider::providers::SnapshotWriter; - use reth_stages::test_utils::{StorageKind, TestStageDB}; - - #[test] - fn run() { - let mut rng = generators::rng(); - - let db = TestStageDB::default(); - - let blocks = random_block_range(&mut rng, 0..=3, B256::ZERO, 2..3); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); - // Unwind headers from snapshots and manually insert them into the database, so we're able - // to check that snapshotter works - db.factory - .snapshot_provider() - .latest_writer(SnapshotSegment::Headers) - .expect("get snapshot writer for headers") - .prune_headers(blocks.len() as u64) - .expect("prune headers"); - let tx = db.factory.db_ref().tx_mut().expect("init tx"); - blocks.iter().for_each(|block| { - TestStageDB::insert_header(None, &tx, &block.header, U256::ZERO) - .expect("insert block header"); - }); - tx.commit().expect("commit tx"); - - let mut receipts = Vec::new(); - for block in &blocks { - for transaction in &block.body { - receipts - .push((receipts.len() as u64, random_receipt(&mut rng, transaction, Some(0)))); - } - } - db.insert_receipts(receipts).expect("insert receipts"); - - let provider_factory = db.factory; - let snapshot_provider = provider_factory.snapshot_provider(); - - let mut snapshotter = - Snapshotter::new(provider_factory, snapshot_provider.clone(), PruneModes::default()); - - let targets = snapshotter - .get_snapshot_targets(HighestSnapshots { - headers: Some(1), - receipts: Some(1), - transactions: Some(1), - }) - .expect("get snapshot targets"); - assert_eq!( - targets, - SnapshotTargets { - headers: Some(0..=1), - receipts: Some(0..=1), - transactions: Some(0..=1) - } - ); - assert_matches!(snapshotter.run(targets), Ok(_)); - assert_eq!( - snapshot_provider.get_highest_snapshots(), - HighestSnapshots { headers: Some(1), receipts: Some(1), transactions: Some(1) } - ); - - let targets = snapshotter - .get_snapshot_targets(HighestSnapshots { - headers: Some(3), - receipts: Some(3), - transactions: Some(3), - }) - .expect("get snapshot targets"); - assert_eq!( - targets, - SnapshotTargets { - headers: Some(2..=3), - receipts: Some(2..=3), - transactions: Some(2..=3) - } - ); - assert_matches!(snapshotter.run(targets), Ok(_)); - assert_eq!( - snapshot_provider.get_highest_snapshots(), - HighestSnapshots { headers: Some(3), receipts: Some(3), transactions: Some(3) } - ); - - let targets = snapshotter - .get_snapshot_targets(HighestSnapshots { - headers: Some(4), - receipts: Some(4), - transactions: Some(4), - }) - .expect("get snapshot targets"); - assert_eq!( - targets, - SnapshotTargets { - headers: Some(4..=4), - receipts: Some(4..=4), - transactions: Some(4..=4) - } - ); - assert_matches!( - snapshotter.run(targets), - Err(RethError::Provider(ProviderError::BlockBodyIndicesNotFound(4))) - ); - assert_eq!( - snapshot_provider.get_highest_snapshots(), - HighestSnapshots { headers: Some(3), receipts: Some(3), transactions: Some(3) } - ); - } -} diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index 9226ab441050..24d90a70084f 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -27,7 +27,7 @@ reth-provider.workspace = true reth-trie.workspace = true reth-tokio-util.workspace = true reth-etl.workspace = true -reth-snapshot.workspace = true +reth-static-file.workspace = true # revm revm.workspace = true diff --git a/crates/stages/src/error.rs b/crates/stages/src/error.rs index b06e03b9a418..fa43c88fadc6 100644 --- a/crates/stages/src/error.rs +++ b/crates/stages/src/error.rs @@ -3,7 +3,7 @@ use reth_interfaces::{ consensus, db::DatabaseError as DbError, executor, p2p::error::DownloadError, provider::ProviderError, RethError, }; -use reth_primitives::{BlockNumber, SealedHeader, SnapshotSegment, TxNumber}; +use reth_primitives::{BlockNumber, SealedHeader, StaticFileSegment, TxNumber}; use thiserror::Error; use tokio::sync::mpsc::error::SendError; @@ -76,21 +76,21 @@ pub enum StageError { /// rely on external downloaders #[error("invalid download response: {0}")] Download(#[from] DownloadError), - /// Database is ahead of snapshot data. - #[error("missing snapshot data for block number: {number}", number = block.number)] - MissingSnapshotData { + /// Database is ahead of static file data. + #[error("missing static file data for block number: {number}", number = block.number)] + MissingStaticFileData { /// Starting block with missing data. block: Box, - /// Snapshot segment - segment: SnapshotSegment, + /// Static File segment + segment: StaticFileSegment, }, /// Unrecoverable inconsistency error related to a transaction number in a static file segment. #[error( "inconsistent transaction number for {segment}. db: {database}, static_file: {static_file}" )] InconsistentTxNumber { - /// Snapshot segment where this error was encountered. - segment: SnapshotSegment, + /// Static File segment where this error was encountered. + segment: StaticFileSegment, /// Expected database transaction number. database: TxNumber, /// Expected static file transaction number. @@ -99,8 +99,8 @@ pub enum StageError { /// Unrecoverable inconsistency error related to a block number in a static file segment. #[error("inconsistent block number for {segment}. db: {database}, static_file: {static_file}")] InconsistentBlockNumber { - /// Snapshot segment where this error was encountered. - segment: SnapshotSegment, + /// Static File segment where this error was encountered. + segment: StaticFileSegment, /// Expected database block number. database: BlockNumber, /// Expected static file block number. diff --git a/crates/stages/src/lib.rs b/crates/stages/src/lib.rs index 7b0575a21320..8f90b080429d 100644 --- a/crates/stages/src/lib.rs +++ b/crates/stages/src/lib.rs @@ -26,7 +26,7 @@ //! # use reth_provider::ProviderFactory; //! # use reth_provider::HeaderSyncMode; //! # use reth_provider::test_utils::create_test_provider_factory; -//! # use reth_snapshot::Snapshotter; +//! # use reth_static_file::StaticFileProducer; //! # //! # let chain_spec = MAINNET.clone(); //! # let consensus: Arc = Arc::new(TestConsensus::default()); @@ -42,9 +42,9 @@ //! # ); //! # let (tip_tx, tip_rx) = watch::channel(B256::default()); //! # let executor_factory = EvmProcessorFactory::new(chain_spec.clone(), EthEvmConfig::default()); -//! # let snapshotter = Snapshotter::new( +//! # let static_file_producer = StaticFileProducer::new( //! # provider_factory.clone(), -//! # provider_factory.snapshot_provider(), +//! # provider_factory.static_file_provider(), //! # PruneModes::default() //! # ); //! // Create a pipeline that can fully sync @@ -59,7 +59,7 @@ //! headers_downloader, //! bodies_downloader, //! executor_factory, -//! snapshotter, +//! static_file_producer, //! ) //! .unwrap(), //! ) diff --git a/crates/stages/src/pipeline/mod.rs b/crates/stages/src/pipeline/mod.rs index ebe9b73f499a..9e4359e4dc84 100644 --- a/crates/stages/src/pipeline/mod.rs +++ b/crates/stages/src/pipeline/mod.rs @@ -9,7 +9,7 @@ use reth_primitives::{ BlockNumber, B256, }; use reth_provider::{ - providers::SnapshotWriter, ProviderFactory, StageCheckpointReader, StageCheckpointWriter, + providers::StaticFileWriter, ProviderFactory, StageCheckpointReader, StageCheckpointWriter, }; use reth_tokio_util::EventListeners; use std::pin::Pin; @@ -281,7 +281,7 @@ where self.listeners .notify(PipelineEvent::Unwound { stage_id, result: unwind_output }); - self.provider_factory.snapshot_provider().commit()?; + self.provider_factory.static_file_provider().commit()?; provider_rw.commit()?; provider_rw = self.provider_factory.provider_rw()?; @@ -375,7 +375,7 @@ where result: out.clone(), }); - self.provider_factory.snapshot_provider().commit()?; + self.provider_factory.static_file_provider().commit()?; provider_rw.commit()?; if done { @@ -433,7 +433,7 @@ fn on_stage_error( StageId::MerkleExecute, prev_checkpoint.unwrap_or_default(), )?; - factory.snapshot_provider().commit()?; + factory.static_file_provider().commit()?; provider_rw.commit()?; // We unwind because of a validation error. If the unwind itself @@ -463,13 +463,13 @@ fn on_stage_error( })) } } - } else if let StageError::MissingSnapshotData { block, segment } = err { + } else if let StageError::MissingStaticFileData { block, segment } = err { error!( target: "sync::pipeline", stage = %stage_id, bad_block = %block.number, segment = %segment, - "Stage is missing snapshot data." + "Stage is missing static file data." ); Ok(Some(ControlFlow::Unwind { target: block.number - 1, bad_block: block })) diff --git a/crates/stages/src/sets.rs b/crates/stages/src/sets.rs index 4db79a40b7fe..60394e30f12e 100644 --- a/crates/stages/src/sets.rs +++ b/crates/stages/src/sets.rs @@ -42,7 +42,7 @@ use crate::{ stages::{ AccountHashingStage, BodyStage, ExecutionStage, FinishStage, HeaderStage, IndexAccountHistoryStage, IndexStorageHistoryStage, MerkleStage, SenderRecoveryStage, - SnapshotStage, StorageHashingStage, TransactionLookupStage, + StaticFileStage, StorageHashingStage, TransactionLookupStage, }, StageError, StageSet, StageSetBuilder, }; @@ -52,7 +52,7 @@ use reth_interfaces::{ p2p::{bodies::downloader::BodyDownloader, headers::downloader::HeaderDownloader}, }; use reth_provider::{ExecutorFactory, HeaderSyncGapProvider, HeaderSyncMode}; -use reth_snapshot::Snapshotter; +use reth_static_file::StaticFileProducer; use std::sync::Arc; use tempfile::TempDir; @@ -60,13 +60,13 @@ use tempfile::TempDir; /// /// A combination of (in order) /// -/// - [`SnapshotStage`] +/// - [`StaticFileStage`] /// - [`OnlineStages`] /// - [`OfflineStages`] /// - [`FinishStage`] /// /// This expands to the following series of stages: -/// - [`SnapshotStage`] +/// - [`StaticFileStage`] /// - [`HeaderStage`] /// - [`BodyStage`] /// - [`SenderRecoveryStage`] @@ -85,7 +85,7 @@ pub struct DefaultStages { online: OnlineStages, /// Executor factory needs for execution stage executor_factory: EF, - snapshotter: Snapshotter, + static_file_producer: StaticFileProducer, } impl DefaultStages { @@ -97,7 +97,7 @@ impl DefaultStages { header_downloader: H, body_downloader: B, executor_factory: EF, - snapshotter: Snapshotter, + static_file_producer: StaticFileProducer, ) -> Result where EF: ExecutorFactory, @@ -112,7 +112,7 @@ impl DefaultStages { Arc::new(TempDir::new()?), ), executor_factory, - snapshotter, + static_file_producer, }) } } @@ -126,10 +126,10 @@ where pub fn add_offline_stages( default_offline: StageSetBuilder, executor_factory: EF, - snapshotter: Snapshotter, + static_file_producer: StaticFileProducer, ) -> StageSetBuilder { StageSetBuilder::default() - .add_stage(SnapshotStage::new(snapshotter)) + .add_stage(StaticFileStage::new(static_file_producer)) .add_set(default_offline) .add_set(OfflineStages::new(executor_factory)) .add_stage(FinishStage) @@ -145,7 +145,11 @@ where DB: Database + 'static, { fn builder(self) -> StageSetBuilder { - Self::add_offline_stages(self.online.builder(), self.executor_factory, self.snapshotter) + Self::add_offline_stages( + self.online.builder(), + self.executor_factory, + self.static_file_producer, + ) } } diff --git a/crates/stages/src/stages/bodies.rs b/crates/stages/src/stages/bodies.rs index a0256bec307e..ea141cb9d4b5 100644 --- a/crates/stages/src/stages/bodies.rs +++ b/crates/stages/src/stages/bodies.rs @@ -13,9 +13,9 @@ use reth_interfaces::{ }; use reth_primitives::{ stage::{EntitiesCheckpoint, StageCheckpoint, StageId}, - SnapshotSegment, + StaticFileSegment, }; -use reth_provider::{providers::SnapshotWriter, DatabaseProviderRW, HeaderProvider, StatsReader}; +use reth_provider::{providers::StaticFileWriter, DatabaseProviderRW, HeaderProvider, StatsReader}; use std::{ cmp::Ordering, task::{ready, Context, Poll}, @@ -124,37 +124,37 @@ impl Stage for BodyStage { // Get id for the next tx_num of zero if there are no transactions. let mut next_tx_num = tx_block_cursor.last()?.map(|(id, _)| id + 1).unwrap_or_default(); - let snapshot_provider = provider.snapshot_provider(); - let mut snapshotter = - snapshot_provider.get_writer(from_block, SnapshotSegment::Transactions)?; + let static_file_provider = provider.static_file_provider(); + let mut static_file_producer = + static_file_provider.get_writer(from_block, StaticFileSegment::Transactions)?; // Make sure Transactions static file is at the same height. If it's further, this // input execution was interrupted previously and we need to unwind the static file. - let next_snapshot_tx_num = snapshot_provider - .get_highest_snapshot_tx(SnapshotSegment::Transactions) + let next_static_file_tx_num = static_file_provider + .get_highest_static_file_tx(StaticFileSegment::Transactions) .map(|id| id + 1) .unwrap_or_default(); - match next_snapshot_tx_num.cmp(&next_tx_num) { + match next_static_file_tx_num.cmp(&next_tx_num) { // If static files are ahead, then we didn't reach the database commit in a previous // stage run. So, our only solution is to unwind the static files and proceed from the // database expected height. - Ordering::Greater => snapshotter - .prune_transactions(next_snapshot_tx_num - next_tx_num, from_block - 1)?, + Ordering::Greater => static_file_producer + .prune_transactions(next_static_file_tx_num - next_tx_num, from_block - 1)?, // If static files are behind, then there was some corruption or loss of files. This // error will trigger an unwind, that will bring the database to the same height as the // static files. Ordering::Less => { - let last_block = snapshot_provider - .get_highest_snapshot_block(SnapshotSegment::Transactions) + let last_block = static_file_provider + .get_highest_static_file_block(StaticFileSegment::Transactions) .unwrap_or_default(); let missing_block = Box::new(provider.sealed_header(last_block + 1)?.unwrap_or_default()); - return Err(StageError::MissingSnapshotData { + return Err(StageError::MissingStaticFileData { block: missing_block, - segment: SnapshotSegment::Transactions, + segment: StaticFileSegment::Transactions, }) } Ordering::Equal => {} @@ -177,16 +177,16 @@ impl Stage for BodyStage { }, }; - // Increment block on snapshot header. + // Increment block on static file header. if block_number > 0 { let appended_block_number = - snapshotter.increment_block(SnapshotSegment::Transactions)?; + static_file_producer.increment_block(StaticFileSegment::Transactions)?; if appended_block_number != block_number { // This scenario indicates a critical error in the logic of adding new // items. It should be treated as an `expect()` failure. return Err(StageError::InconsistentBlockNumber { - segment: SnapshotSegment::Transactions, + segment: StaticFileSegment::Transactions, database: block_number, static_file: appended_block_number, }) @@ -202,14 +202,14 @@ impl Stage for BodyStage { // Write transactions for transaction in block.body { - let appended_tx_number = - snapshotter.append_transaction(next_tx_num, transaction.into())?; + let appended_tx_number = static_file_producer + .append_transaction(next_tx_num, transaction.into())?; if appended_tx_number != next_tx_num { // This scenario indicates a critical error in the logic of adding new // items. It should be treated as an `expect()` failure. return Err(StageError::InconsistentTxNumber { - segment: SnapshotSegment::Transactions, + segment: StaticFileSegment::Transactions, database: next_tx_num, static_file: appended_tx_number, }) @@ -261,7 +261,7 @@ impl Stage for BodyStage { ) -> Result { self.buffer.take(); - let snapshot_provider = provider.snapshot_provider(); + let static_file_provider = provider.static_file_provider(); let tx = provider.tx_ref(); // Cursors to unwind bodies, ommers let mut body_cursor = tx.cursor_write::()?; @@ -297,35 +297,36 @@ impl Stage for BodyStage { rev_walker.delete_current()?; } - let mut snapshotter = snapshot_provider.latest_writer(SnapshotSegment::Transactions)?; + let mut static_file_producer = + static_file_provider.latest_writer(StaticFileSegment::Transactions)?; // Unwind from static files. Get the current last expected transaction from DB, and match it // on static file let db_tx_num = body_cursor.last()?.map(|(_, block_meta)| block_meta.last_tx_num()).unwrap_or_default(); - let snapshot_tx_num: u64 = snapshot_provider - .get_highest_snapshot_tx(SnapshotSegment::Transactions) + let static_file_tx_num: u64 = static_file_provider + .get_highest_static_file_tx(StaticFileSegment::Transactions) .unwrap_or_default(); - // If there are more transactions on database, then we are missing snapshot data and we need - // to unwind further. - if db_tx_num > snapshot_tx_num { - let last_block = snapshot_provider - .get_highest_snapshot_block(SnapshotSegment::Transactions) + // If there are more transactions on database, then we are missing static file data and we + // need to unwind further. + if db_tx_num > static_file_tx_num { + let last_block = static_file_provider + .get_highest_static_file_block(StaticFileSegment::Transactions) .unwrap_or_default(); let missing_block = Box::new(provider.sealed_header(last_block + 1)?.unwrap_or_default()); - return Err(StageError::MissingSnapshotData { + return Err(StageError::MissingStaticFileData { block: missing_block, - segment: SnapshotSegment::Transactions, + segment: StaticFileSegment::Transactions, }) } // Unwinds static file - snapshotter - .prune_transactions(snapshot_tx_num.saturating_sub(db_tx_num), input.unwind_to)?; + static_file_producer + .prune_transactions(static_file_tx_num.saturating_sub(db_tx_num), input.unwind_to)?; Ok(UnwindOutput { checkpoint: StageCheckpoint::new(input.unwind_to) @@ -382,7 +383,7 @@ mod tests { // Check that we only synced around `batch_size` blocks even though the number of blocks // synced by the previous stage is higher let output = rx.await.unwrap(); - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); assert_matches!( output, Ok(ExecOutput { checkpoint: StageCheckpoint { @@ -419,7 +420,7 @@ mod tests { // Check that we synced all blocks successfully, even though our `batch_size` allows us to // sync more (if there were more headers) let output = rx.await.unwrap(); - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); assert_matches!( output, Ok(ExecOutput { @@ -457,7 +458,7 @@ mod tests { // Check that we synced at least 10 blocks let first_run = rx.await.unwrap(); - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); assert_matches!( first_run, Ok(ExecOutput { checkpoint: StageCheckpoint { @@ -478,7 +479,7 @@ mod tests { // Check that we synced more blocks let output = rx.await.unwrap(); - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); assert_matches!( output, Ok(ExecOutput { checkpoint: StageCheckpoint { @@ -519,7 +520,7 @@ mod tests { // Check that we synced all blocks successfully, even though our `batch_size` allows us to // sync more (if there were more headers) let output = rx.await.unwrap(); - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); assert_matches!( output, Ok(ExecOutput { checkpoint: StageCheckpoint { @@ -537,11 +538,11 @@ mod tests { .expect("Written block data invalid"); // Delete a transaction - let snapshot_provider = runner.db().factory.snapshot_provider(); + let static_file_provider = runner.db().factory.static_file_provider(); { - let mut snapshotter = - snapshot_provider.latest_writer(SnapshotSegment::Transactions).unwrap(); - snapshotter.prune_transactions(1, checkpoint.block_number).unwrap(); + let mut static_file_producer = + static_file_provider.latest_writer(StaticFileSegment::Transactions).unwrap(); + static_file_producer.prune_transactions(1, checkpoint.block_number).unwrap(); } // Unwind all of it let unwind_to = 1; @@ -574,7 +575,7 @@ mod tests { use reth_db::{ cursor::DbCursorRO, models::{StoredBlockBodyIndices, StoredBlockOmmers}, - snapshot::HeaderMask, + static_file::HeaderMask, tables, test_utils::TempDatabase, transaction::{DbTx, DbTxMut}, @@ -597,11 +598,11 @@ mod tests { }, }; use reth_primitives::{ - BlockBody, BlockHash, BlockNumber, Header, SealedBlock, SealedHeader, SnapshotSegment, - TxNumber, B256, + BlockBody, BlockHash, BlockNumber, Header, SealedBlock, SealedHeader, + StaticFileSegment, TxNumber, B256, }; use reth_provider::{ - providers::SnapshotWriter, HeaderProvider, ProviderFactory, TransactionsProvider, + providers::StaticFileWriter, HeaderProvider, ProviderFactory, TransactionsProvider, }; use std::{ collections::{HashMap, VecDeque}, @@ -673,7 +674,7 @@ mod tests { let start = input.checkpoint().block_number; let end = input.target(); - let snapshot_provider = self.db.factory.snapshot_provider(); + let static_file_provider = self.db.factory.static_file_provider(); let mut rng = generators::rng(); @@ -684,19 +685,21 @@ mod tests { // Insert last progress data { let tx = self.db.factory.provider_rw()?.into_tx(); - let mut snapshotter = - snapshot_provider.get_writer(start, SnapshotSegment::Transactions)?; + let mut static_file_producer = static_file_provider + .get_writer(start, StaticFileSegment::Transactions)?; let body = StoredBlockBodyIndices { first_tx_num: 0, tx_count: progress.body.len() as u64, }; - snapshotter.set_block_range(0..=progress.number); + static_file_producer.set_block_range(0..=progress.number); body.tx_num_range().try_for_each(|tx_num| { let transaction = random_signed_tx(&mut rng); - snapshotter.append_transaction(tx_num, transaction.into()).map(|_| ()) + static_file_producer + .append_transaction(tx_num, transaction.into()) + .map(|_| ()) })?; if body.tx_count != 0 { @@ -715,7 +718,7 @@ mod tests { )?; } - snapshotter.commit()?; + static_file_producer.commit()?; tx.commit()?; } } @@ -778,7 +781,7 @@ mod tests { prev_progress: BlockNumber, highest_block: BlockNumber, ) -> Result<(), TestRunnerError> { - let snapshot_provider = self.db.factory.snapshot_provider(); + let static_file_provider = self.db.factory.static_file_provider(); self.db.query(|tx| { // Acquire cursors on body related tables @@ -811,7 +814,7 @@ mod tests { "We wrote a block body outside of our synced range. Found block with number {number}, highest block according to stage is {highest_block}", ); - let header = snapshot_provider.header_by_number(number)?.expect("to be present"); + let header = static_file_provider.header_by_number(number)?.expect("to be present"); // Validate that ommers exist if any let stored_ommers = ommers_cursor.seek_exact(number)?; if header.ommers_hash_is_empty() { @@ -828,7 +831,7 @@ mod tests { } for tx_id in body.tx_num_range() { - assert!(snapshot_provider.transaction_by_id(tx_id)?.is_some(), "Transaction is missing."); + assert!(static_file_provider.transaction_by_id(tx_id)?.is_some(), "Transaction is missing."); } prev_number = Some(number); @@ -889,10 +892,10 @@ mod tests { &mut self, range: RangeInclusive, ) -> DownloadResult<()> { - let snapshot_provider = self.provider_factory.snapshot_provider(); + let static_file_provider = self.provider_factory.static_file_provider(); - for header in snapshot_provider.fetch_range_iter( - SnapshotSegment::Headers, + for header in static_file_provider.fetch_range_iter( + StaticFileSegment::Headers, *range.start()..*range.end() + 1, |cursor, number| cursor.get_two::>(number.into()), )? { diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index dcb90ec1af5f..a9dfab787cdc 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -7,7 +7,7 @@ use reth_db::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, database::Database, models::BlockNumberAddress, - snapshot::HeaderMask, + static_file::HeaderMask, tables, transaction::{DbTx, DbTxMut}, }; @@ -15,10 +15,10 @@ use reth_primitives::{ stage::{ CheckpointBlockRange, EntitiesCheckpoint, ExecutionCheckpoint, StageCheckpoint, StageId, }, - BlockNumber, Header, PruneModes, SnapshotSegment, U256, + BlockNumber, Header, PruneModes, StaticFileSegment, U256, }; use reth_provider::{ - providers::{SnapshotProvider, SnapshotProviderRWRefMut, SnapshotWriter}, + providers::{StaticFileProvider, StaticFileProviderRWRefMut, StaticFileWriter}, BlockReader, DatabaseProviderRW, ExecutorFactory, HeaderProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderError, StatsReader, TransactionVariant, }; @@ -122,18 +122,18 @@ impl ExecutionStage { let start_block = input.next_block(); let max_block = input.target(); let prune_modes = self.adjust_prune_modes(provider, start_block, max_block)?; - let snapshot_provider = provider.snapshot_provider(); + let static_file_provider = provider.static_file_provider(); // We only use static files for Receipts, if there is no receipt pruning of any kind. - let mut snapshotter = None; + let mut static_file_producer = None; if self.prune_modes.receipts.is_none() && self.prune_modes.receipts_log_filter.is_empty() { - snapshotter = Some(prepare_snapshotter(provider, start_block)?); + static_file_producer = Some(prepare_static_file_producer(provider, start_block)?); } // Build executor let mut executor = self.executor_factory.with_state(LatestStateProviderRef::new( provider.tx_ref(), - provider.snapshot_provider().clone(), + provider.static_file_provider().clone(), )); executor.set_prune_modes(prune_modes); executor.set_tip(max_block); @@ -141,7 +141,7 @@ impl ExecutionStage { // Progress tracking let mut stage_progress = start_block; let mut stage_checkpoint = - execution_checkpoint(snapshot_provider, start_block, max_block, input.checkpoint())?; + execution_checkpoint(static_file_provider, start_block, max_block, input.checkpoint())?; let mut fetch_block_duration = Duration::default(); let mut execution_duration = Duration::default(); @@ -206,7 +206,11 @@ impl ExecutionStage { let time = Instant::now(); // write output - state.write_to_storage(provider.tx_ref(), snapshotter, OriginalValuesKnown::Yes)?; + state.write_to_storage( + provider.tx_ref(), + static_file_producer, + OriginalValuesKnown::Yes, + )?; let db_write_duration = time.elapsed(); debug!( target: "sync::stages::execution", @@ -258,7 +262,7 @@ impl ExecutionStage { } fn execution_checkpoint( - provider: &SnapshotProvider, + provider: &StaticFileProvider, start_block: BlockNumber, max_block: BlockNumber, checkpoint: StageCheckpoint, @@ -325,7 +329,7 @@ fn execution_checkpoint( } fn calculate_gas_used_from_headers( - provider: &SnapshotProvider, + provider: &StaticFileProvider, range: RangeInclusive, ) -> Result { let mut gas_total = 0; @@ -333,7 +337,7 @@ fn calculate_gas_used_from_headers( let start = Instant::now(); for entry in provider.fetch_range_iter( - SnapshotSegment::Headers, + StaticFileSegment::Headers, *range.start()..*range.end() + 1, |cursor, number| cursor.get_one::>(number.into()), )? { @@ -440,10 +444,10 @@ impl Stage for ExecutionStage { if self.prune_modes.receipts.is_none() && self.prune_modes.receipts_log_filter.is_empty() { // We only use static files for Receipts, if there is no receipt pruning of any kind. - // prepare_snapshotter does a consistency check that will unwind static files if the - // expected highest receipt in the files is higher than the database. Which is - // essentially what happens here when we unwind this stage. - let _snapshotter = prepare_snapshotter(provider, *range.start())?; + // prepare_static_file_producer does a consistency check that will unwind static files + // if the expected highest receipt in the files is higher than the database. + // Which is essentially what happens here when we unwind this stage. + let _static_file_producer = prepare_static_file_producer(provider, *range.start())?; // Update the checkpoint. if let Some(stage_checkpoint) = stage_checkpoint.as_mut() { @@ -530,17 +534,18 @@ impl ExecutionStageThresholds { } } -/// Returns a `SnapshotProviderRWRefMut` snapshotter after performing a consistency check. +/// Returns a `StaticFileProviderRWRefMut` static file producer after performing a consistency +/// check. /// /// This function compares the highest receipt number recorded in the database with that in the /// static file to detect any discrepancies due to unexpected shutdowns or database rollbacks. **If /// the height in the static file is higher**, it rolls back (unwinds) the static file. /// **Conversely, if the height in the database is lower**, it triggers a rollback in the database /// (by returning [`StageError`]) until the heights in both the database and static file match. -fn prepare_snapshotter<'a, 'b, DB: Database>( +fn prepare_static_file_producer<'a, 'b, DB: Database>( provider: &'b DatabaseProviderRW, start_block: u64, -) -> Result, StageError> +) -> Result, StageError> where 'b: 'a, { @@ -553,37 +558,38 @@ where .unwrap_or(0); // Get next expected receipt number in static files - let snapshot_provider = provider.snapshot_provider(); - let mut snapshotter = snapshot_provider.get_writer(start_block, SnapshotSegment::Receipts)?; - let next_snapshot_receipt_num = snapshotter - .get_highest_snapshot_tx(SnapshotSegment::Receipts) + let static_file_provider = provider.static_file_provider(); + let mut static_file_producer = + static_file_provider.get_writer(start_block, StaticFileSegment::Receipts)?; + let next_static_file_receipt_num = static_file_producer + .get_highest_static_file_tx(StaticFileSegment::Receipts) .map(|num| num + 1) .unwrap_or(0); // Check if we had any unexpected shutdown after committing to static files, but // NOT committing to database. - match next_snapshot_receipt_num.cmp(&next_receipt_num) { - Ordering::Greater => snapshotter.prune_receipts( - next_snapshot_receipt_num - next_receipt_num, + match next_static_file_receipt_num.cmp(&next_receipt_num) { + Ordering::Greater => static_file_producer.prune_receipts( + next_static_file_receipt_num - next_receipt_num, start_block.saturating_sub(1), )?, Ordering::Less => { - let last_block = snapshot_provider - .get_highest_snapshot_block(SnapshotSegment::Receipts) + let last_block = static_file_provider + .get_highest_static_file_block(StaticFileSegment::Receipts) .unwrap_or(0); let missing_block = Box::new( tx.get::(last_block + 1)?.unwrap_or_default().seal_slow(), ); - return Err(StageError::MissingSnapshotData { + return Err(StageError::MissingStaticFileData { block: missing_block, - segment: SnapshotSegment::Receipts, + segment: StaticFileSegment::Receipts, }) } Ordering::Equal => {} } - Ok(snapshotter) + Ok(static_file_producer) } #[cfg(test)] @@ -636,7 +642,7 @@ mod tests { }; let stage_checkpoint = execution_checkpoint( - &factory.snapshot_provider(), + &factory.static_file_provider(), previous_stage_checkpoint.block_range.from, previous_stage_checkpoint.block_range.to, previous_checkpoint, @@ -667,8 +673,8 @@ mod tests { .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) .unwrap(); provider - .snapshot_provider() - .latest_writer(SnapshotSegment::Headers) + .static_file_provider() + .latest_writer(StaticFileSegment::Headers) .unwrap() .commit() .unwrap(); @@ -684,7 +690,7 @@ mod tests { }; let stage_checkpoint = - execution_checkpoint(&factory.snapshot_provider(), 1, 1, previous_checkpoint); + execution_checkpoint(&factory.static_file_provider(), 1, 1, previous_checkpoint); assert_matches!(stage_checkpoint, Ok(ExecutionCheckpoint { block_range: CheckpointBlockRange { from: 1, to: 1 }, @@ -710,8 +716,8 @@ mod tests { .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) .unwrap(); provider - .snapshot_provider() - .latest_writer(SnapshotSegment::Headers) + .static_file_provider() + .latest_writer(StaticFileSegment::Headers) .unwrap() .commit() .unwrap(); @@ -727,7 +733,7 @@ mod tests { }; let stage_checkpoint = - execution_checkpoint(&factory.snapshot_provider(), 1, 1, previous_checkpoint); + execution_checkpoint(&factory.static_file_provider(), 1, 1, previous_checkpoint); assert_matches!(stage_checkpoint, Ok(ExecutionCheckpoint { block_range: CheckpointBlockRange { from: 1, to: 1 }, @@ -753,8 +759,8 @@ mod tests { .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) .unwrap(); provider - .snapshot_provider() - .latest_writer(SnapshotSegment::Headers) + .static_file_provider() + .latest_writer(StaticFileSegment::Headers) .unwrap() .commit() .unwrap(); @@ -763,7 +769,7 @@ mod tests { let previous_checkpoint = StageCheckpoint { block_number: 1, stage_checkpoint: None }; let stage_checkpoint = - execution_checkpoint(&factory.snapshot_provider(), 1, 1, previous_checkpoint); + execution_checkpoint(&factory.static_file_provider(), 1, 1, previous_checkpoint); assert_matches!(stage_checkpoint, Ok(ExecutionCheckpoint { block_range: CheckpointBlockRange { from: 1, to: 1 }, @@ -790,8 +796,8 @@ mod tests { .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) .unwrap(); provider - .snapshot_provider() - .latest_writer(SnapshotSegment::Headers) + .static_file_provider() + .latest_writer(StaticFileSegment::Headers) .unwrap() .commit() .unwrap(); @@ -935,8 +941,8 @@ mod tests { .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) .unwrap(); provider - .snapshot_provider() - .latest_writer(SnapshotSegment::Headers) + .static_file_provider() + .latest_writer(StaticFileSegment::Headers) .unwrap() .commit() .unwrap(); @@ -1048,8 +1054,8 @@ mod tests { .insert_historical_block(block.clone().try_seal_with_senders().unwrap(), None) .unwrap(); provider - .snapshot_provider() - .latest_writer(SnapshotSegment::Headers) + .static_file_provider() + .latest_writer(StaticFileSegment::Headers) .unwrap() .commit() .unwrap(); diff --git a/crates/stages/src/stages/finish.rs b/crates/stages/src/stages/finish.rs index b1d9516b2560..e0e0057c3c7e 100644 --- a/crates/stages/src/stages/finish.rs +++ b/crates/stages/src/stages/finish.rs @@ -45,7 +45,7 @@ mod tests { generators::{random_header, random_header_range}, }; use reth_primitives::SealedHeader; - use reth_provider::providers::SnapshotWriter; + use reth_provider::providers::StaticFileWriter; stage_test_suite_ext!(FinishTestRunner, finish); diff --git a/crates/stages/src/stages/hashing_account.rs b/crates/stages/src/stages/hashing_account.rs index d32f45ceba34..148def38ff4f 100644 --- a/crates/stages/src/stages/hashing_account.rs +++ b/crates/stages/src/stages/hashing_account.rs @@ -88,7 +88,7 @@ impl AccountHashingStage { generators::{random_block_range, random_eoa_account_range}, }; use reth_primitives::{Account, B256, U256}; - use reth_provider::providers::SnapshotWriter; + use reth_provider::providers::StaticFileWriter; let mut rng = generators::rng(); @@ -98,8 +98,8 @@ impl AccountHashingStage { provider.insert_historical_block(block.try_seal_with_senders().unwrap(), None).unwrap(); } provider - .snapshot_provider() - .latest_writer(reth_primitives::SnapshotSegment::Headers) + .static_file_provider() + .latest_writer(reth_primitives::StaticFileSegment::Headers) .unwrap() .commit() .unwrap(); @@ -311,7 +311,7 @@ mod tests { }; use assert_matches::assert_matches; use reth_primitives::{stage::StageUnitCheckpoint, Account, U256}; - use reth_provider::providers::SnapshotWriter; + use reth_provider::providers::StaticFileWriter; use test_utils::*; stage_test_suite_ext!(AccountHashingTestRunner, account_hashing); diff --git a/crates/stages/src/stages/hashing_storage.rs b/crates/stages/src/stages/hashing_storage.rs index 926aa961a0ab..d6fde8cc4cd7 100644 --- a/crates/stages/src/stages/hashing_storage.rs +++ b/crates/stages/src/stages/hashing_storage.rs @@ -239,7 +239,7 @@ mod tests { generators::{random_block_range, random_contract_account_range}, }; use reth_primitives::{stage::StageUnitCheckpoint, Address, SealedBlock, B256, U256}; - use reth_provider::providers::SnapshotWriter; + use reth_provider::providers::StaticFileWriter; stage_test_suite_ext!(StorageHashingTestRunner, storage_hashing); diff --git a/crates/stages/src/stages/headers.rs b/crates/stages/src/stages/headers.rs index 06c7966b9916..7b2fb47fbbb2 100644 --- a/crates/stages/src/stages/headers.rs +++ b/crates/stages/src/stages/headers.rs @@ -18,10 +18,10 @@ use reth_primitives::{ stage::{ CheckpointBlockRange, EntitiesCheckpoint, HeadersCheckpoint, StageCheckpoint, StageId, }, - BlockHash, BlockNumber, SealedHeader, SnapshotSegment, + BlockHash, BlockNumber, SealedHeader, StaticFileSegment, }; use reth_provider::{ - providers::{SnapshotProvider, SnapshotWriter}, + providers::{StaticFileProvider, StaticFileWriter}, BlockHashReader, DatabaseProviderRW, HeaderProvider, HeaderSyncGap, HeaderSyncGapProvider, HeaderSyncMode, }; @@ -99,7 +99,7 @@ where fn write_headers( &mut self, tx: &::TXMut, - snapshot_provider: SnapshotProvider, + static_file_provider: StaticFileProvider, ) -> Result { let total_headers = self.header_collector.len(); @@ -107,18 +107,18 @@ where // Consistency check of expected headers in static files vs DB is done on provider::sync_gap // when poll_execute_ready is polled. - let mut last_header_number = snapshot_provider - .get_highest_snapshot_block(SnapshotSegment::Headers) + let mut last_header_number = static_file_provider + .get_highest_static_file_block(StaticFileSegment::Headers) .unwrap_or_default(); // Find the latest total difficulty - let mut td = snapshot_provider + let mut td = static_file_provider .header_td_by_number(last_header_number)? .ok_or(ProviderError::TotalDifficultyNotFound(last_header_number))?; // Although headers were downloaded in reverse order, the collector iterates it in ascending // order - let mut writer = snapshot_provider.latest_writer(SnapshotSegment::Headers)?; + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; let interval = (total_headers / 10).max(1); for (index, header) in self.header_collector.iter()?.enumerate() { let (_, header_buf) = header?; @@ -289,7 +289,7 @@ where // Write the headers and related tables to DB from ETL space let to_be_processed = self.hash_collector.len() as u64; let last_header_number = - self.write_headers::(provider.tx_ref(), provider.snapshot_provider().clone())?; + self.write_headers::(provider.tx_ref(), provider.static_file_provider().clone())?; Ok(ExecOutput { checkpoint: StageCheckpoint::new(last_header_number).with_headers_stage_checkpoint( @@ -318,21 +318,21 @@ where ) -> Result { self.sync_gap.take(); - let snapshot_provider = provider.snapshot_provider(); - let highest_block = snapshot_provider - .get_highest_snapshot_block(SnapshotSegment::Headers) + let static_file_provider = provider.static_file_provider(); + let highest_block = static_file_provider + .get_highest_static_file_block(StaticFileSegment::Headers) .unwrap_or_default(); let unwound_headers = highest_block - input.unwind_to; for block in (input.unwind_to + 1)..=highest_block { - let header_hash = snapshot_provider + let header_hash = static_file_provider .block_hash(block)? .ok_or(ProviderError::HeaderNotFound(block.into()))?; provider.tx_ref().delete::(header_hash, None)?; } - let mut writer = snapshot_provider.latest_writer(SnapshotSegment::Headers)?; + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; writer.prune_headers(unwound_headers)?; let stage_checkpoint = @@ -565,7 +565,7 @@ mod tests { runner.send_tip(tip.hash()); let result = rx.await.unwrap(); - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); assert_matches!( result, Ok(ExecOutput { checkpoint: StageCheckpoint { block_number, stage_checkpoint: Some(StageUnitCheckpoint::Headers(HeadersCheckpoint { diff --git a/crates/stages/src/stages/index_account_history.rs b/crates/stages/src/stages/index_account_history.rs index 8838931ea8f1..37087b4034a5 100644 --- a/crates/stages/src/stages/index_account_history.rs +++ b/crates/stages/src/stages/index_account_history.rs @@ -129,7 +129,7 @@ mod tests { generators::{random_block_range, random_changeset_range, random_contract_account_range}, }; use reth_primitives::{address, Address, BlockNumber, B256}; - use reth_provider::providers::SnapshotWriter; + use reth_provider::providers::StaticFileWriter; use std::collections::BTreeMap; const ADDRESS: Address = address!("0000000000000000000000000000000000000001"); diff --git a/crates/stages/src/stages/index_storage_history.rs b/crates/stages/src/stages/index_storage_history.rs index 54a700b83efa..9819cefd1efa 100644 --- a/crates/stages/src/stages/index_storage_history.rs +++ b/crates/stages/src/stages/index_storage_history.rs @@ -127,7 +127,7 @@ mod tests { generators::{random_block_range, random_changeset_range, random_contract_account_range}, }; use reth_primitives::{address, b256, Address, BlockNumber, StorageEntry, B256, U256}; - use reth_provider::providers::SnapshotWriter; + use reth_provider::providers::StaticFileWriter; use std::collections::BTreeMap; const ADDRESS: Address = address!("0000000000000000000000000000000000000001"); diff --git a/crates/stages/src/stages/merkle.rs b/crates/stages/src/stages/merkle.rs index bf160abe6ac1..2e327835b5d2 100644 --- a/crates/stages/src/stages/merkle.rs +++ b/crates/stages/src/stages/merkle.rs @@ -352,9 +352,9 @@ mod tests { }, }; use reth_primitives::{ - keccak256, stage::StageUnitCheckpoint, SealedBlock, SnapshotSegment, StorageEntry, U256, + keccak256, stage::StageUnitCheckpoint, SealedBlock, StaticFileSegment, StorageEntry, U256, }; - use reth_provider::providers::SnapshotWriter; + use reth_provider::providers::StaticFileWriter; use reth_trie::test_utils::{state_root, state_root_prehashed}; use std::collections::BTreeMap; @@ -552,8 +552,9 @@ mod tests { Ok(state_root_prehashed(accounts.into_iter())) })?; - let snapshot_provider = self.db.factory.snapshot_provider(); - let mut writer = snapshot_provider.latest_writer(SnapshotSegment::Headers).unwrap(); + let static_file_provider = self.db.factory.static_file_provider(); + let mut writer = + static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); let mut last_header = last_block.header().clone(); last_header.state_root = root; diff --git a/crates/stages/src/stages/mod.rs b/crates/stages/src/stages/mod.rs index 71e9ca2fcd0d..a1c0c7c5469e 100644 --- a/crates/stages/src/stages/mod.rs +++ b/crates/stages/src/stages/mod.rs @@ -18,8 +18,8 @@ mod index_storage_history; mod merkle; /// The sender recovery stage. mod sender_recovery; -/// The snapshot stage. -mod snapshot; +/// The static file stage. +mod static_file; /// The transaction lookup stage mod tx_lookup; @@ -33,7 +33,7 @@ pub use index_account_history::*; pub use index_storage_history::*; pub use merkle::*; pub use sender_recovery::*; -pub use snapshot::*; +pub use static_file::*; pub use tx_lookup::*; #[cfg(test)] @@ -53,10 +53,10 @@ mod tests { use reth_node_ethereum::EthEvmConfig; use reth_primitives::{ address, hex_literal::hex, keccak256, Account, Bytecode, ChainSpecBuilder, PruneMode, - PruneModes, SealedBlock, SnapshotSegment, U256, + PruneModes, SealedBlock, StaticFileSegment, U256, }; use reth_provider::{ - providers::SnapshotWriter, AccountExtReader, ProviderFactory, ReceiptProvider, + providers::StaticFileWriter, AccountExtReader, ProviderFactory, ReceiptProvider, StorageReader, }; use reth_revm::EvmProcessorFactory; @@ -92,8 +92,8 @@ mod tests { .unwrap(); } provider_rw - .snapshot_provider() - .latest_writer(SnapshotSegment::Headers) + .static_file_provider() + .latest_writer(StaticFileSegment::Headers) .unwrap() .commit() .unwrap(); diff --git a/crates/stages/src/stages/sender_recovery.rs b/crates/stages/src/stages/sender_recovery.rs index 0aec9e5fa319..a0ccf5f70c6e 100644 --- a/crates/stages/src/stages/sender_recovery.rs +++ b/crates/stages/src/stages/sender_recovery.rs @@ -2,7 +2,7 @@ use crate::{BlockErrorKind, ExecInput, ExecOutput, Stage, StageError, UnwindInpu use reth_db::{ cursor::DbCursorRW, database::Database, - snapshot::TransactionMask, + static_file::TransactionMask, tables, transaction::{DbTx, DbTxMut}, }; @@ -10,7 +10,7 @@ use reth_interfaces::consensus; use reth_primitives::{ keccak256, stage::{EntitiesCheckpoint, StageCheckpoint, StageId}, - Address, PruneSegment, SnapshotSegment, TransactionSignedNoHash, TxNumber, + Address, PruneSegment, StaticFileSegment, TransactionSignedNoHash, TxNumber, }; use reth_provider::{ BlockReader, DatabaseProviderRW, HeaderProvider, ProviderError, PruneCheckpointReader, @@ -104,15 +104,15 @@ impl Stage for SenderRecoveryStage { let (recovered_senders_tx, recovered_senders_rx) = mpsc::channel(); channels.push(recovered_senders_rx); - let snapshot_provider = provider.snapshot_provider().clone(); + let static_file_provider = provider.static_file_provider().clone(); // Spawn the task onto the global rayon pool // This task will send the results through the channel after it has read the transaction // and calculated the sender. rayon::spawn(move || { let mut rlp_buf = Vec::with_capacity(128); - let _ = snapshot_provider.fetch_range_with_predicate( - SnapshotSegment::Transactions, + let _ = static_file_provider.fetch_range_with_predicate( + StaticFileSegment::Transactions, chunk_range, |cursor, number| { Ok(cursor @@ -257,7 +257,7 @@ mod tests { stage::StageUnitCheckpoint, BlockNumber, PruneCheckpoint, PruneMode, SealedBlock, TransactionSigned, B256, }; - use reth_provider::{providers::SnapshotWriter, PruneCheckpointWriter, TransactionsProvider}; + use reth_provider::{providers::StaticFileWriter, PruneCheckpointWriter, TransactionsProvider}; use super::*; use crate::test_utils::{ @@ -335,9 +335,12 @@ mod tests { .insert_blocks(seed.iter(), StorageKind::Static) .expect("failed to seed execution"); - let total_transactions = - runner.db.factory.snapshot_provider().count_entries::().unwrap() - as u64; + let total_transactions = runner + .db + .factory + .static_file_provider() + .count_entries::() + .unwrap() as u64; let first_input = ExecInput { target: Some(previous_stage), diff --git a/crates/stages/src/stages/snapshot.rs b/crates/stages/src/stages/static_file.rs similarity index 58% rename from crates/stages/src/stages/snapshot.rs rename to crates/stages/src/stages/static_file.rs index f4951e2fbeb0..7d8ca87c6241 100644 --- a/crates/stages/src/stages/snapshot.rs +++ b/crates/stages/src/stages/static_file.rs @@ -1,30 +1,30 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput}; use reth_db::database::Database; use reth_primitives::{ - snapshot::HighestSnapshots, stage::{StageCheckpoint, StageId}, + static_file::HighestStaticFiles, }; use reth_provider::{DatabaseProviderRW, StageCheckpointReader}; -use reth_snapshot::Snapshotter; +use reth_static_file::StaticFileProducer; -/// The snapshot stage _copies_ all data from database to static files using [Snapshotter]. The -/// block range for copying is determined by the current highest blocks contained in static files -/// and stage checkpoints for each segment individually. +/// The static file stage _copies_ all data from database to static files using +/// [StaticFileProducer]. The block range for copying is determined by the current highest blocks +/// contained in static files and stage checkpoints for each segment individually. #[derive(Debug)] -pub struct SnapshotStage { - snapshotter: Snapshotter, +pub struct StaticFileStage { + static_file_producer: StaticFileProducer, } -impl SnapshotStage { - /// Creates a new snapshot stage. - pub fn new(snapshotter: Snapshotter) -> Self { - Self { snapshotter } +impl StaticFileStage { + /// Creates a new static file stage. + pub fn new(static_file_producer: StaticFileProducer) -> Self { + Self { static_file_producer } } } -impl Stage for SnapshotStage { +impl Stage for StaticFileStage { fn id(&self) -> StageId { - StageId::Snapshot + StageId::StaticFile } fn execute( @@ -32,7 +32,7 @@ impl Stage for SnapshotStage { provider: &DatabaseProviderRW, input: ExecInput, ) -> Result { - let targets = self.snapshotter.get_snapshot_targets(HighestSnapshots { + let targets = self.static_file_producer.get_static_file_targets(HighestStaticFiles { headers: provider .get_stage_checkpoint(StageId::Headers)? .map(|checkpoint| checkpoint.block_number), @@ -43,7 +43,7 @@ impl Stage for SnapshotStage { .get_stage_checkpoint(StageId::Bodies)? .map(|checkpoint| checkpoint.block_number), })?; - self.snapshotter.run(targets)?; + self.static_file_producer.run(targets)?; Ok(ExecOutput::done(input.checkpoint())) } diff --git a/crates/stages/src/stages/tx_lookup.rs b/crates/stages/src/stages/tx_lookup.rs index 060870d68de6..ed9ddc3fc140 100644 --- a/crates/stages/src/stages/tx_lookup.rs +++ b/crates/stages/src/stages/tx_lookup.rs @@ -179,7 +179,7 @@ impl Stage for TransactionLookupStage { // Cursors to unwind tx hash to number let mut body_cursor = tx.cursor_read::()?; let mut tx_hash_number_cursor = tx.cursor_write::()?; - let snapshot_provider = provider.snapshot_provider(); + let static_file_provider = provider.static_file_provider(); let mut rev_walker = body_cursor.walk_back(Some(*range.end()))?; while let Some((number, body)) = rev_walker.next().transpose()? { if number <= unwind_to { @@ -189,7 +189,7 @@ impl Stage for TransactionLookupStage { // Delete all transactions that belong to this block for tx_id in body.tx_num_range() { // First delete the transaction and hash to id mapping - if let Some(transaction) = snapshot_provider.transaction_by_id(tx_id)? { + if let Some(transaction) = static_file_provider.transaction_by_id(tx_id)? { if tx_hash_number_cursor.seek_exact(transaction.hash())?.is_some() { tx_hash_number_cursor.delete_current()?; } @@ -235,7 +235,7 @@ mod tests { generators::{random_block, random_block_range}, }; use reth_primitives::{stage::StageUnitCheckpoint, BlockNumber, SealedBlock, B256}; - use reth_provider::providers::SnapshotWriter; + use reth_provider::providers::StaticFileWriter; use std::ops::Sub; // Implement stage test suite. @@ -285,7 +285,7 @@ mod tests { total })) }, done: true }) if block_number == previous_stage && processed == total && - total == runner.db.factory.snapshot_provider().count_entries::().unwrap() as u64 + total == runner.db.factory.static_file_provider().count_entries::().unwrap() as u64 ); // Validate the stage execution @@ -328,7 +328,7 @@ mod tests { total })) }, done: true }) if block_number == previous_stage && processed == total && - total == runner.db.factory.snapshot_provider().count_entries::().unwrap() as u64 + total == runner.db.factory.static_file_provider().count_entries::().unwrap() as u64 ); // Validate the stage execution diff --git a/crates/stages/src/test_utils/macros.rs b/crates/stages/src/test_utils/macros.rs index 55d4b3515b4f..0ffb16f08611 100644 --- a/crates/stages/src/test_utils/macros.rs +++ b/crates/stages/src/test_utils/macros.rs @@ -13,7 +13,7 @@ macro_rules! stage_test_suite { // Run stage execution let result = runner.execute(input).await; - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); // Check that the result is returned and the stage does not panic. // The return result with empty db is stage-specific. @@ -46,7 +46,7 @@ macro_rules! stage_test_suite { // Assert the successful result let result = rx.await.unwrap(); - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); assert_matches::assert_matches!( result, @@ -76,7 +76,7 @@ macro_rules! stage_test_suite { // Run stage unwind let rx = runner.unwind(input).await; - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); assert_matches::assert_matches!( rx, @@ -110,7 +110,7 @@ macro_rules! stage_test_suite { // Assert the successful execution result let result = rx.await.unwrap(); - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); assert_matches::assert_matches!( result, @@ -179,7 +179,7 @@ macro_rules! stage_test_suite_ext { // Assert the successful result let result = rx.await.unwrap(); - runner.db().factory.snapshot_provider().commit().unwrap(); + runner.db().factory.static_file_provider().commit().unwrap(); assert_matches::assert_matches!( result, diff --git a/crates/stages/src/test_utils/test_db.rs b/crates/stages/src/test_utils/test_db.rs index 976da6de4b99..391e1ab0bf78 100644 --- a/crates/stages/src/test_utils/test_db.rs +++ b/crates/stages/src/test_utils/test_db.rs @@ -6,7 +6,7 @@ use reth_db::{ table::Table, tables, test_utils::{ - create_test_rw_db, create_test_rw_db_with_path, create_test_snapshots_dir, TempDatabase, + create_test_rw_db, create_test_rw_db_with_path, create_test_static_files_dir, TempDatabase, }, transaction::{DbTx, DbTxMut}, DatabaseEnv, DatabaseError as DbError, @@ -17,7 +17,7 @@ use reth_primitives::{ TxHash, TxNumber, B256, MAINNET, U256, }; use reth_provider::{ - providers::{SnapshotProviderRWRefMut, SnapshotWriter}, + providers::{StaticFileProviderRWRefMut, StaticFileWriter}, HistoryWriter, ProviderError, ProviderFactory, }; use std::{collections::BTreeMap, path::Path, sync::Arc}; @@ -35,7 +35,7 @@ impl Default for TestStageDB { factory: ProviderFactory::new( create_test_rw_db(), MAINNET.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) .unwrap(), } @@ -48,7 +48,7 @@ impl TestStageDB { factory: ProviderFactory::new( create_test_rw_db_with_path(path), MAINNET.clone(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) .unwrap(), } @@ -133,7 +133,7 @@ impl TestStageDB { /// Insert header to static file if `writer` exists, otherwise to DB. pub fn insert_header( - writer: Option<&mut SnapshotProviderRWRefMut<'_>>, + writer: Option<&mut StaticFileProviderRWRefMut<'_>>, tx: &TX, header: &SealedHeader, td: U256, @@ -154,8 +154,8 @@ impl TestStageDB { where I: Iterator, { - let provider = self.factory.snapshot_provider(); - let mut writer = provider.latest_writer(reth_primitives::SnapshotSegment::Headers)?; + let provider = self.factory.static_file_provider(); + let mut writer = provider.latest_writer(reth_primitives::StaticFileSegment::Headers)?; let tx = self.factory.provider_rw()?.into_tx(); let mut td = U256::ZERO; @@ -202,14 +202,14 @@ impl TestStageDB { where I: Iterator, { - let provider = self.factory.snapshot_provider(); + let provider = self.factory.static_file_provider(); let mut txs_writer = storage_kind.is_static().then(|| { - provider.latest_writer(reth_primitives::SnapshotSegment::Transactions).unwrap() + provider.latest_writer(reth_primitives::StaticFileSegment::Transactions).unwrap() }); let mut headers_writer = - provider.latest_writer(reth_primitives::SnapshotSegment::Headers)?; + provider.latest_writer(reth_primitives::StaticFileSegment::Headers)?; let tx = self.factory.provider_rw().unwrap().into_tx(); let mut next_tx_num = storage_kind.tx_offset(); @@ -238,7 +238,7 @@ impl TestStageDB { }); if let Some(txs_writer) = &mut txs_writer { - txs_writer.increment_block(reth_primitives::SnapshotSegment::Transactions)?; + txs_writer.increment_block(reth_primitives::StaticFileSegment::Transactions)?; } res })?; diff --git a/crates/snapshot/Cargo.toml b/crates/static-file/Cargo.toml similarity index 91% rename from crates/snapshot/Cargo.toml rename to crates/static-file/Cargo.toml index df3216c5abb3..8c7128455cb0 100644 --- a/crates/snapshot/Cargo.toml +++ b/crates/static-file/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "reth-snapshot" +name = "reth-static-file" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true homepage.workspace = true repository.workspace = true -description = "Snapshotting implementation" +description = "Static file producer implementation" [lints] workspace = true diff --git a/crates/static-file/README.md b/crates/static-file/README.md new file mode 100644 index 000000000000..1d455475a595 --- /dev/null +++ b/crates/static-file/README.md @@ -0,0 +1,88 @@ +# StaticFile + +## Overview + +Data that has reached a finalized state and won't undergo further changes (essentially frozen) should be read without concerns of modification. This makes it unsuitable for traditional databases. + +This crate aims to copy this data from the current database to multiple static files, aggregated by block ranges. At every 500_000th block new static files are created. + +Below are two diagrams illustrating the processes of creating static files (custom format: `NippyJar`) and querying them. A glossary is also provided to explain the different (linked) components involved in these processes. + +
+ Creation diagram (StaticFileProducer) + +```mermaid +graph TD; + I("BLOCK_HEIGHT % 500_000 == 0")--triggers-->SP(StaticFileProducer) + SP --> |triggers| SH["create_static_file(block_range, StaticFileSegment::Headers)"] + SP --> |triggers| ST["create_static_file(block_range, StaticFileSegment::Transactions)"] + SP --> |triggers| SR["create_static_file(block_range, StaticFileSegment::Receipts)"] + SP --> |triggers| ETC["create_static_file(block_range, ...)"] + SH --> CS["create_static_file::< T >(DatabaseCursor)"] + ST --> CS + SR --> CS + ETC --> CS + CS --> |create| IF(NippyJar::InclusionFilters) + CS -- iterates --> DC(DatabaseCursor) -->HN{HasNext} + HN --> |true| NJC(NippyJar::Compression) + NJC --> HN + NJC --store--> NJ + HN --> |false| NJ + IF --store--> NJ(NippyJar) + NJ --freeze--> F(File) + F--"on success"--> SP1(StaticFileProducer) + SP1 --"sends BLOCK_HEIGHT"--> HST(HighestStaticFileTracker) + HST --"read by"-->Pruner + HST --"read by"-->DatabaseProvider + HST --"read by"-->SnapsotProvider + HST --"read by"-->ProviderFactory + +``` +
+ + +
+ Query diagram (Provider) + +```mermaid +graph TD; + RPC-->P + P("Provider::header(block_number)")-->PF(ProviderFactory) + PF--shares-->SP1("Arc(StaticFileProvider)") + SP1--shares-->PD(DatabaseProvider) + PF--creates-->PD + PD--check `HighestStaticFileTracker`-->PD + PD-->DC1{block_number
>
highest static_file block} + DC1 --> |true| PD1("DatabaseProvider::header(block_number)") + DC1 --> |false| ASP("StaticFileProvider::header(block_number)") + PD1 --> MDBX + ASP --find correct jar and creates--> JP("StaticFileJarProvider::header(block_number)") + JP --"creates"-->SC(StaticFileCursor) + SC --".get_one< HeaderMask< Header > >(number)"--->NJC("NippyJarCursor") + NJC--".row_by_number(row_index, mask)"-->NJ[NippyJar] + NJ--"&[u8]"-->NJC + NJC--"&[u8]"-->SC + SC--"Header"--> JP + JP--"Header"--> ASP +``` +
+ + +### Glossary +In descending order of abstraction hierarchy: + +[`StaticFileProducer`](../../crates/static_file/src/static_file_producer.rs#L20): A `reth` background service that **copies** data from the database to new static-file files when the block height reaches a certain threshold (e.g., `500_000th`). Upon completion, it dispatches a notification about the higher static file block to `HighestStaticFileTracker` channel. **It DOES NOT remove data from the database.** + +[`HighestStaticFileTracker`](../../crates/static_file/src/static_file_producer.rs#L22): A channel utilized by `StaticFileProducer` to announce the newest static_file block to all components with a listener: `Pruner` (to know which additional tables can be pruned) and `DatabaseProvider` (to know which data can be queried from the static files). + +[`StaticFileProvider`](../../crates/storage/provider/src/providers/static_file/manager.rs#L15) A provider similar to `DatabaseProvider`, **managing all existing static_file files** and selecting the optimal one (by range and segment type) to fulfill a request. **A single instance is shared across all components and should be instantiated only once within `ProviderFactory`**. An immutable reference is given everytime `ProviderFactory` creates a new `DatabaseProvider`. + +[`StaticFileJarProvider`](../../crates/storage/provider/src/providers/static_file/jar.rs#L42) A provider similar to `DatabaseProvider` that provides access to a **single static_file file**. + +[`StaticFileCursor`](../../crates/storage/db/src/static_file/cursor.rs#L12) An elevated abstraction of `NippyJarCursor` for simplified access. It associates the bitmasks with type decoding. For instance, `cursor.get_two::>(tx_number)` would yield `Tx` and `Signature`, eliminating the need to manage masks or invoke a decoder/decompressor. + +[`StaticFileSegment`](../../crates/primitives/src/static_file/segment.rs#L10) Each static_file file only contains data of a specific segment, e.g., `Headers`, `Transactions`, or `Receipts`. + +[`NippyJarCursor`](../../crates/storage/nippy-jar/src/cursor.rs#L12) Accessor of data in a `NippyJar` file. It enables queries either by row number (e.g., block number 1) or by a predefined key not part of the file (e.g., transaction hashes). If a file has multiple columns (e.g., `Tx | TxSender | Signature`), and one wishes to access only one of the column values, this can be accomplished by bitmasks. (e.g., for `TxSender`, the mask would be `0b010`). + +[`NippyJar`](../../crates/storage/nippy-jar/src/lib.rs#57) A create-only file format. No data can be appended after creation. It supports multiple columns, compression (e.g., Zstd (with and without dictionaries), lz4, uncompressed) and inclusion filters (e.g., cuckoo filter: `is hash X part of this dataset`). StaticFiles are organized by block ranges. (e.g., `TransactionStaticFile_499_999.jar` contains a transaction per row for all transactions from block `0` to block `499_999`). For more check the struct documentation. diff --git a/crates/static-file/src/event.rs b/crates/static-file/src/event.rs new file mode 100644 index 000000000000..cd95ad5e05f7 --- /dev/null +++ b/crates/static-file/src/event.rs @@ -0,0 +1,14 @@ +use crate::StaticFileTargets; +use std::time::Duration; + +/// An event emitted by a [StaticFileProducer][crate::StaticFileProducer]. +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum StaticFileProducerEvent { + /// Emitted when static_file_producer finished running. + Finished { + /// Targets that were moved to static files + targets: StaticFileTargets, + /// Time it took to run the static_file_producer + elapsed: Duration, + }, +} diff --git a/crates/snapshot/src/lib.rs b/crates/static-file/src/lib.rs similarity index 59% rename from crates/snapshot/src/lib.rs rename to crates/static-file/src/lib.rs index f28b63b326ed..2c6c11dfd4ff 100644 --- a/crates/snapshot/src/lib.rs +++ b/crates/static-file/src/lib.rs @@ -1,4 +1,4 @@ -//! Snapshotting implementation. +//! Static file producer implementation. #![doc( html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", @@ -9,7 +9,9 @@ mod event; pub mod segments; -mod snapshotter; +mod static_file_producer; -pub use event::SnapshotterEvent; -pub use snapshotter::{SnapshotTargets, Snapshotter, SnapshotterResult, SnapshotterWithResult}; +pub use event::StaticFileProducerEvent; +pub use static_file_producer::{ + StaticFileProducer, StaticFileProducerResult, StaticFileProducerWithResult, StaticFileTargets, +}; diff --git a/crates/snapshot/src/segments/headers.rs b/crates/static-file/src/segments/headers.rs similarity index 80% rename from crates/snapshot/src/segments/headers.rs rename to crates/static-file/src/segments/headers.rs index f8b8f2563f83..2e0a2827a831 100644 --- a/crates/snapshot/src/segments/headers.rs +++ b/crates/static-file/src/segments/headers.rs @@ -1,33 +1,33 @@ use crate::segments::{dataset_for_compression, prepare_jar, Segment, SegmentHeader}; use reth_db::{ - cursor::DbCursorRO, database::Database, snapshot::create_snapshot_T1_T2_T3, tables, + cursor::DbCursorRO, database::Database, static_file::create_static_file_T1_T2_T3, tables, transaction::DbTx, RawKey, RawTable, }; use reth_interfaces::provider::ProviderResult; -use reth_primitives::{snapshot::SegmentConfig, BlockNumber, SnapshotSegment}; +use reth_primitives::{static_file::SegmentConfig, BlockNumber, StaticFileSegment}; use reth_provider::{ - providers::{SnapshotProvider, SnapshotWriter}, + providers::{StaticFileProvider, StaticFileWriter}, DatabaseProviderRO, }; use std::{ops::RangeInclusive, path::Path}; -/// Snapshot segment responsible for [SnapshotSegment::Headers] part of data. +/// Static File segment responsible for [StaticFileSegment::Headers] part of data. #[derive(Debug, Default)] pub struct Headers; impl Segment for Headers { - fn segment(&self) -> SnapshotSegment { - SnapshotSegment::Headers + fn segment(&self) -> StaticFileSegment { + StaticFileSegment::Headers } - fn snapshot( + fn copy_to_static_files( &self, provider: DatabaseProviderRO, - snapshot_provider: SnapshotProvider, + static_file_provider: StaticFileProvider, block_range: RangeInclusive, ) -> ProviderResult<()> { - let mut snapshot_writer = - snapshot_provider.get_writer(*block_range.start(), SnapshotSegment::Headers)?; + let mut static_file_writer = + static_file_provider.get_writer(*block_range.start(), StaticFileSegment::Headers)?; let mut headers_cursor = provider.tx_ref().cursor_read::()?; let headers_walker = headers_cursor.walk_range(block_range.clone())?; @@ -49,15 +49,15 @@ impl Segment for Headers { debug_assert_eq!(header_block, header_td_block); debug_assert_eq!(header_td_block, canonical_header_block); - let _snapshot_block = - snapshot_writer.append_header(header, header_td.0, canonical_header)?; - debug_assert_eq!(_snapshot_block, header_block); + let _static_file_block = + static_file_writer.append_header(header, header_td.0, canonical_header)?; + debug_assert_eq!(_static_file_block, header_block); } Ok(()) } - fn create_snapshot_file( + fn create_static_file_file( &self, provider: &DatabaseProviderRO, directory: &Path, @@ -68,7 +68,7 @@ impl Segment for Headers { let mut jar = prepare_jar::( provider, directory, - SnapshotSegment::Headers, + StaticFileSegment::Headers, config, block_range.clone(), range_len, @@ -105,7 +105,7 @@ impl Segment for Headers { ); } - create_snapshot_T1_T2_T3::< + create_static_file_T1_T2_T3::< tables::Headers, tables::HeaderTD, tables::CanonicalHeaders, diff --git a/crates/snapshot/src/segments/mod.rs b/crates/static-file/src/segments/mod.rs similarity index 77% rename from crates/snapshot/src/segments/mod.rs rename to crates/static-file/src/segments/mod.rs index 7f0e3f109e06..7cad895aed7b 100644 --- a/crates/snapshot/src/segments/mod.rs +++ b/crates/static-file/src/segments/mod.rs @@ -1,4 +1,4 @@ -//! Snapshot segment implementations and utilities. +//! StaticFile segment implementations and utilities. mod transactions; pub use transactions::Transactions; @@ -15,34 +15,34 @@ use reth_db::{ use reth_interfaces::provider::ProviderResult; use reth_nippy_jar::NippyJar; use reth_primitives::{ - snapshot::{ + static_file::{ find_fixed_range, Compression, Filters, InclusionFilter, PerfectHashingFunction, SegmentConfig, SegmentHeader, }, - BlockNumber, SnapshotSegment, + BlockNumber, StaticFileSegment, }; -use reth_provider::{providers::SnapshotProvider, DatabaseProviderRO, TransactionsProviderExt}; +use reth_provider::{providers::StaticFileProvider, DatabaseProviderRO, TransactionsProviderExt}; use std::{ops::RangeInclusive, path::Path}; pub(crate) type Rows = [Vec>; COLUMNS]; -/// A segment represents a snapshotting of some portion of the data. +/// A segment represents moving some portion of the data to static files. pub trait Segment: Send + Sync { - /// Returns the [`SnapshotSegment`]. - fn segment(&self) -> SnapshotSegment; + /// Returns the [`StaticFileSegment`]. + fn segment(&self) -> StaticFileSegment; - /// Snapshot data for the provided block range. [SnapshotProvider] will handle the management of - /// and writing to files. - fn snapshot( + /// Move data to static files for the provided block range. [StaticFileProvider] will handle the + /// management of and writing to files. + fn copy_to_static_files( &self, provider: DatabaseProviderRO, - snapshot_provider: SnapshotProvider, + static_file_provider: StaticFileProvider, block_range: RangeInclusive, ) -> ProviderResult<()>; - /// Create a snapshot file of data for the provided block range. The `directory` parameter - /// determines the snapshot file's save location. - fn create_snapshot_file( + /// Create a static file of data for the provided block range. The `directory` parameter + /// determines the static file's save location. + fn create_static_file_file( &self, provider: &DatabaseProviderRO, directory: &Path, @@ -52,19 +52,19 @@ pub trait Segment: Send + Sync { } /// Returns a [`NippyJar`] according to the desired configuration. The `directory` parameter -/// determines the snapshot file's save location. +/// determines the static file's save location. pub(crate) fn prepare_jar( provider: &DatabaseProviderRO, directory: impl AsRef, - segment: SnapshotSegment, + segment: StaticFileSegment, segment_config: SegmentConfig, block_range: RangeInclusive, total_rows: usize, prepare_compression: impl Fn() -> ProviderResult>, ) -> ProviderResult> { let tx_range = match segment { - SnapshotSegment::Headers => None, - SnapshotSegment::Receipts | SnapshotSegment::Transactions => { + StaticFileSegment::Headers => None, + StaticFileSegment::Receipts | StaticFileSegment::Transactions => { Some(provider.transaction_range_by_block_range(block_range.clone())?.into()) } }; diff --git a/crates/snapshot/src/segments/receipts.rs b/crates/static-file/src/segments/receipts.rs similarity index 70% rename from crates/snapshot/src/segments/receipts.rs rename to crates/static-file/src/segments/receipts.rs index ddfe9e0c4589..02693ee1cdb4 100644 --- a/crates/snapshot/src/segments/receipts.rs +++ b/crates/static-file/src/segments/receipts.rs @@ -1,39 +1,41 @@ use crate::segments::{dataset_for_compression, prepare_jar, Segment}; use reth_db::{ - cursor::DbCursorRO, database::Database, snapshot::create_snapshot_T1, tables, transaction::DbTx, + cursor::DbCursorRO, database::Database, static_file::create_static_file_T1, tables, + transaction::DbTx, }; use reth_interfaces::provider::{ProviderError, ProviderResult}; use reth_primitives::{ - snapshot::{SegmentConfig, SegmentHeader}, - BlockNumber, SnapshotSegment, TxNumber, + static_file::{SegmentConfig, SegmentHeader}, + BlockNumber, StaticFileSegment, TxNumber, }; use reth_provider::{ - providers::{SnapshotProvider, SnapshotWriter}, + providers::{StaticFileProvider, StaticFileWriter}, BlockReader, DatabaseProviderRO, TransactionsProviderExt, }; use std::{ops::RangeInclusive, path::Path}; -/// Snapshot segment responsible for [SnapshotSegment::Receipts] part of data. +/// Static File segment responsible for [StaticFileSegment::Receipts] part of data. #[derive(Debug, Default)] pub struct Receipts; impl Segment for Receipts { - fn segment(&self) -> SnapshotSegment { - SnapshotSegment::Receipts + fn segment(&self) -> StaticFileSegment { + StaticFileSegment::Receipts } - fn snapshot( + fn copy_to_static_files( &self, provider: DatabaseProviderRO, - snapshot_provider: SnapshotProvider, + static_file_provider: StaticFileProvider, block_range: RangeInclusive, ) -> ProviderResult<()> { - let mut snapshot_writer = - snapshot_provider.get_writer(*block_range.start(), SnapshotSegment::Receipts)?; + let mut static_file_writer = + static_file_provider.get_writer(*block_range.start(), StaticFileSegment::Receipts)?; for block in block_range { - let _snapshot_block = snapshot_writer.increment_block(SnapshotSegment::Receipts)?; - debug_assert_eq!(_snapshot_block, block); + let _static_file_block = + static_file_writer.increment_block(StaticFileSegment::Receipts)?; + debug_assert_eq!(_static_file_block, block); let block_body_indices = provider .block_body_indices(block)? @@ -45,14 +47,14 @@ impl Segment for Receipts { for entry in receipts_walker { let (tx_number, receipt) = entry?; - snapshot_writer.append_receipt(tx_number, receipt)?; + static_file_writer.append_receipt(tx_number, receipt)?; } } Ok(()) } - fn create_snapshot_file( + fn create_static_file_file( &self, provider: &DatabaseProviderRO, directory: &Path, @@ -65,7 +67,7 @@ impl Segment for Receipts { let mut jar = prepare_jar::( provider, directory, - SnapshotSegment::Receipts, + StaticFileSegment::Receipts, config, block_range, tx_range_len, @@ -89,7 +91,7 @@ impl Segment for Receipts { ); } - create_snapshot_T1::( + create_static_file_T1::( provider.tx_ref(), tx_range, None, diff --git a/crates/snapshot/src/segments/transactions.rs b/crates/static-file/src/segments/transactions.rs similarity index 69% rename from crates/snapshot/src/segments/transactions.rs rename to crates/static-file/src/segments/transactions.rs index f4c8efe977d4..c16d7359e58e 100644 --- a/crates/snapshot/src/segments/transactions.rs +++ b/crates/static-file/src/segments/transactions.rs @@ -1,41 +1,43 @@ use crate::segments::{dataset_for_compression, prepare_jar, Segment}; use reth_db::{ - cursor::DbCursorRO, database::Database, snapshot::create_snapshot_T1, tables, transaction::DbTx, + cursor::DbCursorRO, database::Database, static_file::create_static_file_T1, tables, + transaction::DbTx, }; use reth_interfaces::provider::{ProviderError, ProviderResult}; use reth_primitives::{ - snapshot::{SegmentConfig, SegmentHeader}, - BlockNumber, SnapshotSegment, TxNumber, + static_file::{SegmentConfig, SegmentHeader}, + BlockNumber, StaticFileSegment, TxNumber, }; use reth_provider::{ - providers::{SnapshotProvider, SnapshotWriter}, + providers::{StaticFileProvider, StaticFileWriter}, BlockReader, DatabaseProviderRO, TransactionsProviderExt, }; use std::{ops::RangeInclusive, path::Path}; -/// Snapshot segment responsible for [SnapshotSegment::Transactions] part of data. +/// Static File segment responsible for [StaticFileSegment::Transactions] part of data. #[derive(Debug, Default)] pub struct Transactions; impl Segment for Transactions { - fn segment(&self) -> SnapshotSegment { - SnapshotSegment::Transactions + fn segment(&self) -> StaticFileSegment { + StaticFileSegment::Transactions } /// Write transactions from database table [tables::Transactions] to static files with segment - /// [SnapshotSegment::Transactions] for the provided block range. - fn snapshot( + /// [StaticFileSegment::Transactions] for the provided block range. + fn copy_to_static_files( &self, provider: DatabaseProviderRO, - snapshot_provider: SnapshotProvider, + static_file_provider: StaticFileProvider, block_range: RangeInclusive, ) -> ProviderResult<()> { - let mut snapshot_writer = - snapshot_provider.get_writer(*block_range.start(), SnapshotSegment::Transactions)?; + let mut static_file_writer = static_file_provider + .get_writer(*block_range.start(), StaticFileSegment::Transactions)?; for block in block_range { - let _snapshot_block = snapshot_writer.increment_block(SnapshotSegment::Transactions)?; - debug_assert_eq!(_snapshot_block, block); + let _static_file_block = + static_file_writer.increment_block(StaticFileSegment::Transactions)?; + debug_assert_eq!(_static_file_block, block); let block_body_indices = provider .block_body_indices(block)? @@ -49,14 +51,14 @@ impl Segment for Transactions { for entry in transactions_walker { let (tx_number, transaction) = entry?; - snapshot_writer.append_transaction(tx_number, transaction)?; + static_file_writer.append_transaction(tx_number, transaction)?; } } Ok(()) } - fn create_snapshot_file( + fn create_static_file_file( &self, provider: &DatabaseProviderRO, directory: &Path, @@ -69,7 +71,7 @@ impl Segment for Transactions { let mut jar = prepare_jar::( provider, directory, - SnapshotSegment::Transactions, + StaticFileSegment::Transactions, config, block_range, tx_range_len, @@ -93,7 +95,7 @@ impl Segment for Transactions { ); } - create_snapshot_T1::( + create_static_file_T1::( provider.tx_ref(), tx_range, None, diff --git a/crates/static-file/src/static_file_producer.rs b/crates/static-file/src/static_file_producer.rs new file mode 100644 index 000000000000..15261918ca02 --- /dev/null +++ b/crates/static-file/src/static_file_producer.rs @@ -0,0 +1,325 @@ +//! Support for producing static files. + +use crate::{segments, segments::Segment, StaticFileProducerEvent}; +use rayon::prelude::*; +use reth_db::database::Database; +use reth_interfaces::RethResult; +use reth_primitives::{static_file::HighestStaticFiles, BlockNumber, PruneModes}; +use reth_provider::{ + providers::{StaticFileProvider, StaticFileWriter}, + ProviderFactory, +}; +use reth_tokio_util::EventListeners; +use std::{ops::RangeInclusive, time::Instant}; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tracing::{debug, trace}; + +/// Result of [StaticFileProducer::run] execution. +pub type StaticFileProducerResult = RethResult; + +/// The [StaticFileProducer] instance itself with the result of [StaticFileProducer::run] +pub type StaticFileProducerWithResult = (StaticFileProducer, StaticFileProducerResult); + +/// Static File producer routine. See [StaticFileProducer::run] for more detailed description. +#[derive(Debug, Clone)] +pub struct StaticFileProducer { + /// Provider factory + provider_factory: ProviderFactory, + /// Static File provider + static_file_provider: StaticFileProvider, + /// Pruning configuration for every part of the data that can be pruned. Set by user, and + /// needed in [StaticFileProducer] to prevent attempting to move prunable data to static files. + /// See [StaticFileProducer::get_static_file_targets]. + prune_modes: PruneModes, + listeners: EventListeners, +} + +/// Static File targets, per data part, measured in [`BlockNumber`]. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct StaticFileTargets { + headers: Option>, + receipts: Option>, + transactions: Option>, +} + +impl StaticFileTargets { + /// Returns `true` if any of the targets are [Some]. + pub fn any(&self) -> bool { + self.headers.is_some() || self.receipts.is_some() || self.transactions.is_some() + } + + // Returns `true` if all targets are either [`None`] or has beginning of the range equal to the + // highest static_file. + fn is_contiguous_to_highest_static_files(&self, static_files: HighestStaticFiles) -> bool { + [ + (self.headers.as_ref(), static_files.headers), + (self.receipts.as_ref(), static_files.receipts), + (self.transactions.as_ref(), static_files.transactions), + ] + .iter() + .all(|(target_block_range, highest_static_fileted_block)| { + target_block_range.map_or(true, |target_block_range| { + *target_block_range.start() == + highest_static_fileted_block.map_or(0, |highest_static_fileted_block| { + highest_static_fileted_block + 1 + }) + }) + }) + } +} + +impl StaticFileProducer { + /// Creates a new [StaticFileProducer]. + pub fn new( + provider_factory: ProviderFactory, + static_file_provider: StaticFileProvider, + prune_modes: PruneModes, + ) -> Self { + Self { provider_factory, static_file_provider, prune_modes, listeners: Default::default() } + } + + /// Listen for events on the static_file_producer. + pub fn events(&mut self) -> UnboundedReceiverStream { + self.listeners.new_listener() + } + + /// Run the static_file_producer. + /// + /// For each [Some] target in [StaticFileTargets], initializes a corresponding [Segment] and + /// runs it with the provided block range using [StaticFileProvider] and a read-only + /// database transaction from [ProviderFactory]. All segments are run in parallel. + /// + /// NOTE: it doesn't delete the data from database, and the actual deleting (aka pruning) logic + /// lives in the `prune` crate. + pub fn run(&mut self, targets: StaticFileTargets) -> StaticFileProducerResult { + debug_assert!(targets.is_contiguous_to_highest_static_files( + self.static_file_provider.get_highest_static_files() + )); + + debug!(target: "static_file", ?targets, "StaticFileProducer started"); + let start = Instant::now(); + + let mut segments = Vec::<(Box>, RangeInclusive)>::new(); + + if let Some(block_range) = targets.transactions.clone() { + segments.push((Box::new(segments::Transactions), block_range)); + } + if let Some(block_range) = targets.headers.clone() { + segments.push((Box::new(segments::Headers), block_range)); + } + if let Some(block_range) = targets.receipts.clone() { + segments.push((Box::new(segments::Receipts), block_range)); + } + + segments.par_iter().try_for_each(|(segment, block_range)| -> RethResult<()> { + debug!(target: "static_file", segment = %segment.segment(), ?block_range, "StaticFileProducer segment"); + let start = Instant::now(); + + // Create a new database transaction on every segment to prevent long-lived read-only + // transactions + let provider = self.provider_factory.provider()?.disable_long_read_transaction_safety(); + segment.copy_to_static_files(provider, self.static_file_provider.clone(), block_range.clone())?; + + let elapsed = start.elapsed(); // TODO(alexey): track in metrics + debug!(target: "static_file", segment = %segment.segment(), ?block_range, ?elapsed, "Finished StaticFileProducer segment"); + + Ok(()) + })?; + + self.static_file_provider.commit()?; + for (segment, block_range) in segments { + self.static_file_provider.update_index(segment.segment(), Some(*block_range.end()))?; + } + + let elapsed = start.elapsed(); // TODO(alexey): track in metrics + debug!(target: "static_file", ?targets, ?elapsed, "StaticFileProducer finished"); + + self.listeners + .notify(StaticFileProducerEvent::Finished { targets: targets.clone(), elapsed }); + + Ok(targets) + } + + /// Returns a static file targets at the provided finalized block numbers per segment. + /// The target is determined by the check against highest static_files using + /// [StaticFileProvider::get_highest_static_files]. + pub fn get_static_file_targets( + &self, + finalized_block_numbers: HighestStaticFiles, + ) -> RethResult { + let highest_static_files = self.static_file_provider.get_highest_static_files(); + + let targets = StaticFileTargets { + headers: finalized_block_numbers.headers.and_then(|finalized_block_number| { + self.get_static_file_target(highest_static_files.headers, finalized_block_number) + }), + // StaticFile receipts only if they're not pruned according to the user configuration + receipts: if self.prune_modes.receipts.is_none() && + self.prune_modes.receipts_log_filter.is_empty() + { + finalized_block_numbers.receipts.and_then(|finalized_block_number| { + self.get_static_file_target( + highest_static_files.receipts, + finalized_block_number, + ) + }) + } else { + None + }, + transactions: finalized_block_numbers.transactions.and_then(|finalized_block_number| { + self.get_static_file_target( + highest_static_files.transactions, + finalized_block_number, + ) + }), + }; + + trace!( + target: "static_file", + ?finalized_block_numbers, + ?highest_static_files, + ?targets, + any = %targets.any(), + "StaticFile targets" + ); + + Ok(targets) + } + + fn get_static_file_target( + &self, + highest_static_file: Option, + finalized_block_number: BlockNumber, + ) -> Option> { + let range = highest_static_file.map_or(0, |block| block + 1)..=finalized_block_number; + (!range.is_empty()).then_some(range) + } +} + +#[cfg(test)] +mod tests { + use crate::{static_file_producer::StaticFileTargets, StaticFileProducer}; + use assert_matches::assert_matches; + use reth_db::{database::Database, transaction::DbTx}; + use reth_interfaces::{ + provider::ProviderError, + test_utils::{ + generators, + generators::{random_block_range, random_receipt}, + }, + RethError, + }; + use reth_primitives::{ + static_file::HighestStaticFiles, PruneModes, StaticFileSegment, B256, U256, + }; + use reth_provider::providers::StaticFileWriter; + use reth_stages::test_utils::{StorageKind, TestStageDB}; + + #[test] + fn run() { + let mut rng = generators::rng(); + + let db = TestStageDB::default(); + + let blocks = random_block_range(&mut rng, 0..=3, B256::ZERO, 2..3); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + // Unwind headers from static_files and manually insert them into the database, so we're + // able to check that static_file_producer works + db.factory + .static_file_provider() + .latest_writer(StaticFileSegment::Headers) + .expect("get static file writer for headers") + .prune_headers(blocks.len() as u64) + .expect("prune headers"); + let tx = db.factory.db_ref().tx_mut().expect("init tx"); + blocks.iter().for_each(|block| { + TestStageDB::insert_header(None, &tx, &block.header, U256::ZERO) + .expect("insert block header"); + }); + tx.commit().expect("commit tx"); + + let mut receipts = Vec::new(); + for block in &blocks { + for transaction in &block.body { + receipts + .push((receipts.len() as u64, random_receipt(&mut rng, transaction, Some(0)))); + } + } + db.insert_receipts(receipts).expect("insert receipts"); + + let provider_factory = db.factory; + let static_file_provider = provider_factory.static_file_provider(); + + let mut static_file_producer = StaticFileProducer::new( + provider_factory, + static_file_provider.clone(), + PruneModes::default(), + ); + + let targets = static_file_producer + .get_static_file_targets(HighestStaticFiles { + headers: Some(1), + receipts: Some(1), + transactions: Some(1), + }) + .expect("get static file targets"); + assert_eq!( + targets, + StaticFileTargets { + headers: Some(0..=1), + receipts: Some(0..=1), + transactions: Some(0..=1) + } + ); + assert_matches!(static_file_producer.run(targets), Ok(_)); + assert_eq!( + static_file_provider.get_highest_static_files(), + HighestStaticFiles { headers: Some(1), receipts: Some(1), transactions: Some(1) } + ); + + let targets = static_file_producer + .get_static_file_targets(HighestStaticFiles { + headers: Some(3), + receipts: Some(3), + transactions: Some(3), + }) + .expect("get static file targets"); + assert_eq!( + targets, + StaticFileTargets { + headers: Some(2..=3), + receipts: Some(2..=3), + transactions: Some(2..=3) + } + ); + assert_matches!(static_file_producer.run(targets), Ok(_)); + assert_eq!( + static_file_provider.get_highest_static_files(), + HighestStaticFiles { headers: Some(3), receipts: Some(3), transactions: Some(3) } + ); + + let targets = static_file_producer + .get_static_file_targets(HighestStaticFiles { + headers: Some(4), + receipts: Some(4), + transactions: Some(4), + }) + .expect("get static file targets"); + assert_eq!( + targets, + StaticFileTargets { + headers: Some(4..=4), + receipts: Some(4..=4), + transactions: Some(4..=4) + } + ); + assert_matches!( + static_file_producer.run(targets), + Err(RethError::Provider(ProviderError::BlockBodyIndicesNotFound(4))) + ); + assert_eq!( + static_file_provider.get_highest_static_files(), + HighestStaticFiles { headers: Some(3), receipts: Some(3), transactions: Some(3) } + ); + } +} diff --git a/crates/storage/db/src/lib.rs b/crates/storage/db/src/lib.rs index 9021eeaace7d..ea260eaebf89 100644 --- a/crates/storage/db/src/lib.rs +++ b/crates/storage/db/src/lib.rs @@ -67,7 +67,7 @@ pub mod abstraction; mod implementation; mod metrics; -pub mod snapshot; +pub mod static_file; pub mod tables; mod utils; pub mod version; @@ -164,7 +164,7 @@ pub mod test_utils { /// Error during database creation pub const ERROR_DB_CREATION: &str = "Not able to create the database file."; /// Error during database creation - pub const ERROR_SNAPSHOTS_CREATION: &str = "Not able to create the snapshot path."; + pub const ERROR_STATIC_FILES_CREATION: &str = "Not able to create the static file path."; /// Error during table creation pub const ERROR_TABLE_CREATION: &str = "Not able to create tables in the database."; /// Error during tempdir creation @@ -227,10 +227,10 @@ pub mod test_utils { } } - /// Create snapshots path for testing - pub fn create_test_snapshots_dir() -> PathBuf { + /// Create static_files path for testing + pub fn create_test_static_files_dir() -> PathBuf { let path = tempdir_path(); - let emsg = format!("{}: {:?}", ERROR_SNAPSHOTS_CREATION, path); + let emsg = format!("{}: {:?}", ERROR_STATIC_FILES_CREATION, path); reth_primitives::fs::create_dir_all(path.clone()).expect(&emsg); path diff --git a/crates/storage/db/src/snapshot/masks.rs b/crates/storage/db/src/snapshot/masks.rs deleted file mode 100644 index 53db0212a8f7..000000000000 --- a/crates/storage/db/src/snapshot/masks.rs +++ /dev/null @@ -1,21 +0,0 @@ -use super::{ReceiptMask, TransactionMask}; -use crate::{ - add_snapshot_mask, - snapshot::mask::{ColumnSelectorOne, ColumnSelectorTwo, HeaderMask}, - table::Table, - HeaderTD, Receipts, Transactions, -}; -use reth_primitives::{BlockHash, Header}; - -// HEADER MASKS -add_snapshot_mask!(HeaderMask, Header, 0b001); -add_snapshot_mask!(HeaderMask, ::Value, 0b010); -add_snapshot_mask!(HeaderMask, BlockHash, 0b100); -add_snapshot_mask!(HeaderMask, Header, BlockHash, 0b101); -add_snapshot_mask!(HeaderMask, ::Value, BlockHash, 0b110); - -// RECEIPT MASKS -add_snapshot_mask!(ReceiptMask, ::Value, 0b1); - -// TRANSACTION MASKS -add_snapshot_mask!(TransactionMask, ::Value, 0b1); diff --git a/crates/storage/db/src/snapshot/cursor.rs b/crates/storage/db/src/static_file/cursor.rs similarity index 90% rename from crates/storage/db/src/snapshot/cursor.rs rename to crates/storage/db/src/static_file/cursor.rs index 2302ad343566..5b27ac18de4a 100644 --- a/crates/storage/db/src/snapshot/cursor.rs +++ b/crates/storage/db/src/static_file/cursor.rs @@ -3,21 +3,21 @@ use crate::table::Decompress; use derive_more::{Deref, DerefMut}; use reth_interfaces::provider::ProviderResult; use reth_nippy_jar::{DataReader, NippyJar, NippyJarCursor}; -use reth_primitives::{snapshot::SegmentHeader, B256}; +use reth_primitives::{static_file::SegmentHeader, B256}; use std::sync::Arc; -/// Cursor of a snapshot segment. +/// Cursor of a static file segment. #[derive(Debug, Deref, DerefMut)] -pub struct SnapshotCursor<'a>(NippyJarCursor<'a, SegmentHeader>); +pub struct StaticFileCursor<'a>(NippyJarCursor<'a, SegmentHeader>); -impl<'a> SnapshotCursor<'a> { - /// Returns a new [`SnapshotCursor`]. +impl<'a> StaticFileCursor<'a> { + /// Returns a new [`StaticFileCursor`]. pub fn new(jar: &'a NippyJar, reader: Arc) -> ProviderResult { Ok(Self(NippyJarCursor::with_reader(jar, reader)?)) } /// Returns the current `BlockNumber` or `TxNumber` of the cursor depending on the kind of - /// snapshot segment. + /// static file segment. pub fn number(&self) -> Option { self.jar().user_header().start().map(|start| self.row_index() + start) } @@ -29,7 +29,7 @@ impl<'a> SnapshotCursor<'a> { mask: usize, ) -> ProviderResult>> { if self.jar().rows() == 0 { - return Ok(None) + return Ok(None); } let row = match key_or_num { @@ -37,7 +37,7 @@ impl<'a> SnapshotCursor<'a> { KeyOrNumber::Number(n) => match self.jar().user_header().start() { Some(offset) => { if offset > n { - return Ok(None) + return Ok(None); } self.row_by_number_with_cols((n - offset) as usize, mask) } diff --git a/crates/storage/db/src/snapshot/generation.rs b/crates/storage/db/src/static_file/generation.rs similarity index 70% rename from crates/storage/db/src/snapshot/generation.rs rename to crates/storage/db/src/static_file/generation.rs index ea1c1e65431d..82afa7e118fd 100644 --- a/crates/storage/db/src/snapshot/generation.rs +++ b/crates/storage/db/src/static_file/generation.rs @@ -10,16 +10,16 @@ use reth_nippy_jar::{ColumnResult, NippyJar, NippyJarHeader, PHFKey}; use reth_tracing::tracing::*; use std::{error::Error as StdError, ops::RangeInclusive}; -/// Macro that generates snapshot creation functions that take an arbitratry number of [`Table`] and -/// creates a [`NippyJar`] file out of their [`Table::Value`]. Each list of [`Table::Value`] from a -/// table is a column of values. +/// Macro that generates static file creation functions that take an arbitratry number of [`Table`] +/// and creates a [`NippyJar`] file out of their [`Table::Value`]. Each list of [`Table::Value`] +/// from a table is a column of values. /// /// Has membership filter set and compression dictionary support. -macro_rules! generate_snapshot_func { +macro_rules! generate_static_file_func { ($(($($tbl:ident),+)),+ $(,)? ) => { $( paste::item! { - /// Creates a snapshot from specified tables. Each table's `Value` iterator represents a column. + /// Creates a static file from specified tables. Each table's `Value` iterator represents a column. /// /// **Ensure the range contains the same number of rows.** /// @@ -29,9 +29,9 @@ macro_rules! generate_snapshot_func { /// * `keys`: Iterator of keys (eg. `TxHash` or `BlockHash`) with length equal to `row_count` and ordered by future column insertion from `range`. /// * `dict_compression_set`: Sets of column data for compression dictionaries. Max size is 2GB. Row count is independent. /// * `row_count`: Total rows to add to `NippyJar`. Must match row count in `range`. - /// * `nippy_jar`: Snapshot object responsible for file generation. + /// * `nippy_jar`: Static File object responsible for file generation. #[allow(non_snake_case)] - pub fn []< + pub fn []< $($tbl: Table,)+ K, H: NippyJarHeader @@ -48,22 +48,22 @@ macro_rules! generate_snapshot_func { where K: Key + Copy { let additional = additional.unwrap_or_default(); - debug!(target: "reth::snapshot", ?range, "Creating snapshot {:?} and {} more columns.", vec![$($tbl::NAME,)+], additional.len()); + debug!(target: "reth::static_file", ?range, "Creating static file {:?} and {} more columns.", vec![$($tbl::NAME,)+], additional.len()); let range: RangeInclusive> = RawKey::new(*range.start())..=RawKey::new(*range.end()); // Create PHF and Filter if required if let Some(keys) = keys { - debug!(target: "reth::snapshot", "Calculating Filter, PHF and offset index list"); + debug!(target: "reth::static_file", "Calculating Filter, PHF and offset index list"); nippy_jar.prepare_index(keys, row_count)?; - debug!(target: "reth::snapshot", "Filter, PHF and offset index list calculated."); + debug!(target: "reth::static_file", "Filter, PHF and offset index list calculated."); } // Create compression dictionaries if required if let Some(data_sets) = dict_compression_set { - debug!(target: "reth::snapshot", "Creating compression dictionaries."); + debug!(target: "reth::static_file", "Creating compression dictionaries."); nippy_jar.prepare_compression(data_sets)?; - debug!(target: "reth::snapshot", "Compression dictionaries created."); + debug!(target: "reth::static_file", "Compression dictionaries created."); } // Creates the cursors for the columns @@ -80,17 +80,17 @@ macro_rules! generate_snapshot_func { )+ - // Create the snapshot from the data + // Create the static file from the data let col_iterators: Vec,_>>>> = vec![ $(Box::new([< $tbl _iter>]),)+ ]; - debug!(target: "reth::snapshot", jar=?nippy_jar, "Generating snapshot file."); + debug!(target: "reth::static_file", jar=?nippy_jar, "Generating static file."); nippy_jar.freeze(col_iterators.into_iter().chain(additional).collect(), row_count as u64)?; - debug!(target: "reth::snapshot", jar=?nippy_jar, "Snapshot file generated."); + debug!(target: "reth::static_file", jar=?nippy_jar, "StaticFile file generated."); Ok(()) } @@ -99,4 +99,4 @@ macro_rules! generate_snapshot_func { }; } -generate_snapshot_func!((T1), (T1, T2), (T1, T2, T3), (T1, T2, T3, T4), (T1, T2, T3, T4, T5),); +generate_static_file_func!((T1), (T1, T2), (T1, T2, T3), (T1, T2, T3, T4), (T1, T2, T3, T4, T5),); diff --git a/crates/storage/db/src/snapshot/mask.rs b/crates/storage/db/src/static_file/mask.rs similarity index 77% rename from crates/storage/db/src/snapshot/mask.rs rename to crates/storage/db/src/static_file/mask.rs index 2507dcd77ed3..de5932ea9ac3 100644 --- a/crates/storage/db/src/snapshot/mask.rs +++ b/crates/storage/db/src/static_file/mask.rs @@ -4,14 +4,14 @@ use crate::table::Decompress; /// /// #### Explanation: /// -/// A `NippyJar` snapshot row can contain multiple column values. To specify the column values +/// A `NippyJar` static file row can contain multiple column values. To specify the column values /// to be read, a mask is utilized. /// -/// For example, a snapshot with three columns, if the first and last columns are queried, the mask -/// `0b101` would be passed. To select only the second column, the mask `0b010` would be used. +/// For example, a static file with three columns, if the first and last columns are queried, the +/// mask `0b101` would be passed. To select only the second column, the mask `0b010` would be used. /// -/// Since each snapshot has its own column distribution, different wrapper types are necessary. For -/// instance, `B256` might be the third column in the `Header` segment, while being the second +/// Since each static file has its own column distribution, different wrapper types are necessary. +/// For instance, `B256` might be the third column in the `Header` segment, while being the second /// column in another segment. Hence, `Mask` would only be applicable to one of these /// scenarios. /// @@ -24,7 +24,7 @@ macro_rules! add_segments { ($($segment:tt),+) => { paste::paste! { $( - #[doc = concat!("Mask for ", stringify!($segment), " snapshot segment. See [`Mask`] for more.")] + #[doc = concat!("Mask for ", stringify!($segment), " static file segment. See [`Mask`] for more.")] #[derive(Debug)] pub struct [<$segment Mask>](Mask); )+ @@ -37,7 +37,7 @@ add_segments!(Header, Receipt, Transaction); pub trait ColumnSelectorOne { /// First desired column value type FIRST: Decompress; - /// Mask to obtain desired values, should correspond to the order of columns in a snapshot. + /// Mask to obtain desired values, should correspond to the order of columns in a static_file. const MASK: usize; } @@ -47,7 +47,7 @@ pub trait ColumnSelectorTwo { type FIRST: Decompress; /// Second desired column value type SECOND: Decompress; - /// Mask to obtain desired values, should correspond to the order of columns in a snapshot. + /// Mask to obtain desired values, should correspond to the order of columns in a static_file. const MASK: usize; } @@ -59,13 +59,13 @@ pub trait ColumnSelectorThree { type SECOND: Decompress; /// Third desired column value type THIRD: Decompress; - /// Mask to obtain desired values, should correspond to the order of columns in a snapshot. + /// Mask to obtain desired values, should correspond to the order of columns in a static_file. const MASK: usize; } #[macro_export] -/// Add mask to select `N` column values from a specific snapshot segment row. -macro_rules! add_snapshot_mask { +/// Add mask to select `N` column values from a specific static file segment row. +macro_rules! add_static_file_mask { ($mask_struct:tt, $type1:ty, $mask:expr) => { impl ColumnSelectorOne for $mask_struct<$type1> { type FIRST = $type1; diff --git a/crates/storage/db/src/static_file/masks.rs b/crates/storage/db/src/static_file/masks.rs new file mode 100644 index 000000000000..ee19ed4e03ef --- /dev/null +++ b/crates/storage/db/src/static_file/masks.rs @@ -0,0 +1,21 @@ +use super::{ReceiptMask, TransactionMask}; +use crate::{ + add_static_file_mask, + static_file::mask::{ColumnSelectorOne, ColumnSelectorTwo, HeaderMask}, + table::Table, + HeaderTD, Receipts, Transactions, +}; +use reth_primitives::{BlockHash, Header}; + +// HEADER MASKS +add_static_file_mask!(HeaderMask, Header, 0b001); +add_static_file_mask!(HeaderMask, ::Value, 0b010); +add_static_file_mask!(HeaderMask, BlockHash, 0b100); +add_static_file_mask!(HeaderMask, Header, BlockHash, 0b101); +add_static_file_mask!(HeaderMask, ::Value, BlockHash, 0b110); + +// RECEIPT MASKS +add_static_file_mask!(ReceiptMask, ::Value, 0b1); + +// TRANSACTION MASKS +add_static_file_mask!(TransactionMask, ::Value, 0b1); diff --git a/crates/storage/db/src/snapshot/mod.rs b/crates/storage/db/src/static_file/mod.rs similarity index 66% rename from crates/storage/db/src/snapshot/mod.rs rename to crates/storage/db/src/static_file/mod.rs index 3c65955b507f..eed27e0de954 100644 --- a/crates/storage/db/src/snapshot/mod.rs +++ b/crates/storage/db/src/static_file/mod.rs @@ -1,4 +1,4 @@ -//! reth's snapshot database table import and access +//! reth's static file database table import and access mod generation; use std::{ @@ -9,33 +9,33 @@ use std::{ pub use generation::*; mod cursor; -pub use cursor::SnapshotCursor; +pub use cursor::StaticFileCursor; mod mask; pub use mask::*; use reth_nippy_jar::{NippyJar, NippyJarError}; use reth_primitives::{ - snapshot::{SegmentHeader, SegmentRangeInclusive}, - SnapshotSegment, + static_file::{SegmentHeader, SegmentRangeInclusive}, + StaticFileSegment, }; mod masks; -/// Alias type for a map of [`SnapshotSegment`] and sorted lists of existing snapshot ranges. -type SortedSnapshots = - HashMap)>>; +/// Alias type for a map of [`StaticFileSegment`] and sorted lists of existing static file ranges. +type SortedStaticFiles = + HashMap)>>; -/// Given the snapshots directory path, it returns a list over the existing snapshots organized by -/// [`SnapshotSegment`]. Each segment has a sorted list of block ranges and transaction ranges as -/// presented in the file configuration. -pub fn iter_snapshots(path: impl AsRef) -> Result { +/// Given the static_files directory path, it returns a list over the existing static_files +/// organized by [`StaticFileSegment`]. Each segment has a sorted list of block ranges and +/// transaction ranges as presented in the file configuration. +pub fn iter_static_files(path: impl AsRef) -> Result { let path = path.as_ref(); if !path.exists() { reth_primitives::fs::create_dir_all(path) .map_err(|err| NippyJarError::Custom(err.to_string()))?; } - let mut static_files = SortedSnapshots::default(); + let mut static_files = SortedStaticFiles::default(); let entries = reth_primitives::fs::read_dir(path) .map_err(|err| NippyJarError::Custom(err.to_string()))? .filter_map(Result::ok) @@ -44,7 +44,7 @@ pub fn iter_snapshots(path: impl AsRef) -> Result::load(&entry.path())?; diff --git a/crates/storage/libmdbx-rs/src/environment.rs b/crates/storage/libmdbx-rs/src/environment.rs index 8cd2eb08470a..2903a6b01791 100644 --- a/crates/storage/libmdbx-rs/src/environment.rs +++ b/crates/storage/libmdbx-rs/src/environment.rs @@ -941,7 +941,8 @@ mod tests { .open(tempdir.path()) .unwrap(); - // Insert some data in the database, so the read transaction can lock on the snapshot of it + // Insert some data in the database, so the read transaction can lock on the static file of + // it { let tx = env.begin_rw_txn().unwrap(); let db = tx.open_db(None).unwrap(); @@ -954,7 +955,8 @@ mod tests { // Create a read transaction let _tx_ro = env.begin_ro_txn().unwrap(); - // Change previously inserted data, so the read transaction would use the previous snapshot + // Change previously inserted data, so the read transaction would use the previous static + // file { let tx = env.begin_rw_txn().unwrap(); let db = tx.open_db(None).unwrap(); @@ -965,7 +967,7 @@ mod tests { } // Insert more data in the database, so we hit the DB size limit error, and MDBX tries to - // kick long-lived readers and delete their snapshots + // kick long-lived readers and delete their static_files { let tx = env.begin_rw_txn().unwrap(); let db = tx.open_db(None).unwrap(); diff --git a/crates/storage/libmdbx-rs/src/flags.rs b/crates/storage/libmdbx-rs/src/flags.rs index ad88c1fbedc9..f984ffcaf024 100644 --- a/crates/storage/libmdbx-rs/src/flags.rs +++ b/crates/storage/libmdbx-rs/src/flags.rs @@ -25,7 +25,7 @@ pub enum SyncMode { /// /// [SyncMode::UtterlyNoSync] the [SyncMode::SafeNoSync] flag disable similarly flush system /// buffers to disk when committing a transaction. But there is a huge difference in how - /// are recycled the MVCC snapshots corresponding to previous "steady" transactions (see + /// are recycled the MVCC static_files corresponding to previous "steady" transactions (see /// below). /// /// With [crate::EnvironmentKind::WriteMap] the [SyncMode::SafeNoSync] instructs MDBX to use diff --git a/crates/storage/nippy-jar/src/lib.rs b/crates/storage/nippy-jar/src/lib.rs index 12513a469f61..0ff3382f441d 100644 --- a/crates/storage/nippy-jar/src/lib.rs +++ b/crates/storage/nippy-jar/src/lib.rs @@ -455,9 +455,9 @@ impl PerfectHashingFunction for NippyJar { } } -/// Manages the reading of snapshot data using memory-mapped files. +/// Manages the reading of static file data using memory-mapped files. /// -/// Holds file and mmap descriptors of the data and offsets files of a snapshot. +/// Holds file and mmap descriptors of the data and offsets files of a static_file. #[derive(Debug)] pub struct DataReader { /// Data file descriptor. Needs to be kept alive as long as `data_mmap` handle. @@ -930,7 +930,7 @@ mod tests { let mut data = col1.iter().zip(col2.iter()).enumerate().collect::>(); data.shuffle(&mut rand::thread_rng()); - // Imagine `Blocks` snapshot file has two columns: `Block | StoredWithdrawals` + // Imagine `Blocks` static file has two columns: `Block | StoredWithdrawals` const BLOCKS_FULL_MASK: usize = 0b11; // Read both columns diff --git a/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs b/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs index dacc1df7d2e7..bda63002f210 100644 --- a/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs +++ b/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs @@ -1,4 +1,4 @@ -use crate::{providers::SnapshotProviderRWRefMut, StateChanges, StateReverts}; +use crate::{providers::StaticFileProviderRWRefMut, StateChanges, StateReverts}; use reth_db::{ cursor::{DbCursorRO, DbCursorRW}, tables, @@ -8,7 +8,7 @@ use reth_interfaces::provider::{ProviderError, ProviderResult}; use reth_primitives::{ logs_bloom, revm::compat::{into_reth_acc, into_revm_acc}, - Account, Address, BlockNumber, Bloom, Bytecode, Log, Receipt, Receipts, SnapshotSegment, + Account, Address, BlockNumber, Bloom, Bytecode, Log, Receipt, Receipts, StaticFileSegment, StorageEntry, B256, U256, }; use reth_trie::HashedPostState; @@ -286,7 +286,7 @@ impl BundleStateWithReceipts { } /// Write the [BundleStateWithReceipts] to database and receipts to either database or static - /// files if `snapshotter` is `Some`. It should be none if there is any kind of + /// files if `static_file_producer` is `Some`. It should be none if there is any kind of /// pruning/filtering over the receipts. /// /// `omit_changed_check` should be set to true of bundle has some of it data @@ -294,7 +294,7 @@ impl BundleStateWithReceipts { pub fn write_to_storage( self, tx: &TX, - mut snapshotter: Option>, + mut static_file_producer: Option>, is_value_known: OriginalValuesKnown, ) -> ProviderResult<()> where @@ -315,14 +315,14 @@ impl BundleStateWithReceipts { .map(|(_, indices)| indices.first_tx_num()) .ok_or_else(|| ProviderError::BlockBodyIndicesNotFound(block_number))?; - if let Some(snapshotter) = &mut snapshotter { + if let Some(static_file_producer) = &mut static_file_producer { // Increment block on static file header. - snapshotter.increment_block(SnapshotSegment::Receipts)?; + static_file_producer.increment_block(StaticFileSegment::Receipts)?; for (tx_idx, receipt) in receipts.into_iter().enumerate() { let receipt = receipt .expect("receipt should not be filtered when saving to static files."); - snapshotter.append_receipt(first_tx_index + tx_idx as u64, receipt)?; + static_file_producer.append_receipt(first_tx_index + tx_idx as u64, receipt)?; } } else if !receipts.is_empty() { for (tx_idx, receipt) in receipts.into_iter().enumerate() { diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 6fa23e826eac..6287406f3029 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -1,7 +1,7 @@ use crate::{ providers::{ state::{historical::HistoricalStateProvider, latest::LatestStateProvider}, - SnapshotProvider, + StaticFileProvider, }, to_range, traits::{BlockSource, ReceiptProvider}, @@ -17,7 +17,7 @@ use reth_primitives::{ stage::{StageCheckpoint, StageId}, Address, Block, BlockHash, BlockHashOrNumber, BlockNumber, BlockWithSenders, ChainInfo, ChainSpec, Header, PruneCheckpoint, PruneSegment, Receipt, SealedBlock, SealedBlockWithSenders, - SealedHeader, SnapshotSegment, TransactionMeta, TransactionSigned, TransactionSignedNoHash, + SealedHeader, StaticFileSegment, TransactionMeta, TransactionSigned, TransactionSignedNoHash, TxHash, TxNumber, Withdrawal, Withdrawals, B256, U256, }; use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg}; @@ -43,8 +43,8 @@ pub struct ProviderFactory { db: DB, /// Chain spec chain_spec: Arc, - /// Snapshot Provider - snapshot_provider: SnapshotProvider, + /// Static File Provider + static_file_provider: StaticFileProvider, } impl ProviderFactory { @@ -52,9 +52,13 @@ impl ProviderFactory { pub fn new( db: DB, chain_spec: Arc, - snapshots_path: PathBuf, + static_files_path: PathBuf, ) -> RethResult> { - Ok(Self { db, chain_spec, snapshot_provider: SnapshotProvider::new(snapshots_path)? }) + Ok(Self { + db, + chain_spec, + static_file_provider: StaticFileProvider::new(static_files_path)?, + }) } /// Create new database provider by passing a path. [`ProviderFactory`] will own the database @@ -63,18 +67,18 @@ impl ProviderFactory { path: P, chain_spec: Arc, args: DatabaseArguments, - snapshots_path: PathBuf, + static_files_path: PathBuf, ) -> RethResult> { Ok(ProviderFactory:: { db: init_db(path, args).map_err(|e| RethError::Custom(e.to_string()))?, chain_spec, - snapshot_provider: SnapshotProvider::new(snapshots_path)?, + static_file_provider: StaticFileProvider::new(static_files_path)?, }) } - /// Enables metrics on the snapshot provider. - pub fn with_snapshots_metrics(mut self) -> Self { - self.snapshot_provider = self.snapshot_provider.with_metrics(); + /// Enables metrics on the static file provider. + pub fn with_static_files_metrics(mut self) -> Self { + self.static_file_provider = self.static_file_provider.with_metrics(); self } @@ -83,9 +87,9 @@ impl ProviderFactory { &self.db } - /// Returns snapshot provider - pub fn snapshot_provider(&self) -> SnapshotProvider { - self.snapshot_provider.clone() + /// Returns static file provider + pub fn static_file_provider(&self) -> StaticFileProvider { + self.static_file_provider.clone() } #[cfg(any(test, feature = "test-utils"))] @@ -104,7 +108,7 @@ impl ProviderFactory { Ok(DatabaseProvider::new( self.db.tx()?, self.chain_spec.clone(), - self.snapshot_provider.clone(), + self.static_file_provider.clone(), )) } @@ -117,7 +121,7 @@ impl ProviderFactory { Ok(DatabaseProviderRW(DatabaseProvider::new_rw( self.db.tx_mut()?, self.chain_spec.clone(), - self.snapshot_provider.clone(), + self.static_file_provider.clone(), ))) } @@ -125,7 +129,7 @@ impl ProviderFactory { #[track_caller] pub fn latest(&self) -> ProviderResult { trace!(target: "providers::db", "Returning latest state provider"); - Ok(Box::new(LatestStateProvider::new(self.db.tx()?, self.snapshot_provider()))) + Ok(Box::new(LatestStateProvider::new(self.db.tx()?, self.static_file_provider()))) } /// Storage provider for state at that given block @@ -139,7 +143,7 @@ impl ProviderFactory { { return Ok(Box::new(LatestStateProvider::new( provider.into_tx(), - self.snapshot_provider(), + self.static_file_provider(), ))) } @@ -154,7 +158,7 @@ impl ProviderFactory { let mut state_provider = HistoricalStateProvider::new( provider.into_tx(), block_number, - self.snapshot_provider(), + self.static_file_provider(), ); // If we pruned account or storage history, we can't return state on every historical block. @@ -218,10 +222,10 @@ impl HeaderProvider for ProviderFactory { } fn header_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Headers, num, - |snapshot| snapshot.header_by_number(num), + |static_file| static_file.header_by_number(num), || self.provider()?.header_by_number(num), ) } @@ -237,29 +241,29 @@ impl HeaderProvider for ProviderFactory { return Ok(Some(td)) } - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Headers, number, - |snapshot| snapshot.header_td_by_number(number), + |static_file| static_file.header_td_by_number(number), || self.provider()?.header_td_by_number(number), ) } fn headers_range(&self, range: impl RangeBounds) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Headers, to_range(range), - |snapshot, range, _| snapshot.headers_range(range), + |static_file, range, _| static_file.headers_range(range), |range, _| self.provider()?.headers_range(range), |_| true, ) } fn sealed_header(&self, number: BlockNumber) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Headers, number, - |snapshot| snapshot.sealed_header(number), + |static_file| static_file.sealed_header(number), || self.provider()?.sealed_header(number), ) } @@ -276,10 +280,10 @@ impl HeaderProvider for ProviderFactory { range: impl RangeBounds, predicate: impl FnMut(&SealedHeader) -> bool, ) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Headers, to_range(range), - |snapshot, range, predicate| snapshot.sealed_headers_while(range, predicate), + |static_file, range, predicate| static_file.sealed_headers_while(range, predicate), |range, predicate| self.provider()?.sealed_headers_while(range, predicate), predicate, ) @@ -288,10 +292,10 @@ impl HeaderProvider for ProviderFactory { impl BlockHashReader for ProviderFactory { fn block_hash(&self, number: u64) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Headers, number, - |snapshot| snapshot.block_hash(number), + |static_file| static_file.block_hash(number), || self.provider()?.block_hash(number), ) } @@ -301,10 +305,10 @@ impl BlockHashReader for ProviderFactory { start: BlockNumber, end: BlockNumber, ) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Headers, start..end, - |snapshot, range, _| snapshot.canonical_hashes_range(range.start, range.end), + |static_file, range, _| static_file.canonical_hashes_range(range.start, range.end), |range, _| self.provider()?.canonical_hashes_range(range.start, range.end), |_| true, ) @@ -380,10 +384,10 @@ impl TransactionsProvider for ProviderFactory { } fn transaction_by_id(&self, id: TxNumber) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Transactions, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Transactions, id, - |snapshot| snapshot.transaction_by_id(id), + |static_file| static_file.transaction_by_id(id), || self.provider()?.transaction_by_id(id), ) } @@ -392,10 +396,10 @@ impl TransactionsProvider for ProviderFactory { &self, id: TxNumber, ) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Transactions, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Transactions, id, - |snapshot| snapshot.transaction_by_id_no_hash(id), + |static_file| static_file.transaction_by_id_no_hash(id), || self.provider()?.transaction_by_id_no_hash(id), ) } @@ -457,10 +461,10 @@ impl TransactionsProvider for ProviderFactory { impl ReceiptProvider for ProviderFactory { fn receipt(&self, id: TxNumber) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Receipts, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Receipts, id, - |snapshot| snapshot.receipt(id), + |static_file| static_file.receipt(id), || self.provider()?.receipt(id), ) } @@ -477,10 +481,10 @@ impl ReceiptProvider for ProviderFactory { &self, range: impl RangeBounds, ) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Receipts, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Receipts, to_range(range), - |snapshot, range, _| snapshot.receipts_by_tx_range(range), + |static_file, range, _| static_file.receipts_by_tx_range(range), |range, _| self.provider()?.receipts_by_tx_range(range), |_| true, ) @@ -601,7 +605,7 @@ impl PruneCheckpointReader for ProviderFactory { mod tests { use super::ProviderFactory; use crate::{ - providers::SnapshotWriter, test_utils::create_test_provider_factory, BlockHashReader, + providers::StaticFileWriter, test_utils::create_test_provider_factory, BlockHashReader, BlockNumReader, BlockWriter, HeaderSyncGapProvider, HeaderSyncMode, TransactionsProvider, }; use alloy_rlp::Decodable; @@ -609,7 +613,7 @@ mod tests { use rand::Rng; use reth_db::{ tables, - test_utils::{create_test_snapshots_dir, ERROR_TEMPDIR}, + test_utils::{create_test_static_files_dir, ERROR_TEMPDIR}, DatabaseEnv, }; use reth_interfaces::{ @@ -621,7 +625,7 @@ mod tests { RethError, }; use reth_primitives::{ - hex_literal::hex, ChainSpecBuilder, PruneMode, PruneModes, SealedBlock, SnapshotSegment, + hex_literal::hex, ChainSpecBuilder, PruneMode, PruneModes, SealedBlock, StaticFileSegment, TxNumber, B256, U256, }; use std::{ops::RangeInclusive, sync::Arc}; @@ -660,7 +664,7 @@ mod tests { tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path(), Arc::new(chain_spec), Default::default(), - create_test_snapshots_dir(), + create_test_static_files_dir(), ) .unwrap(); @@ -773,11 +777,11 @@ mod tests { ); // Checkpoint and no gap - let mut snapshot_writer = - provider.snapshot_provider().latest_writer(SnapshotSegment::Headers).unwrap(); - snapshot_writer.append_header(head.header().clone(), U256::ZERO, head.hash()).unwrap(); - snapshot_writer.commit().unwrap(); - drop(snapshot_writer); + let mut static_file_writer = + provider.static_file_provider().latest_writer(StaticFileSegment::Headers).unwrap(); + static_file_writer.append_header(head.header().clone(), U256::ZERO, head.hash()).unwrap(); + static_file_writer.commit().unwrap(); + drop(static_file_writer); let gap = provider.sync_gap(mode.clone(), checkpoint).unwrap(); assert_eq!(gap.local_head, head); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index b77e7a149314..e456e497b463 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1,6 +1,6 @@ use crate::{ bundle_state::{BundleStateInit, BundleStateWithReceipts, HashedStateChanges, RevertsInit}, - providers::{database::metrics, snapshot::SnapshotWriter, SnapshotProvider}, + providers::{database::metrics, static_file::StaticFileWriter, StaticFileProvider}, to_range, traits::{ AccountExtReader, BlockSource, ChangeSetReader, ReceiptProvider, StageCheckpointWriter, @@ -38,7 +38,7 @@ use reth_primitives::{ trie::Nibbles, Account, Address, Block, BlockHash, BlockHashOrNumber, BlockNumber, BlockWithSenders, ChainInfo, ChainSpec, GotExpected, Hardfork, Head, Header, PruneCheckpoint, PruneModes, - PruneSegment, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, SnapshotSegment, + PruneSegment, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, StaticFileSegment, StorageEntry, TransactionMeta, TransactionSigned, TransactionSignedEcRecovered, TransactionSignedNoHash, TxHash, TxNumber, Withdrawal, Withdrawals, B256, U256, }; @@ -83,7 +83,7 @@ impl DerefMut for DatabaseProviderRW { } impl DatabaseProviderRW { - /// Commit database transaction and snapshot if it exists. + /// Commit database transaction and static file if it exists. pub fn commit(self) -> ProviderResult { self.0.commit() } @@ -102,21 +102,25 @@ pub struct DatabaseProvider { tx: TX, /// Chain spec chain_spec: Arc, - /// Snapshot provider - snapshot_provider: SnapshotProvider, + /// Static File provider + static_file_provider: StaticFileProvider, } impl DatabaseProvider { - /// Returns a snapshot provider - pub fn snapshot_provider(&self) -> &SnapshotProvider { - &self.snapshot_provider + /// Returns a static file provider + pub fn static_file_provider(&self) -> &StaticFileProvider { + &self.static_file_provider } } impl DatabaseProvider { /// Creates a provider with an inner read-write transaction. - pub fn new_rw(tx: TX, chain_spec: Arc, snapshot_provider: SnapshotProvider) -> Self { - Self { tx, chain_spec, snapshot_provider } + pub fn new_rw( + tx: TX, + chain_spec: Arc, + static_file_provider: StaticFileProvider, + ) -> Self { + Self { tx, chain_spec, static_file_provider } } } @@ -184,7 +188,7 @@ impl DatabaseProvider { parent_ttd + block.difficulty }; - let mut writer = self.snapshot_provider.latest_writer(SnapshotSegment::Headers)?; + let mut writer = self.static_file_provider.latest_writer(StaticFileSegment::Headers)?; writer.append_header(block.header.as_ref().clone(), ttd, block.hash())?; self.insert_block(block, prune_modes) @@ -241,8 +245,12 @@ where impl DatabaseProvider { /// Creates a provider with an inner read-only transaction. - pub fn new(tx: TX, chain_spec: Arc, snapshot_provider: SnapshotProvider) -> Self { - Self { tx, chain_spec, snapshot_provider } + pub fn new( + tx: TX, + chain_spec: Arc, + static_file_provider: StaticFileProvider, + ) -> Self { + Self { tx, chain_spec, static_file_provider } } /// Consume `DbTx` or `DbTxMut`. @@ -290,10 +298,10 @@ impl DatabaseProvider { where C: DbCursorRO, { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Transactions, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Transactions, to_range(range), - |snapshot, range, _| snapshot.transactions_by_tx_range(range), + |static_file, range, _| static_file.transactions_by_tx_range(range), |range, _| self.cursor_collect(cursor, range, Ok), |_| true, ) @@ -979,31 +987,32 @@ impl HeaderSyncGapProvider for DatabaseProvider { mode: HeaderSyncMode, highest_uninterrupted_block: BlockNumber, ) -> RethResult { - let snapshot_provider = self.snapshot_provider(); + let static_file_provider = self.static_file_provider(); // Make sure Headers static file is at the same height. If it's further, this // input execution was interrupted previously and we need to unwind the static file. - let next_snapshot_block_num = snapshot_provider - .get_highest_snapshot_block(SnapshotSegment::Headers) + let next_static_file_block_num = static_file_provider + .get_highest_static_file_block(StaticFileSegment::Headers) .map(|id| id + 1) .unwrap_or_default(); let next_block = highest_uninterrupted_block + 1; - match next_snapshot_block_num.cmp(&next_block) { + match next_static_file_block_num.cmp(&next_block) { // The node shutdown between an executed static file commit and before the database // commit, so we need to unwind the static files. Ordering::Greater => { - let mut snapshotter = snapshot_provider.latest_writer(SnapshotSegment::Headers)?; - snapshotter.prune_headers(next_snapshot_block_num - next_block)? + let mut static_file_producer = + static_file_provider.latest_writer(StaticFileSegment::Headers)?; + static_file_producer.prune_headers(next_static_file_block_num - next_block)? } Ordering::Less => { // There's either missing or corrupted files. - return Err(ProviderError::HeaderNotFound(next_snapshot_block_num.into()).into()) + return Err(ProviderError::HeaderNotFound(next_static_file_block_num.into()).into()) } Ordering::Equal => {} } - let local_head = snapshot_provider + let local_head = static_file_provider .sealed_header(highest_uninterrupted_block)? .ok_or_else(|| ProviderError::HeaderNotFound(highest_uninterrupted_block.into()))?; @@ -1026,10 +1035,10 @@ impl HeaderProvider for DatabaseProvider { } fn header_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Headers, num, - |snapshot| snapshot.header_by_number(num), + |static_file| static_file.header_by_number(num), || Ok(self.tx.get::(num)?), ) } @@ -1049,19 +1058,19 @@ impl HeaderProvider for DatabaseProvider { return Ok(Some(td)) } - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Headers, number, - |snapshot| snapshot.header_td_by_number(number), + |static_file| static_file.header_td_by_number(number), || Ok(self.tx.get::(number)?.map(|td| td.0)), ) } fn headers_range(&self, range: impl RangeBounds) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Headers, to_range(range), - |snapshot, range, _| snapshot.headers_range(range), + |static_file, range, _| static_file.headers_range(range), |range, _| { self.cursor_read_collect::(range, Ok).map_err(Into::into) }, @@ -1070,10 +1079,10 @@ impl HeaderProvider for DatabaseProvider { } fn sealed_header(&self, number: BlockNumber) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Headers, number, - |snapshot| snapshot.sealed_header(number), + |static_file| static_file.sealed_header(number), || { if let Some(header) = self.header_by_number(number)? { let hash = self @@ -1092,10 +1101,10 @@ impl HeaderProvider for DatabaseProvider { range: impl RangeBounds, predicate: impl FnMut(&SealedHeader) -> bool, ) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Headers, to_range(range), - |snapshot, range, predicate| snapshot.sealed_headers_while(range, predicate), + |static_file, range, predicate| static_file.sealed_headers_while(range, predicate), |range, mut predicate| { let mut headers = vec![]; for entry in self.tx.cursor_read::()?.walk_range(range)? { @@ -1118,10 +1127,10 @@ impl HeaderProvider for DatabaseProvider { impl BlockHashReader for DatabaseProvider { fn block_hash(&self, number: u64) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Headers, number, - |snapshot| snapshot.block_hash(number), + |static_file| static_file.block_hash(number), || Ok(self.tx.get::(number)?), ) } @@ -1131,10 +1140,10 @@ impl BlockHashReader for DatabaseProvider { start: BlockNumber, end: BlockNumber, ) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Headers, start..end, - |snapshot, range, _| snapshot.canonical_hashes_range(range.start, range.end), + |static_file, range, _| static_file.canonical_hashes_range(range.start, range.end), |range, _| { self.cursor_read_collect::(range, Ok) .map_err(Into::into) @@ -1360,10 +1369,10 @@ impl TransactionsProviderExt for DatabaseProvider { &self, tx_range: Range, ) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Transactions, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Transactions, tx_range, - |snapshot, range, _| snapshot.transaction_hashes_by_range(range), + |static_file, range, _| static_file.transaction_hashes_by_range(range), |tx_range, _| { let mut tx_cursor = self.tx.cursor_read::()?; let tx_range_size = tx_range.clone().count(); @@ -1428,10 +1437,10 @@ impl TransactionsProvider for DatabaseProvider { } fn transaction_by_id(&self, id: TxNumber) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Transactions, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Transactions, id, - |snapshot| snapshot.transaction_by_id(id), + |static_file| static_file.transaction_by_id(id), || Ok(self.tx.get::(id)?.map(Into::into)), ) } @@ -1440,10 +1449,10 @@ impl TransactionsProvider for DatabaseProvider { &self, id: TxNumber, ) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Transactions, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Transactions, id, - |snapshot| snapshot.transaction_by_id_no_hash(id), + |static_file| static_file.transaction_by_id_no_hash(id), || Ok(self.tx.get::(id)?), ) } @@ -1571,10 +1580,10 @@ impl TransactionsProvider for DatabaseProvider { &self, range: impl RangeBounds, ) -> ProviderResult>> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Transactions, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Transactions, to_range(range), - |snapshot, range, _| snapshot.raw_transactions_by_tx_range(range), + |static_file, range, _| static_file.raw_transactions_by_tx_range(range), |range, _| { self.cursor_collect_with_capacity( &mut self.tx.cursor_read::>()?, @@ -1601,10 +1610,10 @@ impl TransactionsProvider for DatabaseProvider { impl ReceiptProvider for DatabaseProvider { fn receipt(&self, id: TxNumber) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Receipts, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Receipts, id, - |snapshot| snapshot.receipt(id), + |static_file| static_file.receipt(id), || Ok(self.tx.get::(id)?), ) } @@ -1635,10 +1644,10 @@ impl ReceiptProvider for DatabaseProvider { &self, range: impl RangeBounds, ) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Receipts, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Receipts, to_range(range), - |snapshot, range, _| snapshot.receipts_by_tx_range(range), + |static_file, range, _| static_file.receipts_by_tx_range(range), |range, _| { self.cursor_read_collect::(range, Ok).map_err(Into::into) }, @@ -2514,13 +2523,13 @@ impl PruneCheckpointWriter for DatabaseProvider { impl StatsReader for DatabaseProvider { fn count_entries(&self) -> ProviderResult { let db_entries = self.tx.entries::()?; - let snapshot_entries = match self.snapshot_provider.count_entries::() { + let static_file_entries = match self.static_file_provider.count_entries::() { Ok(entries) => entries, Err(ProviderError::UnsupportedProvider) => 0, Err(err) => return Err(err), }; - Ok(db_entries + snapshot_entries) + Ok(db_entries + static_file_entries) } } diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index d74696f65bac..8e5ed06311d8 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -39,10 +39,10 @@ pub use state::{ mod bundle_state_provider; mod chain_info; mod database; -mod snapshot; -pub use snapshot::{ - SnapshotJarProvider, SnapshotProvider, SnapshotProviderRW, SnapshotProviderRWRefMut, - SnapshotWriter, +mod static_file; +pub use static_file::{ + StaticFileJarProvider, StaticFileProvider, StaticFileProviderRW, StaticFileProviderRWRefMut, + StaticFileWriter, }; mod state; use crate::{providers::chain_info::ChainInfoTracker, traits::BlockSource}; @@ -607,7 +607,7 @@ where if let Some(block) = self.tree.pending_block_num_hash() { if let Ok(pending) = self.tree.pending_state_provider(block.hash) { - return self.pending_with_provider(pending) + return self.pending_with_provider(pending); } } @@ -617,7 +617,7 @@ where fn pending_state_by_hash(&self, block_hash: B256) -> ProviderResult> { if let Some(state) = self.tree.find_pending_state_provider(block_hash) { - return Ok(Some(self.pending_with_provider(state)?)) + return Ok(Some(self.pending_with_provider(state)?)); } Ok(None) } diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index c9a9a5d57846..e506192ed620 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -1,5 +1,5 @@ use crate::{ - providers::{state::macros::delegate_provider_impls, SnapshotProvider}, + providers::{state::macros::delegate_provider_impls, StaticFileProvider}, AccountReader, BlockHashReader, BundleStateWithReceipts, ProviderError, StateProvider, StateRootProvider, }; @@ -14,7 +14,7 @@ use reth_db::{ use reth_interfaces::provider::ProviderResult; use reth_primitives::{ constants::EPOCH_SLOTS, trie::AccountProof, Account, Address, BlockNumber, Bytecode, - SnapshotSegment, StorageKey, StorageValue, B256, + StaticFileSegment, StorageKey, StorageValue, B256, }; use reth_trie::{updates::TrieUpdates, HashedPostState}; @@ -37,8 +37,8 @@ pub struct HistoricalStateProviderRef<'b, TX: DbTx> { block_number: BlockNumber, /// Lowest blocks at which different parts of the state are available. lowest_available_blocks: LowestAvailableBlocks, - /// Snapshot provider - snapshot_provider: SnapshotProvider, + /// Static File provider + static_file_provider: StaticFileProvider, } #[derive(Debug, Eq, PartialEq)] @@ -51,8 +51,12 @@ pub enum HistoryInfo { impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> { /// Create new StateProvider for historical block number - pub fn new(tx: &'b TX, block_number: BlockNumber, snapshot_provider: SnapshotProvider) -> Self { - Self { tx, block_number, lowest_available_blocks: Default::default(), snapshot_provider } + pub fn new( + tx: &'b TX, + block_number: BlockNumber, + static_file_provider: StaticFileProvider, + ) -> Self { + Self { tx, block_number, lowest_available_blocks: Default::default(), static_file_provider } } /// Create new StateProvider for historical block number and lowest block numbers at which @@ -61,9 +65,9 @@ impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> { tx: &'b TX, block_number: BlockNumber, lowest_available_blocks: LowestAvailableBlocks, - snapshot_provider: SnapshotProvider, + static_file_provider: StaticFileProvider, ) -> Self { - Self { tx, block_number, lowest_available_blocks, snapshot_provider } + Self { tx, block_number, lowest_available_blocks, static_file_provider } } /// Lookup an account in the AccountHistory table @@ -113,7 +117,9 @@ impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> { .cursor_read::()? .last()? .map(|(tip, _)| tip) - .or_else(|| self.snapshot_provider.get_highest_snapshot_block(SnapshotSegment::Headers)) + .or_else(|| { + self.static_file_provider.get_highest_static_file_block(StaticFileSegment::Headers) + }) .ok_or(ProviderError::BestBlockNotFound)?; if tip.saturating_sub(self.block_number) > EPOCH_SLOTS { @@ -209,10 +215,10 @@ impl<'b, TX: DbTx> AccountReader for HistoricalStateProviderRef<'b, TX> { impl<'b, TX: DbTx> BlockHashReader for HistoricalStateProviderRef<'b, TX> { /// Get block hash by number. fn block_hash(&self, number: u64) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Headers, number, - |snapshot| snapshot.block_hash(number), + |static_file| static_file.block_hash(number), || Ok(self.tx.get::(number)?), ) } @@ -222,10 +228,10 @@ impl<'b, TX: DbTx> BlockHashReader for HistoricalStateProviderRef<'b, TX> { start: BlockNumber, end: BlockNumber, ) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Headers, start..end, - |snapshot, range, _| snapshot.canonical_hashes_range(range.start, range.end), + |static_file, range, _| static_file.canonical_hashes_range(range.start, range.end), |range, _| { self.tx .cursor_read::() @@ -313,14 +319,18 @@ pub struct HistoricalStateProvider { block_number: BlockNumber, /// Lowest blocks at which different parts of the state are available. lowest_available_blocks: LowestAvailableBlocks, - /// Snapshot provider - snapshot_provider: SnapshotProvider, + /// Static File provider + static_file_provider: StaticFileProvider, } impl HistoricalStateProvider { /// Create new StateProvider for historical block number - pub fn new(tx: TX, block_number: BlockNumber, snapshot_provider: SnapshotProvider) -> Self { - Self { tx, block_number, lowest_available_blocks: Default::default(), snapshot_provider } + pub fn new( + tx: TX, + block_number: BlockNumber, + static_file_provider: StaticFileProvider, + ) -> Self { + Self { tx, block_number, lowest_available_blocks: Default::default(), static_file_provider } } /// Set the lowest block number at which the account history is available. @@ -348,7 +358,7 @@ impl HistoricalStateProvider { &self.tx, self.block_number, self.lowest_available_blocks, - self.snapshot_provider.clone(), + self.static_file_provider.clone(), ) } } @@ -414,7 +424,7 @@ mod tests { fn history_provider_get_account() { let factory = create_test_provider_factory(); let tx = factory.provider_rw().unwrap().into_tx(); - let snapshot_provider = factory.snapshot_provider(); + let static_file_provider = factory.static_file_provider(); tx.put::( ShardedKey { key: ADDRESS, highest_block_number: 7 }, @@ -478,59 +488,59 @@ mod tests { // run assert_eq!( - HistoricalStateProviderRef::new(&tx, 1, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 1, static_file_provider.clone()) .basic_account(ADDRESS) .clone(), Ok(None) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 2, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 2, static_file_provider.clone()) .basic_account(ADDRESS), Ok(Some(acc_at3)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 3, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 3, static_file_provider.clone()) .basic_account(ADDRESS), Ok(Some(acc_at3)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 4, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 4, static_file_provider.clone()) .basic_account(ADDRESS), Ok(Some(acc_at7)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 7, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 7, static_file_provider.clone()) .basic_account(ADDRESS), Ok(Some(acc_at7)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 9, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 9, static_file_provider.clone()) .basic_account(ADDRESS), Ok(Some(acc_at10)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 10, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 10, static_file_provider.clone()) .basic_account(ADDRESS), Ok(Some(acc_at10)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 11, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 11, static_file_provider.clone()) .basic_account(ADDRESS), Ok(Some(acc_at15)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 16, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 16, static_file_provider.clone()) .basic_account(ADDRESS), Ok(Some(acc_plain)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 1, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 1, static_file_provider.clone()) .basic_account(HIGHER_ADDRESS), Ok(None) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 1000, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 1000, static_file_provider.clone()) .basic_account(HIGHER_ADDRESS), Ok(Some(higher_acc_plain)) ); @@ -540,7 +550,7 @@ mod tests { fn history_provider_get_storage() { let factory = create_test_provider_factory(); let tx = factory.provider_rw().unwrap().into_tx(); - let snapshot_provider = factory.snapshot_provider(); + let static_file_provider = factory.static_file_provider(); tx.put::( StorageShardedKey { @@ -591,52 +601,52 @@ mod tests { // run assert_eq!( - HistoricalStateProviderRef::new(&tx, 0, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 0, static_file_provider.clone()) .storage(ADDRESS, STORAGE), Ok(None) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 3, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 3, static_file_provider.clone()) .storage(ADDRESS, STORAGE), Ok(Some(U256::ZERO)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 4, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 4, static_file_provider.clone()) .storage(ADDRESS, STORAGE), Ok(Some(entry_at7.value)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 7, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 7, static_file_provider.clone()) .storage(ADDRESS, STORAGE), Ok(Some(entry_at7.value)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 9, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 9, static_file_provider.clone()) .storage(ADDRESS, STORAGE), Ok(Some(entry_at10.value)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 10, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 10, static_file_provider.clone()) .storage(ADDRESS, STORAGE), Ok(Some(entry_at10.value)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 11, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 11, static_file_provider.clone()) .storage(ADDRESS, STORAGE), Ok(Some(entry_at15.value)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 16, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 16, static_file_provider.clone()) .storage(ADDRESS, STORAGE), Ok(Some(entry_plain.value)) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 1, snapshot_provider.clone()) + HistoricalStateProviderRef::new(&tx, 1, static_file_provider.clone()) .storage(HIGHER_ADDRESS, STORAGE), Ok(None) ); assert_eq!( - HistoricalStateProviderRef::new(&tx, 1000, snapshot_provider) + HistoricalStateProviderRef::new(&tx, 1000, static_file_provider) .storage(HIGHER_ADDRESS, STORAGE), Ok(Some(higher_entry_plain.value)) ); @@ -646,7 +656,7 @@ mod tests { fn history_provider_unavailable() { let factory = create_test_provider_factory(); let tx = factory.provider_rw().unwrap().into_tx(); - let snapshot_provider = factory.snapshot_provider(); + let static_file_provider = factory.static_file_provider(); // provider block_number < lowest available block number, // i.e. state at provider block is pruned @@ -657,7 +667,7 @@ mod tests { account_history_block_number: Some(3), storage_history_block_number: Some(3), }, - snapshot_provider.clone(), + static_file_provider.clone(), ); assert_eq!( provider.account_history_lookup(ADDRESS), @@ -677,7 +687,7 @@ mod tests { account_history_block_number: Some(2), storage_history_block_number: Some(2), }, - snapshot_provider.clone(), + static_file_provider.clone(), ); assert_eq!(provider.account_history_lookup(ADDRESS), Ok(HistoryInfo::MaybeInPlainState)); assert_eq!( @@ -694,7 +704,7 @@ mod tests { account_history_block_number: Some(1), storage_history_block_number: Some(1), }, - snapshot_provider.clone(), + static_file_provider.clone(), ); assert_eq!(provider.account_history_lookup(ADDRESS), Ok(HistoryInfo::MaybeInPlainState)); assert_eq!( diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index 53d59ec5726b..29441f220579 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -1,5 +1,5 @@ use crate::{ - providers::{state::macros::delegate_provider_impls, SnapshotProvider}, + providers::{state::macros::delegate_provider_impls, StaticFileProvider}, AccountReader, BlockHashReader, BundleStateWithReceipts, StateProvider, StateRootProvider, }; use reth_db::{ @@ -9,7 +9,7 @@ use reth_db::{ }; use reth_interfaces::provider::{ProviderError, ProviderResult}; use reth_primitives::{ - trie::AccountProof, Account, Address, BlockNumber, Bytecode, SnapshotSegment, StorageKey, + trie::AccountProof, Account, Address, BlockNumber, Bytecode, StaticFileSegment, StorageKey, StorageValue, B256, }; use reth_trie::{proof::Proof, updates::TrieUpdates}; @@ -19,14 +19,14 @@ use reth_trie::{proof::Proof, updates::TrieUpdates}; pub struct LatestStateProviderRef<'b, TX: DbTx> { /// database transaction db: &'b TX, - /// Snapshot provider - snapshot_provider: SnapshotProvider, + /// Static File provider + static_file_provider: StaticFileProvider, } impl<'b, TX: DbTx> LatestStateProviderRef<'b, TX> { /// Create new state provider - pub fn new(db: &'b TX, snapshot_provider: SnapshotProvider) -> Self { - Self { db, snapshot_provider } + pub fn new(db: &'b TX, static_file_provider: StaticFileProvider) -> Self { + Self { db, static_file_provider } } } @@ -40,10 +40,10 @@ impl<'b, TX: DbTx> AccountReader for LatestStateProviderRef<'b, TX> { impl<'b, TX: DbTx> BlockHashReader for LatestStateProviderRef<'b, TX> { /// Get block hash by number. fn block_hash(&self, number: u64) -> ProviderResult> { - self.snapshot_provider.get_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_with_static_file_or_database( + StaticFileSegment::Headers, number, - |snapshot| snapshot.block_hash(number), + |static_file| static_file.block_hash(number), || Ok(self.db.get::(number)?), ) } @@ -53,10 +53,10 @@ impl<'b, TX: DbTx> BlockHashReader for LatestStateProviderRef<'b, TX> { start: BlockNumber, end: BlockNumber, ) -> ProviderResult> { - self.snapshot_provider.get_range_with_snapshot_or_database( - SnapshotSegment::Headers, + self.static_file_provider.get_range_with_static_file_or_database( + StaticFileSegment::Headers, start..end, - |snapshot, range, _| snapshot.canonical_hashes_range(range.start, range.end), + |static_file, range, _| static_file.canonical_hashes_range(range.start, range.end), |range, _| { self.db .cursor_read::() @@ -125,20 +125,20 @@ impl<'b, TX: DbTx> StateProvider for LatestStateProviderRef<'b, TX> { pub struct LatestStateProvider { /// database transaction db: TX, - /// Snapshot provider - snapshot_provider: SnapshotProvider, + /// Static File provider + static_file_provider: StaticFileProvider, } impl LatestStateProvider { /// Create new state provider - pub fn new(db: TX, snapshot_provider: SnapshotProvider) -> Self { - Self { db, snapshot_provider } + pub fn new(db: TX, static_file_provider: StaticFileProvider) -> Self { + Self { db, static_file_provider } } /// Returns a new provider that takes the `TX` as reference #[inline(always)] fn as_ref(&self) -> LatestStateProviderRef<'_, TX> { - LatestStateProviderRef::new(&self.db, self.snapshot_provider.clone()) + LatestStateProviderRef::new(&self.db, self.static_file_provider.clone()) } } diff --git a/crates/storage/provider/src/providers/snapshot/jar.rs b/crates/storage/provider/src/providers/static_file/jar.rs similarity index 85% rename from crates/storage/provider/src/providers/snapshot/jar.rs rename to crates/storage/provider/src/providers/static_file/jar.rs index 44a9a2aa6816..356757b9d388 100644 --- a/crates/storage/provider/src/providers/snapshot/jar.rs +++ b/crates/storage/provider/src/providers/static_file/jar.rs @@ -1,5 +1,5 @@ use super::{ - metrics::{SnapshotProviderMetrics, SnapshotProviderOperation}, + metrics::{StaticFileProviderMetrics, StaticFileProviderOperation}, LoadedJarRef, }; use crate::{ @@ -8,7 +8,7 @@ use crate::{ }; use reth_db::{ codecs::CompactU256, - snapshot::{ColumnSelectorOne, HeaderMask, ReceiptMask, SnapshotCursor, TransactionMask}, + static_file::{ColumnSelectorOne, HeaderMask, ReceiptMask, StaticFileCursor, TransactionMask}, RawValue, }; use reth_interfaces::provider::{ProviderError, ProviderResult}; @@ -23,39 +23,39 @@ use std::{ /// Provider over a specific `NippyJar` and range. #[derive(Debug)] -pub struct SnapshotJarProvider<'a> { - /// Main snapshot segment +pub struct StaticFileJarProvider<'a> { + /// Main static file segment jar: LoadedJarRef<'a>, - /// Another kind of snapshot segment to help query data from the main one. + /// Another kind of static file segment to help query data from the main one. auxiliar_jar: Option>, - metrics: Option>, + metrics: Option>, } -impl<'a> Deref for SnapshotJarProvider<'a> { +impl<'a> Deref for StaticFileJarProvider<'a> { type Target = LoadedJarRef<'a>; fn deref(&self) -> &Self::Target { &self.jar } } -impl<'a> From> for SnapshotJarProvider<'a> { +impl<'a> From> for StaticFileJarProvider<'a> { fn from(value: LoadedJarRef<'a>) -> Self { - SnapshotJarProvider { jar: value, auxiliar_jar: None, metrics: None } + StaticFileJarProvider { jar: value, auxiliar_jar: None, metrics: None } } } -impl<'a> SnapshotJarProvider<'a> { +impl<'a> StaticFileJarProvider<'a> { /// Provides a cursor for more granular data access. - pub fn cursor<'b>(&'b self) -> ProviderResult> + pub fn cursor<'b>(&'b self) -> ProviderResult> where 'b: 'a, { - let result = SnapshotCursor::new(self.value(), self.mmap_handle())?; + let result = StaticFileCursor::new(self.value(), self.mmap_handle())?; if let Some(metrics) = &self.metrics { metrics.record_segment_operation( self.segment(), - SnapshotProviderOperation::InitCursor, + StaticFileProviderOperation::InitCursor, None, ); } @@ -63,20 +63,20 @@ impl<'a> SnapshotJarProvider<'a> { Ok(result) } - /// Adds a new auxiliar snapshot to help query data from the main one - pub fn with_auxiliar(mut self, auxiliar_jar: SnapshotJarProvider<'a>) -> Self { + /// Adds a new auxiliar static file to help query data from the main one + pub fn with_auxiliar(mut self, auxiliar_jar: StaticFileJarProvider<'a>) -> Self { self.auxiliar_jar = Some(Box::new(auxiliar_jar)); self } /// Enables metrics on the provider. - pub fn with_metrics(mut self, metrics: Arc) -> Self { + pub fn with_metrics(mut self, metrics: Arc) -> Self { self.metrics = Some(metrics); self } } -impl<'a> HeaderProvider for SnapshotJarProvider<'a> { +impl<'a> HeaderProvider for StaticFileJarProvider<'a> { fn header(&self, block_hash: &BlockHash) -> ProviderResult> { Ok(self .cursor()? @@ -139,7 +139,7 @@ impl<'a> HeaderProvider for SnapshotJarProvider<'a> { { let sealed = header.seal(hash); if !predicate(&sealed) { - break + break; } headers.push(sealed); } @@ -148,7 +148,7 @@ impl<'a> HeaderProvider for SnapshotJarProvider<'a> { } } -impl<'a> BlockHashReader for SnapshotJarProvider<'a> { +impl<'a> BlockHashReader for StaticFileJarProvider<'a> { fn block_hash(&self, number: u64) -> ProviderResult> { self.cursor()?.get_one::>(number.into()) } @@ -170,7 +170,7 @@ impl<'a> BlockHashReader for SnapshotJarProvider<'a> { } } -impl<'a> BlockNumReader for SnapshotJarProvider<'a> { +impl<'a> BlockNumReader for StaticFileJarProvider<'a> { fn chain_info(&self) -> ProviderResult { // Information on live database Err(ProviderError::UnsupportedProvider) @@ -195,7 +195,7 @@ impl<'a> BlockNumReader for SnapshotJarProvider<'a> { } } -impl<'a> TransactionsProvider for SnapshotJarProvider<'a> { +impl<'a> TransactionsProvider for StaticFileJarProvider<'a> { fn transaction_id(&self, hash: TxHash) -> ProviderResult> { let mut cursor = self.cursor()?; @@ -242,7 +242,7 @@ impl<'a> TransactionsProvider for SnapshotJarProvider<'a> { &self, _block_id: BlockHashOrNumber, ) -> ProviderResult>> { - // Related to indexing tables. Live database should get the tx_range and call snapshot + // Related to indexing tables. Live database should get the tx_range and call static file // provider with `transactions_by_tx_range` instead. Err(ProviderError::UnsupportedProvider) } @@ -251,7 +251,7 @@ impl<'a> TransactionsProvider for SnapshotJarProvider<'a> { &self, _range: impl RangeBounds, ) -> ProviderResult>> { - // Related to indexing tables. Live database should get the tx_range and call snapshot + // Related to indexing tables. Live database should get the tx_range and call static file // provider with `transactions_by_tx_range` instead. Err(ProviderError::UnsupportedProvider) } @@ -310,22 +310,22 @@ impl<'a> TransactionsProvider for SnapshotJarProvider<'a> { } } -impl<'a> ReceiptProvider for SnapshotJarProvider<'a> { +impl<'a> ReceiptProvider for StaticFileJarProvider<'a> { fn receipt(&self, num: TxNumber) -> ProviderResult> { self.cursor()?.get_one::>(num.into()) } fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult> { - if let Some(tx_snapshot) = &self.auxiliar_jar { - if let Some(num) = tx_snapshot.transaction_id(hash)? { - return self.receipt(num) + if let Some(tx_static_file) = &self.auxiliar_jar { + if let Some(num) = tx_static_file.transaction_id(hash)? { + return self.receipt(num); } } Ok(None) } fn receipts_by_block(&self, _block: BlockHashOrNumber) -> ProviderResult>> { - // Related to indexing tables. Snapshot should get the tx_range and call snapshot + // Related to indexing tables. StaticFile should get the tx_range and call static file // provider with `receipt()` instead for each Err(ProviderError::UnsupportedProvider) } diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs similarity index 68% rename from crates/storage/provider/src/providers/snapshot/manager.rs rename to crates/storage/provider/src/providers/static_file/manager.rs index b4d1372f9d51..67abb7636f59 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -1,6 +1,6 @@ use super::{ - metrics::SnapshotProviderMetrics, LoadedJar, SnapshotJarProvider, SnapshotProviderRW, - SnapshotProviderRWRefMut, BLOCKS_PER_SNAPSHOT, + metrics::StaticFileProviderMetrics, LoadedJar, StaticFileJarProvider, StaticFileProviderRW, + StaticFileProviderRWRefMut, BLOCKS_PER_STATIC_FILE, }; use crate::{ to_range, BlockHashReader, BlockNumReader, BlockReader, BlockSource, HeaderProvider, @@ -12,8 +12,9 @@ use parking_lot::RwLock; use reth_db::{ codecs::CompactU256, models::StoredBlockBodyIndices, - snapshot::{ - iter_snapshots, ColumnSelectorOne, HeaderMask, ReceiptMask, SnapshotCursor, TransactionMask, + static_file::{ + iter_static_files, ColumnSelectorOne, HeaderMask, ReceiptMask, StaticFileCursor, + TransactionMask, }, table::Table, tables, RawValue, @@ -22,9 +23,9 @@ use reth_interfaces::provider::{ProviderError, ProviderResult}; use reth_nippy_jar::NippyJar; use reth_primitives::{ keccak256, - snapshot::{find_fixed_range, HighestSnapshots, SegmentHeader, SegmentRangeInclusive}, + static_file::{find_fixed_range, HighestStaticFiles, SegmentHeader, SegmentRangeInclusive}, Address, Block, BlockHash, BlockHashOrNumber, BlockNumber, BlockWithSenders, ChainInfo, Header, - Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, SnapshotSegment, TransactionMeta, + Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, StaticFileSegment, TransactionMeta, TransactionSigned, TransactionSignedNoHash, TxHash, TxNumber, Withdrawal, Withdrawals, B256, U256, }; @@ -37,58 +38,59 @@ use std::{ use tracing::warn; /// Alias type for a map that can be queried for block ranges from a transaction -/// segment respectively. It uses `TxNumber` to represent the transaction end of a snapshot range. -type SegmentRanges = HashMap>; +/// segment respectively. It uses `TxNumber` to represent the transaction end of a static file +/// range. +type SegmentRanges = HashMap>; -/// [`SnapshotProvider`] manages all existing [`SnapshotJarProvider`]. +/// [`StaticFileProvider`] manages all existing [`StaticFileJarProvider`]. #[derive(Debug, Default, Clone)] -pub struct SnapshotProvider(Arc); +pub struct StaticFileProvider(Arc); -impl SnapshotProvider { - /// Creates a new [`SnapshotProvider`]. +impl StaticFileProvider { + /// Creates a new [`StaticFileProvider`]. pub fn new(path: impl AsRef) -> ProviderResult { - let provider = Self(Arc::new(SnapshotProviderInner::new(path)?)); + let provider = Self(Arc::new(StaticFileProviderInner::new(path)?)); provider.initialize_index()?; Ok(provider) } } -impl Deref for SnapshotProvider { - type Target = SnapshotProviderInner; +impl Deref for StaticFileProvider { + type Target = StaticFileProviderInner; fn deref(&self) -> &Self::Target { &self.0 } } -/// [`SnapshotProviderInner`] manages all existing [`SnapshotJarProvider`]. +/// [`StaticFileProviderInner`] manages all existing [`StaticFileJarProvider`]. #[derive(Debug, Default)] -pub struct SnapshotProviderInner { +pub struct StaticFileProviderInner { /// Maintains a map which allows for concurrent access to different `NippyJars`, over different /// segments and ranges. - map: DashMap<(BlockNumber, SnapshotSegment), LoadedJar>, - /// Max snapshotted block for each snapshot segment - snapshots_max_block: RwLock>, - /// Available snapshot block ranges on disk indexed by max transactions. - snapshots_tx_index: RwLock, - /// Directory where snapshots are located + map: DashMap<(BlockNumber, StaticFileSegment), LoadedJar>, + /// Max static file block for each segment + static_files_max_block: RwLock>, + /// Available static file block ranges on disk indexed by max transactions. + static_files_tx_index: RwLock, + /// Directory where static_files are located path: PathBuf, - /// Whether [`SnapshotJarProvider`] loads filters into memory. If not, `by_hash` queries won't - /// be able to be queried directly. + /// Whether [`StaticFileJarProvider`] loads filters into memory. If not, `by_hash` queries + /// won't be able to be queried directly. load_filters: bool, - /// Maintains a map of Snapshot writers for each [`SnapshotSegment`] - writers: DashMap>, - metrics: Option>, + /// Maintains a map of StaticFile writers for each [`StaticFileSegment`] + writers: DashMap>, + metrics: Option>, } -impl SnapshotProviderInner { - /// Creates a new [`SnapshotProviderInner`]. +impl StaticFileProviderInner { + /// Creates a new [`StaticFileProviderInner`]. fn new(path: impl AsRef) -> ProviderResult { let provider = Self { map: Default::default(), writers: Default::default(), - snapshots_max_block: Default::default(), - snapshots_tx_index: Default::default(), + static_files_max_block: Default::default(), + static_files_tx_index: Default::default(), path: path.as_ref().to_path_buf(), load_filters: false, metrics: None, @@ -98,8 +100,8 @@ impl SnapshotProviderInner { } } -impl SnapshotProvider { - /// Loads filters into memory when creating a [`SnapshotJarProvider`]. +impl StaticFileProvider { + /// Loads filters into memory when creating a [`StaticFileJarProvider`]. pub fn with_filters(self) -> Self { let mut provider = Arc::try_unwrap(self.0).expect("should be called when initializing only"); @@ -107,65 +109,67 @@ impl SnapshotProvider { Self(Arc::new(provider)) } - /// Enables metrics on the [`SnapshotProvider`]. + /// Enables metrics on the [`StaticFileProvider`]. pub fn with_metrics(self) -> Self { let mut provider = Arc::try_unwrap(self.0).expect("should be called when initializing only"); - provider.metrics = Some(Arc::new(SnapshotProviderMetrics::default())); + provider.metrics = Some(Arc::new(StaticFileProviderMetrics::default())); Self(Arc::new(provider)) } - /// Gets the [`SnapshotJarProvider`] of the requested segment and block. + /// Gets the [`StaticFileJarProvider`] of the requested segment and block. pub fn get_segment_provider_from_block( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, block: BlockNumber, path: Option<&Path>, - ) -> ProviderResult> { + ) -> ProviderResult> { self.get_segment_provider( segment, || self.get_segment_ranges_from_block(segment, block), path, )? - .ok_or_else(|| ProviderError::MissingSnapshotBlock(segment, block)) + .ok_or_else(|| ProviderError::MissingStaticFileBlock(segment, block)) } - /// Gets the [`SnapshotJarProvider`] of the requested segment and transaction. + /// Gets the [`StaticFileJarProvider`] of the requested segment and transaction. pub fn get_segment_provider_from_transaction( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, tx: TxNumber, path: Option<&Path>, - ) -> ProviderResult> { + ) -> ProviderResult> { self.get_segment_provider( segment, || self.get_segment_ranges_from_transaction(segment, tx), path, )? - .ok_or_else(|| ProviderError::MissingSnapshotTx(segment, tx)) + .ok_or_else(|| ProviderError::MissingStaticFileTx(segment, tx)) } - /// Gets the [`SnapshotJarProvider`] of the requested segment and block or transaction. + /// Gets the [`StaticFileJarProvider`] of the requested segment and block or transaction. /// /// `fn_range` should make sure the range goes through `find_fixed_range`. pub fn get_segment_provider( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, fn_range: impl Fn() -> Option, path: Option<&Path>, - ) -> ProviderResult>> { + ) -> ProviderResult>> { // If we have a path, then get the block range from its name. - // Otherwise, check `self.available_snapshots` + // Otherwise, check `self.available_static_files` let block_range = match path { - Some(path) => SnapshotSegment::parse_filename( + Some(path) => StaticFileSegment::parse_filename( &path .file_name() - .ok_or_else(|| ProviderError::MissingSnapshotPath(segment, path.to_path_buf()))? + .ok_or_else(|| { + ProviderError::MissingStaticFilePath(segment, path.to_path_buf()) + })? .to_string_lossy(), ) .and_then(|(parsed_segment, block_range)| { if parsed_segment == segment { - return Some(block_range) + return Some(block_range); } None }), @@ -174,7 +178,7 @@ impl SnapshotProvider { // Return cached `LoadedJar` or insert it for the first time, and then, return it. if let Some(block_range) = block_range { - return Ok(Some(self.get_or_create_jar_provider(segment, &block_range)?)) + return Ok(Some(self.get_or_create_jar_provider(segment, &block_range)?)); } Ok(None) @@ -183,7 +187,7 @@ impl SnapshotProvider { /// Given a segment and block range it removes the cached provider from the map. pub fn remove_cached_provider( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, fixed_block_range_end: BlockNumber, ) { self.map.remove(&(fixed_block_range_end, segment)); @@ -194,7 +198,7 @@ impl SnapshotProvider { /// CAUTION: destructive. Deletes files on disk. pub fn delete_jar( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, fixed_block_range: SegmentRangeInclusive, ) -> ProviderResult<()> { let key = (fixed_block_range.end(), segment); @@ -222,17 +226,17 @@ impl SnapshotProvider { } /// Given a segment and block range it returns a cached - /// [`SnapshotJarProvider`]. TODO(joshie): we should check the size and pop N if there's too + /// [`StaticFileJarProvider`]. TODO(joshie): we should check the size and pop N if there's too /// many. fn get_or_create_jar_provider( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, fixed_block_range: &SegmentRangeInclusive, - ) -> ProviderResult> { + ) -> ProviderResult> { let key = (fixed_block_range.end(), segment); // Avoid using `entry` directly to avoid a write lock in the common case. - let mut provider: SnapshotJarProvider<'_> = if let Some(jar) = self.map.get(&key) { + let mut provider: StaticFileJarProvider<'_> = if let Some(jar) = self.map.get(&key) { jar.into() } else { let path = self.path.join(segment.filename(fixed_block_range)); @@ -250,42 +254,42 @@ impl SnapshotProvider { Ok(provider) } - /// Gets a snapshot segment's block range from the provider inner block + /// Gets a static file segment's block range from the provider inner block /// index. fn get_segment_ranges_from_block( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, block: u64, ) -> Option { - self.snapshots_max_block + self.static_files_max_block .read() .get(&segment) .filter(|max| **max >= block) .map(|_| find_fixed_range(block)) } - /// Gets a snapshot segment's fixed block range from the provider inner + /// Gets a static file segment's fixed block range from the provider inner /// transaction index. fn get_segment_ranges_from_transaction( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, tx: u64, ) -> Option { - let snapshots = self.snapshots_tx_index.read(); - let segment_snapshots = snapshots.get(&segment)?; + let static_files = self.static_files_tx_index.read(); + let segment_static_files = static_files.get(&segment)?; // It's more probable that the request comes from a newer tx height, so we iterate - // the snapshots in reverse. - let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable(); + // the static_files in reverse. + let mut static_files_rev_iter = segment_static_files.iter().rev().peekable(); - while let Some((tx_end, block_range)) = snapshots_rev_iter.next() { + while let Some((tx_end, block_range)) = static_files_rev_iter.next() { if tx > *tx_end { - // request tx is higher than highest snapshot tx - return None + // request tx is higher than highest static file tx + return None; } - let tx_start = snapshots_rev_iter.peek().map(|(tx_end, _)| *tx_end + 1).unwrap_or(0); + let tx_start = static_files_rev_iter.peek().map(|(tx_end, _)| *tx_end + 1).unwrap_or(0); if tx_start <= tx { - return Some(find_fixed_range(block_range.end())) + return Some(find_fixed_range(block_range.end())); } } None @@ -299,11 +303,11 @@ impl SnapshotProvider { /// If `segment_max_block` is None it means there's no static file for this segment. pub fn update_index( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, segment_max_block: Option, ) -> ProviderResult<()> { - let mut max_block = self.snapshots_max_block.write(); - let mut tx_index = self.snapshots_tx_index.write(); + let mut max_block = self.static_files_max_block.write(); + let mut tx_index = self.static_files_tx_index.write(); match segment_max_block { Some(segment_max_block) => { @@ -347,7 +351,7 @@ impl SnapshotProvider { if jar.user_header().expected_block_start() == 0 && matches!( segment, - SnapshotSegment::Receipts | SnapshotSegment::Transactions + StaticFileSegment::Receipts | StaticFileSegment::Transactions ) { tx_index.remove(&segment); @@ -371,12 +375,12 @@ impl SnapshotProvider { /// Initializes the inner transaction and block index pub fn initialize_index(&self) -> ProviderResult<()> { - let mut max_block = self.snapshots_max_block.write(); - let mut tx_index = self.snapshots_tx_index.write(); + let mut max_block = self.static_files_max_block.write(); + let mut tx_index = self.static_files_tx_index.write(); tx_index.clear(); - for (segment, ranges) in iter_snapshots(&self.path)? { + for (segment, ranges) in iter_static_files(&self.path)? { // Update last block for each segment if let Some((block_range, _)) = ranges.last() { max_block.insert(segment, block_range.end()); @@ -402,44 +406,44 @@ impl SnapshotProvider { Ok(()) } - /// Gets the highest snapshot block if it exists for a snapshot segment. - pub fn get_highest_snapshot_block(&self, segment: SnapshotSegment) -> Option { - self.snapshots_max_block.read().get(&segment).copied() + /// Gets the highest static file block if it exists for a static file segment. + pub fn get_highest_static_file_block(&self, segment: StaticFileSegment) -> Option { + self.static_files_max_block.read().get(&segment).copied() } - /// Gets the highest snapshotted transaction. - pub fn get_highest_snapshot_tx(&self, segment: SnapshotSegment) -> Option { - self.snapshots_tx_index + /// Gets the highest static file transaction. + pub fn get_highest_static_file_tx(&self, segment: StaticFileSegment) -> Option { + self.static_files_tx_index .read() .get(&segment) .and_then(|index| index.last_key_value().map(|(last_tx, _)| *last_tx)) } - /// Gets the highest snapshotted blocks for all segments. - pub fn get_highest_snapshots(&self) -> HighestSnapshots { - HighestSnapshots { - headers: self.get_highest_snapshot_block(SnapshotSegment::Headers), - receipts: self.get_highest_snapshot_block(SnapshotSegment::Receipts), - transactions: self.get_highest_snapshot_block(SnapshotSegment::Transactions), + /// Gets the highest static file block for all segments. + pub fn get_highest_static_files(&self) -> HighestStaticFiles { + HighestStaticFiles { + headers: self.get_highest_static_file_block(StaticFileSegment::Headers), + receipts: self.get_highest_static_file_block(StaticFileSegment::Receipts), + transactions: self.get_highest_static_file_block(StaticFileSegment::Transactions), } } - /// Iterates through segment snapshots in reverse order, executing a function until it returns - /// some object. Useful for finding objects by [`TxHash`] or [`BlockHash`]. - pub fn find_snapshot( + /// Iterates through segment static_files in reverse order, executing a function until it + /// returns some object. Useful for finding objects by [`TxHash`] or [`BlockHash`]. + pub fn find_static_file( &self, - segment: SnapshotSegment, - func: impl Fn(SnapshotJarProvider<'_>) -> ProviderResult>, + segment: StaticFileSegment, + func: impl Fn(StaticFileJarProvider<'_>) -> ProviderResult>, ) -> ProviderResult> { - if let Some(highest_block) = self.get_highest_snapshot_block(segment) { + if let Some(highest_block) = self.get_highest_static_file_block(segment) { let mut range = find_fixed_range(highest_block); while range.end() > 0 { if let Some(res) = func(self.get_or_create_jar_provider(segment, &range)?)? { - return Ok(Some(res)) + return Ok(Some(res)); } range = SegmentRangeInclusive::new( - range.start().saturating_sub(BLOCKS_PER_SNAPSHOT), - range.end().saturating_sub(BLOCKS_PER_SNAPSHOT), + range.start().saturating_sub(BLOCKS_PER_STATIC_FILE), + range.end().saturating_sub(BLOCKS_PER_STATIC_FILE), ); } } @@ -447,25 +451,27 @@ impl SnapshotProvider { Ok(None) } - /// Fetches data within a specified range across multiple snapshot files. + /// Fetches data within a specified range across multiple static files. /// /// This function iteratively retrieves data using `get_fn` for each item in the given range. /// It continues fetching until the end of the range is reached or the provided `predicate` /// returns false. pub fn fetch_range_with_predicate( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, range: Range, mut get_fn: F, mut predicate: P, ) -> ProviderResult> where - F: FnMut(&mut SnapshotCursor<'_>, u64) -> ProviderResult>, + F: FnMut(&mut StaticFileCursor<'_>, u64) -> ProviderResult>, P: FnMut(&T) -> bool, { let get_provider = |start: u64| match segment { - SnapshotSegment::Headers => self.get_segment_provider_from_block(segment, start, None), - SnapshotSegment::Transactions | SnapshotSegment::Receipts => { + StaticFileSegment::Headers => { + self.get_segment_provider_from_block(segment, start, None) + } + StaticFileSegment::Transactions | StaticFileSegment::Receipts => { self.get_segment_provider_from_transaction(segment, start, None) } }; @@ -481,15 +487,15 @@ impl SnapshotProvider { // an error, effectively preventing infinite retry loops. let mut retrying = false; - // advances snapshot files if `get_fn` returns None + // advances static files if `get_fn` returns None 'inner: loop { match get_fn(&mut cursor, number)? { Some(res) => { if !predicate(&res) { - break 'outer + break 'outer; } result.push(res); - break 'inner + break 'inner; } None => { if retrying { @@ -501,11 +507,11 @@ impl SnapshotProvider { ); let err = if segment.is_headers() { - ProviderError::MissingSnapshotBlock(segment, number) + ProviderError::MissingStaticFileBlock(segment, number) } else { - ProviderError::MissingSnapshotTx(segment, number) + ProviderError::MissingStaticFileTx(segment, number) }; - return Err(err) + return Err(err); } provider = get_provider(number)?; cursor = provider.cursor()?; @@ -518,22 +524,24 @@ impl SnapshotProvider { Ok(result) } - /// Fetches data within a specified range across multiple snapshot files. + /// Fetches data within a specified range across multiple static files. /// /// Returns an iterator over the data pub fn fetch_range_iter<'a, T, F>( &'a self, - segment: SnapshotSegment, + segment: StaticFileSegment, range: Range, get_fn: F, ) -> ProviderResult> + 'a> where - F: Fn(&mut SnapshotCursor<'_>, u64) -> ProviderResult> + 'a, + F: Fn(&mut StaticFileCursor<'_>, u64) -> ProviderResult> + 'a, T: std::fmt::Debug, { let get_provider = move |start: u64| match segment { - SnapshotSegment::Headers => self.get_segment_provider_from_block(segment, start, None), - SnapshotSegment::Transactions | SnapshotSegment::Receipts => { + StaticFileSegment::Headers => { + self.get_segment_provider_from_block(segment, start, None) + } + StaticFileSegment::Transactions | StaticFileSegment::Receipts => { self.get_segment_provider_from_transaction(segment, start, None) } }; @@ -550,81 +558,83 @@ impl SnapshotProvider { })) } - /// Returns directory where snapshots are located. + /// Returns directory where static_files are located. pub fn directory(&self) -> &Path { &self.path } - /// Retrieves data from the database or snapshot, wherever it's available. + /// Retrieves data from the database or static file, wherever it's available. /// /// # Arguments - /// * `segment` - The segment of the snapshot to check against. + /// * `segment` - The segment of the static file to check against. /// * `index_key` - Requested index key, usually a block or transaction number. - /// * `fetch_from_snapshot` - A closure that defines how to fetch the data from the snapshot - /// provider. + /// * `fetch_from_static_file` - A closure that defines how to fetch the data from the static + /// file provider. /// * `fetch_from_database` - A closure that defines how to fetch the data from the database - /// when the snapshot doesn't contain the required data or is not available. - pub fn get_with_snapshot_or_database( + /// when the static file doesn't contain the required data or is not available. + pub fn get_with_static_file_or_database( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, number: u64, - fetch_from_snapshot: FS, + fetch_from_static_file: FS, fetch_from_database: FD, ) -> ProviderResult> where - FS: Fn(&SnapshotProvider) -> ProviderResult>, + FS: Fn(&StaticFileProvider) -> ProviderResult>, FD: Fn() -> ProviderResult>, { // If there is, check the maximum block or transaction number of the segment. - let snapshot_upper_bound = match segment { - SnapshotSegment::Headers => self.get_highest_snapshot_block(segment), - SnapshotSegment::Transactions | SnapshotSegment::Receipts => { - self.get_highest_snapshot_tx(segment) + let static_file_upper_bound = match segment { + StaticFileSegment::Headers => self.get_highest_static_file_block(segment), + StaticFileSegment::Transactions | StaticFileSegment::Receipts => { + self.get_highest_static_file_tx(segment) } }; - if snapshot_upper_bound.map_or(false, |snapshot_upper_bound| snapshot_upper_bound >= number) + if static_file_upper_bound + .map_or(false, |static_file_upper_bound| static_file_upper_bound >= number) { - return fetch_from_snapshot(self) + return fetch_from_static_file(self); } fetch_from_database() } - /// Gets data within a specified range, potentially spanning different snapshots and database. + /// Gets data within a specified range, potentially spanning different static_files and + /// database. /// /// # Arguments - /// * `segment` - The segment of the snapshot to query. + /// * `segment` - The segment of the static file to query. /// * `block_range` - The range of data to fetch. - /// * `fetch_from_snapshot` - A function to fetch data from the snapshot. + /// * `fetch_from_static_file` - A function to fetch data from the static_file. /// * `fetch_from_database` - A function to fetch data from the database. /// * `predicate` - A function used to evaluate each item in the fetched data. Fetching is /// terminated when this function returns false, thereby filtering the data based on the /// provided condition. - pub fn get_range_with_snapshot_or_database( + pub fn get_range_with_static_file_or_database( &self, - segment: SnapshotSegment, + segment: StaticFileSegment, mut block_or_tx_range: Range, - fetch_from_snapshot: FS, + fetch_from_static_file: FS, mut fetch_from_database: FD, mut predicate: P, ) -> ProviderResult> where - FS: Fn(&SnapshotProvider, Range, &mut P) -> ProviderResult>, + FS: Fn(&StaticFileProvider, Range, &mut P) -> ProviderResult>, FD: FnMut(Range, P) -> ProviderResult>, P: FnMut(&T) -> bool, { let mut data = Vec::new(); // If there is, check the maximum block or transaction number of the segment. - if let Some(snapshot_upper_bound) = match segment { - SnapshotSegment::Headers => self.get_highest_snapshot_block(segment), - SnapshotSegment::Transactions | SnapshotSegment::Receipts => { - self.get_highest_snapshot_tx(segment) + if let Some(static_file_upper_bound) = match segment { + StaticFileSegment::Headers => self.get_highest_static_file_block(segment), + StaticFileSegment::Transactions | StaticFileSegment::Receipts => { + self.get_highest_static_file_tx(segment) } } { - if block_or_tx_range.start <= snapshot_upper_bound { - let end = block_or_tx_range.end.min(snapshot_upper_bound + 1); - data.extend(fetch_from_snapshot( + if block_or_tx_range.start <= static_file_upper_bound { + let end = block_or_tx_range.end.min(static_file_upper_bound + 1); + data.extend(fetch_from_static_file( self, block_or_tx_range.start..end, &mut predicate, @@ -641,43 +651,44 @@ impl SnapshotProvider { } #[cfg(any(test, feature = "test-utils"))] - /// Returns snapshots directory + /// Returns static_files directory pub fn path(&self) -> &Path { &self.path } } -/// Helper trait to manage different [`SnapshotProviderRW`] of an `Arc ProviderResult>; + segment: StaticFileSegment, + ) -> ProviderResult>; - /// Returns a mutable reference to a [`SnapshotProviderRW`] of the latest [`SnapshotSegment`]. + /// Returns a mutable reference to a [`StaticFileProviderRW`] of the latest + /// [`StaticFileSegment`]. fn latest_writer( &self, - segment: SnapshotSegment, - ) -> ProviderResult>; + segment: StaticFileSegment, + ) -> ProviderResult>; - /// Commits all changes of all [`SnapshotProviderRW`] of all [`SnapshotSegment`]. + /// Commits all changes of all [`StaticFileProviderRW`] of all [`StaticFileSegment`]. fn commit(&self) -> ProviderResult<()>; } -impl SnapshotWriter for SnapshotProvider { +impl StaticFileWriter for StaticFileProvider { fn get_writer( &self, block: BlockNumber, - segment: SnapshotSegment, - ) -> ProviderResult> { + segment: StaticFileSegment, + ) -> ProviderResult> { tracing::trace!(target: "providers::static_file", ?block, ?segment, "Getting static file writer."); Ok(match self.writers.entry(segment) { DashMapEntry::Occupied(entry) => entry.into_ref(), DashMapEntry::Vacant(entry) => { let writer = - SnapshotProviderRW::new(segment, block, self.clone(), self.metrics.clone())?; + StaticFileProviderRW::new(segment, block, self.clone(), self.metrics.clone())?; entry.insert(writer) } }) @@ -685,9 +696,9 @@ impl SnapshotWriter for SnapshotProvider { fn latest_writer( &self, - segment: SnapshotSegment, - ) -> ProviderResult> { - self.get_writer(self.get_highest_snapshot_block(segment).unwrap_or_default(), segment) + segment: StaticFileSegment, + ) -> ProviderResult> { + self.get_writer(self.get_highest_static_file_block(segment).unwrap_or_default(), segment) } fn commit(&self) -> ProviderResult<()> { @@ -698,15 +709,15 @@ impl SnapshotWriter for SnapshotProvider { } } -impl HeaderProvider for SnapshotProvider { +impl HeaderProvider for StaticFileProvider { fn header(&self, block_hash: &BlockHash) -> ProviderResult> { - self.find_snapshot(SnapshotSegment::Headers, |jar_provider| { + self.find_static_file(StaticFileSegment::Headers, |jar_provider| { Ok(jar_provider .cursor()? .get_two::>(block_hash.into())? .and_then(|(header, hash)| { if &hash == block_hash { - return Some(header) + return Some(header); } None })) @@ -714,12 +725,12 @@ impl HeaderProvider for SnapshotProvider { } fn header_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)? + self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None)? .header_by_number(num) } fn header_td(&self, block_hash: &BlockHash) -> ProviderResult> { - self.find_snapshot(SnapshotSegment::Headers, |jar_provider| { + self.find_static_file(StaticFileSegment::Headers, |jar_provider| { Ok(jar_provider .cursor()? .get_two::>(block_hash.into())? @@ -728,13 +739,13 @@ impl HeaderProvider for SnapshotProvider { } fn header_td_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)? + self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None)? .header_td_by_number(num) } fn headers_range(&self, range: impl RangeBounds) -> ProviderResult> { self.fetch_range_with_predicate( - SnapshotSegment::Headers, + StaticFileSegment::Headers, to_range(range), |cursor, number| cursor.get_one::>(number.into()), |_| true, @@ -742,7 +753,7 @@ impl HeaderProvider for SnapshotProvider { } fn sealed_header(&self, num: BlockNumber) -> ProviderResult> { - self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)? + self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None)? .sealed_header(num) } @@ -752,7 +763,7 @@ impl HeaderProvider for SnapshotProvider { predicate: impl FnMut(&SealedHeader) -> bool, ) -> ProviderResult> { self.fetch_range_with_predicate( - SnapshotSegment::Headers, + StaticFileSegment::Headers, to_range(range), |cursor, number| { Ok(cursor @@ -764,9 +775,9 @@ impl HeaderProvider for SnapshotProvider { } } -impl BlockHashReader for SnapshotProvider { +impl BlockHashReader for StaticFileProvider { fn block_hash(&self, num: u64) -> ProviderResult> { - self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)?.block_hash(num) + self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None)?.block_hash(num) } fn canonical_hashes_range( @@ -775,7 +786,7 @@ impl BlockHashReader for SnapshotProvider { end: BlockNumber, ) -> ProviderResult> { self.fetch_range_with_predicate( - SnapshotSegment::Headers, + StaticFileSegment::Headers, start..end, |cursor, number| cursor.get_one::>(number.into()), |_| true, @@ -783,15 +794,15 @@ impl BlockHashReader for SnapshotProvider { } } -impl ReceiptProvider for SnapshotProvider { +impl ReceiptProvider for StaticFileProvider { fn receipt(&self, num: TxNumber) -> ProviderResult> { - self.get_segment_provider_from_transaction(SnapshotSegment::Receipts, num, None)? + self.get_segment_provider_from_transaction(StaticFileSegment::Receipts, num, None)? .receipt(num) } fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult> { if let Some(num) = self.transaction_id(hash)? { - return self.receipt(num) + return self.receipt(num); } Ok(None) } @@ -805,7 +816,7 @@ impl ReceiptProvider for SnapshotProvider { range: impl RangeBounds, ) -> ProviderResult> { self.fetch_range_with_predicate( - SnapshotSegment::Receipts, + StaticFileSegment::Receipts, to_range(range), |cursor, number| cursor.get_one::>(number.into()), |_| true, @@ -813,7 +824,7 @@ impl ReceiptProvider for SnapshotProvider { } } -impl TransactionsProviderExt for SnapshotProvider { +impl TransactionsProviderExt for StaticFileProvider { fn transaction_hashes_by_range( &self, tx_range: Range, @@ -853,7 +864,7 @@ impl TransactionsProviderExt for SnapshotProvider { rayon::spawn(move || { let mut rlp_buf = Vec::with_capacity(128); let _ = manager.fetch_range_with_predicate( - SnapshotSegment::Transactions, + StaticFileSegment::Transactions, chunk_range, |cursor, number| { Ok(cursor @@ -883,9 +894,9 @@ impl TransactionsProviderExt for SnapshotProvider { } } -impl TransactionsProvider for SnapshotProvider { +impl TransactionsProvider for StaticFileProvider { fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult> { - self.find_snapshot(SnapshotSegment::Transactions, |jar_provider| { + self.find_static_file(StaticFileSegment::Transactions, |jar_provider| { let mut cursor = jar_provider.cursor()?; if cursor .get_one::>((&tx_hash).into())? @@ -900,7 +911,7 @@ impl TransactionsProvider for SnapshotProvider { } fn transaction_by_id(&self, num: TxNumber) -> ProviderResult> { - self.get_segment_provider_from_transaction(SnapshotSegment::Transactions, num, None)? + self.get_segment_provider_from_transaction(StaticFileSegment::Transactions, num, None)? .transaction_by_id(num) } @@ -908,12 +919,12 @@ impl TransactionsProvider for SnapshotProvider { &self, num: TxNumber, ) -> ProviderResult> { - self.get_segment_provider_from_transaction(SnapshotSegment::Transactions, num, None)? + self.get_segment_provider_from_transaction(StaticFileSegment::Transactions, num, None)? .transaction_by_id_no_hash(num) } fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult> { - self.find_snapshot(SnapshotSegment::Transactions, |jar_provider| { + self.find_static_file(StaticFileSegment::Transactions, |jar_provider| { Ok(jar_provider .cursor()? .get_one::>((&hash).into())? @@ -926,12 +937,12 @@ impl TransactionsProvider for SnapshotProvider { &self, _hash: TxHash, ) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn transaction_block(&self, _id: TxNumber) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } @@ -939,7 +950,7 @@ impl TransactionsProvider for SnapshotProvider { &self, _block_id: BlockHashOrNumber, ) -> ProviderResult>> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } @@ -947,7 +958,7 @@ impl TransactionsProvider for SnapshotProvider { &self, _range: impl RangeBounds, ) -> ProviderResult>> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } @@ -965,7 +976,7 @@ impl TransactionsProvider for SnapshotProvider { range: impl RangeBounds, ) -> ProviderResult> { self.fetch_range_with_predicate( - SnapshotSegment::Transactions, + StaticFileSegment::Transactions, to_range(range), |cursor, number| { cursor.get_one::>(number.into()) @@ -979,7 +990,7 @@ impl TransactionsProvider for SnapshotProvider { range: impl RangeBounds, ) -> ProviderResult>> { self.fetch_range_with_predicate( - SnapshotSegment::Transactions, + StaticFileSegment::Transactions, to_range(range), |cursor, number| { cursor.get(number.into(), >::MASK).map( @@ -1001,65 +1012,65 @@ impl TransactionsProvider for SnapshotProvider { /* Cannot be successfully implemented but must exist for trait requirements */ -impl BlockNumReader for SnapshotProvider { +impl BlockNumReader for StaticFileProvider { fn chain_info(&self) -> ProviderResult { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn best_block_number(&self) -> ProviderResult { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn last_block_number(&self) -> ProviderResult { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn block_number(&self, _hash: B256) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } } -impl BlockReader for SnapshotProvider { +impl BlockReader for StaticFileProvider { fn find_block_by_hash( &self, _hash: B256, _source: BlockSource, ) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn block(&self, _id: BlockHashOrNumber) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn pending_block(&self) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn pending_block_with_senders(&self) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn pending_block_and_receipts(&self) -> ProviderResult)>> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn ommers(&self, _id: BlockHashOrNumber) -> ProviderResult>> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn block_body_indices(&self, _num: u64) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } @@ -1068,47 +1079,47 @@ impl BlockReader for SnapshotProvider { _id: BlockHashOrNumber, _transaction_kind: TransactionVariant, ) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn block_range(&self, _range: RangeInclusive) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } } -impl WithdrawalsProvider for SnapshotProvider { +impl WithdrawalsProvider for StaticFileProvider { fn withdrawals_by_block( &self, _id: BlockHashOrNumber, _timestamp: u64, ) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn latest_withdrawal(&self) -> ProviderResult> { - // Required data not present in snapshots + // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } } -impl StatsReader for SnapshotProvider { +impl StatsReader for StaticFileProvider { fn count_entries(&self) -> ProviderResult { match T::NAME { tables::CanonicalHeaders::NAME | tables::Headers::NAME | tables::HeaderTD::NAME => { Ok(self - .get_highest_snapshot_block(SnapshotSegment::Headers) + .get_highest_static_file_block(StaticFileSegment::Headers) .map(|block| block + 1) .unwrap_or_default() as usize) } tables::Receipts::NAME => Ok(self - .get_highest_snapshot_tx(SnapshotSegment::Receipts) + .get_highest_static_file_tx(StaticFileSegment::Receipts) .map(|receipts| receipts + 1) .unwrap_or_default() as usize), tables::Transactions::NAME => Ok(self - .get_highest_snapshot_tx(SnapshotSegment::Transactions) + .get_highest_static_file_tx(StaticFileSegment::Transactions) .map(|txs| txs + 1) .unwrap_or_default() as usize), _ => Err(ProviderError::UnsupportedProvider), diff --git a/crates/storage/provider/src/providers/snapshot/metrics.rs b/crates/storage/provider/src/providers/static_file/metrics.rs similarity index 64% rename from crates/storage/provider/src/providers/snapshot/metrics.rs rename to crates/storage/provider/src/providers/static_file/metrics.rs index fa2380fd13e6..497620b64b87 100644 --- a/crates/storage/provider/src/providers/snapshot/metrics.rs +++ b/crates/storage/provider/src/providers/static_file/metrics.rs @@ -3,25 +3,27 @@ use std::{collections::HashMap, time::Duration}; use itertools::Itertools; use metrics::{Counter, Histogram}; use reth_metrics::Metrics; -use reth_primitives::SnapshotSegment; +use reth_primitives::StaticFileSegment; use strum::{EnumIter, IntoEnumIterator}; -/// Metrics for the snapshot provider. +/// Metrics for the static file provider. #[derive(Debug)] -pub struct SnapshotProviderMetrics { - segment_operations: - HashMap<(SnapshotSegment, SnapshotProviderOperation), SnapshotProviderOperationMetrics>, +pub struct StaticFileProviderMetrics { + segment_operations: HashMap< + (StaticFileSegment, StaticFileProviderOperation), + StaticFileProviderOperationMetrics, + >, } -impl Default for SnapshotProviderMetrics { +impl Default for StaticFileProviderMetrics { fn default() -> Self { Self { - segment_operations: SnapshotSegment::iter() - .cartesian_product(SnapshotProviderOperation::iter()) + segment_operations: StaticFileSegment::iter() + .cartesian_product(StaticFileProviderOperation::iter()) .map(|(segment, operation)| { ( (segment, operation), - SnapshotProviderOperationMetrics::new_with_labels(&[ + StaticFileProviderOperationMetrics::new_with_labels(&[ ("segment", segment.as_str()), ("operation", operation.as_str()), ]), @@ -32,11 +34,11 @@ impl Default for SnapshotProviderMetrics { } } -impl SnapshotProviderMetrics { +impl StaticFileProviderMetrics { pub(crate) fn record_segment_operation( &self, - segment: SnapshotSegment, - operation: SnapshotProviderOperation, + segment: StaticFileSegment, + operation: StaticFileProviderOperation, duration: Option, ) { self.segment_operations @@ -56,7 +58,7 @@ impl SnapshotProviderMetrics { } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumIter)] -pub(crate) enum SnapshotProviderOperation { +pub(crate) enum StaticFileProviderOperation { InitCursor, OpenWriter, Append, @@ -65,7 +67,7 @@ pub(crate) enum SnapshotProviderOperation { CommitWriter, } -impl SnapshotProviderOperation { +impl StaticFileProviderOperation { const fn as_str(&self) -> &'static str { match self { Self::InitCursor => "init-cursor", @@ -79,10 +81,10 @@ impl SnapshotProviderOperation { } #[derive(Metrics)] -#[metrics(scope = "snapshots.jar_provider")] -pub(crate) struct SnapshotProviderOperationMetrics { - /// Total number of snapshot jar provider operations made. +#[metrics(scope = "static_files.jar_provider")] +pub(crate) struct StaticFileProviderOperationMetrics { + /// Total number of static file jar provider operations made. calls_total: Counter, - /// The time it took to execute the snapshot jar provider operation that writes data. + /// The time it took to execute the static file jar provider operation that writes data. write_duration_seconds: Histogram, } diff --git a/crates/storage/provider/src/providers/snapshot/mod.rs b/crates/storage/provider/src/providers/static_file/mod.rs similarity index 79% rename from crates/storage/provider/src/providers/snapshot/mod.rs rename to crates/storage/provider/src/providers/static_file/mod.rs index f4ff986e89d4..ee318ac8e1ff 100644 --- a/crates/storage/provider/src/providers/snapshot/mod.rs +++ b/crates/storage/provider/src/providers/static_file/mod.rs @@ -1,25 +1,25 @@ mod manager; -pub use manager::{SnapshotProvider, SnapshotWriter}; +pub use manager::{StaticFileProvider, StaticFileWriter}; mod jar; -pub use jar::SnapshotJarProvider; +pub use jar::StaticFileJarProvider; mod writer; -pub use writer::{SnapshotProviderRW, SnapshotProviderRWRefMut}; +pub use writer::{StaticFileProviderRW, StaticFileProviderRWRefMut}; mod metrics; use reth_interfaces::provider::ProviderResult; use reth_nippy_jar::NippyJar; -use reth_primitives::{snapshot::SegmentHeader, SnapshotSegment}; +use reth_primitives::{static_file::SegmentHeader, StaticFileSegment}; use std::{ops::Deref, sync::Arc}; -const BLOCKS_PER_SNAPSHOT: u64 = 500_000; +const BLOCKS_PER_STATIC_FILE: u64 = 500_000; /// Alias type for each specific `NippyJar`. -type LoadedJarRef<'a> = dashmap::mapref::one::Ref<'a, (u64, SnapshotSegment), LoadedJar>; +type LoadedJarRef<'a> = dashmap::mapref::one::Ref<'a, (u64, StaticFileSegment), LoadedJar>; -/// Helper type to reuse an associated snapshot mmap handle on created cursors. +/// Helper type to reuse an associated static file mmap handle on created cursors. #[derive(Debug)] pub struct LoadedJar { jar: NippyJar, @@ -37,7 +37,7 @@ impl LoadedJar { self.mmap_handle.clone() } - fn segment(&self) -> SnapshotSegment { + fn segment(&self) -> StaticFileSegment { self.jar.user_header().segment() } } @@ -56,12 +56,12 @@ mod tests { use rand::seq::SliceRandom; use reth_db::{ cursor::DbCursorRO, - snapshot::create_snapshot_T1_T2_T3, + static_file::create_static_file_T1_T2_T3, transaction::{DbTx, DbTxMut}, CanonicalHeaders, HeaderNumbers, HeaderTD, Headers, RawTable, }; use reth_interfaces::test_utils::generators::{self, random_header_range}; - use reth_primitives::{snapshot::find_fixed_range, BlockNumber, B256, U256}; + use reth_primitives::{static_file::find_fixed_range, BlockNumber, B256, U256}; #[test] fn test_snap() { @@ -72,15 +72,15 @@ mod tests { range.clone().into(), Some(range.clone().into()), Some(range.clone().into()), - SnapshotSegment::Headers, + StaticFileSegment::Headers, ); // Data sources let factory = create_test_provider_factory(); - let snap_path = tempfile::tempdir().unwrap(); - let snap_file = snap_path + let static_files_path = tempfile::tempdir().unwrap(); + let static_file = static_files_path .path() - .join(SnapshotSegment::Headers.filename(&find_fixed_range(*range.end()))); + .join(StaticFileSegment::Headers.filename(&find_fixed_range(*range.end()))); // Setup data let mut headers = random_header_range( @@ -103,12 +103,12 @@ mod tests { } provider_rw.commit().unwrap(); - // Create Snapshot + // Create StaticFile { let with_compression = true; let with_filter = true; - let mut nippy_jar = NippyJar::new(3, snap_file.as_path(), segment_header); + let mut nippy_jar = NippyJar::new(3, static_file.as_path(), segment_header); if with_compression { nippy_jar = nippy_jar.with_zstd(false, 0); @@ -132,7 +132,7 @@ mod tests { .unwrap() .map(|row| row.map(|(_key, value)| value.into_value()).map_err(|e| e.into())); - create_snapshot_T1_T2_T3::< + create_static_file_T1_T2_T3::< Headers, HeaderTD, CanonicalHeaders, @@ -147,9 +147,9 @@ mod tests { // Use providers to query Header data and compare if it matches { let db_provider = factory.provider().unwrap(); - let manager = SnapshotProvider::new(snap_path.path()).unwrap().with_filters(); + let manager = StaticFileProvider::new(static_files_path.path()).unwrap().with_filters(); let jar_provider = manager - .get_segment_provider_from_block(SnapshotSegment::Headers, 0, Some(&snap_file)) + .get_segment_provider_from_block(StaticFileSegment::Headers, 0, Some(&static_file)) .unwrap(); assert!(!headers.is_empty()); diff --git a/crates/storage/provider/src/providers/snapshot/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs similarity index 76% rename from crates/storage/provider/src/providers/snapshot/writer.rs rename to crates/storage/provider/src/providers/static_file/writer.rs index b7f5f8fc6869..0d86941da8f4 100644 --- a/crates/storage/provider/src/providers/snapshot/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -1,14 +1,14 @@ -use crate::providers::snapshot::metrics::SnapshotProviderOperation; +use crate::providers::static_file::metrics::StaticFileProviderOperation; -use super::{metrics::SnapshotProviderMetrics, SnapshotProvider}; +use super::{metrics::StaticFileProviderMetrics, StaticFileProvider}; use dashmap::mapref::one::RefMut; use reth_codecs::Compact; use reth_db::codecs::CompactU256; use reth_interfaces::provider::{ProviderError, ProviderResult}; use reth_nippy_jar::{NippyJar, NippyJarError, NippyJarWriter}; use reth_primitives::{ - snapshot::{find_fixed_range, SegmentHeader, SegmentRangeInclusive}, - BlockHash, BlockNumber, Header, Receipt, SnapshotSegment, TransactionSignedNoHash, TxNumber, + static_file::{find_fixed_range, SegmentHeader, SegmentRangeInclusive}, + BlockHash, BlockNumber, Header, Receipt, StaticFileSegment, TransactionSignedNoHash, TxNumber, U256, }; use std::{ @@ -19,36 +19,37 @@ use std::{ }; use tracing::debug; -/// Mutable reference to a dashmap element of [`SnapshotProviderRW`]. -pub type SnapshotProviderRWRefMut<'a> = RefMut<'a, SnapshotSegment, SnapshotProviderRW<'static>>; +/// Mutable reference to a dashmap element of [`StaticFileProviderRW`]. +pub type StaticFileProviderRWRefMut<'a> = + RefMut<'a, StaticFileSegment, StaticFileProviderRW<'static>>; #[derive(Debug)] -/// Extends `SnapshotProvider` with writing capabilities -pub struct SnapshotProviderRW<'a> { - reader: SnapshotProvider, +/// Extends `StaticFileProvider` with writing capabilities +pub struct StaticFileProviderRW<'a> { + reader: StaticFileProvider, writer: NippyJarWriter<'a, SegmentHeader>, data_path: PathBuf, buf: Vec, - metrics: Option>, + metrics: Option>, } -impl<'a> SnapshotProviderRW<'a> { - /// Creates a new [`SnapshotProviderRW`] for a [`SnapshotSegment`]. +impl<'a> StaticFileProviderRW<'a> { + /// Creates a new [`StaticFileProviderRW`] for a [`StaticFileSegment`]. pub fn new( - segment: SnapshotSegment, + segment: StaticFileSegment, block: BlockNumber, - reader: SnapshotProvider, - metrics: Option>, + reader: StaticFileProvider, + metrics: Option>, ) -> ProviderResult { let (writer, data_path) = Self::open(segment, block, reader.clone(), metrics.clone())?; Ok(Self { writer, data_path, buf: Vec::with_capacity(100), reader, metrics }) } fn open( - segment: SnapshotSegment, + segment: StaticFileSegment, block: u64, - reader: SnapshotProvider, - metrics: Option>, + reader: StaticFileProvider, + metrics: Option>, ) -> ProviderResult<(NippyJarWriter<'a, SegmentHeader>, PathBuf)> { let start = Instant::now(); @@ -58,7 +59,7 @@ impl<'a> SnapshotProviderRW<'a> { Ok(provider) => { (NippyJar::load(provider.data_path())?, provider.data_path().into()) } - Err(ProviderError::MissingSnapshotBlock(_, _)) => { + Err(ProviderError::MissingStaticFileBlock(_, _)) => { let path = reader.directory().join(segment.filename(&block_range)); (create_jar(segment, &path, block_range), path) } @@ -68,8 +69,8 @@ impl<'a> SnapshotProviderRW<'a> { let result = match NippyJarWriter::from_owned(jar) { Ok(writer) => Ok((writer, path)), Err(NippyJarError::FrozenJar) => { - // This snapshot has been frozen, so we should - Err(ProviderError::FinalizedSnapshot(segment, block)) + // This static file has been frozen, so we should + Err(ProviderError::FinalizedStaticFile(segment, block)) } Err(e) => Err(e.into()), }?; @@ -77,7 +78,7 @@ impl<'a> SnapshotProviderRW<'a> { if let Some(metrics) = &metrics { metrics.record_segment_operation( segment, - SnapshotProviderOperation::OpenWriter, + StaticFileProviderOperation::OpenWriter, Some(start.elapsed()), ); } @@ -95,7 +96,7 @@ impl<'a> SnapshotProviderRW<'a> { if let Some(metrics) = &self.metrics { metrics.record_segment_operation( self.writer.user_header().segment(), - SnapshotProviderOperation::CommitWriter, + StaticFileProviderOperation::CommitWriter, Some(start.elapsed()), ); } @@ -122,7 +123,7 @@ impl<'a> SnapshotProviderRW<'a> { // previous file. // // If that expected block start is 0, then it means that there's no actual block data, and - // there's no snapshotted block data. + // there's no block data in static files. let segment_max_block = match self.writer.user_header().block_range() { Some(block_range) => Some(block_range.end()), None => { @@ -137,19 +138,19 @@ impl<'a> SnapshotProviderRW<'a> { self.reader.update_index(self.writer.user_header().segment(), segment_max_block) } - /// Allows to increment the [`SegmentHeader`] end block. It will commit the current snapshot, + /// Allows to increment the [`SegmentHeader`] end block. It will commit the current static file, /// and create the next one if we are past the end range. /// /// Returns the current [`BlockNumber`] as seen in the static file. - pub fn increment_block(&mut self, segment: SnapshotSegment) -> ProviderResult { + pub fn increment_block(&mut self, segment: StaticFileSegment) -> ProviderResult { let start = Instant::now(); if let Some(last_block) = self.writer.user_header().block_end() { - // We have finished the previous snapshot and must freeze it + // We have finished the previous static file and must freeze it if last_block == self.writer.user_header().expected_block_end() { // Commits offsets and new user_header to disk self.commit()?; - // Opens the new snapshot + // Opens the new static file let (writer, data_path) = Self::open(segment, last_block + 1, self.reader.clone(), self.metrics.clone())?; self.writer = writer; @@ -164,7 +165,7 @@ impl<'a> SnapshotProviderRW<'a> { if let Some(metrics) = &self.metrics { metrics.record_segment_operation( segment, - SnapshotProviderOperation::IncrementBlock, + StaticFileProviderOperation::IncrementBlock, Some(start.elapsed()), ); } @@ -172,7 +173,7 @@ impl<'a> SnapshotProviderRW<'a> { Ok(block) } - /// Truncates a number of rows from disk. It deletes and loads an older snapshot file if block + /// Truncates a number of rows from disk. It deletes and loads an older static file if block /// goes beyond the start of the current block range. /// /// **last_block** should be passed only with transaction based segments. @@ -181,23 +182,23 @@ impl<'a> SnapshotProviderRW<'a> { /// Commits to the configuration file at the end. fn truncate( &mut self, - segment: SnapshotSegment, + segment: StaticFileSegment, mut num_rows: u64, last_block: Option, ) -> ProviderResult<()> { while num_rows > 0 { let len = match segment { - SnapshotSegment::Headers => { + StaticFileSegment::Headers => { self.writer.user_header().block_len().unwrap_or_default() } - SnapshotSegment::Transactions | SnapshotSegment::Receipts => { + StaticFileSegment::Transactions | StaticFileSegment::Receipts => { self.writer.user_header().tx_len().unwrap_or_default() } }; if num_rows >= len { - // If there's more rows to delete than this snapshot contains, then just - // delete the whole file and go to the next snapshot + // If there's more rows to delete than this static file contains, then just + // delete the whole file and go to the next static file let previous_snap = self.data_path.clone(); let block_start = self.writer.user_header().expected_block_start(); @@ -216,7 +217,7 @@ impl<'a> SnapshotProviderRW<'a> { // Update `SegmentHeader` self.writer.user_header_mut().prune(len); self.writer.prune_rows(len as usize)?; - break + break; } num_rows -= len; @@ -242,7 +243,7 @@ impl<'a> SnapshotProviderRW<'a> { Ok(()) } - /// Appends column to snapshot file. + /// Appends column to static file. fn append_column(&mut self, column: T) -> ProviderResult<()> { self.buf.clear(); column.to_compact(&mut self.buf); @@ -251,12 +252,12 @@ impl<'a> SnapshotProviderRW<'a> { Ok(()) } - /// Appends to tx number-based snapshot file. + /// Appends to tx number-based static file. /// /// Returns the current [`TxNumber`] as seen in the static file. fn append_with_tx_number( &mut self, - segment: SnapshotSegment, + segment: StaticFileSegment, tx_num: TxNumber, value: V, ) -> ProviderResult { @@ -273,7 +274,7 @@ impl<'a> SnapshotProviderRW<'a> { Ok(self.writer.user_header().tx_end().expect("qed")) } - /// Appends header to snapshot file. + /// Appends header to static file. /// /// It **CALLS** `increment_block()` since the number of headers is equal to the number of /// blocks. @@ -287,9 +288,9 @@ impl<'a> SnapshotProviderRW<'a> { ) -> ProviderResult { let start = Instant::now(); - debug_assert!(self.writer.user_header().segment() == SnapshotSegment::Headers); + debug_assert!(self.writer.user_header().segment() == StaticFileSegment::Headers); - let block_number = self.increment_block(SnapshotSegment::Headers)?; + let block_number = self.increment_block(StaticFileSegment::Headers)?; self.append_column(header)?; self.append_column(CompactU256::from(terminal_difficulty))?; @@ -297,8 +298,8 @@ impl<'a> SnapshotProviderRW<'a> { if let Some(metrics) = &self.metrics { metrics.record_segment_operation( - SnapshotSegment::Headers, - SnapshotProviderOperation::Append, + StaticFileSegment::Headers, + StaticFileProviderOperation::Append, Some(start.elapsed()), ); } @@ -306,7 +307,7 @@ impl<'a> SnapshotProviderRW<'a> { Ok(block_number) } - /// Appends transaction to snapshot file. + /// Appends transaction to static file. /// /// It **DOES NOT CALL** `increment_block()`, it should be handled elsewhere. There might be /// empty blocks and this function wouldn't be called. @@ -319,12 +320,12 @@ impl<'a> SnapshotProviderRW<'a> { ) -> ProviderResult { let start = Instant::now(); - let result = self.append_with_tx_number(SnapshotSegment::Transactions, tx_num, tx)?; + let result = self.append_with_tx_number(StaticFileSegment::Transactions, tx_num, tx)?; if let Some(metrics) = &self.metrics { metrics.record_segment_operation( - SnapshotSegment::Transactions, - SnapshotProviderOperation::Append, + StaticFileSegment::Transactions, + StaticFileProviderOperation::Append, Some(start.elapsed()), ); } @@ -332,7 +333,7 @@ impl<'a> SnapshotProviderRW<'a> { Ok(result) } - /// Appends receipt to snapshot file. + /// Appends receipt to static file. /// /// It **DOES NOT** call `increment_block()`, it should be handled elsewhere. There might be /// empty blocks and this function wouldn't be called. @@ -345,12 +346,12 @@ impl<'a> SnapshotProviderRW<'a> { ) -> ProviderResult { let start = Instant::now(); - let result = self.append_with_tx_number(SnapshotSegment::Receipts, tx_num, receipt)?; + let result = self.append_with_tx_number(StaticFileSegment::Receipts, tx_num, receipt)?; if let Some(metrics) = &self.metrics { metrics.record_segment_operation( - SnapshotSegment::Receipts, - SnapshotProviderOperation::Append, + StaticFileSegment::Receipts, + StaticFileProviderOperation::Append, Some(start.elapsed()), ); } @@ -369,15 +370,15 @@ impl<'a> SnapshotProviderRW<'a> { ) -> ProviderResult<()> { let start = Instant::now(); - let segment = SnapshotSegment::Transactions; + let segment = StaticFileSegment::Transactions; debug_assert!(self.writer.user_header().segment() == segment); self.truncate(segment, number, Some(last_block))?; if let Some(metrics) = &self.metrics { metrics.record_segment_operation( - SnapshotSegment::Transactions, - SnapshotProviderOperation::Prune, + StaticFileSegment::Transactions, + StaticFileProviderOperation::Prune, Some(start.elapsed()), ); } @@ -385,7 +386,7 @@ impl<'a> SnapshotProviderRW<'a> { Ok(()) } - /// Prunes `to_delete` number of receipts from snapshots. + /// Prunes `to_delete` number of receipts from static_files. /// /// # Note /// Commits to the configuration file at the end. @@ -396,15 +397,15 @@ impl<'a> SnapshotProviderRW<'a> { ) -> ProviderResult<()> { let start = Instant::now(); - let segment = SnapshotSegment::Receipts; + let segment = StaticFileSegment::Receipts; debug_assert!(self.writer.user_header().segment() == segment); self.truncate(segment, to_delete, Some(last_block))?; if let Some(metrics) = &self.metrics { metrics.record_segment_operation( - SnapshotSegment::Receipts, - SnapshotProviderOperation::Prune, + StaticFileSegment::Receipts, + StaticFileProviderOperation::Prune, Some(start.elapsed()), ); } @@ -412,22 +413,22 @@ impl<'a> SnapshotProviderRW<'a> { Ok(()) } - /// Prunes `to_delete` number of headers from snapshots. + /// Prunes `to_delete` number of headers from static_files. /// /// # Note /// Commits to the configuration file at the end. pub fn prune_headers(&mut self, to_delete: u64) -> ProviderResult<()> { let start = Instant::now(); - let segment = SnapshotSegment::Headers; + let segment = StaticFileSegment::Headers; debug_assert!(self.writer.user_header().segment() == segment); self.truncate(segment, to_delete, None)?; if let Some(metrics) = &self.metrics { metrics.record_segment_operation( - SnapshotSegment::Headers, - SnapshotProviderOperation::Prune, + StaticFileSegment::Headers, + StaticFileProviderOperation::Prune, Some(start.elapsed()), ); } @@ -442,15 +443,15 @@ impl<'a> SnapshotProviderRW<'a> { } } -impl<'a> Deref for SnapshotProviderRW<'a> { - type Target = SnapshotProvider; +impl<'a> Deref for StaticFileProviderRW<'a> { + type Target = StaticFileProvider; fn deref(&self) -> &Self::Target { &self.reader } } fn create_jar( - segment: SnapshotSegment, + segment: StaticFileSegment, path: &Path, expected_block_range: SegmentRangeInclusive, ) -> NippyJar { diff --git a/crates/storage/provider/src/test_utils/mod.rs b/crates/storage/provider/src/test_utils/mod.rs index f19db949d65f..7857b8c2144a 100644 --- a/crates/storage/provider/src/test_utils/mod.rs +++ b/crates/storage/provider/src/test_utils/mod.rs @@ -1,6 +1,6 @@ use crate::ProviderFactory; use reth_db::{ - test_utils::{create_test_rw_db, create_test_snapshots_dir, TempDatabase}, + test_utils::{create_test_rw_db, create_test_static_files_dir, TempDatabase}, DatabaseEnv, }; use reth_primitives::{ChainSpec, MAINNET}; @@ -27,6 +27,6 @@ pub fn create_test_provider_factory_with_chain_spec( chain_spec: Arc, ) -> ProviderFactory>> { let db = create_test_rw_db(); - ProviderFactory::new(db, chain_spec, create_test_snapshots_dir()) - .expect("create provider factory with snapshots") + ProviderFactory::new(db, chain_spec, create_test_static_files_dir()) + .expect("create provider factory with static_files") } diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index 027dfa6b6c79..0ce59dc3de10 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -85,7 +85,7 @@ pub(crate) struct BestTransactions { /// There might be the case where a yielded transactions is invalid, this will track it. pub(crate) invalid: HashSet, /// Used to receive any new pending transactions that have been added to the pool after this - /// iterator was snapshotted + /// iterator was static fileted /// /// These new pending transactions are inserted into this iterator's pool before yielding the /// next value diff --git a/crates/transaction-pool/src/pool/pending.rs b/crates/transaction-pool/src/pool/pending.rs index 9476f3c6cb1b..90ae13cd6449 100644 --- a/crates/transaction-pool/src/pool/pending.rs +++ b/crates/transaction-pool/src/pool/pending.rs @@ -51,7 +51,7 @@ pub struct PendingPool { /// See also [`PoolTransaction::size`](crate::traits::PoolTransaction::size). size_of: SizeTracker, /// Used to broadcast new transactions that have been added to the PendingPool to existing - /// snapshots of this pool. + /// static_files of this pool. new_transaction_notifier: broadcast::Sender>, } @@ -309,7 +309,7 @@ impl PendingPool { self.update_independents_and_highest_nonces(&tx, &tx_id); self.all.insert(tx.clone()); - // send the new transaction to any existing pendingpool snapshot iterators + // send the new transaction to any existing pendingpool static file iterators if self.new_transaction_notifier.receiver_count() > 0 { let _ = self.new_transaction_notifier.send(tx.clone()); } diff --git a/examples/db-access.rs b/examples/db-access.rs index c5ece0f77e8f..6edfc6afe166 100644 --- a/examples/db-access.rs +++ b/examples/db-access.rs @@ -25,7 +25,7 @@ fn main() -> eyre::Result<()> { // Instantiate a provider factory for Ethereum mainnet using the provided DB. // TODO: Should the DB version include the spec so that you do not need to specify it here? let spec = ChainSpecBuilder::mainnet().build(); - let factory = ProviderFactory::new(db, spec.into(), db_path.join("snapshots"))?; + let factory = ProviderFactory::new(db, spec.into(), db_path.join("static_files"))?; // This call opens a RO transaction on the database. To write to the DB you'd need to call // the `provider_rw` function and look for the `Writer` variants of the traits. diff --git a/examples/rpc-db/src/main.rs b/examples/rpc-db/src/main.rs index 9f6fe99829b8..13fbc6223728 100644 --- a/examples/rpc-db/src/main.rs +++ b/examples/rpc-db/src/main.rs @@ -40,7 +40,7 @@ async fn main() -> eyre::Result<()> { let db_path = Path::new(&db_path); let db = Arc::new(open_db_read_only(db_path.join("db").as_path(), Default::default())?); let spec = Arc::new(ChainSpecBuilder::mainnet().build()); - let factory = ProviderFactory::new(db.clone(), spec.clone(), db_path.join("snapshots"))?; + let factory = ProviderFactory::new(db.clone(), spec.clone(), db_path.join("static_files"))?; // 2. Setup the blockchain provider using only the database provider and a noop for the tree to // satisfy trait bounds. Tree is not used in this example since we are only operating on the diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index c7c0df4489f6..22f5e8b85ee2 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -5,10 +5,10 @@ use crate::{ Case, Error, Suite, }; use alloy_rlp::Decodable; -use reth_db::test_utils::{create_test_rw_db, create_test_snapshots_dir}; +use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; use reth_node_ethereum::EthEvmConfig; -use reth_primitives::{BlockBody, SealedBlock, SnapshotSegment}; -use reth_provider::{providers::SnapshotWriter, HashingWriter, ProviderFactory}; +use reth_primitives::{BlockBody, SealedBlock, StaticFileSegment}; +use reth_provider::{providers::StaticFileWriter, HashingWriter, ProviderFactory}; use reth_stages::{stages::ExecutionStage, ExecInput, Stage}; use std::{collections::BTreeMap, fs, path::Path, sync::Arc}; @@ -81,7 +81,7 @@ impl Case for BlockchainTestCase { let provider = ProviderFactory::new( db.as_ref(), Arc::new(case.network.clone().into()), - create_test_snapshots_dir(), + create_test_static_files_dir(), )? .provider_rw() .unwrap(); @@ -109,8 +109,8 @@ impl Case for BlockchainTestCase { Ok::, Error>(Some(decoded)) })?; provider - .snapshot_provider() - .latest_writer(SnapshotSegment::Headers) + .static_file_provider() + .latest_writer(StaticFileSegment::Headers) .unwrap() .commit() .unwrap();