Merge remote-tracking branch 'origin/feat/static-files' into alexey/s…

…ender-recovery-snapshot
paradigmxyz · Jan 26, 2024 · 7d6bb56 · 7d6bb56
2 parents 019d1bf + 3dcb835
commit 7d6bb56
Show file tree

Hide file tree

Showing 51 changed files with 1,811 additions and 1,286 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bin/reth/src/commands/db/snapshots/headers.rs b/bin/reth/src/commands/db/snapshots/headers.rs
@@ -11,7 +11,6 @@ use reth_primitives::{
 };
 use reth_provider::{
     providers::SnapshotProvider, BlockNumReader, HeaderProvider, ProviderError, ProviderFactory,
-    TransactionsProviderExt,
 };
 use std::{
     path::{Path, PathBuf},
@@ -45,12 +44,8 @@ impl Command {
         let mut row_indexes = block_range.clone().collect::<Vec<_>>();
         let mut rng = rand::thread_rng();
 
-        let tx_range = ProviderFactory::new(open_db_read_only(db_path, db_args)?, chain.clone())
-            .provider()?
-            .transaction_range_by_block_range(block_range.clone())?;
-
         let path: PathBuf = SnapshotSegment::Headers
-            .filename_with_configuration(filters, compression, &block_range, &tx_range)
+            .filename_with_configuration(filters, compression, &block_range)
             .into();
         let provider = SnapshotProvider::new(PathBuf::default())?;
         let jar_provider = provider.get_segment_provider_from_block(

diff --git a/bin/reth/src/commands/db/snapshots/mod.rs b/bin/reth/src/commands/db/snapshots/mod.rs
@@ -10,10 +10,12 @@ use reth_db::{
 use reth_interfaces::db::LogLevel;
 use reth_nippy_jar::{NippyJar, NippyJarCursor};
 use reth_primitives::{
-    snapshot::{Compression, Filters, InclusionFilter, PerfectHashingFunction, SegmentHeader},
+    snapshot::{
+        Compression, Filters, InclusionFilter, PerfectHashingFunction, SegmentConfig, SegmentHeader,
+    },
     BlockNumber, ChainSpec, SnapshotSegment,
 };
-use reth_provider::{BlockNumReader, ProviderFactory, TransactionsProviderExt};
+use reth_provider::{BlockNumReader, ProviderFactory};
 use reth_snapshot::{segments as snap_segments, segments::Segment};
 use std::{
     ops::RangeInclusive,
@@ -83,14 +85,15 @@ impl Command {
         log_level: Option<LogLevel>,
         chain: Arc<ChainSpec>,
     ) -> eyre::Result<()> {
-        let all_combinations =
-            self.segments.iter().cartesian_product(self.compression.iter()).cartesian_product(
-                if self.phf.is_empty() {
-                    vec![None]
-                } else {
-                    self.phf.iter().copied().map(Some).collect::<Vec<_>>()
-                },
-            );
+        let all_combinations = self
+            .segments
+            .iter()
+            .cartesian_product(self.compression.iter().copied())
+            .cartesian_product(if self.phf.is_empty() {
+                vec![None]
+            } else {
+                self.phf.iter().copied().map(Some).collect::<Vec<_>>()
+            });
 
         {
             let db = open_db_read_only(
@@ -111,45 +114,48 @@ impl Command {
                     match mode {
                         SnapshotSegment::Headers => self.generate_snapshot::<DatabaseEnv>(
                             factory.clone(),
-                            snap_segments::Headers::new(*compression, filters),
+                            snap_segments::Headers,
+                            SegmentConfig { filters, compression },
                         )?,
                         SnapshotSegment::Transactions => self.generate_snapshot::<DatabaseEnv>(
                             factory.clone(),
-                            snap_segments::Transactions::new(*compression, filters),
+                            snap_segments::Transactions,
+                            SegmentConfig { filters, compression },
                         )?,
                         SnapshotSegment::Receipts => self.generate_snapshot::<DatabaseEnv>(
                             factory.clone(),
-                            snap_segments::Receipts::new(*compression, filters),
+                            snap_segments::Receipts,
+                            SegmentConfig { filters, compression },
                         )?,
                     }
                 }
             }
         }
 
         if self.only_bench || self.bench {
-            for ((mode, compression), phf) in all_combinations.clone() {
+            for ((mode, compression), phf) in all_combinations {
                 match mode {
                     SnapshotSegment::Headers => self.bench_headers_snapshot(
                         db_path,
                         log_level,
                         chain.clone(),
-                        *compression,
+                        compression,
                         InclusionFilter::Cuckoo,
                         phf,
                     )?,
                     SnapshotSegment::Transactions => self.bench_transactions_snapshot(
                         db_path,
                         log_level,
                         chain.clone(),
-                        *compression,
+                        compression,
                         InclusionFilter::Cuckoo,
                         phf,
                     )?,
                     SnapshotSegment::Receipts => self.bench_receipts_snapshot(
                         db_path,
                         log_level,
                         chain.clone(),
-                        *compression,
+                        compression,
                         InclusionFilter::Cuckoo,
                         phf,
                     )?,
@@ -179,7 +185,8 @@ impl Command {
     fn generate_snapshot<DB: Database>(
         &self,
         factory: Arc<ProviderFactory<DB>>,
-        segment: impl Segment + Send + Sync,
+        segment: impl Segment<DB> + Send + Sync,
+        config: SegmentConfig,
     ) -> eyre::Result<()> {
         let dir = PathBuf::default();
         let ranges = self.block_ranges(factory.best_block_number()?);
@@ -194,13 +201,15 @@ impl Command {
                     let provider = factory.provider()?;
 
                     if !self.only_stats {
-                        segment.snapshot::<DB>(&provider, &dir, block_range.clone())?;
+                        segment.create_snapshot_file(
+                            &provider,
+                            dir.as_path(),
+                            config,
+                            block_range.clone(),
+                        )?;
                     }
 
-                    let tx_range =
-                        provider.transaction_range_by_block_range(block_range.clone())?;
-
-                    Ok(segment.segment().filename(block_range, &tx_range))
+                    Ok(segment.segment().filename(block_range))
                 })
                 .collect::<Result<Vec<_>, eyre::Report>>()?;
 

diff --git a/bin/reth/src/commands/db/snapshots/receipts.rs b/bin/reth/src/commands/db/snapshots/receipts.rs
@@ -53,7 +53,7 @@ impl Command {
         let mut row_indexes = tx_range.clone().collect::<Vec<_>>();
 
         let path: PathBuf = SnapshotSegment::Receipts
-            .filename_with_configuration(filters, compression, &block_range, &tx_range)
+            .filename_with_configuration(filters, compression, &block_range)
             .into();
 
         let provider = SnapshotProvider::new(PathBuf::default())?;

diff --git a/bin/reth/src/commands/db/snapshots/transactions.rs b/bin/reth/src/commands/db/snapshots/transactions.rs
@@ -51,7 +51,7 @@ impl Command {
         let mut row_indexes = tx_range.clone().collect::<Vec<_>>();
 
         let path: PathBuf = SnapshotSegment::Transactions
-            .filename_with_configuration(filters, compression, &block_range, &tx_range)
+            .filename_with_configuration(filters, compression, &block_range)
             .into();
         let provider = SnapshotProvider::new(PathBuf::default())?;
         let jar_provider = provider.get_segment_provider_from_block(