From a0bb1eb3e8bca88e0669b6e5a59ed2e113e15c02 Mon Sep 17 00:00:00 2001 From: Georgios Konstantopoulos Date: Thu, 20 Jun 2024 02:32:51 +0300 Subject: [PATCH 1/6] feat: reth stage unwind --offline this only adds the offline stages to the pipeline and should allow advancing the log ahead of the state, incl. skipping sender recovery (cherry picked from commit b5b42cf68815d34e74983231269236887f6d250d) --- bin/reth/src/commands/stage/unwind.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/bin/reth/src/commands/stage/unwind.rs b/bin/reth/src/commands/stage/unwind.rs index 157f33bff7cb..99b9dd9d65ee 100644 --- a/bin/reth/src/commands/stage/unwind.rs +++ b/bin/reth/src/commands/stage/unwind.rs @@ -15,7 +15,7 @@ use reth_provider::{ }; use reth_prune_types::PruneModes; use reth_stages::{ - sets::DefaultStages, + sets::{DefaultStages, OfflineStages}, stages::{ExecutionStage, ExecutionStageThresholds}, Pipeline, StageSet, }; @@ -40,6 +40,8 @@ pub struct Command { #[command(subcommand)] command: Subcommands, + + offline: bool, } impl Command { @@ -105,9 +107,16 @@ impl Command { let (tip_tx, tip_rx) = watch::channel(B256::ZERO); let executor = block_executor!(provider_factory.chain_spec()); - let pipeline = Pipeline::builder() - .with_tip_sender(tip_tx) - .add_stages( + let builder = if self.offline { + let stages = OfflineStages::new(executor, config.stages, PruneModes::default()); + let stages = stages.builder().disable(reth_stages::StageId::SenderRecovery).build(); + let mut builder = Pipeline::builder(); + for stage in stages { + builder = builder.add_stage(stage); + } + builder + } else { + Pipeline::builder().with_tip_sender(tip_tx).add_stages( DefaultStages::new( provider_factory.clone(), tip_rx, @@ -131,10 +140,12 @@ impl Command { ExExManagerHandle::empty(), )), ) - .build( - provider_factory.clone(), - StaticFileProducer::new(provider_factory, PruneModes::default()), - ); + }; + + let pipeline = builder.build( + provider_factory.clone(), + StaticFileProducer::new(provider_factory, PruneModes::default()), + ); Ok(pipeline) } } From 842201bd3b0d080aabb5c916f77dd68cdbf5e5a8 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 25 Jun 2024 12:26:37 -0400 Subject: [PATCH 2/6] chore: add proper logging, use pipeline if offline is specified --- bin/reth/src/commands/stage/unwind.rs | 31 ++++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/bin/reth/src/commands/stage/unwind.rs b/bin/reth/src/commands/stage/unwind.rs index 99b9dd9d65ee..25906e4f89e6 100644 --- a/bin/reth/src/commands/stage/unwind.rs +++ b/bin/reth/src/commands/stage/unwind.rs @@ -41,6 +41,7 @@ pub struct Command { #[command(subcommand)] command: Subcommands, + /// If this is enabled, then _only_ the offline stages (headers, bodies) will be unwound. offline: bool, } @@ -54,16 +55,30 @@ impl Command { eyre::bail!("Cannot unwind genesis block") } - // Only execute a pipeline unwind if the start of the range overlaps the existing static - // files. If that's the case, then copy all available data from MDBX to static files, and - // only then, proceed with the unwind. - if let Some(highest_static_block) = provider_factory + let highest_static_file_block = provider_factory .static_file_provider() .get_highest_static_files() .max() - .filter(|highest_static_file_block| highest_static_file_block >= range.start()) - { - info!(target: "reth::cli", ?range, ?highest_static_block, "Executing a pipeline unwind."); + .filter(|highest_static_file_block| highest_static_file_block >= range.start()); + + // Execute a pipeline unwind if the start of the range overlaps the existing static + // files. If that's the case, then copy all available data from MDBX to static files, and + // only then, proceed with the unwind. + // + // We also execute a pipeline unwind if `offline` is specified, because we need to only + // unwind the data associated with offline stages. + if highest_static_file_block.is_some() || self.offline { + if self.offline { + info!(target: "reth::cli", "Performing an unwind for offline-only data!"); + } + + if let Some(highest_static_file_block) = highest_static_file_block { + info!(target: "reth::cli", ?range, ?highest_static_file_block, "Executing a pipeline unwind."); + } else { + info!(target: "reth::cli", ?range, "Executing a pipeline unwind."); + } + + // This will build an offline-only pipeline if the `offline` flag is enabled let mut pipeline = self.build_pipeline(config, provider_factory.clone()).await?; // Move all applicable data from database to static files. @@ -89,7 +104,7 @@ impl Command { provider.commit()?; } - println!("Unwound {} blocks", range.count()); + info!(target: "reth::cli", range=?range.clone(), count=range.count(), "Unwound blocks"); Ok(()) } From 3d5dbb5645d0b8ee82a6dd305940da98adc2038d Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 20 Jun 2024 12:01:38 +0200 Subject: [PATCH 3/6] refactor: clean up stage mgmt (#8980) (cherry picked from commit 7229f93ff88cec9b29bc3c220684d8f6c385ce0a) --- bin/reth/src/commands/stage/unwind.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/bin/reth/src/commands/stage/unwind.rs b/bin/reth/src/commands/stage/unwind.rs index 25906e4f89e6..48e410559714 100644 --- a/bin/reth/src/commands/stage/unwind.rs +++ b/bin/reth/src/commands/stage/unwind.rs @@ -123,13 +123,11 @@ impl Command { let executor = block_executor!(provider_factory.chain_spec()); let builder = if self.offline { - let stages = OfflineStages::new(executor, config.stages, PruneModes::default()); - let stages = stages.builder().disable(reth_stages::StageId::SenderRecovery).build(); - let mut builder = Pipeline::builder(); - for stage in stages { - builder = builder.add_stage(stage); - } - builder + Pipeline::builder().add_stages( + OfflineStages::new(executor, config.stages, PruneModes::default()) + .builder() + .disable(reth_stages::StageId::SenderRecovery), + ) } else { Pipeline::builder().with_tip_sender(tip_tx).add_stages( DefaultStages::new( From bfb1bff1085d26e80f95e306899db9de8c8f76c9 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 25 Jun 2024 12:36:26 -0400 Subject: [PATCH 4/6] fix: use long arg --- bin/reth/src/commands/stage/unwind.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/reth/src/commands/stage/unwind.rs b/bin/reth/src/commands/stage/unwind.rs index 48e410559714..69d29c5b6723 100644 --- a/bin/reth/src/commands/stage/unwind.rs +++ b/bin/reth/src/commands/stage/unwind.rs @@ -42,6 +42,7 @@ pub struct Command { command: Subcommands, /// If this is enabled, then _only_ the offline stages (headers, bodies) will be unwound. + #[arg(long)] offline: bool, } From 75524a2ae0ac7afd622039f99782debf6be92ea0 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 25 Jun 2024 12:44:14 -0400 Subject: [PATCH 5/6] update book --- book/cli/reth/stage/unwind.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/book/cli/reth/stage/unwind.md b/book/cli/reth/stage/unwind.md index a1a538f3b1dc..dd27004620c8 100644 --- a/book/cli/reth/stage/unwind.md +++ b/book/cli/reth/stage/unwind.md @@ -204,6 +204,9 @@ Networking: [default: 131072] + --offline + If this is enabled, then _only_ the offline stages (headers, bodies) will be unwound + Logging: --log.stdout.format The format to use for logs written to stdout From 72333f5ebfbfb04e76cff36261b6d196f265b64f Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 25 Jun 2024 13:08:01 -0400 Subject: [PATCH 6/6] fix comment --- bin/reth/src/commands/stage/unwind.rs | 3 ++- book/cli/reth/stage/unwind.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/reth/src/commands/stage/unwind.rs b/bin/reth/src/commands/stage/unwind.rs index 69d29c5b6723..5e6b62537572 100644 --- a/bin/reth/src/commands/stage/unwind.rs +++ b/bin/reth/src/commands/stage/unwind.rs @@ -41,7 +41,8 @@ pub struct Command { #[command(subcommand)] command: Subcommands, - /// If this is enabled, then _only_ the offline stages (headers, bodies) will be unwound. + /// If this is enabled, then all stages except headers, bodies, and sender recovery will be + /// unwound. #[arg(long)] offline: bool, } diff --git a/book/cli/reth/stage/unwind.md b/book/cli/reth/stage/unwind.md index dd27004620c8..3af76e1d567e 100644 --- a/book/cli/reth/stage/unwind.md +++ b/book/cli/reth/stage/unwind.md @@ -205,7 +205,7 @@ Networking: [default: 131072] --offline - If this is enabled, then _only_ the offline stages (headers, bodies) will be unwound + If this is enabled, then all stages except headers, bodies, and sender recovery will be unwound Logging: --log.stdout.format